From 0ef58820517730ac8920d76a24dae5fa7be2e3a5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jind=C5=99ich=20B=C3=A4r?= Date: Wed, 13 May 2026 16:05:34 +0200 Subject: [PATCH 1/2] docs: regenerate documentation snapshots --- .../version-0.2/api-packages.json | 2 +- .../version-0.2/api-typedoc.json | 58632 ++++++++++++-- .../version-1.7/api-packages.json | 2 +- .../version-1.7/api-typedoc.json | 63652 +++++++++++++-- .../version-2.7/api-packages.json | 2 +- .../version-2.7/api-typedoc.json | 64644 ++++++++++++++-- 6 files changed, 172168 insertions(+), 14766 deletions(-) diff --git a/website/versioned_docs/version-0.2/api-packages.json b/website/versioned_docs/version-0.2/api-packages.json index e2e41f0d..765ea7e7 100644 --- a/website/versioned_docs/version-0.2/api-packages.json +++ b/website/versioned_docs/version-0.2/api-packages.json @@ -1 +1 @@ -[{"entryPoints":{"index":{"label":"Index","path":"src/index.ts"}},"packageRoot":".","packagePath":".","packageSlug":".","packageName":"apify-sdk-python"}] +[{"entryPoints":{"index":{"label":"Index","path":"src/index.ts"}},"packageRoot":".","packagePath":".","packageSlug":".","packageName":"apify-sdk-python-website"}] \ No newline at end of file diff --git a/website/versioned_docs/version-0.2/api-typedoc.json b/website/versioned_docs/version-0.2/api-typedoc.json index 262edb81..76739ea2 100644 --- a/website/versioned_docs/version-0.2/api-typedoc.json +++ b/website/versioned_docs/version-0.2/api-typedoc.json @@ -1,6356 +1,56284 @@ { - "id": 0, - "name": "apify", - "kind": 1, - "kindString": "Project", - "flags": {}, - "originalName": "", "children": [ { - "id": 1, - "name": "Actor", - "module": "actor", "kind": 128, "kindString": "Class", - "flags": {}, - "comment": { - "summary": [ - { - "kind": "text", - "text": "{\"content\": [\"The main class of the SDK, through which all the actor operations should be done.\"]}" - } - ] - }, "children": [ { - "id": 2, - "name": "__init__", - "module": "actor", - "kind": 512, - "kindString": "Constructor", - "flags": {}, + "kind": 2048, + "kindString": "Method", + "children": [], "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Create an Actor instance.\\n\\nNote that you don't have to do this, all the methods on this class function as classmethods too,\\nand that is their preferred usage.\\n\\nArgs:\\n config (Configuration, optional): The actor configuration to be used. If not passed, a new Configuration instance will be created.\"]}" + "text": "Create a `Configuration` instance.\n\nAll the parameters are loaded by default from environment variables when running on the Apify platform.\nYou can override them here in the Configuration constructor, which might be useful for local testing of your actors.\n" } ] }, - "children": [], + "decorations": [], + "flags": {}, "groups": [], + "id": 2, + "module": "config", + "name": "__init__", + "parsedDocstring": { + "text": "Create a `Configuration` instance.\n\nAll the parameters are loaded by default from environment variables when running on the Apify platform.\nYou can override them here in the Configuration constructor, which might be useful for local testing of your actors.\n", + "args": { + "api_base_url": "The URL of the Apify API.\nThis is the URL actually used for connecting to the API, so it can contain an IP address when running in a container on the platform.", + "api_public_base_url": "The public URL of the Apify API.\nThis will always contain the public URL of the API, even when running in a container on the platform.\nUseful for generating shareable URLs to key-value store records or datasets.", + "container_port": "The port on which the container can listen for HTTP requests.", + "container_url": "The URL on which the container can listen for HTTP requests.", + "default_dataset_id": "The ID of the default dataset for the actor.", + "default_key_value_store_id": "The ID of the default key-value store for the actor.", + "default_request_queue_id": "The ID of the default request queue for the actor.", + "input_key": "The key of the input record in the actor's default key-value store", + "max_used_cpu_ratio": "The CPU usage above which the SYSTEM_INFO event will report the CPU is overloaded.", + "metamorph_after_sleep_millis": "How long should the actor sleep after calling metamorph.", + "persist_state_interval_millis": "How often should the actor emit the PERSIST_STATE event.", + "persist_storage": "Whether the actor should persist its used storages to the filesystem when running locally.", + "proxy_hostname": "The hostname of Apify Proxy.", + "proxy_password": "The password for Apify Proxy.", + "proxy_port": "The port of Apify Proxy.", + "proxy_status_url": "The URL on which the Apify Proxy status page is available.", + "purge_on_start": "Whether the actor should purge its default storages on startup, when running locally.", + "token": "The API token for the Apify API this actor should use.", + "system_info_interval_millis": "How often should the actor emit the SYSTEM_INFO event when running locally." + } + }, "sources": [ { - "filename": "src/apify/actor.py", - "line": 76, "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v0.2.0/src/apify/actor.py#L76" + "fileName": "/src/apify/config.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 16 } ], + "type": { + "name": "Undefined", + "type": "reference" + }, "signatures": [ { - "id": 4, - "name": "__init__", - "kind": 4096, - "kindString": "Call signature", - "flags": {}, "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Create an Actor instance.\\n\\nNote that you don't have to do this, all the methods on this class function as classmethods too,\\nand that is their preferred usage.\\n\\nArgs:\\n config (Configuration, optional): The actor configuration to be used. If not passed, a new Configuration instance will be created.\"]}" + "text": "Create a `Configuration` instance.\n\nAll the parameters are loaded by default from environment variables when running on the Apify platform.\nYou can override them here in the Configuration constructor, which might be useful for local testing of your actors.\n" } ] }, - "type": { - "type": "reference", - "name": "None" - }, + "flags": {}, + "id": 3, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "__init__", "parameters": [ { - "id": 3, - "name": "config", + "comment": { + "summary": [ + { + "kind": "text", + "text": "The URL of the Apify API.\nThis is the URL actually used for connecting to the API, so it can contain an IP address when running in a container on the platform." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 4, + "kind": 32768, + "kindString": "Parameter", + "name": "api_base_url", + "type": { + "name": "Optional", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "str" + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The public URL of the Apify API.\nThis will always contain the public URL of the API, even when running in a container on the platform.\nUseful for generating shareable URLs to key-value store records or datasets." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 5, + "kind": 32768, + "kindString": "Parameter", + "name": "api_public_base_url", + "type": { + "name": "Optional", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "str" + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The port on which the container can listen for HTTP requests." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 6, + "kind": 32768, + "kindString": "Parameter", + "name": "container_port", + "type": { + "name": "Optional", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "int" + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The URL on which the container can listen for HTTP requests." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 7, + "kind": 32768, + "kindString": "Parameter", + "name": "container_url", + "type": { + "name": "Optional", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "str" + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The ID of the default dataset for the actor." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 8, + "kind": 32768, + "kindString": "Parameter", + "name": "default_dataset_id", + "type": { + "name": "Optional", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "str" + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The ID of the default key-value store for the actor." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 9, + "kind": 32768, + "kindString": "Parameter", + "name": "default_key_value_store_id", + "type": { + "name": "Optional", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "str" + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The ID of the default request queue for the actor." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 10, + "kind": 32768, + "kindString": "Parameter", + "name": "default_request_queue_id", + "type": { + "name": "Optional", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "str" + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The key of the input record in the actor's default key-value store" + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 11, + "kind": 32768, + "kindString": "Parameter", + "name": "input_key", + "type": { + "name": "Optional", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "str" + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The CPU usage above which the SYSTEM_INFO event will report the CPU is overloaded." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 12, "kind": 32768, "kindString": "Parameter", - "flags": {}, + "name": "max_used_cpu_ratio", + "type": { + "name": "Optional", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "float" + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "How long should the actor sleep after calling metamorph." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 13, + "kind": 32768, + "kindString": "Parameter", + "name": "metamorph_after_sleep_millis", + "type": { + "name": "Optional", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "int" + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "How often should the actor emit the PERSIST_STATE event." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 14, + "kind": 32768, + "kindString": "Parameter", + "name": "persist_state_interval_millis", + "type": { + "name": "Optional", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "int" + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Whether the actor should persist its used storages to the filesystem when running locally." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 15, + "kind": 32768, + "kindString": "Parameter", + "name": "persist_storage", + "type": { + "name": "Optional", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "bool" + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The hostname of Apify Proxy." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 16, + "kind": 32768, + "kindString": "Parameter", + "name": "proxy_hostname", + "type": { + "name": "Optional", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "str" + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The password for Apify Proxy." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 17, + "kind": 32768, + "kindString": "Parameter", + "name": "proxy_password", + "type": { + "name": "Optional", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "str" + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The port of Apify Proxy." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 18, + "kind": 32768, + "kindString": "Parameter", + "name": "proxy_port", + "type": { + "name": "Optional", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "int" + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The URL on which the Apify Proxy status page is available." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 19, + "kind": 32768, + "kindString": "Parameter", + "name": "proxy_status_url", + "type": { + "name": "Optional", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "str" + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Whether the actor should purge its default storages on startup, when running locally." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 20, + "kind": 32768, + "kindString": "Parameter", + "name": "purge_on_start", + "type": { + "name": "Optional", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "bool" + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The API token for the Apify API this actor should use." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 21, + "kind": 32768, + "kindString": "Parameter", + "name": "token", "type": { + "name": "Optional", "type": "reference", - "name": "Optional[Configuration]" + "typeArguments": [ + { + "type": "reference", + "name": "str" + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "How often should the actor emit the SYSTEM_INFO event when running locally." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true }, - "defaultValue": "None" + "id": 22, + "kind": 32768, + "kindString": "Parameter", + "name": "system_info_interval_millis", + "type": { + "name": "Optional", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "int" + } + ] + } } - ] + ], + "type": { + "name": "None", + "type": "literal", + "value": null + } } ] }, { - "id": 5, - "name": "apify_client", - "module": "actor", "kind": 2048, "kindString": "Method", - "flags": {}, + "children": [], "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"The ApifyClientAsync instance the Actor instance uses.\"]}" + "text": "Retrive the global configuration.\n\nThe global configuration applies when you call actor methods via their static versions, e.g. `Actor.init()`.\nAlso accessible via `Actor.config`." } ] }, - "children": [], + "decorations": [ + { + "name": "classmethod" + } + ], + "flags": {}, "groups": [], + "id": 23, + "module": "config", + "name": "get_global_configuration", + "parsedDocstring": { + "text": "Retrive the global configuration.\n\nThe global configuration applies when you call actor methods via their static versions, e.g. `Actor.init()`.\nAlso accessible via `Actor.config`." + }, "sources": [ { - "filename": "src/apify/actor.py", - "line": 169, "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v0.2.0/src/apify/actor.py#L169" + "fileName": "/src/apify/config.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 117 } ], + "type": { + "name": "Undefined", + "type": "reference" + }, "signatures": [ { - "id": 7, - "name": "apify_client", - "kind": 4096, - "kindString": "Call signature", - "flags": {}, "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"The ApifyClientAsync instance the Actor instance uses.\"]}" + "text": "Retrive the global configuration.\n\nThe global configuration applies when you call actor methods via their static versions, e.g. `Actor.init()`.\nAlso accessible via `Actor.config`." } ] }, - "type": { - "type": "reference", - "name": "ApifyClientAsync" - }, - "parameters": [ - { - "id": 6, - "name": "self_or_cls", - "kind": 32768, - "kindString": "Parameter", - "flags": {}, - "type": { - "type": "reference", - "name": "Any" - } - } - ] + "flags": {}, + "id": 24, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "get_global_configuration", + "parameters": [], + "type": { + "name": "'Configuration'", + "type": "literal", + "value": "Configuration" + } } ] - }, + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "A class for specifying the configuration of an actor.\n\nCan be used either globally via `Configuration.get_global_configuration()`,\nor it can be specific to each `Actor` instance on the `actor.config` property." + } + ] + }, + "flags": {}, + "groups": [ + { + "children": [ + 2, + 23 + ], + "title": "Methods" + } + ], + "id": 1, + "module": "config", + "name": "Configuration", + "parsedDocstring": { + "text": "A class for specifying the configuration of an actor.\n\nCan be used either globally via `Configuration.get_global_configuration()`,\nor it can be specific to each `Actor` instance on the `actor.config` property." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/config.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 7 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 25, + "module": "actor", + "name": "MainReturnType", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/actor.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 34 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 128, + "kindString": "Class", + "children": [ { - "id": 8, - "name": "config", - "module": "actor", "kind": 2048, "kindString": "Method", - "flags": {}, + "children": [], "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"The Configuration instance the Actor instance uses.\"]}" + "text": "" } ] }, - "children": [], + "decorations": [ + { + "name": "staticmethod" + } + ], + "flags": {}, "groups": [], + "id": 27, + "module": "actor", + "name": "__aenter__", + "parsedDocstring": { + "text": "" + }, "sources": [ { - "filename": "src/apify/actor.py", - "line": 177, "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v0.2.0/src/apify/actor.py#L177" + "fileName": "/src/apify/actor.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 43 } ], + "type": { + "name": "Undefined", + "type": "reference" + }, "signatures": [ { - "id": 10, - "name": "config", + "flags": {}, + "id": 28, "kind": 4096, "kindString": "Call signature", - "flags": {}, - "comment": { - "summary": [ - { - "kind": "text", - "text": "{\"content\": [\"The Configuration instance the Actor instance uses.\"]}" - } - ] - }, + "modifiers": [ + "async" + ], + "name": "__aenter__", + "parameters": [], "type": { + "name": "Type", "type": "reference", - "name": "Configuration" - }, - "parameters": [ - { - "id": 9, - "name": "self_or_cls", - "kind": 32768, - "kindString": "Parameter", - "flags": {}, - "type": { - "type": "reference", - "name": "Any" + "typeArguments": [ + { + "type": "literal", + "value": "Actor" } - } - ] + ] + } } ] }, { - "id": 11, - "name": "event_manager", - "module": "actor", "kind": 2048, "kindString": "Method", - "flags": {}, + "children": [], "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"The EventManager instance the Actor instance uses.\"]}" + "text": "" } ] }, - "children": [], + "decorations": [ + { + "name": "staticmethod" + } + ], + "flags": {}, "groups": [], + "id": 29, + "module": "actor", + "name": "__aexit__", + "parsedDocstring": { + "text": "" + }, "sources": [ { - "filename": "src/apify/actor.py", - "line": 185, "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v0.2.0/src/apify/actor.py#L185" + "fileName": "/src/apify/actor.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 48 } ], + "type": { + "name": "Undefined", + "type": "reference" + }, "signatures": [ { - "id": 13, - "name": "event_manager", + "flags": {}, + "id": 30, "kind": 4096, "kindString": "Call signature", - "flags": {}, - "comment": { - "summary": [ - { - "kind": "text", - "text": "{\"content\": [\"The EventManager instance the Actor instance uses.\"]}" - } - ] - }, - "type": { - "type": "reference", - "name": "EventManager" - }, + "modifiers": [ + "async" + ], + "name": "__aexit__", "parameters": [ { - "id": 12, - "name": "self_or_cls", + "flags": { + "isOptional": true, + "keyword-only": false + }, + "id": 31, + "kind": 32768, + "kindString": "Parameter", + "name": "_exc_type", + "type": { + "name": "Optional", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "Type", + "typeArguments": [ + { + "type": "reference", + "name": "BaseException" + } + ] + } + ] + } + }, + { + "flags": { + "isOptional": true, + "keyword-only": false + }, + "id": 32, + "kind": 32768, + "kindString": "Parameter", + "name": "exc_value", + "type": { + "name": "Optional", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "BaseException" + } + ] + } + }, + { + "flags": { + "isOptional": true, + "keyword-only": false + }, + "id": 33, "kind": 32768, "kindString": "Parameter", - "flags": {}, + "name": "_exc_traceback", "type": { + "name": "Optional", "type": "reference", - "name": "Any" + "typeArguments": [ + { + "type": "reference", + "name": "TracebackType" + } + ] } } - ] + ], + "type": { + "name": "None", + "type": "literal", + "value": null + } } ] - }, + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [ + { + "children": [ + 27, + 29 + ], + "title": "Methods" + } + ], + "id": 26, + "module": "actor", + "name": "_ActorContextManager", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/actor.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 41 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 128, + "kindString": "Class", + "children": [ { - "id": 14, - "name": "log", - "module": "actor", "kind": 2048, "kindString": "Method", - "flags": {}, + "children": [], "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"The logging.Logger instance the Actor uses.\"]}" + "text": "Create an Actor instance.\n\nNote that you don't have to do this, all the methods on this class function as classmethods too,\nand that is their preferred usage.\n" } ] }, - "children": [], + "decorations": [], + "flags": {}, "groups": [], + "id": 35, + "module": "actor", + "name": "__init__", + "parsedDocstring": { + "text": "Create an Actor instance.\n\nNote that you don't have to do this, all the methods on this class function as classmethods too,\nand that is their preferred usage.\n", + "args": { + "config": "The actor configuration to be used. If not passed, a new Configuration instance will be created." + } + }, "sources": [ { - "filename": "src/apify/actor.py", - "line": 193, "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v0.2.0/src/apify/actor.py#L193" + "fileName": "/src/apify/actor.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 76 } ], + "type": { + "name": "Undefined", + "type": "reference" + }, "signatures": [ { - "id": 16, - "name": "log", - "kind": 4096, - "kindString": "Call signature", - "flags": {}, "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"The logging.Logger instance the Actor uses.\"]}" + "text": "Create an Actor instance.\n\nNote that you don't have to do this, all the methods on this class function as classmethods too,\nand that is their preferred usage.\n" } ] }, - "type": { - "type": "reference", - "name": "logging.Logger" - }, + "flags": {}, + "id": 36, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "__init__", "parameters": [ { - "id": 15, - "name": "_self_or_cls", + "comment": { + "summary": [ + { + "kind": "text", + "text": "The actor configuration to be used. If not passed, a new Configuration instance will be created." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": false + }, + "id": 37, "kind": 32768, "kindString": "Parameter", - "flags": {}, + "name": "config", "type": { + "name": "Optional", "type": "reference", - "name": "Any" + "typeArguments": [ + { + "type": "reference", + "name": "Configuration", + "target": "1" + } + ] } } - ] + ], + "type": { + "name": "None", + "type": "literal", + "value": null + } } ] }, { - "id": 17, - "name": "init", - "module": "actor", - "kind": 2048, - "kindString": "Method", - "flags": {}, + "kind": 1024, + "kindString": "Property", + "children": [], "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Initialize the actor instance.\\n\\nThis initializes the Actor instance.\\nIt configures the right storage client based on whether the actor is running locally or on the Apify platform,\\nit initializes the event manager for processing actor events,\\nand starts an interval for regularly sending `PERSIST_STATE` events,\\nso that the actor can regularly persist its state in response to these events.\\n\\nThis method should be called immediately before performing any additional actor actions,\\nand it should be called only once.\"]}" + "text": "The ApifyClientAsync instance the Actor instance uses." } ] }, - "children": [], + "decorations": [ + { + "name": "dualproperty" + } + ], + "flags": {}, "groups": [], + "id": 38, + "module": "actor", + "name": "apify_client", + "parsedDocstring": { + "text": "The ApifyClientAsync instance the Actor instance uses." + }, "sources": [ { - "filename": "src/apify/actor.py", - "line": 202, "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v0.2.0/src/apify/actor.py#L202" + "fileName": "/src/apify/actor.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 169 } ], - "signatures": [ - { - "id": 18, - "name": "init", - "kind": 4096, - "kindString": "Call signature", - "flags": {}, - "comment": { - "summary": [ - { - "kind": "text", - "text": "{\"content\": [\"Initialize the actor instance.\\n\\nThis initializes the Actor instance.\\nIt configures the right storage client based on whether the actor is running locally or on the Apify platform,\\nit initializes the event manager for processing actor events,\\nand starts an interval for regularly sending `PERSIST_STATE` events,\\nso that the actor can regularly persist its state in response to these events.\\n\\nThis method should be called immediately before performing any additional actor actions,\\nand it should be called only once.\"]}" - } - ] - }, - "type": { - "type": "reference", - "name": "None" - }, - "parameters": [], - "modifiers": [ - "async" - ] - } - ] + "type": { + "name": "ApifyClientAsync", + "type": "reference" + } }, { - "id": 19, - "name": "exit", + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "The Configuration instance the Actor instance uses." + } + ] + }, + "decorations": [ + { + "name": "dualproperty" + } + ], + "flags": {}, + "groups": [], + "id": 39, "module": "actor", - "kind": 2048, - "kindString": "Method", + "name": "config", + "parsedDocstring": { + "text": "The Configuration instance the Actor instance uses." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/actor.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 177 + } + ], + "type": { + "name": "Configuration", + "type": "reference", + "target": "1" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "The EventManager instance the Actor instance uses." + } + ] + }, + "decorations": [ + { + "name": "dualproperty" + } + ], "flags": {}, + "groups": [], + "id": 40, + "module": "actor", + "name": "event_manager", + "parsedDocstring": { + "text": "The EventManager instance the Actor instance uses." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/actor.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 185 + } + ], + "type": { + "name": "EventManager", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Exit the actor instance.\\n\\nThis stops the Actor instance.\\nIt cancels all the intervals for regularly sending `PERSIST_STATE` events,\\nsends a final `PERSIST_STATE` event,\\nwaits for all the event listeners to finish,\\nand stops the event manager.\\n\\nArgs:\\n exit_code (int, optional): The exit code with which the actor should fail (defaults to `0`).\\n event_listeners_timeout_secs (float, optional): How long should the actor wait for actor event listeners to finish before exiting\"]}" + "text": "The logging.Logger instance the Actor uses." } ] }, + "decorations": [ + { + "name": "dualproperty" + } + ], + "flags": {}, + "groups": [], + "id": 41, + "module": "actor", + "name": "log", + "parsedDocstring": { + "text": "The logging.Logger instance the Actor uses." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/actor.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 193 + } + ], + "type": { + "name": "logging.Logger", + "type": "reference" + } + }, + { + "kind": 2048, + "kindString": "Method", "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Initialize the actor instance.\n\nThis initializes the Actor instance.\nIt configures the right storage client based on whether the actor is running locally or on the Apify platform,\nit initializes the event manager for processing actor events,\nand starts an interval for regularly sending `PERSIST_STATE` events,\nso that the actor can regularly persist its state in response to these events.\n\nThis method should be called immediately before performing any additional actor actions,\nand it should be called only once." + } + ] + }, + "decorations": [ + { + "name": "classmethod" + } + ], + "flags": {}, "groups": [], + "id": 42, + "module": "actor", + "name": "init", + "parsedDocstring": { + "text": "Initialize the actor instance.\n\nThis initializes the Actor instance.\nIt configures the right storage client based on whether the actor is running locally or on the Apify platform,\nit initializes the event manager for processing actor events,\nand starts an interval for regularly sending `PERSIST_STATE` events,\nso that the actor can regularly persist its state in response to these events.\n\nThis method should be called immediately before performing any additional actor actions,\nand it should be called only once." + }, "sources": [ { - "filename": "src/apify/actor.py", - "line": 299, "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v0.2.0/src/apify/actor.py#L299" + "fileName": "/src/apify/actor.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 202 } ], + "type": { + "name": "Undefined", + "type": "reference" + }, "signatures": [ { - "id": 22, - "name": "exit", - "kind": 4096, - "kindString": "Call signature", - "flags": {}, "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Exit the actor instance.\\n\\nThis stops the Actor instance.\\nIt cancels all the intervals for regularly sending `PERSIST_STATE` events,\\nsends a final `PERSIST_STATE` event,\\nwaits for all the event listeners to finish,\\nand stops the event manager.\\n\\nArgs:\\n exit_code (int, optional): The exit code with which the actor should fail (defaults to `0`).\\n event_listeners_timeout_secs (float, optional): How long should the actor wait for actor event listeners to finish before exiting\"]}" + "text": "Initialize the actor instance.\n\nThis initializes the Actor instance.\nIt configures the right storage client based on whether the actor is running locally or on the Apify platform,\nit initializes the event manager for processing actor events,\nand starts an interval for regularly sending `PERSIST_STATE` events,\nso that the actor can regularly persist its state in response to these events.\n\nThis method should be called immediately before performing any additional actor actions,\nand it should be called only once." } ] }, - "type": { - "type": "reference", - "name": "None" - }, - "parameters": [ - { - "id": 20, - "name": "exit_code", - "kind": 32768, - "kindString": "Parameter", - "flags": { - "keyword-only": "true" - }, - "type": { - "type": "reference", - "name": "int" - }, - "defaultValue": "0" - }, - { - "id": 21, - "name": "event_listeners_timeout_secs", - "kind": 32768, - "kindString": "Parameter", - "flags": { - "keyword-only": "true" - }, - "type": { - "type": "reference", - "name": "Optional[float]" - }, - "defaultValue": "EVENT_LISTENERS_TIMEOUT_SECS" - } - ], + "flags": {}, + "id": 43, + "kind": 4096, + "kindString": "Call signature", "modifiers": [ "async" - ] + ], + "name": "init", + "parameters": [], + "type": { + "name": "None", + "type": "literal", + "value": null + } } ] }, { - "id": 23, - "name": "fail", - "module": "actor", "kind": 2048, "kindString": "Method", - "flags": {}, + "children": [], "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Fail the actor instance.\\n\\nThis performs all the same steps as Actor.exit(),\\nbut it additionally sets the exit code to `1` (by default).\\n\\nArgs:\\n exit_code (int, optional): The exit code with which the actor should fail (defaults to `1`).\\n exception (BaseException, optional): The exception with which the actor failed.\"]}" + "text": "Exit the actor instance.\n\nThis stops the Actor instance.\nIt cancels all the intervals for regularly sending `PERSIST_STATE` events,\nsends a final `PERSIST_STATE` event,\nwaits for all the event listeners to finish,\nand stops the event manager.\n" } ] }, - "children": [], + "decorations": [ + { + "name": "classmethod" + } + ], + "flags": {}, "groups": [], + "id": 44, + "module": "actor", + "name": "exit", + "parsedDocstring": { + "text": "Exit the actor instance.\n\nThis stops the Actor instance.\nIt cancels all the intervals for regularly sending `PERSIST_STATE` events,\nsends a final `PERSIST_STATE` event,\nwaits for all the event listeners to finish,\nand stops the event manager.\n", + "args": { + "exit_code": "The exit code with which the actor should fail (defaults to `0`).", + "event_listeners_timeout_secs": "How long should the actor wait for actor event listeners to finish before exiting" + } + }, "sources": [ { - "filename": "src/apify/actor.py", - "line": 360, "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v0.2.0/src/apify/actor.py#L360" + "fileName": "/src/apify/actor.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 299 } ], + "type": { + "name": "Undefined", + "type": "reference" + }, "signatures": [ { - "id": 26, - "name": "fail", - "kind": 4096, - "kindString": "Call signature", - "flags": {}, "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Fail the actor instance.\\n\\nThis performs all the same steps as Actor.exit(),\\nbut it additionally sets the exit code to `1` (by default).\\n\\nArgs:\\n exit_code (int, optional): The exit code with which the actor should fail (defaults to `1`).\\n exception (BaseException, optional): The exception with which the actor failed.\"]}" + "text": "Exit the actor instance.\n\nThis stops the Actor instance.\nIt cancels all the intervals for regularly sending `PERSIST_STATE` events,\nsends a final `PERSIST_STATE` event,\nwaits for all the event listeners to finish,\nand stops the event manager.\n" } ] }, - "type": { - "type": "reference", - "name": "None" - }, + "flags": {}, + "id": 45, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "exit", "parameters": [ { - "id": 24, - "name": "exit_code", - "kind": 32768, - "kindString": "Parameter", + "comment": { + "summary": [ + { + "kind": "text", + "text": "The exit code with which the actor should fail (defaults to `0`)." + } + ] + }, + "defaultValue": "0", "flags": { - "keyword-only": "true" + "isOptional": true, + "keyword-only": true }, + "id": 46, + "kind": 32768, + "kindString": "Parameter", + "name": "exit_code", "type": { - "type": "reference", - "name": "int" - }, - "defaultValue": "1" + "name": "int", + "type": "reference" + } }, { - "id": 25, - "name": "exception", - "kind": 32768, - "kindString": "Parameter", + "comment": { + "summary": [ + { + "kind": "text", + "text": "How long should the actor wait for actor event listeners to finish before exiting" + } + ] + }, + "defaultValue": "EVENT_LISTENERS_TIMEOUT_SECS", "flags": { - "keyword-only": "true" + "isOptional": true, + "keyword-only": true }, + "id": 47, + "kind": 32768, + "kindString": "Parameter", + "name": "event_listeners_timeout_secs", "type": { + "name": "Optional", "type": "reference", - "name": "Optional[BaseException]" - }, - "defaultValue": "None" + "typeArguments": [ + { + "type": "reference", + "name": "float" + } + ] + } } ], - "modifiers": [ - "async" - ] + "type": { + "name": "None", + "type": "literal", + "value": null + } } ] }, { - "id": 27, - "name": "main", - "module": "actor", "kind": 2048, "kindString": "Method", - "flags": {}, + "children": [], "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Initialize the actor, run the passed function and finish the actor cleanly.\\n\\n**The `Actor.main()` function is optional** and is provided merely for your convenience.\\nIt is mainly useful when you're running your code as an actor on the [Apify platform](https://apify.com/actors).\\n\\nThe `Actor.main()` function performs the following actions:\\n\\n- When running on the Apify platform (i.e. `APIFY_IS_AT_HOME` environment variable is set),\\n it sets up a connection to listen for platform events.\\n For example, to get a notification about an imminent migration to another server.\\n- It invokes the user function passed as the `main_actor_function` parameter.\\n- If the user function was an async function, it awaits it.\\n- If the user function throws an exception or some other error is encountered,\\n it prints error details to console so that they are stored to the log,\\n and finishes the actor cleanly.\\n- Finally, it exits the Python process, with zero exit code on success and non-zero on errors.\\n\\nArgs:\\n main_actor_function (Callable): The user function which should be run in the actor\"]}" + "text": "Fail the actor instance.\n\nThis performs all the same steps as Actor.exit(),\nbut it additionally sets the exit code to `1` (by default).\n" } ] }, - "children": [], + "decorations": [ + { + "name": "classmethod" + } + ], + "flags": {}, "groups": [], + "id": 48, + "module": "actor", + "name": "fail", + "parsedDocstring": { + "text": "Fail the actor instance.\n\nThis performs all the same steps as Actor.exit(),\nbut it additionally sets the exit code to `1` (by default).\n", + "args": { + "exit_code": "The exit code with which the actor should fail (defaults to `1`).", + "exception": "The exception with which the actor failed." + } + }, "sources": [ { - "filename": "src/apify/actor.py", - "line": 396, "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v0.2.0/src/apify/actor.py#L396" + "fileName": "/src/apify/actor.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 360 } ], + "type": { + "name": "Undefined", + "type": "reference" + }, "signatures": [ { - "id": 29, - "name": "main", - "kind": 4096, - "kindString": "Call signature", - "flags": {}, "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Initialize the actor, run the passed function and finish the actor cleanly.\\n\\n**The `Actor.main()` function is optional** and is provided merely for your convenience.\\nIt is mainly useful when you're running your code as an actor on the [Apify platform](https://apify.com/actors).\\n\\nThe `Actor.main()` function performs the following actions:\\n\\n- When running on the Apify platform (i.e. `APIFY_IS_AT_HOME` environment variable is set),\\n it sets up a connection to listen for platform events.\\n For example, to get a notification about an imminent migration to another server.\\n- It invokes the user function passed as the `main_actor_function` parameter.\\n- If the user function was an async function, it awaits it.\\n- If the user function throws an exception or some other error is encountered,\\n it prints error details to console so that they are stored to the log,\\n and finishes the actor cleanly.\\n- Finally, it exits the Python process, with zero exit code on success and non-zero on errors.\\n\\nArgs:\\n main_actor_function (Callable): The user function which should be run in the actor\"]}" + "text": "Fail the actor instance.\n\nThis performs all the same steps as Actor.exit(),\nbut it additionally sets the exit code to `1` (by default).\n" } ] }, - "type": { - "type": "reference", - "name": "Optional[MainReturnType]" - }, + "flags": {}, + "id": 49, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "fail", "parameters": [ { - "id": 28, - "name": "main_actor_function", + "comment": { + "summary": [ + { + "kind": "text", + "text": "The exit code with which the actor should fail (defaults to `1`)." + } + ] + }, + "defaultValue": "1", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 50, + "kind": 32768, + "kindString": "Parameter", + "name": "exit_code", + "type": { + "name": "int", + "type": "reference" + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The exception with which the actor failed." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 51, "kind": 32768, "kindString": "Parameter", - "flags": {}, + "name": "exception", "type": { + "name": "Optional", "type": "reference", - "name": "Callable[[], MainReturnType]" + "typeArguments": [ + { + "type": "reference", + "name": "BaseException" + } + ] } } ], - "modifiers": [ - "async" - ] + "type": { + "name": "None", + "type": "literal", + "value": null + } } ] }, { - "id": 30, - "name": "new_client", - "module": "actor", "kind": 2048, "kindString": "Method", - "flags": {}, + "children": [], "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Return a new instance of the Apify API client.\\n\\nThe `ApifyClientAsync` class is provided by the [apify-client](https://github.com/apify/apify-client-python) package,\\nand it is automatically configured using the `APIFY_API_BASE_URL` and `APIFY_TOKEN` environment variables.\\n\\nYou can override the token via the available options.\\nThat's useful if you want to use the client as a different Apify user than the SDK internals are using.\\n\\nArgs:\\n token (str, optional): The Apify API token\\n api_url (str, optional): The URL of the Apify API server to which to connect to. Defaults to https://api.apify.com\\n max_retries (int, optional): How many times to retry a failed request at most\\n min_delay_between_retries_millis (int, optional): How long will the client wait between retrying requests\\n (increases exponentially from this value)\\n timeout_secs (int, optional): The socket timeout of the HTTP requests sent to the Apify API\"]}" + "text": "Initialize the actor, run the passed function and finish the actor cleanly.\n\n**The `Actor.main()` function is optional** and is provided merely for your convenience.\nIt is mainly useful when you're running your code as an actor on the [Apify platform](https://apify.com/actors).\n\nThe `Actor.main()` function performs the following actions:\n\n- When running on the Apify platform (i.e. `APIFY_IS_AT_HOME` environment variable is set),\nit sets up a connection to listen for platform events.\nFor example, to get a notification about an imminent migration to another server.\n- It invokes the user function passed as the `main_actor_function` parameter.\n- If the user function was an async function, it awaits it.\n- If the user function throws an exception or some other error is encountered,\nit prints error details to console so that they are stored to the log,\nand finishes the actor cleanly.\n- Finally, it exits the Python process, with zero exit code on success and non-zero on errors.\n" } ] }, - "children": [], + "decorations": [ + { + "name": "classmethod" + } + ], + "flags": {}, "groups": [], + "id": 52, + "module": "actor", + "name": "main", + "parsedDocstring": { + "text": "Initialize the actor, run the passed function and finish the actor cleanly.\n\n**The `Actor.main()` function is optional** and is provided merely for your convenience.\nIt is mainly useful when you're running your code as an actor on the [Apify platform](https://apify.com/actors).\n\nThe `Actor.main()` function performs the following actions:\n\n- When running on the Apify platform (i.e. `APIFY_IS_AT_HOME` environment variable is set),\nit sets up a connection to listen for platform events.\nFor example, to get a notification about an imminent migration to another server.\n- It invokes the user function passed as the `main_actor_function` parameter.\n- If the user function was an async function, it awaits it.\n- If the user function throws an exception or some other error is encountered,\nit prints error details to console so that they are stored to the log,\nand finishes the actor cleanly.\n- Finally, it exits the Python process, with zero exit code on success and non-zero on errors.\n", + "args": { + "main_actor_function": "The user function which should be run in the actor" + } + }, "sources": [ { - "filename": "src/apify/actor.py", - "line": 441, "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v0.2.0/src/apify/actor.py#L441" + "fileName": "/src/apify/actor.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 396 } ], + "type": { + "name": "Undefined", + "type": "reference" + }, "signatures": [ { - "id": 36, - "name": "new_client", - "kind": 4096, - "kindString": "Call signature", - "flags": {}, "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Return a new instance of the Apify API client.\\n\\nThe `ApifyClientAsync` class is provided by the [apify-client](https://github.com/apify/apify-client-python) package,\\nand it is automatically configured using the `APIFY_API_BASE_URL` and `APIFY_TOKEN` environment variables.\\n\\nYou can override the token via the available options.\\nThat's useful if you want to use the client as a different Apify user than the SDK internals are using.\\n\\nArgs:\\n token (str, optional): The Apify API token\\n api_url (str, optional): The URL of the Apify API server to which to connect to. Defaults to https://api.apify.com\\n max_retries (int, optional): How many times to retry a failed request at most\\n min_delay_between_retries_millis (int, optional): How long will the client wait between retrying requests\\n (increases exponentially from this value)\\n timeout_secs (int, optional): The socket timeout of the HTTP requests sent to the Apify API\"]}" + "text": "Initialize the actor, run the passed function and finish the actor cleanly.\n\n**The `Actor.main()` function is optional** and is provided merely for your convenience.\nIt is mainly useful when you're running your code as an actor on the [Apify platform](https://apify.com/actors).\n\nThe `Actor.main()` function performs the following actions:\n\n- When running on the Apify platform (i.e. `APIFY_IS_AT_HOME` environment variable is set),\nit sets up a connection to listen for platform events.\nFor example, to get a notification about an imminent migration to another server.\n- It invokes the user function passed as the `main_actor_function` parameter.\n- If the user function was an async function, it awaits it.\n- If the user function throws an exception or some other error is encountered,\nit prints error details to console so that they are stored to the log,\nand finishes the actor cleanly.\n- Finally, it exits the Python process, with zero exit code on success and non-zero on errors.\n" } ] }, - "type": { - "type": "reference", - "name": "ApifyClientAsync" - }, + "flags": {}, + "id": 53, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "main", "parameters": [ { - "id": 31, - "name": "token", - "kind": 32768, - "kindString": "Parameter", - "flags": { - "keyword-only": "true" - }, - "type": { - "type": "reference", - "name": "Optional[str]" - }, - "defaultValue": "None" - }, - { - "id": 32, - "name": "api_url", - "kind": 32768, - "kindString": "Parameter", - "flags": { - "keyword-only": "true" - }, - "type": { - "type": "reference", - "name": "Optional[str]" - }, - "defaultValue": "None" - }, - { - "id": 33, - "name": "max_retries", - "kind": 32768, - "kindString": "Parameter", - "flags": { - "keyword-only": "true" - }, - "type": { - "type": "reference", - "name": "Optional[int]" + "comment": { + "summary": [ + { + "kind": "text", + "text": "The user function which should be run in the actor" + } + ] }, - "defaultValue": "None" - }, - { - "id": 34, - "name": "min_delay_between_retries_millis", - "kind": 32768, - "kindString": "Parameter", "flags": { - "keyword-only": "true" - }, - "type": { - "type": "reference", - "name": "Optional[int]" + "isOptional": false, + "keyword-only": false }, - "defaultValue": "None" - }, - { - "id": 35, - "name": "timeout_secs", + "id": 54, "kind": 32768, "kindString": "Parameter", - "flags": { - "keyword-only": "true" - }, + "name": "main_actor_function", "type": { + "name": "Callable", "type": "reference", - "name": "Optional[int]" - }, - "defaultValue": "None" + "typeArguments": [ + { + "type": "reference", + "name": "[]" + }, + { + "type": "reference", + "name": "MainReturnType", + "target": "25" + } + ] + } } - ] + ], + "type": { + "name": "Optional", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "MainReturnType", + "target": "25" + } + ] + } } ] }, { - "id": 37, - "name": "open_dataset", - "module": "actor", "kind": 2048, "kindString": "Method", - "flags": {}, + "children": [], "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Open a dataset.\\n\\nDatasets are used to store structured data where each object stored has the same attributes,\\nsuch as online store products or real estate offers.\\nThe actual data is stored either on the local filesystem or in the Apify cloud.\\n\\nArgs:\\n id (str, optional): ID of the dataset to be opened.\\n If neither `id` nor `name` are provided, the method returns the default dataset associated with the actor run.\\n name (str, optional): Name of the dataset to be opened.\\n If neither `id` nor `name` are provided, the method returns the default dataset associated with the actor run.\\n force_cloud (bool, optional): If set to `True` then the Apify cloud storage is always used.\\n This way it is possible to combine local and cloud storage.\\n\\nReturns:\\n Dataset: An instance of the `Dataset` class for the given ID or name.\"]}" + "text": "Return a new instance of the Apify API client.\n\nThe `ApifyClientAsync` class is provided by the [apify-client](https://github.com/apify/apify-client-python) package,\nand it is automatically configured using the `APIFY_API_BASE_URL` and `APIFY_TOKEN` environment variables.\n\nYou can override the token via the available options.\nThat's useful if you want to use the client as a different Apify user than the SDK internals are using.\n" } ] }, - "children": [], + "decorations": [ + { + "name": "classmethod" + } + ], + "flags": {}, "groups": [], + "id": 55, + "module": "actor", + "name": "new_client", + "parsedDocstring": { + "text": "Return a new instance of the Apify API client.\n\nThe `ApifyClientAsync` class is provided by the [apify-client](https://github.com/apify/apify-client-python) package,\nand it is automatically configured using the `APIFY_API_BASE_URL` and `APIFY_TOKEN` environment variables.\n\nYou can override the token via the available options.\nThat's useful if you want to use the client as a different Apify user than the SDK internals are using.\n", + "args": { + "token": "The Apify API token", + "api_url": "The URL of the Apify API server to which to connect to. Defaults to https://api.apify.com", + "max_retries": "How many times to retry a failed request at most", + "min_delay_between_retries_millis": "How long will the client wait between retrying requests\n(increases exponentially from this value)", + "timeout_secs": "The socket timeout of the HTTP requests sent to the Apify API" + } + }, "sources": [ { - "filename": "src/apify/actor.py", - "line": 497, "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v0.2.0/src/apify/actor.py#L497" + "fileName": "/src/apify/actor.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 441 } ], + "type": { + "name": "Undefined", + "type": "reference" + }, "signatures": [ { - "id": 41, - "name": "open_dataset", - "kind": 4096, - "kindString": "Call signature", - "flags": {}, "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Open a dataset.\\n\\nDatasets are used to store structured data where each object stored has the same attributes,\\nsuch as online store products or real estate offers.\\nThe actual data is stored either on the local filesystem or in the Apify cloud.\\n\\nArgs:\\n id (str, optional): ID of the dataset to be opened.\\n If neither `id` nor `name` are provided, the method returns the default dataset associated with the actor run.\\n name (str, optional): Name of the dataset to be opened.\\n If neither `id` nor `name` are provided, the method returns the default dataset associated with the actor run.\\n force_cloud (bool, optional): If set to `True` then the Apify cloud storage is always used.\\n This way it is possible to combine local and cloud storage.\\n\\nReturns:\\n Dataset: An instance of the `Dataset` class for the given ID or name.\"]}" + "text": "Return a new instance of the Apify API client.\n\nThe `ApifyClientAsync` class is provided by the [apify-client](https://github.com/apify/apify-client-python) package,\nand it is automatically configured using the `APIFY_API_BASE_URL` and `APIFY_TOKEN` environment variables.\n\nYou can override the token via the available options.\nThat's useful if you want to use the client as a different Apify user than the SDK internals are using.\n" } ] }, - "type": { - "type": "reference", - "name": "Dataset" - }, + "flags": {}, + "id": 56, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "new_client", "parameters": [ { - "id": 38, - "name": "id", - "kind": 32768, - "kindString": "Parameter", + "comment": { + "summary": [ + { + "kind": "text", + "text": "The Apify API token" + } + ] + }, + "defaultValue": "None", "flags": { - "keyword-only": "true" + "isOptional": true, + "keyword-only": true }, + "id": 57, + "kind": 32768, + "kindString": "Parameter", + "name": "token", "type": { + "name": "Optional", "type": "reference", - "name": "Optional[str]" - }, - "defaultValue": "None" + "typeArguments": [ + { + "type": "reference", + "name": "str" + } + ] + } }, { - "id": 39, - "name": "name", + "comment": { + "summary": [ + { + "kind": "text", + "text": "The URL of the Apify API server to which to connect to. Defaults to https://api.apify.com" + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 58, "kind": 32768, "kindString": "Parameter", + "name": "api_url", + "type": { + "name": "Optional", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "str" + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "How many times to retry a failed request at most" + } + ] + }, + "defaultValue": "None", "flags": { - "keyword-only": "true" + "isOptional": true, + "keyword-only": true }, + "id": 59, + "kind": 32768, + "kindString": "Parameter", + "name": "max_retries", "type": { + "name": "Optional", "type": "reference", - "name": "Optional[str]" - }, - "defaultValue": "None" + "typeArguments": [ + { + "type": "reference", + "name": "int" + } + ] + } }, { - "id": 40, - "name": "force_cloud", + "comment": { + "summary": [ + { + "kind": "text", + "text": "How long will the client wait between retrying requests\n(increases exponentially from this value)" + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 60, "kind": 32768, "kindString": "Parameter", + "name": "min_delay_between_retries_millis", + "type": { + "name": "Optional", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "int" + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The socket timeout of the HTTP requests sent to the Apify API" + } + ] + }, + "defaultValue": "None", "flags": { - "keyword-only": "true" + "isOptional": true, + "keyword-only": true }, + "id": 61, + "kind": 32768, + "kindString": "Parameter", + "name": "timeout_secs", "type": { + "name": "Optional", "type": "reference", - "name": "bool" - }, - "defaultValue": "False" + "typeArguments": [ + { + "type": "reference", + "name": "int" + } + ] + } } ], - "modifiers": [ - "async" - ] + "type": { + "name": "ApifyClientAsync", + "type": "reference" + } } ] }, { - "id": 42, - "name": "open_key_value_store", - "module": "actor", "kind": 2048, "kindString": "Method", - "flags": {}, + "children": [], "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Open a key-value store.\\n\\nKey-value stores are used to store records or files, along with their MIME content type.\\nThe records are stored and retrieved using a unique key.\\nThe actual data is stored either on a local filesystem or in the Apify cloud.\\n\\nArgs:\\n id (str, optional): ID of the key-value store to be opened.\\n If neither `id` nor `name` are provided, the method returns the default key-value store associated with the actor run.\\n name (str, optional): Name of the key-value store to be opened.\\n If neither `id` nor `name` are provided, the method returns the default key-value store associated with the actor run.\\n force_cloud (bool, optional): If set to `True` then the Apify cloud storage is always used.\\n This way it is possible to combine local and cloud storage.\\n\\nReturns:\\n KeyValueStore: An instance of the `KeyValueStore` class for the given ID or name.\"]}" + "text": "Open a dataset.\n\nDatasets are used to store structured data where each object stored has the same attributes,\nsuch as online store products or real estate offers.\nThe actual data is stored either on the local filesystem or in the Apify cloud.\n" } ] }, - "children": [], + "decorations": [ + { + "name": "classmethod" + } + ], + "flags": {}, "groups": [], + "id": 62, + "module": "actor", + "name": "open_dataset", + "parsedDocstring": { + "text": "Open a dataset.\n\nDatasets are used to store structured data where each object stored has the same attributes,\nsuch as online store products or real estate offers.\nThe actual data is stored either on the local filesystem or in the Apify cloud.\n", + "args": { + "id": "ID of the dataset to be opened.\nIf neither `id` nor `name` are provided, the method returns the default dataset associated with the actor run.", + "name": "Name of the dataset to be opened.\nIf neither `id` nor `name` are provided, the method returns the default dataset associated with the actor run.", + "force_cloud": "If set to `True` then the Apify cloud storage is always used.\nThis way it is possible to combine local and cloud storage.\n" + }, + "returns": "Dataset: An instance of the `Dataset` class for the given ID or name." + }, "sources": [ { - "filename": "src/apify/actor.py", - "line": 524, "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v0.2.0/src/apify/actor.py#L524" + "fileName": "/src/apify/actor.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 497 } ], + "type": { + "name": "Undefined", + "type": "reference" + }, "signatures": [ { - "id": 46, - "name": "open_key_value_store", - "kind": 4096, - "kindString": "Call signature", - "flags": {}, "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "Dataset: An instance of the `Dataset` class for the given ID or name." + } + ], + "tag": "@returns" + } + ], "summary": [ { "kind": "text", - "text": "{\"content\": [\"Open a key-value store.\\n\\nKey-value stores are used to store records or files, along with their MIME content type.\\nThe records are stored and retrieved using a unique key.\\nThe actual data is stored either on a local filesystem or in the Apify cloud.\\n\\nArgs:\\n id (str, optional): ID of the key-value store to be opened.\\n If neither `id` nor `name` are provided, the method returns the default key-value store associated with the actor run.\\n name (str, optional): Name of the key-value store to be opened.\\n If neither `id` nor `name` are provided, the method returns the default key-value store associated with the actor run.\\n force_cloud (bool, optional): If set to `True` then the Apify cloud storage is always used.\\n This way it is possible to combine local and cloud storage.\\n\\nReturns:\\n KeyValueStore: An instance of the `KeyValueStore` class for the given ID or name.\"]}" + "text": "Open a dataset.\n\nDatasets are used to store structured data where each object stored has the same attributes,\nsuch as online store products or real estate offers.\nThe actual data is stored either on the local filesystem or in the Apify cloud.\n" } ] }, - "type": { - "type": "reference", - "name": "KeyValueStore" - }, + "flags": {}, + "id": 63, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "open_dataset", "parameters": [ { - "id": 43, - "name": "id", - "kind": 32768, - "kindString": "Parameter", + "comment": { + "summary": [ + { + "kind": "text", + "text": "ID of the dataset to be opened.\nIf neither `id` nor `name` are provided, the method returns the default dataset associated with the actor run." + } + ] + }, + "defaultValue": "None", "flags": { - "keyword-only": "true" + "isOptional": true, + "keyword-only": true }, + "id": 64, + "kind": 32768, + "kindString": "Parameter", + "name": "id", "type": { + "name": "Optional", "type": "reference", - "name": "Optional[str]" - }, - "defaultValue": "None" + "typeArguments": [ + { + "type": "reference", + "name": "str" + } + ] + } }, { - "id": 44, - "name": "name", - "kind": 32768, - "kindString": "Parameter", + "comment": { + "summary": [ + { + "kind": "text", + "text": "Name of the dataset to be opened.\nIf neither `id` nor `name` are provided, the method returns the default dataset associated with the actor run." + } + ] + }, + "defaultValue": "None", "flags": { - "keyword-only": "true" + "isOptional": true, + "keyword-only": true }, + "id": 65, + "kind": 32768, + "kindString": "Parameter", + "name": "name", "type": { + "name": "Optional", "type": "reference", - "name": "Optional[str]" - }, - "defaultValue": "None" + "typeArguments": [ + { + "type": "reference", + "name": "str" + } + ] + } }, { - "id": 45, - "name": "force_cloud", - "kind": 32768, - "kindString": "Parameter", + "comment": { + "summary": [ + { + "kind": "text", + "text": "If set to `True` then the Apify cloud storage is always used.\nThis way it is possible to combine local and cloud storage.\n" + } + ] + }, + "defaultValue": "False", "flags": { - "keyword-only": "true" + "isOptional": true, + "keyword-only": true }, + "id": 66, + "kind": 32768, + "kindString": "Parameter", + "name": "force_cloud", "type": { - "type": "reference", - "name": "bool" - }, - "defaultValue": "False" + "name": "bool", + "type": "reference" + } } ], - "modifiers": [ - "async" - ] + "type": { + "name": "Dataset", + "type": "reference", + "target": "511" + } } ] }, { - "id": 47, - "name": "open_request_queue", - "module": "actor", "kind": 2048, "kindString": "Method", - "flags": {}, + "children": [], "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Open a request queue.\\n\\nRequest queue represents a queue of URLs to crawl, which is stored either on local filesystem or in the Apify cloud.\\nThe queue is used for deep crawling of websites, where you start with several URLs and then\\nrecursively follow links to other pages. The data structure supports both breadth-first\\nand depth-first crawling orders.\\n\\nArgs:\\n id (str, optional): ID of the request queue to be opened.\\n If neither `id` nor `name` are provided, the method returns the default request queue associated with the actor run.\\n name (str, optional): Name of the request queue to be opened.\\n If neither `id` nor `name` are provided, the method returns the default request queue associated with the actor run.\\n force_cloud (bool, optional): If set to `True` then the Apify cloud storage is always used.\\n This way it is possible to combine local and cloud storage.\\n\\nReturns:\\n RequestQueue: An instance of the `RequestQueue` class for the given ID or name.\"]}" + "text": "Open a key-value store.\n\nKey-value stores are used to store records or files, along with their MIME content type.\nThe records are stored and retrieved using a unique key.\nThe actual data is stored either on a local filesystem or in the Apify cloud.\n" } ] }, - "children": [], + "decorations": [ + { + "name": "classmethod" + } + ], + "flags": {}, "groups": [], + "id": 67, + "module": "actor", + "name": "open_key_value_store", + "parsedDocstring": { + "text": "Open a key-value store.\n\nKey-value stores are used to store records or files, along with their MIME content type.\nThe records are stored and retrieved using a unique key.\nThe actual data is stored either on a local filesystem or in the Apify cloud.\n", + "args": { + "id": "ID of the key-value store to be opened.\nIf neither `id` nor `name` are provided, the method returns the default key-value store associated with the actor run.", + "name": "Name of the key-value store to be opened.\nIf neither `id` nor `name` are provided, the method returns the default key-value store associated with the actor run.", + "force_cloud": "If set to `True` then the Apify cloud storage is always used.\nThis way it is possible to combine local and cloud storage.\n" + }, + "returns": "KeyValueStore: An instance of the `KeyValueStore` class for the given ID or name." + }, "sources": [ { - "filename": "src/apify/actor.py", - "line": 556, "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v0.2.0/src/apify/actor.py#L556" + "fileName": "/src/apify/actor.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 524 } ], + "type": { + "name": "Undefined", + "type": "reference" + }, "signatures": [ { - "id": 51, - "name": "open_request_queue", - "kind": 4096, - "kindString": "Call signature", - "flags": {}, "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "KeyValueStore: An instance of the `KeyValueStore` class for the given ID or name." + } + ], + "tag": "@returns" + } + ], "summary": [ { "kind": "text", - "text": "{\"content\": [\"Open a request queue.\\n\\nRequest queue represents a queue of URLs to crawl, which is stored either on local filesystem or in the Apify cloud.\\nThe queue is used for deep crawling of websites, where you start with several URLs and then\\nrecursively follow links to other pages. The data structure supports both breadth-first\\nand depth-first crawling orders.\\n\\nArgs:\\n id (str, optional): ID of the request queue to be opened.\\n If neither `id` nor `name` are provided, the method returns the default request queue associated with the actor run.\\n name (str, optional): Name of the request queue to be opened.\\n If neither `id` nor `name` are provided, the method returns the default request queue associated with the actor run.\\n force_cloud (bool, optional): If set to `True` then the Apify cloud storage is always used.\\n This way it is possible to combine local and cloud storage.\\n\\nReturns:\\n RequestQueue: An instance of the `RequestQueue` class for the given ID or name.\"]}" + "text": "Open a key-value store.\n\nKey-value stores are used to store records or files, along with their MIME content type.\nThe records are stored and retrieved using a unique key.\nThe actual data is stored either on a local filesystem or in the Apify cloud.\n" } ] }, - "type": { - "type": "reference", - "name": "RequestQueue" - }, + "flags": {}, + "id": 68, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "open_key_value_store", "parameters": [ { - "id": 48, - "name": "id", - "kind": 32768, - "kindString": "Parameter", + "comment": { + "summary": [ + { + "kind": "text", + "text": "ID of the key-value store to be opened.\nIf neither `id` nor `name` are provided, the method returns the default key-value store associated with the actor run." + } + ] + }, + "defaultValue": "None", "flags": { - "keyword-only": "true" + "isOptional": true, + "keyword-only": true }, + "id": 69, + "kind": 32768, + "kindString": "Parameter", + "name": "id", "type": { + "name": "Optional", "type": "reference", - "name": "Optional[str]" - }, - "defaultValue": "None" + "typeArguments": [ + { + "type": "reference", + "name": "str" + } + ] + } }, { - "id": 49, - "name": "name", - "kind": 32768, - "kindString": "Parameter", + "comment": { + "summary": [ + { + "kind": "text", + "text": "Name of the key-value store to be opened.\nIf neither `id` nor `name` are provided, the method returns the default key-value store associated with the actor run." + } + ] + }, + "defaultValue": "None", "flags": { - "keyword-only": "true" + "isOptional": true, + "keyword-only": true }, + "id": 70, + "kind": 32768, + "kindString": "Parameter", + "name": "name", "type": { + "name": "Optional", "type": "reference", - "name": "Optional[str]" - }, - "defaultValue": "None" + "typeArguments": [ + { + "type": "reference", + "name": "str" + } + ] + } }, { - "id": 50, - "name": "force_cloud", - "kind": 32768, - "kindString": "Parameter", + "comment": { + "summary": [ + { + "kind": "text", + "text": "If set to `True` then the Apify cloud storage is always used.\nThis way it is possible to combine local and cloud storage.\n" + } + ] + }, + "defaultValue": "False", "flags": { - "keyword-only": "true" + "isOptional": true, + "keyword-only": true }, + "id": 71, + "kind": 32768, + "kindString": "Parameter", + "name": "force_cloud", "type": { - "type": "reference", - "name": "bool" - }, - "defaultValue": "False" + "name": "bool", + "type": "reference" + } } ], - "modifiers": [ - "async" - ] + "type": { + "name": "KeyValueStore", + "type": "reference", + "target": "572" + } } ] }, { - "id": 52, - "name": "push_data", - "module": "actor", "kind": 2048, "kindString": "Method", - "flags": {}, + "children": [], "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Store an object or a list of objects to the default dataset of the current actor run.\\n\\nArgs:\\n data (object or list of objects, optional): The data to push to the default dataset.\"]}" + "text": "Open a request queue.\n\nRequest queue represents a queue of URLs to crawl, which is stored either on local filesystem or in the Apify cloud.\nThe queue is used for deep crawling of websites, where you start with several URLs and then\nrecursively follow links to other pages. The data structure supports both breadth-first\nand depth-first crawling orders.\n" } ] }, - "children": [], - "groups": [], + "decorations": [ + { + "name": "classmethod" + } + ], + "flags": {}, + "groups": [], + "id": 72, + "module": "actor", + "name": "open_request_queue", + "parsedDocstring": { + "text": "Open a request queue.\n\nRequest queue represents a queue of URLs to crawl, which is stored either on local filesystem or in the Apify cloud.\nThe queue is used for deep crawling of websites, where you start with several URLs and then\nrecursively follow links to other pages. The data structure supports both breadth-first\nand depth-first crawling orders.\n", + "args": { + "id": "ID of the request queue to be opened.\nIf neither `id` nor `name` are provided, the method returns the default request queue associated with the actor run.", + "name": "Name of the request queue to be opened.\nIf neither `id` nor `name` are provided, the method returns the default request queue associated with the actor run.", + "force_cloud": "If set to `True` then the Apify cloud storage is always used.\nThis way it is possible to combine local and cloud storage.\n" + }, + "returns": "RequestQueue: An instance of the `RequestQueue` class for the given ID or name." + }, "sources": [ { - "filename": "src/apify/actor.py", - "line": 589, "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v0.2.0/src/apify/actor.py#L589" + "fileName": "/src/apify/actor.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 556 } ], + "type": { + "name": "Undefined", + "type": "reference" + }, "signatures": [ { - "id": 54, - "name": "push_data", - "kind": 4096, - "kindString": "Call signature", - "flags": {}, "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "RequestQueue: An instance of the `RequestQueue` class for the given ID or name." + } + ], + "tag": "@returns" + } + ], "summary": [ { "kind": "text", - "text": "{\"content\": [\"Store an object or a list of objects to the default dataset of the current actor run.\\n\\nArgs:\\n data (object or list of objects, optional): The data to push to the default dataset.\"]}" + "text": "Open a request queue.\n\nRequest queue represents a queue of URLs to crawl, which is stored either on local filesystem or in the Apify cloud.\nThe queue is used for deep crawling of websites, where you start with several URLs and then\nrecursively follow links to other pages. The data structure supports both breadth-first\nand depth-first crawling orders.\n" } ] }, - "type": { - "type": "reference", - "name": "None" - }, + "flags": {}, + "id": 73, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "open_request_queue", "parameters": [ { - "id": 53, - "name": "data", + "comment": { + "summary": [ + { + "kind": "text", + "text": "ID of the request queue to be opened.\nIf neither `id` nor `name` are provided, the method returns the default request queue associated with the actor run." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 74, + "kind": 32768, + "kindString": "Parameter", + "name": "id", + "type": { + "name": "Optional", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "str" + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Name of the request queue to be opened.\nIf neither `id` nor `name` are provided, the method returns the default request queue associated with the actor run." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 75, "kind": 32768, "kindString": "Parameter", - "flags": {}, + "name": "name", "type": { + "name": "Optional", "type": "reference", - "name": "Any" + "typeArguments": [ + { + "type": "reference", + "name": "str" + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "If set to `True` then the Apify cloud storage is always used.\nThis way it is possible to combine local and cloud storage.\n" + } + ] + }, + "defaultValue": "False", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 76, + "kind": 32768, + "kindString": "Parameter", + "name": "force_cloud", + "type": { + "name": "bool", + "type": "reference" } } ], - "modifiers": [ - "async" - ] + "type": { + "name": "RequestQueue", + "type": "reference", + "target": "636" + } } ] }, { - "id": 55, - "name": "get_input", - "module": "actor", "kind": 2048, "kindString": "Method", - "flags": {}, + "children": [], "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Get the actor input value from the default key-value store associated with the current actor run.\"]}" + "text": "Store an object or a list of objects to the default dataset of the current actor run.\n" } ] }, - "children": [], + "decorations": [ + { + "name": "classmethod" + } + ], + "flags": {}, "groups": [], + "id": 77, + "module": "actor", + "name": "push_data", + "parsedDocstring": { + "text": "Store an object or a list of objects to the default dataset of the current actor run.\n", + "args": { + "data": "The data to push to the default dataset." + } + }, "sources": [ { - "filename": "src/apify/actor.py", - "line": 607, "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v0.2.0/src/apify/actor.py#L607" + "fileName": "/src/apify/actor.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 589 } ], + "type": { + "name": "Undefined", + "type": "reference" + }, "signatures": [ { - "id": 56, - "name": "get_input", - "kind": 4096, - "kindString": "Call signature", - "flags": {}, "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Get the actor input value from the default key-value store associated with the current actor run.\"]}" + "text": "Store an object or a list of objects to the default dataset of the current actor run.\n" } ] }, - "type": { - "type": "reference", - "name": "Any" - }, - "parameters": [], + "flags": {}, + "id": 78, + "kind": 4096, + "kindString": "Call signature", "modifiers": [ "async" - ] + ], + "name": "push_data", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The data to push to the default dataset." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 79, + "kind": 32768, + "kindString": "Parameter", + "name": "data", + "type": { + "name": "Any", + "type": "reference" + } + } + ], + "type": { + "name": "None", + "type": "literal", + "value": null + } } ] }, { - "id": 57, - "name": "get_value", - "module": "actor", "kind": 2048, "kindString": "Method", - "flags": {}, + "children": [], "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Get a value from the default key-value store associated with the current actor run.\\n\\nArgs:\\n key (str): The key of the record which to retrieve.\"]}" + "text": "Get the actor input value from the default key-value store associated with the current actor run." } ] }, - "children": [], + "decorations": [ + { + "name": "classmethod" + } + ], + "flags": {}, "groups": [], + "id": 80, + "module": "actor", + "name": "get_input", + "parsedDocstring": { + "text": "Get the actor input value from the default key-value store associated with the current actor run." + }, "sources": [ { - "filename": "src/apify/actor.py", - "line": 627, "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v0.2.0/src/apify/actor.py#L627" + "fileName": "/src/apify/actor.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 607 } ], + "type": { + "name": "Undefined", + "type": "reference" + }, "signatures": [ { - "id": 59, - "name": "get_value", - "kind": 4096, - "kindString": "Call signature", - "flags": {}, "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Get a value from the default key-value store associated with the current actor run.\\n\\nArgs:\\n key (str): The key of the record which to retrieve.\"]}" + "text": "Get the actor input value from the default key-value store associated with the current actor run." } ] }, - "type": { - "type": "reference", - "name": "Any" - }, - "parameters": [ - { - "id": 58, - "name": "key", - "kind": 32768, - "kindString": "Parameter", - "flags": {}, - "type": { - "type": "reference", - "name": "str" - } - } - ], + "flags": {}, + "id": 81, + "kind": 4096, + "kindString": "Call signature", "modifiers": [ "async" - ] + ], + "name": "get_input", + "parameters": [], + "type": { + "name": "Any", + "type": "reference" + } } ] }, { - "id": 60, - "name": "set_value", - "module": "actor", "kind": 2048, "kindString": "Method", - "flags": {}, + "children": [], "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Set or delete a value in the default key-value store associated with the current actor run.\\n\\nArgs:\\n key (str): The key of the record which to set.\\n value (any): The value of the record which to set, or None, if the record should be deleted.\\n content_type (str, optional): The content type which should be set to the value.\"]}" + "text": "Get a value from the default key-value store associated with the current actor run.\n" } ] }, - "children": [], + "decorations": [ + { + "name": "classmethod" + } + ], + "flags": {}, "groups": [], + "id": 82, + "module": "actor", + "name": "get_value", + "parsedDocstring": { + "text": "Get a value from the default key-value store associated with the current actor run.\n", + "args": { + "key": "The key of the record which to retrieve." + } + }, "sources": [ { - "filename": "src/apify/actor.py", - "line": 643, "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v0.2.0/src/apify/actor.py#L643" + "fileName": "/src/apify/actor.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 627 } ], + "type": { + "name": "Undefined", + "type": "reference" + }, "signatures": [ { - "id": 64, - "name": "set_value", - "kind": 4096, - "kindString": "Call signature", - "flags": {}, "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Set or delete a value in the default key-value store associated with the current actor run.\\n\\nArgs:\\n key (str): The key of the record which to set.\\n value (any): The value of the record which to set, or None, if the record should be deleted.\\n content_type (str, optional): The content type which should be set to the value.\"]}" + "text": "Get a value from the default key-value store associated with the current actor run.\n" } ] }, - "type": { - "type": "reference", - "name": "None" - }, + "flags": {}, + "id": 83, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "get_value", "parameters": [ { - "id": 61, - "name": "key", - "kind": 32768, - "kindString": "Parameter", - "flags": {}, - "type": { - "type": "reference", - "name": "str" - } - }, - { - "id": 62, - "name": "value", + "comment": { + "summary": [ + { + "kind": "text", + "text": "The key of the record which to retrieve." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 84, "kind": 32768, "kindString": "Parameter", - "flags": {}, + "name": "key", "type": { - "type": "reference", - "name": "Any" + "name": "str", + "type": "reference" } - }, - { - "id": 63, - "name": "content_type", - "kind": 32768, - "kindString": "Parameter", - "flags": { - "keyword-only": "true" - }, - "type": { - "type": "reference", - "name": "Optional[str]" - }, - "defaultValue": "None" } ], - "modifiers": [ - "async" - ] + "type": { + "name": "Any", + "type": "reference" + } } ] }, { - "id": 65, - "name": "on", - "module": "actor", "kind": 2048, "kindString": "Method", - "flags": {}, + "children": [], "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Add an event listener to the actor's event manager.\\n\\nThe following events can be emitted:\\n - `ActorEventTypes.SYSTEM_INFO`:\\n Emitted every minute, the event data contains info about the resource usage of the actor.\\n - `ActorEventTypes.MIGRATING`:\\n Emitted when the actor running on the Apify platform is going to be migrated to another worker server soon.\\n You can use it to persist the state of the actor and abort the run, to speed up the migration.\\n - `ActorEventTypes.PERSIST_STATE`:\\n Emitted in regular intervals (by default 60 seconds) to notify the actor that it should persist its state,\\n in order to avoid repeating all work when the actor restarts.\\n This event is automatically emitted together with the migrating event,\\n in which case the `isMigrating` flag in the event data is set to True, otherwise the flag is False.\\n Note that this event is provided merely for your convenience,\\n you can achieve the same effect using an interval and listening for the migrating event.\\n - `ActorEventTypes.ABORTING`:\\n When a user aborts an actor run on the Apify platform,\\n they can choose to abort it gracefully, to allow the actor some time before getting terminated.\\n This graceful abort emits the aborting event, which you can use to clean up the actor state.\\n\\nArgs:\\n event_name (ActorEventTypes): The actor event for which to listen to.\\n listener (Callable): The function which is to be called when the event is emitted (can be async).\"]}" + "text": "Set or delete a value in the default key-value store associated with the current actor run.\n" } ] }, - "children": [], + "decorations": [ + { + "name": "classmethod" + } + ], + "flags": {}, "groups": [], + "id": 85, + "module": "actor", + "name": "set_value", + "parsedDocstring": { + "text": "Set or delete a value in the default key-value store associated with the current actor run.\n", + "args": { + "key": "The key of the record which to set.", + "value": "The value of the record which to set, or None, if the record should be deleted.", + "content_type": "The content type which should be set to the value." + } + }, "sources": [ { - "filename": "src/apify/actor.py", - "line": 676, "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v0.2.0/src/apify/actor.py#L676" + "fileName": "/src/apify/actor.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 643 } ], + "type": { + "name": "Undefined", + "type": "reference" + }, "signatures": [ { - "id": 68, - "name": "on", - "kind": 4096, - "kindString": "Call signature", - "flags": {}, "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Add an event listener to the actor's event manager.\\n\\nThe following events can be emitted:\\n - `ActorEventTypes.SYSTEM_INFO`:\\n Emitted every minute, the event data contains info about the resource usage of the actor.\\n - `ActorEventTypes.MIGRATING`:\\n Emitted when the actor running on the Apify platform is going to be migrated to another worker server soon.\\n You can use it to persist the state of the actor and abort the run, to speed up the migration.\\n - `ActorEventTypes.PERSIST_STATE`:\\n Emitted in regular intervals (by default 60 seconds) to notify the actor that it should persist its state,\\n in order to avoid repeating all work when the actor restarts.\\n This event is automatically emitted together with the migrating event,\\n in which case the `isMigrating` flag in the event data is set to True, otherwise the flag is False.\\n Note that this event is provided merely for your convenience,\\n you can achieve the same effect using an interval and listening for the migrating event.\\n - `ActorEventTypes.ABORTING`:\\n When a user aborts an actor run on the Apify platform,\\n they can choose to abort it gracefully, to allow the actor some time before getting terminated.\\n This graceful abort emits the aborting event, which you can use to clean up the actor state.\\n\\nArgs:\\n event_name (ActorEventTypes): The actor event for which to listen to.\\n listener (Callable): The function which is to be called when the event is emitted (can be async).\"]}" + "text": "Set or delete a value in the default key-value store associated with the current actor run.\n" } ] }, - "type": { - "type": "reference", - "name": "Callable" - }, + "flags": {}, + "id": 86, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "set_value", "parameters": [ { - "id": 66, - "name": "event_name", + "comment": { + "summary": [ + { + "kind": "text", + "text": "The key of the record which to set." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 87, "kind": 32768, "kindString": "Parameter", - "flags": {}, + "name": "key", "type": { - "type": "reference", - "name": "ActorEventTypes" + "name": "str", + "type": "reference" } }, { - "id": 67, - "name": "listener", + "comment": { + "summary": [ + { + "kind": "text", + "text": "The value of the record which to set, or None, if the record should be deleted." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 88, + "kind": 32768, + "kindString": "Parameter", + "name": "value", + "type": { + "name": "Any", + "type": "reference" + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The content type which should be set to the value." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 89, "kind": 32768, "kindString": "Parameter", - "flags": {}, + "name": "content_type", "type": { + "name": "Optional", "type": "reference", - "name": "Callable" + "typeArguments": [ + { + "type": "reference", + "name": "str" + } + ] } } - ] + ], + "type": { + "name": "None", + "type": "literal", + "value": null + } } ] }, { - "id": 69, - "name": "off", - "module": "actor", "kind": 2048, "kindString": "Method", - "flags": {}, + "children": [], "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Remove a listener, or all listeners, from an actor event.\\n\\nArgs:\\n event_name (ActorEventTypes): The actor event for which to remove listeners.\\n listener (Callable, optional): The listener which is supposed to be removed. If not passed, all listeners of this event are removed.\"]}" + "text": "Add an event listener to the actor's event manager.\n\nThe following events can be emitted:\n- `ActorEventTypes.SYSTEM_INFO`:\nEmitted every minute, the event data contains info about the resource usage of the actor.\n- `ActorEventTypes.MIGRATING`:\nEmitted when the actor running on the Apify platform is going to be migrated to another worker server soon.\nYou can use it to persist the state of the actor and abort the run, to speed up the migration.\n- `ActorEventTypes.PERSIST_STATE`:\nEmitted in regular intervals (by default 60 seconds) to notify the actor that it should persist its state,\nin order to avoid repeating all work when the actor restarts.\nThis event is automatically emitted together with the migrating event,\nin which case the `isMigrating` flag in the event data is set to True, otherwise the flag is False.\nNote that this event is provided merely for your convenience,\nyou can achieve the same effect using an interval and listening for the migrating event.\n- `ActorEventTypes.ABORTING`:\nWhen a user aborts an actor run on the Apify platform,\nthey can choose to abort it gracefully, to allow the actor some time before getting terminated.\nThis graceful abort emits the aborting event, which you can use to clean up the actor state.\n" } ] }, - "children": [], + "decorations": [ + { + "name": "classmethod" + } + ], + "flags": {}, "groups": [], + "id": 90, + "module": "actor", + "name": "on", + "parsedDocstring": { + "text": "Add an event listener to the actor's event manager.\n\nThe following events can be emitted:\n- `ActorEventTypes.SYSTEM_INFO`:\nEmitted every minute, the event data contains info about the resource usage of the actor.\n- `ActorEventTypes.MIGRATING`:\nEmitted when the actor running on the Apify platform is going to be migrated to another worker server soon.\nYou can use it to persist the state of the actor and abort the run, to speed up the migration.\n- `ActorEventTypes.PERSIST_STATE`:\nEmitted in regular intervals (by default 60 seconds) to notify the actor that it should persist its state,\nin order to avoid repeating all work when the actor restarts.\nThis event is automatically emitted together with the migrating event,\nin which case the `isMigrating` flag in the event data is set to True, otherwise the flag is False.\nNote that this event is provided merely for your convenience,\nyou can achieve the same effect using an interval and listening for the migrating event.\n- `ActorEventTypes.ABORTING`:\nWhen a user aborts an actor run on the Apify platform,\nthey can choose to abort it gracefully, to allow the actor some time before getting terminated.\nThis graceful abort emits the aborting event, which you can use to clean up the actor state.\n", + "args": { + "event_name": "The actor event for which to listen to.", + "listener": "The function which is to be called when the event is emitted (can be async)." + } + }, "sources": [ { - "filename": "src/apify/actor.py", - "line": 709, "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v0.2.0/src/apify/actor.py#L709" + "fileName": "/src/apify/actor.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 676 } ], + "type": { + "name": "Undefined", + "type": "reference" + }, "signatures": [ { - "id": 72, - "name": "off", - "kind": 4096, - "kindString": "Call signature", - "flags": {}, "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Remove a listener, or all listeners, from an actor event.\\n\\nArgs:\\n event_name (ActorEventTypes): The actor event for which to remove listeners.\\n listener (Callable, optional): The listener which is supposed to be removed. If not passed, all listeners of this event are removed.\"]}" + "text": "Add an event listener to the actor's event manager.\n\nThe following events can be emitted:\n- `ActorEventTypes.SYSTEM_INFO`:\nEmitted every minute, the event data contains info about the resource usage of the actor.\n- `ActorEventTypes.MIGRATING`:\nEmitted when the actor running on the Apify platform is going to be migrated to another worker server soon.\nYou can use it to persist the state of the actor and abort the run, to speed up the migration.\n- `ActorEventTypes.PERSIST_STATE`:\nEmitted in regular intervals (by default 60 seconds) to notify the actor that it should persist its state,\nin order to avoid repeating all work when the actor restarts.\nThis event is automatically emitted together with the migrating event,\nin which case the `isMigrating` flag in the event data is set to True, otherwise the flag is False.\nNote that this event is provided merely for your convenience,\nyou can achieve the same effect using an interval and listening for the migrating event.\n- `ActorEventTypes.ABORTING`:\nWhen a user aborts an actor run on the Apify platform,\nthey can choose to abort it gracefully, to allow the actor some time before getting terminated.\nThis graceful abort emits the aborting event, which you can use to clean up the actor state.\n" } ] }, - "type": { - "type": "reference", - "name": "None" - }, + "flags": {}, + "id": 91, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "on", "parameters": [ { - "id": 70, - "name": "event_name", + "comment": { + "summary": [ + { + "kind": "text", + "text": "The actor event for which to listen to." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 92, "kind": 32768, "kindString": "Parameter", - "flags": {}, + "name": "event_name", "type": { + "name": "ActorEventTypes", "type": "reference", - "name": "ActorEventTypes" + "target": "195" } }, { - "id": 71, + "comment": { + "summary": [ + { + "kind": "text", + "text": "The function which is to be called when the event is emitted (can be async)." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 93, + "kind": 32768, + "kindString": "Parameter", "name": "listener", + "type": { + "name": "Callable", + "type": "reference" + } + } + ], + "type": { + "name": "Callable", + "type": "reference" + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Remove a listener, or all listeners, from an actor event.\n" + } + ] + }, + "decorations": [ + { + "name": "classmethod" + } + ], + "flags": {}, + "groups": [], + "id": 94, + "module": "actor", + "name": "off", + "parsedDocstring": { + "text": "Remove a listener, or all listeners, from an actor event.\n", + "args": { + "event_name": "The actor event for which to remove listeners.", + "listener": "The listener which is supposed to be removed. If not passed, all listeners of this event are removed." + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/actor.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 709 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Remove a listener, or all listeners, from an actor event.\n" + } + ] + }, + "flags": {}, + "id": 95, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "off", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The actor event for which to remove listeners." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 96, "kind": 32768, "kindString": "Parameter", - "flags": {}, + "name": "event_name", "type": { + "name": "ActorEventTypes", "type": "reference", - "name": "Optional[Callable]" + "target": "195" + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The listener which is supposed to be removed. If not passed, all listeners of this event are removed." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": false }, - "defaultValue": "None" + "id": 97, + "kind": 32768, + "kindString": "Parameter", + "name": "listener", + "type": { + "name": "Optional", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "Callable" + } + ] + } } - ] + ], + "type": { + "name": "None", + "type": "literal", + "value": null + } } ] }, { - "id": 73, - "name": "is_at_home", - "module": "actor", "kind": 2048, "kindString": "Method", - "flags": {}, + "children": [], "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Return `True` when the actor is running on the Apify platform, and `False` otherwise (for example when running locally).\"]}" + "text": "Return `True` when the actor is running on the Apify platform, and `False` otherwise (for example when running locally)." } ] }, - "children": [], + "decorations": [ + { + "name": "classmethod" + } + ], + "flags": {}, "groups": [], + "id": 98, + "module": "actor", + "name": "is_at_home", + "parsedDocstring": { + "text": "Return `True` when the actor is running on the Apify platform, and `False` otherwise (for example when running locally)." + }, "sources": [ { - "filename": "src/apify/actor.py", - "line": 724, "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v0.2.0/src/apify/actor.py#L724" + "fileName": "/src/apify/actor.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 724 } ], + "type": { + "name": "Undefined", + "type": "reference" + }, "signatures": [ { - "id": 74, - "name": "is_at_home", - "kind": 4096, - "kindString": "Call signature", - "flags": {}, "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Return `True` when the actor is running on the Apify platform, and `False` otherwise (for example when running locally).\"]}" + "text": "Return `True` when the actor is running on the Apify platform, and `False` otherwise (for example when running locally)." } ] }, + "flags": {}, + "id": 99, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "is_at_home", + "parameters": [], "type": { - "type": "reference", - "name": "bool" - }, - "parameters": [] + "name": "bool", + "type": "reference" + } } ] }, { - "id": 75, - "name": "get_env", - "module": "actor", "kind": 2048, "kindString": "Method", - "flags": {}, + "children": [], "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Return a dictionary with information parsed from all the `APIFY_XXX` environment variables.\\n\\nFor a list of all the environment variables,\\nsee the [Actor documentation](https://docs.apify.com/actors/development/environment-variables).\\nIf some variables are not defined or are invalid, the corresponding value in the resulting dictionary will be None.\"]}" + "text": "Return a dictionary with information parsed from all the `APIFY_XXX` environment variables.\n\nFor a list of all the environment variables,\nsee the [Actor documentation](https://docs.apify.com/actors/development/environment-variables).\nIf some variables are not defined or are invalid, the corresponding value in the resulting dictionary will be None." } ] }, - "children": [], + "decorations": [ + { + "name": "classmethod" + } + ], + "flags": {}, "groups": [], + "id": 100, + "module": "actor", + "name": "get_env", + "parsedDocstring": { + "text": "Return a dictionary with information parsed from all the `APIFY_XXX` environment variables.\n\nFor a list of all the environment variables,\nsee the [Actor documentation](https://docs.apify.com/actors/development/environment-variables).\nIf some variables are not defined or are invalid, the corresponding value in the resulting dictionary will be None." + }, "sources": [ { - "filename": "src/apify/actor.py", - "line": 732, "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v0.2.0/src/apify/actor.py#L732" + "fileName": "/src/apify/actor.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 732 } ], + "type": { + "name": "Undefined", + "type": "reference" + }, "signatures": [ { - "id": 76, - "name": "get_env", - "kind": 4096, - "kindString": "Call signature", - "flags": {}, "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Return a dictionary with information parsed from all the `APIFY_XXX` environment variables.\\n\\nFor a list of all the environment variables,\\nsee the [Actor documentation](https://docs.apify.com/actors/development/environment-variables).\\nIf some variables are not defined or are invalid, the corresponding value in the resulting dictionary will be None.\"]}" + "text": "Return a dictionary with information parsed from all the `APIFY_XXX` environment variables.\n\nFor a list of all the environment variables,\nsee the [Actor documentation](https://docs.apify.com/actors/development/environment-variables).\nIf some variables are not defined or are invalid, the corresponding value in the resulting dictionary will be None." } ] }, + "flags": {}, + "id": 101, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "get_env", + "parameters": [], "type": { - "type": "reference", - "name": "Dict" - }, - "parameters": [] + "name": "Dict", + "type": "reference" + } } ] }, { - "id": 77, - "name": "start", - "module": "actor", "kind": 2048, "kindString": "Method", - "flags": {}, + "children": [], "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Run an actor on the Apify platform.\\n\\nUnlike `Actor.call`, this method just starts the run without waiting for finish.\\n\\nArgs:\\n actor_id (str): The ID of the actor to be run.\\n run_input (Any, optional): The input to pass to the actor run.\\n token (str, optional): The Apify API token to use for this request (defaults to the `APIFY_TOKEN` environment variable).\\n content_type (str, optional): The content type of the input.\\n build (str, optional): Specifies the actor build to run. It can be either a build tag or build number.\\n By default, the run uses the build specified in the default run configuration for the actor (typically latest).\\n memory_mbytes (int, optional): Memory limit for the run, in megabytes.\\n By default, the run uses a memory limit specified in the default run configuration for the actor.\\n timeout_secs (int, optional): Optional timeout for the run, in seconds.\\n By default, the run uses timeout specified in the default run configuration for the actor.\\n wait_for_finish (int, optional): The maximum number of seconds the server waits for the run to finish.\\n By default, it is 0, the maximum value is 300.\\n webhooks (list of dict, optional): Optional ad-hoc webhooks (https://docs.apify.com/webhooks/ad-hoc-webhooks)\\n associated with the actor run which can be used to receive a notification,\\n e.g. when the actor finished or failed.\\n If you already have a webhook set up for the actor or task, you do not have to add it again here.\\n Each webhook is represented by a dictionary containing these items:\\n * ``event_types``: list of ``WebhookEventType`` values which trigger the webhook\\n * ``request_url``: URL to which to send the webhook HTTP request\\n * ``payload_template`` (optional): Optional template for the request payload\\n\\nReturns:\\n dict: Info about the started actor run\"]}" + "text": "Run an actor on the Apify platform.\n\nUnlike `Actor.call`, this method just starts the run without waiting for finish.\n" } ] }, - "children": [], + "decorations": [ + { + "name": "classmethod" + } + ], + "flags": {}, "groups": [], + "id": 102, + "module": "actor", + "name": "start", + "parsedDocstring": { + "text": "Run an actor on the Apify platform.\n\nUnlike `Actor.call`, this method just starts the run without waiting for finish.\n", + "args": { + "actor_id": "The ID of the actor to be run.", + "run_input": "The input to pass to the actor run.", + "token": "The Apify API token to use for this request (defaults to the `APIFY_TOKEN` environment variable).", + "content_type": "The content type of the input.", + "build": "Specifies the actor build to run. It can be either a build tag or build number.\nBy default, the run uses the build specified in the default run configuration for the actor (typically latest).", + "memory_mbytes": "Memory limit for the run, in megabytes.\nBy default, the run uses a memory limit specified in the default run configuration for the actor.", + "timeout_secs": "Optional timeout for the run, in seconds.\nBy default, the run uses timeout specified in the default run configuration for the actor.", + "wait_for_finish": "The maximum number of seconds the server waits for the run to finish.\nBy default, it is 0, the maximum value is 300.", + "webhooks": "Optional ad-hoc webhooks (https://docs.apify.com/webhooks/ad-hoc-webhooks)\nassociated with the actor run which can be used to receive a notification,\ne.g. when the actor finished or failed.\nIf you already have a webhook set up for the actor or task, you do not have to add it again here.\nEach webhook is represented by a dictionary containing these items:\n* ``event_types``: list of ``WebhookEventType`` values which trigger the webhook\n* ``request_url``: URL to which to send the webhook HTTP request\n* ``payload_template`` (optional): Optional template for the request payload\n" + }, + "returns": "dict: Info about the started actor run" + }, "sources": [ { - "filename": "src/apify/actor.py", - "line": 749, "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v0.2.0/src/apify/actor.py#L749" + "fileName": "/src/apify/actor.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 749 } ], + "type": { + "name": "Undefined", + "type": "reference" + }, "signatures": [ { - "id": 87, - "name": "start", - "kind": 4096, - "kindString": "Call signature", - "flags": {}, "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "dict: Info about the started actor run" + } + ], + "tag": "@returns" + } + ], "summary": [ { "kind": "text", - "text": "{\"content\": [\"Run an actor on the Apify platform.\\n\\nUnlike `Actor.call`, this method just starts the run without waiting for finish.\\n\\nArgs:\\n actor_id (str): The ID of the actor to be run.\\n run_input (Any, optional): The input to pass to the actor run.\\n token (str, optional): The Apify API token to use for this request (defaults to the `APIFY_TOKEN` environment variable).\\n content_type (str, optional): The content type of the input.\\n build (str, optional): Specifies the actor build to run. It can be either a build tag or build number.\\n By default, the run uses the build specified in the default run configuration for the actor (typically latest).\\n memory_mbytes (int, optional): Memory limit for the run, in megabytes.\\n By default, the run uses a memory limit specified in the default run configuration for the actor.\\n timeout_secs (int, optional): Optional timeout for the run, in seconds.\\n By default, the run uses timeout specified in the default run configuration for the actor.\\n wait_for_finish (int, optional): The maximum number of seconds the server waits for the run to finish.\\n By default, it is 0, the maximum value is 300.\\n webhooks (list of dict, optional): Optional ad-hoc webhooks (https://docs.apify.com/webhooks/ad-hoc-webhooks)\\n associated with the actor run which can be used to receive a notification,\\n e.g. when the actor finished or failed.\\n If you already have a webhook set up for the actor or task, you do not have to add it again here.\\n Each webhook is represented by a dictionary containing these items:\\n * ``event_types``: list of ``WebhookEventType`` values which trigger the webhook\\n * ``request_url``: URL to which to send the webhook HTTP request\\n * ``payload_template`` (optional): Optional template for the request payload\\n\\nReturns:\\n dict: Info about the started actor run\"]}" + "text": "Run an actor on the Apify platform.\n\nUnlike `Actor.call`, this method just starts the run without waiting for finish.\n" } ] }, - "type": { - "type": "reference", - "name": "Dict" - }, + "flags": {}, + "id": 103, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "start", "parameters": [ { - "id": 78, - "name": "actor_id", + "comment": { + "summary": [ + { + "kind": "text", + "text": "The ID of the actor to be run." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 104, "kind": 32768, "kindString": "Parameter", - "flags": {}, + "name": "actor_id", "type": { - "type": "reference", - "name": "str" + "name": "str", + "type": "reference" } }, { - "id": 79, - "name": "run_input", + "comment": { + "summary": [ + { + "kind": "text", + "text": "The input to pass to the actor run." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": false + }, + "id": 105, "kind": 32768, "kindString": "Parameter", - "flags": {}, + "name": "run_input", "type": { + "name": "Optional", "type": "reference", - "name": "Optional[Any]" - }, - "defaultValue": "None" + "typeArguments": [ + { + "type": "reference", + "name": "Any" + } + ] + } }, { - "id": 80, - "name": "token", - "kind": 32768, - "kindString": "Parameter", + "comment": { + "summary": [ + { + "kind": "text", + "text": "The Apify API token to use for this request (defaults to the `APIFY_TOKEN` environment variable)." + } + ] + }, + "defaultValue": "None", "flags": { - "keyword-only": "true" + "isOptional": true, + "keyword-only": true }, + "id": 106, + "kind": 32768, + "kindString": "Parameter", + "name": "token", "type": { + "name": "Optional", "type": "reference", - "name": "Optional[str]" - }, - "defaultValue": "None" + "typeArguments": [ + { + "type": "reference", + "name": "str" + } + ] + } }, { - "id": 81, - "name": "content_type", - "kind": 32768, - "kindString": "Parameter", + "comment": { + "summary": [ + { + "kind": "text", + "text": "The content type of the input." + } + ] + }, + "defaultValue": "None", "flags": { - "keyword-only": "true" + "isOptional": true, + "keyword-only": true }, + "id": 107, + "kind": 32768, + "kindString": "Parameter", + "name": "content_type", "type": { + "name": "Optional", "type": "reference", - "name": "Optional[str]" - }, - "defaultValue": "None" + "typeArguments": [ + { + "type": "reference", + "name": "str" + } + ] + } }, { - "id": 82, - "name": "build", - "kind": 32768, - "kindString": "Parameter", + "comment": { + "summary": [ + { + "kind": "text", + "text": "Specifies the actor build to run. It can be either a build tag or build number.\nBy default, the run uses the build specified in the default run configuration for the actor (typically latest)." + } + ] + }, + "defaultValue": "None", "flags": { - "keyword-only": "true" + "isOptional": true, + "keyword-only": true }, + "id": 108, + "kind": 32768, + "kindString": "Parameter", + "name": "build", "type": { + "name": "Optional", "type": "reference", - "name": "Optional[str]" - }, - "defaultValue": "None" + "typeArguments": [ + { + "type": "reference", + "name": "str" + } + ] + } }, { - "id": 83, - "name": "memory_mbytes", - "kind": 32768, - "kindString": "Parameter", + "comment": { + "summary": [ + { + "kind": "text", + "text": "Memory limit for the run, in megabytes.\nBy default, the run uses a memory limit specified in the default run configuration for the actor." + } + ] + }, + "defaultValue": "None", "flags": { - "keyword-only": "true" + "isOptional": true, + "keyword-only": true }, + "id": 109, + "kind": 32768, + "kindString": "Parameter", + "name": "memory_mbytes", "type": { + "name": "Optional", "type": "reference", - "name": "Optional[int]" - }, - "defaultValue": "None" + "typeArguments": [ + { + "type": "reference", + "name": "int" + } + ] + } }, { - "id": 84, - "name": "timeout_secs", - "kind": 32768, - "kindString": "Parameter", + "comment": { + "summary": [ + { + "kind": "text", + "text": "Optional timeout for the run, in seconds.\nBy default, the run uses timeout specified in the default run configuration for the actor." + } + ] + }, + "defaultValue": "None", "flags": { - "keyword-only": "true" + "isOptional": true, + "keyword-only": true }, + "id": 110, + "kind": 32768, + "kindString": "Parameter", + "name": "timeout_secs", "type": { + "name": "Optional", "type": "reference", - "name": "Optional[int]" - }, - "defaultValue": "None" + "typeArguments": [ + { + "type": "reference", + "name": "int" + } + ] + } }, { - "id": 85, - "name": "wait_for_finish", - "kind": 32768, - "kindString": "Parameter", + "comment": { + "summary": [ + { + "kind": "text", + "text": "The maximum number of seconds the server waits for the run to finish.\nBy default, it is 0, the maximum value is 300." + } + ] + }, + "defaultValue": "None", "flags": { - "keyword-only": "true" + "isOptional": true, + "keyword-only": true }, + "id": 111, + "kind": 32768, + "kindString": "Parameter", + "name": "wait_for_finish", "type": { + "name": "Optional", "type": "reference", - "name": "Optional[int]" - }, - "defaultValue": "None" + "typeArguments": [ + { + "type": "reference", + "name": "int" + } + ] + } }, { - "id": 86, - "name": "webhooks", - "kind": 32768, - "kindString": "Parameter", + "comment": { + "summary": [ + { + "kind": "text", + "text": "Optional ad-hoc webhooks (https://docs.apify.com/webhooks/ad-hoc-webhooks)\nassociated with the actor run which can be used to receive a notification,\ne.g. when the actor finished or failed.\nIf you already have a webhook set up for the actor or task, you do not have to add it again here.\nEach webhook is represented by a dictionary containing these items:\n* ``event_types``: list of ``WebhookEventType`` values which trigger the webhook\n* ``request_url``: URL to which to send the webhook HTTP request\n* ``payload_template`` (optional): Optional template for the request payload\n" + } + ] + }, + "defaultValue": "None", "flags": { - "keyword-only": "true" + "isOptional": true, + "keyword-only": true }, + "id": 112, + "kind": 32768, + "kindString": "Parameter", + "name": "webhooks", "type": { + "name": "Optional", "type": "reference", - "name": "Optional[List[Dict]]" - }, - "defaultValue": "None" + "typeArguments": [ + { + "type": "reference", + "name": "List", + "typeArguments": [ + { + "type": "reference", + "name": "Dict" + } + ] + } + ] + } } ], - "modifiers": [ - "async" - ] + "type": { + "name": "Dict", + "type": "reference" + } } ] }, { - "id": 88, - "name": "abort", - "module": "actor", "kind": 2048, "kindString": "Method", - "flags": {}, + "children": [], "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Abort given actor run on the Apify platform using the current user account (determined by the `APIFY_TOKEN` environment variable).\\n\\nArgs:\\n run_id (str): The ID of the actor run to be aborted.\\n token (str, optional): The Apify API token to use for this request (defaults to the `APIFY_TOKEN` environment variable).\\n gracefully (bool, optional): If True, the actor run will abort gracefully.\\n It will send ``aborting`` and ``persistStates`` events into the run and force-stop the run after 30 seconds.\\n It is helpful in cases where you plan to resurrect the run later.\\n\\nReturns:\\n dict: Info about the aborted actor run\"]}" + "text": "Abort given actor run on the Apify platform using the current user account (determined by the `APIFY_TOKEN` environment variable).\n" } ] }, - "children": [], + "decorations": [ + { + "name": "classmethod" + } + ], + "flags": {}, "groups": [], + "id": 113, + "module": "actor", + "name": "abort", + "parsedDocstring": { + "text": "Abort given actor run on the Apify platform using the current user account (determined by the `APIFY_TOKEN` environment variable).\n", + "args": { + "run_id": "The ID of the actor run to be aborted.", + "token": "The Apify API token to use for this request (defaults to the `APIFY_TOKEN` environment variable).", + "gracefully": "If True, the actor run will abort gracefully.\nIt will send ``aborting`` and ``persistStates`` events into the run and force-stop the run after 30 seconds.\nIt is helpful in cases where you plan to resurrect the run later.\n" + }, + "returns": "dict: Info about the aborted actor run" + }, "sources": [ { - "filename": "src/apify/actor.py", - "line": 834, "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v0.2.0/src/apify/actor.py#L834" + "fileName": "/src/apify/actor.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 834 } ], + "type": { + "name": "Undefined", + "type": "reference" + }, "signatures": [ { - "id": 92, - "name": "abort", - "kind": 4096, - "kindString": "Call signature", - "flags": {}, "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "dict: Info about the aborted actor run" + } + ], + "tag": "@returns" + } + ], "summary": [ { "kind": "text", - "text": "{\"content\": [\"Abort given actor run on the Apify platform using the current user account (determined by the `APIFY_TOKEN` environment variable).\\n\\nArgs:\\n run_id (str): The ID of the actor run to be aborted.\\n token (str, optional): The Apify API token to use for this request (defaults to the `APIFY_TOKEN` environment variable).\\n gracefully (bool, optional): If True, the actor run will abort gracefully.\\n It will send ``aborting`` and ``persistStates`` events into the run and force-stop the run after 30 seconds.\\n It is helpful in cases where you plan to resurrect the run later.\\n\\nReturns:\\n dict: Info about the aborted actor run\"]}" + "text": "Abort given actor run on the Apify platform using the current user account (determined by the `APIFY_TOKEN` environment variable).\n" } ] }, - "type": { - "type": "reference", - "name": "Dict" - }, + "flags": {}, + "id": 114, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "abort", "parameters": [ { - "id": 89, - "name": "run_id", + "comment": { + "summary": [ + { + "kind": "text", + "text": "The ID of the actor run to be aborted." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 115, "kind": 32768, "kindString": "Parameter", - "flags": {}, + "name": "run_id", "type": { - "type": "reference", - "name": "str" + "name": "str", + "type": "reference" } }, { - "id": 90, - "name": "token", - "kind": 32768, - "kindString": "Parameter", + "comment": { + "summary": [ + { + "kind": "text", + "text": "The Apify API token to use for this request (defaults to the `APIFY_TOKEN` environment variable)." + } + ] + }, + "defaultValue": "None", "flags": { - "keyword-only": "true" + "isOptional": true, + "keyword-only": true }, + "id": 116, + "kind": 32768, + "kindString": "Parameter", + "name": "token", "type": { + "name": "Optional", "type": "reference", - "name": "Optional[str]" - }, - "defaultValue": "None" + "typeArguments": [ + { + "type": "reference", + "name": "str" + } + ] + } }, { - "id": 91, - "name": "gracefully", - "kind": 32768, - "kindString": "Parameter", + "comment": { + "summary": [ + { + "kind": "text", + "text": "If True, the actor run will abort gracefully.\nIt will send ``aborting`` and ``persistStates`` events into the run and force-stop the run after 30 seconds.\nIt is helpful in cases where you plan to resurrect the run later.\n" + } + ] + }, + "defaultValue": "None", "flags": { - "keyword-only": "true" + "isOptional": true, + "keyword-only": true }, + "id": 117, + "kind": 32768, + "kindString": "Parameter", + "name": "gracefully", "type": { + "name": "Optional", "type": "reference", - "name": "Optional[bool]" - }, - "defaultValue": "None" + "typeArguments": [ + { + "type": "reference", + "name": "bool" + } + ] + } } ], - "modifiers": [ - "async" - ] + "type": { + "name": "Dict", + "type": "reference" + } } ] }, { - "id": 93, - "name": "call", - "module": "actor", "kind": 2048, "kindString": "Method", - "flags": {}, + "children": [], "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Start an actor on the Apify Platform and wait for it to finish before returning.\\n\\nIt waits indefinitely, unless the wait_secs argument is provided.\\n\\nArgs:\\n actor_id (str): The ID of the actor to be run.\\n run_input (Any, optional): The input to pass to the actor run.\\n token (str, optional): The Apify API token to use for this request (defaults to the `APIFY_TOKEN` environment variable).\\n content_type (str, optional): The content type of the input.\\n build (str, optional): Specifies the actor build to run. It can be either a build tag or build number.\\n By default, the run uses the build specified in the default run configuration for the actor (typically latest).\\n memory_mbytes (int, optional): Memory limit for the run, in megabytes.\\n By default, the run uses a memory limit specified in the default run configuration for the actor.\\n timeout_secs (int, optional): Optional timeout for the run, in seconds.\\n By default, the run uses timeout specified in the default run configuration for the actor.\\n webhooks (list, optional): Optional webhooks (https://docs.apify.com/webhooks) associated with the actor run,\\n which can be used to receive a notification, e.g. when the actor finished or failed.\\n If you already have a webhook set up for the actor, you do not have to add it again here.\\n wait_secs (int, optional): The maximum number of seconds the server waits for the run to finish. If not provided, waits indefinitely.\\n\\nReturns:\\n dict: Info about the started actor run\"]}" + "text": "Start an actor on the Apify Platform and wait for it to finish before returning.\n\nIt waits indefinitely, unless the wait_secs argument is provided.\n" } ] }, - "children": [], + "decorations": [ + { + "name": "classmethod" + } + ], + "flags": {}, "groups": [], + "id": 118, + "module": "actor", + "name": "call", + "parsedDocstring": { + "text": "Start an actor on the Apify Platform and wait for it to finish before returning.\n\nIt waits indefinitely, unless the wait_secs argument is provided.\n", + "args": { + "actor_id": "The ID of the actor to be run.", + "run_input": "The input to pass to the actor run.", + "token": "The Apify API token to use for this request (defaults to the `APIFY_TOKEN` environment variable).", + "content_type": "The content type of the input.", + "build": "Specifies the actor build to run. It can be either a build tag or build number.\nBy default, the run uses the build specified in the default run configuration for the actor (typically latest).", + "memory_mbytes": "Memory limit for the run, in megabytes.\nBy default, the run uses a memory limit specified in the default run configuration for the actor.", + "timeout_secs": "Optional timeout for the run, in seconds.\nBy default, the run uses timeout specified in the default run configuration for the actor.", + "webhooks": "Optional webhooks (https://docs.apify.com/webhooks) associated with the actor run,\nwhich can be used to receive a notification, e.g. when the actor finished or failed.\nIf you already have a webhook set up for the actor, you do not have to add it again here.", + "wait_secs": "The maximum number of seconds the server waits for the run to finish. If not provided, waits indefinitely.\n" + }, + "returns": "dict: Info about the started actor run" + }, "sources": [ { - "filename": "src/apify/actor.py", - "line": 880, "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v0.2.0/src/apify/actor.py#L880" + "fileName": "/src/apify/actor.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 880 } ], + "type": { + "name": "Undefined", + "type": "reference" + }, "signatures": [ { - "id": 103, - "name": "call", - "kind": 4096, - "kindString": "Call signature", - "flags": {}, "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "dict: Info about the started actor run" + } + ], + "tag": "@returns" + } + ], "summary": [ { "kind": "text", - "text": "{\"content\": [\"Start an actor on the Apify Platform and wait for it to finish before returning.\\n\\nIt waits indefinitely, unless the wait_secs argument is provided.\\n\\nArgs:\\n actor_id (str): The ID of the actor to be run.\\n run_input (Any, optional): The input to pass to the actor run.\\n token (str, optional): The Apify API token to use for this request (defaults to the `APIFY_TOKEN` environment variable).\\n content_type (str, optional): The content type of the input.\\n build (str, optional): Specifies the actor build to run. It can be either a build tag or build number.\\n By default, the run uses the build specified in the default run configuration for the actor (typically latest).\\n memory_mbytes (int, optional): Memory limit for the run, in megabytes.\\n By default, the run uses a memory limit specified in the default run configuration for the actor.\\n timeout_secs (int, optional): Optional timeout for the run, in seconds.\\n By default, the run uses timeout specified in the default run configuration for the actor.\\n webhooks (list, optional): Optional webhooks (https://docs.apify.com/webhooks) associated with the actor run,\\n which can be used to receive a notification, e.g. when the actor finished or failed.\\n If you already have a webhook set up for the actor, you do not have to add it again here.\\n wait_secs (int, optional): The maximum number of seconds the server waits for the run to finish. If not provided, waits indefinitely.\\n\\nReturns:\\n dict: Info about the started actor run\"]}" + "text": "Start an actor on the Apify Platform and wait for it to finish before returning.\n\nIt waits indefinitely, unless the wait_secs argument is provided.\n" } ] }, - "type": { - "type": "reference", - "name": "Optional[Dict]" - }, + "flags": {}, + "id": 119, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "call", "parameters": [ { - "id": 94, - "name": "actor_id", + "comment": { + "summary": [ + { + "kind": "text", + "text": "The ID of the actor to be run." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 120, "kind": 32768, "kindString": "Parameter", - "flags": {}, + "name": "actor_id", "type": { - "type": "reference", - "name": "str" + "name": "str", + "type": "reference" } }, { - "id": 95, - "name": "run_input", + "comment": { + "summary": [ + { + "kind": "text", + "text": "The input to pass to the actor run." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": false + }, + "id": 121, "kind": 32768, "kindString": "Parameter", - "flags": {}, + "name": "run_input", "type": { + "name": "Optional", "type": "reference", - "name": "Optional[Any]" - }, - "defaultValue": "None" + "typeArguments": [ + { + "type": "reference", + "name": "Any" + } + ] + } }, { - "id": 96, - "name": "token", - "kind": 32768, - "kindString": "Parameter", + "comment": { + "summary": [ + { + "kind": "text", + "text": "The Apify API token to use for this request (defaults to the `APIFY_TOKEN` environment variable)." + } + ] + }, + "defaultValue": "None", "flags": { - "keyword-only": "true" + "isOptional": true, + "keyword-only": true }, + "id": 122, + "kind": 32768, + "kindString": "Parameter", + "name": "token", "type": { + "name": "Optional", "type": "reference", - "name": "Optional[str]" - }, - "defaultValue": "None" + "typeArguments": [ + { + "type": "reference", + "name": "str" + } + ] + } }, { - "id": 97, - "name": "content_type", - "kind": 32768, - "kindString": "Parameter", + "comment": { + "summary": [ + { + "kind": "text", + "text": "The content type of the input." + } + ] + }, + "defaultValue": "None", "flags": { - "keyword-only": "true" + "isOptional": true, + "keyword-only": true }, + "id": 123, + "kind": 32768, + "kindString": "Parameter", + "name": "content_type", "type": { + "name": "Optional", "type": "reference", - "name": "Optional[str]" - }, - "defaultValue": "None" + "typeArguments": [ + { + "type": "reference", + "name": "str" + } + ] + } }, { - "id": 98, - "name": "build", - "kind": 32768, - "kindString": "Parameter", + "comment": { + "summary": [ + { + "kind": "text", + "text": "Specifies the actor build to run. It can be either a build tag or build number.\nBy default, the run uses the build specified in the default run configuration for the actor (typically latest)." + } + ] + }, + "defaultValue": "None", "flags": { - "keyword-only": "true" + "isOptional": true, + "keyword-only": true }, + "id": 124, + "kind": 32768, + "kindString": "Parameter", + "name": "build", "type": { + "name": "Optional", "type": "reference", - "name": "Optional[str]" - }, - "defaultValue": "None" + "typeArguments": [ + { + "type": "reference", + "name": "str" + } + ] + } }, { - "id": 99, - "name": "memory_mbytes", - "kind": 32768, - "kindString": "Parameter", + "comment": { + "summary": [ + { + "kind": "text", + "text": "Memory limit for the run, in megabytes.\nBy default, the run uses a memory limit specified in the default run configuration for the actor." + } + ] + }, + "defaultValue": "None", "flags": { - "keyword-only": "true" + "isOptional": true, + "keyword-only": true }, + "id": 125, + "kind": 32768, + "kindString": "Parameter", + "name": "memory_mbytes", "type": { + "name": "Optional", "type": "reference", - "name": "Optional[int]" - }, - "defaultValue": "None" + "typeArguments": [ + { + "type": "reference", + "name": "int" + } + ] + } }, { - "id": 100, - "name": "timeout_secs", - "kind": 32768, - "kindString": "Parameter", + "comment": { + "summary": [ + { + "kind": "text", + "text": "Optional timeout for the run, in seconds.\nBy default, the run uses timeout specified in the default run configuration for the actor." + } + ] + }, + "defaultValue": "None", "flags": { - "keyword-only": "true" + "isOptional": true, + "keyword-only": true }, + "id": 126, + "kind": 32768, + "kindString": "Parameter", + "name": "timeout_secs", "type": { + "name": "Optional", "type": "reference", - "name": "Optional[int]" - }, - "defaultValue": "None" + "typeArguments": [ + { + "type": "reference", + "name": "int" + } + ] + } }, { - "id": 101, - "name": "webhooks", - "kind": 32768, - "kindString": "Parameter", + "comment": { + "summary": [ + { + "kind": "text", + "text": "Optional webhooks (https://docs.apify.com/webhooks) associated with the actor run,\nwhich can be used to receive a notification, e.g. when the actor finished or failed.\nIf you already have a webhook set up for the actor, you do not have to add it again here." + } + ] + }, + "defaultValue": "None", "flags": { - "keyword-only": "true" + "isOptional": true, + "keyword-only": true }, + "id": 127, + "kind": 32768, + "kindString": "Parameter", + "name": "webhooks", "type": { + "name": "Optional", "type": "reference", - "name": "Optional[List[Dict]]" - }, - "defaultValue": "None" + "typeArguments": [ + { + "type": "reference", + "name": "List", + "typeArguments": [ + { + "type": "reference", + "name": "Dict" + } + ] + } + ] + } }, { - "id": 102, - "name": "wait_secs", - "kind": 32768, - "kindString": "Parameter", + "comment": { + "summary": [ + { + "kind": "text", + "text": "The maximum number of seconds the server waits for the run to finish. If not provided, waits indefinitely.\n" + } + ] + }, + "defaultValue": "None", "flags": { - "keyword-only": "true" + "isOptional": true, + "keyword-only": true }, + "id": 128, + "kind": 32768, + "kindString": "Parameter", + "name": "wait_secs", "type": { + "name": "Optional", "type": "reference", - "name": "Optional[int]" - }, - "defaultValue": "None" + "typeArguments": [ + { + "type": "reference", + "name": "int" + } + ] + } } ], - "modifiers": [ - "async" - ] + "type": { + "name": "Optional", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "Dict" + } + ] + } } ] }, { - "id": 104, - "name": "call_task", - "module": "actor", "kind": 2048, "kindString": "Method", - "flags": {}, + "children": [], "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Start an actor task on the Apify Platform and wait for it to finish before returning.\\n\\nIt waits indefinitely, unless the wait_secs argument is provided.\\n\\nNote that an actor task is a saved input configuration and options for an actor.\\nIf you want to run an actor directly rather than an actor task, please use the `Actor.call`\\n\\nArgs:\\n task_id (str): The ID of the actor to be run.\\n task_input (Any, optional): Overrides the input to pass to the actor run.\\n token (str, optional): The Apify API token to use for this request (defaults to the `APIFY_TOKEN` environment variable).\\n content_type (str, optional): The content type of the input.\\n build (str, optional): Specifies the actor build to run. It can be either a build tag or build number.\\n By default, the run uses the build specified in the default run configuration for the actor (typically latest).\\n memory_mbytes (int, optional): Memory limit for the run, in megabytes.\\n By default, the run uses a memory limit specified in the default run configuration for the actor.\\n timeout_secs (int, optional): Optional timeout for the run, in seconds.\\n By default, the run uses timeout specified in the default run configuration for the actor.\\n webhooks (list, optional): Optional webhooks (https://docs.apify.com/webhooks) associated with the actor run,\\n which can be used to receive a notification, e.g. when the actor finished or failed.\\n If you already have a webhook set up for the actor, you do not have to add it again here.\\n wait_secs (int, optional): The maximum number of seconds the server waits for the run to finish. If not provided, waits indefinitely.\\n\\nReturns:\\n dict: Info about the started actor run\"]}" + "text": "Start an actor task on the Apify Platform and wait for it to finish before returning.\n\nIt waits indefinitely, unless the wait_secs argument is provided.\n\nNote that an actor task is a saved input configuration and options for an actor.\nIf you want to run an actor directly rather than an actor task, please use the `Actor.call`\n" } ] }, - "children": [], + "decorations": [ + { + "name": "classmethod" + } + ], + "flags": {}, "groups": [], + "id": 129, + "module": "actor", + "name": "call_task", + "parsedDocstring": { + "text": "Start an actor task on the Apify Platform and wait for it to finish before returning.\n\nIt waits indefinitely, unless the wait_secs argument is provided.\n\nNote that an actor task is a saved input configuration and options for an actor.\nIf you want to run an actor directly rather than an actor task, please use the `Actor.call`\n", + "args": { + "task_id": "The ID of the actor to be run.", + "task_input": "Overrides the input to pass to the actor run.", + "token": "The Apify API token to use for this request (defaults to the `APIFY_TOKEN` environment variable).", + "content_type": "The content type of the input.", + "build": "Specifies the actor build to run. It can be either a build tag or build number.\nBy default, the run uses the build specified in the default run configuration for the actor (typically latest).", + "memory_mbytes": "Memory limit for the run, in megabytes.\nBy default, the run uses a memory limit specified in the default run configuration for the actor.", + "timeout_secs": "Optional timeout for the run, in seconds.\nBy default, the run uses timeout specified in the default run configuration for the actor.", + "webhooks": "Optional webhooks (https://docs.apify.com/webhooks) associated with the actor run,\nwhich can be used to receive a notification, e.g. when the actor finished or failed.\nIf you already have a webhook set up for the actor, you do not have to add it again here.", + "wait_secs": "The maximum number of seconds the server waits for the run to finish. If not provided, waits indefinitely.\n" + }, + "returns": "dict: Info about the started actor run" + }, "sources": [ { - "filename": "src/apify/actor.py", - "line": 959, "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v0.2.0/src/apify/actor.py#L959" + "fileName": "/src/apify/actor.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 959 } ], + "type": { + "name": "Undefined", + "type": "reference" + }, "signatures": [ { - "id": 113, - "name": "call_task", - "kind": 4096, - "kindString": "Call signature", - "flags": {}, "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "dict: Info about the started actor run" + } + ], + "tag": "@returns" + } + ], "summary": [ { "kind": "text", - "text": "{\"content\": [\"Start an actor task on the Apify Platform and wait for it to finish before returning.\\n\\nIt waits indefinitely, unless the wait_secs argument is provided.\\n\\nNote that an actor task is a saved input configuration and options for an actor.\\nIf you want to run an actor directly rather than an actor task, please use the `Actor.call`\\n\\nArgs:\\n task_id (str): The ID of the actor to be run.\\n task_input (Any, optional): Overrides the input to pass to the actor run.\\n token (str, optional): The Apify API token to use for this request (defaults to the `APIFY_TOKEN` environment variable).\\n content_type (str, optional): The content type of the input.\\n build (str, optional): Specifies the actor build to run. It can be either a build tag or build number.\\n By default, the run uses the build specified in the default run configuration for the actor (typically latest).\\n memory_mbytes (int, optional): Memory limit for the run, in megabytes.\\n By default, the run uses a memory limit specified in the default run configuration for the actor.\\n timeout_secs (int, optional): Optional timeout for the run, in seconds.\\n By default, the run uses timeout specified in the default run configuration for the actor.\\n webhooks (list, optional): Optional webhooks (https://docs.apify.com/webhooks) associated with the actor run,\\n which can be used to receive a notification, e.g. when the actor finished or failed.\\n If you already have a webhook set up for the actor, you do not have to add it again here.\\n wait_secs (int, optional): The maximum number of seconds the server waits for the run to finish. If not provided, waits indefinitely.\\n\\nReturns:\\n dict: Info about the started actor run\"]}" + "text": "Start an actor task on the Apify Platform and wait for it to finish before returning.\n\nIt waits indefinitely, unless the wait_secs argument is provided.\n\nNote that an actor task is a saved input configuration and options for an actor.\nIf you want to run an actor directly rather than an actor task, please use the `Actor.call`\n" } ] }, - "type": { - "type": "reference", - "name": "Optional[Dict]" - }, + "flags": {}, + "id": 130, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "call_task", "parameters": [ { - "id": 105, - "name": "task_id", + "comment": { + "summary": [ + { + "kind": "text", + "text": "The ID of the actor to be run." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 131, "kind": 32768, "kindString": "Parameter", - "flags": {}, + "name": "task_id", "type": { - "type": "reference", - "name": "str" + "name": "str", + "type": "reference" } }, { - "id": 106, - "name": "task_input", + "comment": { + "summary": [ + { + "kind": "text", + "text": "Overrides the input to pass to the actor run." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": false + }, + "id": 132, "kind": 32768, "kindString": "Parameter", - "flags": {}, + "name": "task_input", "type": { + "name": "Optional", "type": "reference", - "name": "Optional[Dict[str, Any]]" - }, - "defaultValue": "None" + "typeArguments": [ + { + "type": "reference", + "name": "Dict", + "typeArguments": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "reference", + "name": "Any" + } + ] + } + ] + } }, { - "id": 107, - "name": "build", - "kind": 32768, - "kindString": "Parameter", + "comment": { + "summary": [ + { + "kind": "text", + "text": "Specifies the actor build to run. It can be either a build tag or build number.\nBy default, the run uses the build specified in the default run configuration for the actor (typically latest)." + } + ] + }, + "defaultValue": "None", "flags": { - "keyword-only": "true" + "isOptional": true, + "keyword-only": true }, + "id": 133, + "kind": 32768, + "kindString": "Parameter", + "name": "build", "type": { + "name": "Optional", "type": "reference", - "name": "Optional[str]" - }, - "defaultValue": "None" + "typeArguments": [ + { + "type": "reference", + "name": "str" + } + ] + } }, { - "id": 108, - "name": "memory_mbytes", - "kind": 32768, - "kindString": "Parameter", + "comment": { + "summary": [ + { + "kind": "text", + "text": "Memory limit for the run, in megabytes.\nBy default, the run uses a memory limit specified in the default run configuration for the actor." + } + ] + }, + "defaultValue": "None", "flags": { - "keyword-only": "true" + "isOptional": true, + "keyword-only": true }, + "id": 134, + "kind": 32768, + "kindString": "Parameter", + "name": "memory_mbytes", "type": { + "name": "Optional", "type": "reference", - "name": "Optional[int]" - }, - "defaultValue": "None" + "typeArguments": [ + { + "type": "reference", + "name": "int" + } + ] + } }, { - "id": 109, - "name": "timeout_secs", - "kind": 32768, - "kindString": "Parameter", + "comment": { + "summary": [ + { + "kind": "text", + "text": "Optional timeout for the run, in seconds.\nBy default, the run uses timeout specified in the default run configuration for the actor." + } + ] + }, + "defaultValue": "None", "flags": { - "keyword-only": "true" + "isOptional": true, + "keyword-only": true }, + "id": 135, + "kind": 32768, + "kindString": "Parameter", + "name": "timeout_secs", "type": { + "name": "Optional", "type": "reference", - "name": "Optional[int]" - }, - "defaultValue": "None" + "typeArguments": [ + { + "type": "reference", + "name": "int" + } + ] + } }, { - "id": 110, - "name": "webhooks", - "kind": 32768, - "kindString": "Parameter", + "comment": { + "summary": [ + { + "kind": "text", + "text": "Optional webhooks (https://docs.apify.com/webhooks) associated with the actor run,\nwhich can be used to receive a notification, e.g. when the actor finished or failed.\nIf you already have a webhook set up for the actor, you do not have to add it again here." + } + ] + }, + "defaultValue": "None", "flags": { - "keyword-only": "true" + "isOptional": true, + "keyword-only": true }, + "id": 136, + "kind": 32768, + "kindString": "Parameter", + "name": "webhooks", "type": { + "name": "Optional", "type": "reference", - "name": "Optional[List[Dict]]" - }, - "defaultValue": "None" + "typeArguments": [ + { + "type": "reference", + "name": "List", + "typeArguments": [ + { + "type": "reference", + "name": "Dict" + } + ] + } + ] + } }, { - "id": 111, - "name": "wait_secs", - "kind": 32768, - "kindString": "Parameter", + "comment": { + "summary": [ + { + "kind": "text", + "text": "The maximum number of seconds the server waits for the run to finish. If not provided, waits indefinitely.\n" + } + ] + }, + "defaultValue": "None", "flags": { - "keyword-only": "true" + "isOptional": true, + "keyword-only": true }, + "id": 137, + "kind": 32768, + "kindString": "Parameter", + "name": "wait_secs", "type": { + "name": "Optional", "type": "reference", - "name": "Optional[int]" - }, - "defaultValue": "None" + "typeArguments": [ + { + "type": "reference", + "name": "int" + } + ] + } }, { - "id": 112, - "name": "token", - "kind": 32768, - "kindString": "Parameter", + "comment": { + "summary": [ + { + "kind": "text", + "text": "The Apify API token to use for this request (defaults to the `APIFY_TOKEN` environment variable)." + } + ] + }, + "defaultValue": "None", "flags": { - "keyword-only": "true" + "isOptional": true, + "keyword-only": true }, + "id": 138, + "kind": 32768, + "kindString": "Parameter", + "name": "token", "type": { + "name": "Optional", "type": "reference", - "name": "Optional[str]" - }, - "defaultValue": "None" + "typeArguments": [ + { + "type": "reference", + "name": "str" + } + ] + } } ], - "modifiers": [ - "async" - ] + "type": { + "name": "Optional", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "Dict" + } + ] + } } ] }, { - "id": 114, - "name": "metamorph", - "module": "actor", "kind": 2048, "kindString": "Method", - "flags": {}, + "children": [], "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Transform this actor run to an actor run of a different actor.\\n\\nThe platform stops the current actor container and starts a new container with the new actor instead.\\nAll the default storages are preserved,\\nand the new input is stored under the `INPUT-METAMORPH-1` key in the same default key-value store.\\n\\nArgs:\\n target_actor_id (str): ID of the target actor that the run should be transformed into\\n run_input (Any, optional): The input to pass to the new run.\\n target_actor_build (str, optional): The build of the target actor. It can be either a build tag or build number.\\n By default, the run uses the build specified in the default run configuration for the target actor (typically the latest build).\\n content_type (str, optional): The content type of the input.\\n custom_after_sleep_millis (int, optional): How long to sleep for after the metamorph, to wait for the container to be stopped.\\n\\nReturns:\\n dict: The actor run data.\"]}" + "text": "Transform this actor run to an actor run of a different actor.\n\nThe platform stops the current actor container and starts a new container with the new actor instead.\nAll the default storages are preserved,\nand the new input is stored under the `INPUT-METAMORPH-1` key in the same default key-value store.\n" } ] }, - "children": [], + "decorations": [ + { + "name": "classmethod" + } + ], + "flags": {}, "groups": [], + "id": 139, + "module": "actor", + "name": "metamorph", + "parsedDocstring": { + "text": "Transform this actor run to an actor run of a different actor.\n\nThe platform stops the current actor container and starts a new container with the new actor instead.\nAll the default storages are preserved,\nand the new input is stored under the `INPUT-METAMORPH-1` key in the same default key-value store.\n", + "args": { + "target_actor_id": "ID of the target actor that the run should be transformed into", + "run_input": "The input to pass to the new run.", + "target_actor_build": "The build of the target actor. It can be either a build tag or build number.\nBy default, the run uses the build specified in the default run configuration for the target actor (typically the latest build).", + "content_type": "The content type of the input.", + "custom_after_sleep_millis": "How long to sleep for after the metamorph, to wait for the container to be stopped.\n" + }, + "returns": "dict: The actor run data." + }, "sources": [ { - "filename": "src/apify/actor.py", - "line": 1037, "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v0.2.0/src/apify/actor.py#L1037" + "fileName": "/src/apify/actor.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 1037 } ], + "type": { + "name": "Undefined", + "type": "reference" + }, "signatures": [ { - "id": 120, - "name": "metamorph", - "kind": 4096, - "kindString": "Call signature", - "flags": {}, "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "dict: The actor run data." + } + ], + "tag": "@returns" + } + ], "summary": [ { "kind": "text", - "text": "{\"content\": [\"Transform this actor run to an actor run of a different actor.\\n\\nThe platform stops the current actor container and starts a new container with the new actor instead.\\nAll the default storages are preserved,\\nand the new input is stored under the `INPUT-METAMORPH-1` key in the same default key-value store.\\n\\nArgs:\\n target_actor_id (str): ID of the target actor that the run should be transformed into\\n run_input (Any, optional): The input to pass to the new run.\\n target_actor_build (str, optional): The build of the target actor. It can be either a build tag or build number.\\n By default, the run uses the build specified in the default run configuration for the target actor (typically the latest build).\\n content_type (str, optional): The content type of the input.\\n custom_after_sleep_millis (int, optional): How long to sleep for after the metamorph, to wait for the container to be stopped.\\n\\nReturns:\\n dict: The actor run data.\"]}" + "text": "Transform this actor run to an actor run of a different actor.\n\nThe platform stops the current actor container and starts a new container with the new actor instead.\nAll the default storages are preserved,\nand the new input is stored under the `INPUT-METAMORPH-1` key in the same default key-value store.\n" } ] }, - "type": { - "type": "reference", - "name": "None" - }, + "flags": {}, + "id": 140, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "metamorph", "parameters": [ { - "id": 115, - "name": "target_actor_id", + "comment": { + "summary": [ + { + "kind": "text", + "text": "ID of the target actor that the run should be transformed into" + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 141, "kind": 32768, "kindString": "Parameter", - "flags": {}, + "name": "target_actor_id", "type": { - "type": "reference", - "name": "str" + "name": "str", + "type": "reference" } }, { - "id": 116, - "name": "run_input", + "comment": { + "summary": [ + { + "kind": "text", + "text": "The input to pass to the new run." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": false + }, + "id": 142, "kind": 32768, "kindString": "Parameter", - "flags": {}, + "name": "run_input", "type": { + "name": "Optional", "type": "reference", - "name": "Optional[Any]" - }, - "defaultValue": "None" + "typeArguments": [ + { + "type": "reference", + "name": "Any" + } + ] + } }, { - "id": 117, - "name": "target_actor_build", - "kind": 32768, - "kindString": "Parameter", + "comment": { + "summary": [ + { + "kind": "text", + "text": "The build of the target actor. It can be either a build tag or build number.\nBy default, the run uses the build specified in the default run configuration for the target actor (typically the latest build)." + } + ] + }, + "defaultValue": "None", "flags": { - "keyword-only": "true" + "isOptional": true, + "keyword-only": true }, + "id": 143, + "kind": 32768, + "kindString": "Parameter", + "name": "target_actor_build", "type": { + "name": "Optional", "type": "reference", - "name": "Optional[str]" - }, - "defaultValue": "None" + "typeArguments": [ + { + "type": "reference", + "name": "str" + } + ] + } }, { - "id": 118, - "name": "content_type", - "kind": 32768, - "kindString": "Parameter", + "comment": { + "summary": [ + { + "kind": "text", + "text": "The content type of the input." + } + ] + }, + "defaultValue": "None", "flags": { - "keyword-only": "true" + "isOptional": true, + "keyword-only": true }, + "id": 144, + "kind": 32768, + "kindString": "Parameter", + "name": "content_type", "type": { + "name": "Optional", "type": "reference", - "name": "Optional[str]" - }, - "defaultValue": "None" + "typeArguments": [ + { + "type": "reference", + "name": "str" + } + ] + } }, { - "id": 119, - "name": "custom_after_sleep_millis", - "kind": 32768, - "kindString": "Parameter", + "comment": { + "summary": [ + { + "kind": "text", + "text": "How long to sleep for after the metamorph, to wait for the container to be stopped.\n" + } + ] + }, + "defaultValue": "None", "flags": { - "keyword-only": "true" + "isOptional": true, + "keyword-only": true }, + "id": 145, + "kind": 32768, + "kindString": "Parameter", + "name": "custom_after_sleep_millis", "type": { + "name": "Optional", "type": "reference", - "name": "Optional[int]" - }, - "defaultValue": "None" + "typeArguments": [ + { + "type": "reference", + "name": "int" + } + ] + } } ], - "modifiers": [ - "async" - ] + "type": { + "name": "None", + "type": "literal", + "value": null + } } ] }, { - "id": 121, - "name": "reboot", - "module": "actor", "kind": 2048, "kindString": "Method", - "flags": {}, + "children": [], "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Internally reboot this actor.\\n\\nThe system stops the current container and starts a new one, with the same run ID and default storages.\\n\\nArgs:\\n event_listeners_timeout_secs (int, optional): How long should the actor wait for actor event listeners to finish before exiting\"]}" + "text": "Internally reboot this actor.\n\nThe system stops the current container and starts a new one, with the same run ID and default storages.\n" } ] }, - "children": [], + "decorations": [ + { + "name": "classmethod" + } + ], + "flags": {}, "groups": [], + "id": 146, + "module": "actor", + "name": "reboot", + "parsedDocstring": { + "text": "Internally reboot this actor.\n\nThe system stops the current container and starts a new one, with the same run ID and default storages.\n", + "args": { + "event_listeners_timeout_secs": "How long should the actor wait for actor event listeners to finish before exiting" + } + }, "sources": [ { - "filename": "src/apify/actor.py", - "line": 1103, "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v0.2.0/src/apify/actor.py#L1103" + "fileName": "/src/apify/actor.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 1103 } ], + "type": { + "name": "Undefined", + "type": "reference" + }, "signatures": [ { - "id": 123, - "name": "reboot", - "kind": 4096, - "kindString": "Call signature", - "flags": {}, "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Internally reboot this actor.\\n\\nThe system stops the current container and starts a new one, with the same run ID and default storages.\\n\\nArgs:\\n event_listeners_timeout_secs (int, optional): How long should the actor wait for actor event listeners to finish before exiting\"]}" + "text": "Internally reboot this actor.\n\nThe system stops the current container and starts a new one, with the same run ID and default storages.\n" } ] }, - "type": { - "type": "reference", - "name": "None" - }, + "flags": {}, + "id": 147, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "reboot", "parameters": [ { - "id": 122, - "name": "event_listeners_timeout_secs", - "kind": 32768, - "kindString": "Parameter", + "comment": { + "summary": [ + { + "kind": "text", + "text": "How long should the actor wait for actor event listeners to finish before exiting" + } + ] + }, + "defaultValue": "EVENT_LISTENERS_TIMEOUT_SECS", "flags": { - "keyword-only": "true" + "isOptional": true, + "keyword-only": true }, + "id": 148, + "kind": 32768, + "kindString": "Parameter", + "name": "event_listeners_timeout_secs", "type": { + "name": "Optional", "type": "reference", - "name": "Optional[int]" - }, - "defaultValue": "EVENT_LISTENERS_TIMEOUT_SECS" + "typeArguments": [ + { + "type": "reference", + "name": "int" + } + ] + } } ], - "modifiers": [ - "async" - ] + "type": { + "name": "None", + "type": "literal", + "value": null + } } ] }, { - "id": 124, - "name": "add_webhook", - "module": "actor", "kind": 2048, "kindString": "Method", - "flags": {}, + "children": [], "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Create an ad-hoc webhook for the current actor run.\\n\\nThis webhook lets you receive a notification when the actor run finished or failed.\\n\\nNote that webhooks are only supported for actors running on the Apify platform.\\nWhen running the actor locally, the function will print a warning and have no effect.\\n\\nFor more information about Apify actor webhooks, please see the [documentation](https://docs.apify.com/webhooks).\\n\\nArgs:\\n event_types (list of WebhookEventType): List of event types that should trigger the webhook. At least one is required.\\n request_url (str): URL that will be invoked once the webhook is triggered.\\n payload_template (str, optional): Specification of the payload that will be sent to request_url\\n ignore_ssl_errors (bool, optional): Whether the webhook should ignore SSL errors returned by request_url\\n do_not_retry (bool, optional): Whether the webhook should retry sending the payload to request_url upon\\n failure.\\n idempotency_key (str, optional): A unique identifier of a webhook. You can use it to ensure that you won't\\n create the same webhook multiple times.\\n\\nReturns:\\n dict: The created webhook\"]}" + "text": "Create an ad-hoc webhook for the current actor run.\n\nThis webhook lets you receive a notification when the actor run finished or failed.\n\nNote that webhooks are only supported for actors running on the Apify platform.\nWhen running the actor locally, the function will print a warning and have no effect.\n\nFor more information about Apify actor webhooks, please see the [documentation](https://docs.apify.com/webhooks).\n" } ] }, - "children": [], + "decorations": [ + { + "name": "classmethod" + } + ], + "flags": {}, "groups": [], + "id": 149, + "module": "actor", + "name": "add_webhook", + "parsedDocstring": { + "text": "Create an ad-hoc webhook for the current actor run.\n\nThis webhook lets you receive a notification when the actor run finished or failed.\n\nNote that webhooks are only supported for actors running on the Apify platform.\nWhen running the actor locally, the function will print a warning and have no effect.\n\nFor more information about Apify actor webhooks, please see the [documentation](https://docs.apify.com/webhooks).\n", + "args": { + "event_types": "List of event types that should trigger the webhook. At least one is required.", + "request_url": "URL that will be invoked once the webhook is triggered.", + "payload_template": "Specification of the payload that will be sent to request_url", + "ignore_ssl_errors": "Whether the webhook should ignore SSL errors returned by request_url", + "do_not_retry": "Whether the webhook should retry sending the payload to request_url upon\nfailure.", + "idempotency_key": "A unique identifier of a webhook. You can use it to ensure that you won't\ncreate the same webhook multiple times.\n" + }, + "returns": "dict: The created webhook" + }, "sources": [ { - "filename": "src/apify/actor.py", - "line": 1141, "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v0.2.0/src/apify/actor.py#L1141" + "fileName": "/src/apify/actor.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 1141 } ], + "type": { + "name": "Undefined", + "type": "reference" + }, "signatures": [ { - "id": 131, - "name": "add_webhook", - "kind": 4096, - "kindString": "Call signature", - "flags": {}, "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "dict: The created webhook" + } + ], + "tag": "@returns" + } + ], "summary": [ { "kind": "text", - "text": "{\"content\": [\"Create an ad-hoc webhook for the current actor run.\\n\\nThis webhook lets you receive a notification when the actor run finished or failed.\\n\\nNote that webhooks are only supported for actors running on the Apify platform.\\nWhen running the actor locally, the function will print a warning and have no effect.\\n\\nFor more information about Apify actor webhooks, please see the [documentation](https://docs.apify.com/webhooks).\\n\\nArgs:\\n event_types (list of WebhookEventType): List of event types that should trigger the webhook. At least one is required.\\n request_url (str): URL that will be invoked once the webhook is triggered.\\n payload_template (str, optional): Specification of the payload that will be sent to request_url\\n ignore_ssl_errors (bool, optional): Whether the webhook should ignore SSL errors returned by request_url\\n do_not_retry (bool, optional): Whether the webhook should retry sending the payload to request_url upon\\n failure.\\n idempotency_key (str, optional): A unique identifier of a webhook. You can use it to ensure that you won't\\n create the same webhook multiple times.\\n\\nReturns:\\n dict: The created webhook\"]}" + "text": "Create an ad-hoc webhook for the current actor run.\n\nThis webhook lets you receive a notification when the actor run finished or failed.\n\nNote that webhooks are only supported for actors running on the Apify platform.\nWhen running the actor locally, the function will print a warning and have no effect.\n\nFor more information about Apify actor webhooks, please see the [documentation](https://docs.apify.com/webhooks).\n" } ] }, - "type": { - "type": "reference", - "name": "Dict" - }, + "flags": {}, + "id": 150, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "add_webhook", "parameters": [ { - "id": 125, - "name": "event_types", - "kind": 32768, - "kindString": "Parameter", + "comment": { + "summary": [ + { + "kind": "text", + "text": "List of event types that should trigger the webhook. At least one is required." + } + ] + }, "flags": { - "keyword-only": "true" + "isOptional": false, + "keyword-only": true }, + "id": 151, + "kind": 32768, + "kindString": "Parameter", + "name": "event_types", "type": { + "name": "List", "type": "reference", - "name": "List[WebhookEventType]" + "typeArguments": [ + { + "type": "reference", + "name": "WebhookEventType" + } + ] } }, { - "id": 126, - "name": "request_url", - "kind": 32768, - "kindString": "Parameter", + "comment": { + "summary": [ + { + "kind": "text", + "text": "URL that will be invoked once the webhook is triggered." + } + ] + }, "flags": { - "keyword-only": "true" + "isOptional": false, + "keyword-only": true }, + "id": 152, + "kind": 32768, + "kindString": "Parameter", + "name": "request_url", "type": { - "type": "reference", - "name": "str" + "name": "str", + "type": "reference" } }, { - "id": 127, - "name": "payload_template", - "kind": 32768, - "kindString": "Parameter", + "comment": { + "summary": [ + { + "kind": "text", + "text": "Specification of the payload that will be sent to request_url" + } + ] + }, + "defaultValue": "None", "flags": { - "keyword-only": "true" + "isOptional": true, + "keyword-only": true }, + "id": 153, + "kind": 32768, + "kindString": "Parameter", + "name": "payload_template", "type": { + "name": "Optional", "type": "reference", - "name": "Optional[str]" - }, - "defaultValue": "None" + "typeArguments": [ + { + "type": "reference", + "name": "str" + } + ] + } }, { - "id": 128, - "name": "ignore_ssl_errors", - "kind": 32768, - "kindString": "Parameter", + "comment": { + "summary": [ + { + "kind": "text", + "text": "Whether the webhook should ignore SSL errors returned by request_url" + } + ] + }, + "defaultValue": "None", "flags": { - "keyword-only": "true" + "isOptional": true, + "keyword-only": true }, + "id": 154, + "kind": 32768, + "kindString": "Parameter", + "name": "ignore_ssl_errors", "type": { + "name": "Optional", "type": "reference", - "name": "Optional[bool]" - }, - "defaultValue": "None" + "typeArguments": [ + { + "type": "reference", + "name": "bool" + } + ] + } }, { - "id": 129, - "name": "do_not_retry", - "kind": 32768, - "kindString": "Parameter", + "comment": { + "summary": [ + { + "kind": "text", + "text": "Whether the webhook should retry sending the payload to request_url upon\nfailure." + } + ] + }, + "defaultValue": "None", "flags": { - "keyword-only": "true" + "isOptional": true, + "keyword-only": true }, + "id": 155, + "kind": 32768, + "kindString": "Parameter", + "name": "do_not_retry", "type": { + "name": "Optional", "type": "reference", - "name": "Optional[bool]" - }, - "defaultValue": "None" + "typeArguments": [ + { + "type": "reference", + "name": "bool" + } + ] + } }, { - "id": 130, - "name": "idempotency_key", - "kind": 32768, - "kindString": "Parameter", + "comment": { + "summary": [ + { + "kind": "text", + "text": "A unique identifier of a webhook. You can use it to ensure that you won't\ncreate the same webhook multiple times.\n" + } + ] + }, + "defaultValue": "None", "flags": { - "keyword-only": "true" + "isOptional": true, + "keyword-only": true }, + "id": 156, + "kind": 32768, + "kindString": "Parameter", + "name": "idempotency_key", "type": { + "name": "Optional", "type": "reference", - "name": "Optional[str]" - }, - "defaultValue": "None" + "typeArguments": [ + { + "type": "reference", + "name": "str" + } + ] + } } ], - "modifiers": [ - "async" - ] + "type": { + "name": "Dict", + "type": "reference" + } } ] }, { - "id": 132, - "name": "set_status_message", - "module": "actor", "kind": 2048, "kindString": "Method", - "flags": {}, + "children": [], "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Set the status message for the current actor run.\\n\\nArgs:\\n status_message (str): The status message to set to the run.\\n\\nReturns:\\n dict: The updated actor run object\"]}" + "text": "Set the status message for the current actor run.\n" } ] }, - "children": [], + "decorations": [ + { + "name": "classmethod" + } + ], + "flags": {}, "groups": [], + "id": 157, + "module": "actor", + "name": "set_status_message", + "parsedDocstring": { + "text": "Set the status message for the current actor run.\n", + "args": { + "status_message": "The status message to set to the run.\n" + }, + "returns": "dict: The updated actor run object" + }, "sources": [ { - "filename": "src/apify/actor.py", - "line": 1212, "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v0.2.0/src/apify/actor.py#L1212" + "fileName": "/src/apify/actor.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 1212 } ], + "type": { + "name": "Undefined", + "type": "reference" + }, "signatures": [ { - "id": 134, - "name": "set_status_message", - "kind": 4096, - "kindString": "Call signature", - "flags": {}, "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "dict: The updated actor run object" + } + ], + "tag": "@returns" + } + ], "summary": [ { "kind": "text", - "text": "{\"content\": [\"Set the status message for the current actor run.\\n\\nArgs:\\n status_message (str): The status message to set to the run.\\n\\nReturns:\\n dict: The updated actor run object\"]}" + "text": "Set the status message for the current actor run.\n" } ] }, - "type": { - "type": "reference", - "name": "Optional[Dict]" - }, + "flags": {}, + "id": 158, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "set_status_message", "parameters": [ { - "id": 133, - "name": "status_message", + "comment": { + "summary": [ + { + "kind": "text", + "text": "The status message to set to the run.\n" + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 159, "kind": 32768, "kindString": "Parameter", - "flags": {}, + "name": "status_message", "type": { - "type": "reference", - "name": "str" + "name": "str", + "type": "reference" } } ], - "modifiers": [ - "async" - ] + "type": { + "name": "Optional", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "Dict" + } + ] + } } ] }, { - "id": 135, - "name": "create_proxy_configuration", - "module": "actor", "kind": 2048, "kindString": "Method", - "flags": {}, + "children": [], "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Create a ProxyConfiguration object with the passed proxy configuration.\\n\\nConfigures connection to a proxy server with the provided options.\\nProxy servers are used to prevent target websites from blocking your crawlers based on IP address rate limits or blacklists.\\n\\nFor more details and code examples, see the `ProxyConfiguration` class.\\n\\nArgs:\\n password (str, optional): Password for the Apify Proxy. If not provided, will use os.environ['APIFY_PROXY_PASSWORD'], if available.\\n groups (list of str, optional): Proxy groups which the Apify Proxy should use, if provided.\\n country_code (str, optional): Country which the Apify Proxy should use, if provided.\\n proxy_urls (list of str, optional): Custom proxy server URLs which should be rotated through.\\n new_url_function (Callable, optional): Function which returns a custom proxy URL to be used.\\n actor_proxy_input (dict, optional): Proxy configuration field from the actor input, if actor has such input field.\\n\\nReturns:\\n ProxyConfiguration, optional: ProxyConfiguration object with the passed configuration,\\n or None, if no proxy should be used based on the configuration.\"]}" + "text": "Create a ProxyConfiguration object with the passed proxy configuration.\n\nConfigures connection to a proxy server with the provided options.\nProxy servers are used to prevent target websites from blocking your crawlers based on IP address rate limits or blacklists.\n\nFor more details and code examples, see the `ProxyConfiguration` class.\n" } ] }, - "children": [], + "decorations": [ + { + "name": "classmethod" + } + ], + "flags": {}, "groups": [], + "id": 160, + "module": "actor", + "name": "create_proxy_configuration", + "parsedDocstring": { + "text": "Create a ProxyConfiguration object with the passed proxy configuration.\n\nConfigures connection to a proxy server with the provided options.\nProxy servers are used to prevent target websites from blocking your crawlers based on IP address rate limits or blacklists.\n\nFor more details and code examples, see the `ProxyConfiguration` class.\n", + "args": { + "password": "Password for the Apify Proxy. If not provided, will use os.environ['APIFY_PROXY_PASSWORD'], if available.", + "groups": "Proxy groups which the Apify Proxy should use, if provided.", + "country_code": "Country which the Apify Proxy should use, if provided.", + "proxy_urls": "Custom proxy server URLs which should be rotated through.", + "new_url_function": "Function which returns a custom proxy URL to be used.", + "actor_proxy_input": "Proxy configuration field from the actor input, if actor has such input field.\n" + }, + "returns": "ProxyConfiguration, optional: ProxyConfiguration object with the passed configuration,\nor None, if no proxy should be used based on the configuration." + }, "sources": [ { - "filename": "src/apify/actor.py", - "line": 1236, "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v0.2.0/src/apify/actor.py#L1236" + "fileName": "/src/apify/actor.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 1236 } ], + "type": { + "name": "Undefined", + "type": "reference" + }, "signatures": [ { - "id": 142, - "name": "create_proxy_configuration", - "kind": 4096, - "kindString": "Call signature", - "flags": {}, "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "ProxyConfiguration, optional: ProxyConfiguration object with the passed configuration,\nor None, if no proxy should be used based on the configuration." + } + ], + "tag": "@returns" + } + ], "summary": [ { "kind": "text", - "text": "{\"content\": [\"Create a ProxyConfiguration object with the passed proxy configuration.\\n\\nConfigures connection to a proxy server with the provided options.\\nProxy servers are used to prevent target websites from blocking your crawlers based on IP address rate limits or blacklists.\\n\\nFor more details and code examples, see the `ProxyConfiguration` class.\\n\\nArgs:\\n password (str, optional): Password for the Apify Proxy. If not provided, will use os.environ['APIFY_PROXY_PASSWORD'], if available.\\n groups (list of str, optional): Proxy groups which the Apify Proxy should use, if provided.\\n country_code (str, optional): Country which the Apify Proxy should use, if provided.\\n proxy_urls (list of str, optional): Custom proxy server URLs which should be rotated through.\\n new_url_function (Callable, optional): Function which returns a custom proxy URL to be used.\\n actor_proxy_input (dict, optional): Proxy configuration field from the actor input, if actor has such input field.\\n\\nReturns:\\n ProxyConfiguration, optional: ProxyConfiguration object with the passed configuration,\\n or None, if no proxy should be used based on the configuration.\"]}" + "text": "Create a ProxyConfiguration object with the passed proxy configuration.\n\nConfigures connection to a proxy server with the provided options.\nProxy servers are used to prevent target websites from blocking your crawlers based on IP address rate limits or blacklists.\n\nFor more details and code examples, see the `ProxyConfiguration` class.\n" } ] }, - "type": { - "type": "reference", - "name": "Optional[ProxyConfiguration]" - }, + "flags": {}, + "id": 161, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "create_proxy_configuration", "parameters": [ { - "id": 136, - "name": "password", - "kind": 32768, - "kindString": "Parameter", + "comment": { + "summary": [ + { + "kind": "text", + "text": "Password for the Apify Proxy. If not provided, will use os.environ['APIFY_PROXY_PASSWORD'], if available." + } + ] + }, + "defaultValue": "None", "flags": { - "keyword-only": "true" + "isOptional": true, + "keyword-only": true }, + "id": 162, + "kind": 32768, + "kindString": "Parameter", + "name": "password", "type": { + "name": "Optional", "type": "reference", - "name": "Optional[str]" - }, - "defaultValue": "None" + "typeArguments": [ + { + "type": "reference", + "name": "str" + } + ] + } }, { - "id": 137, - "name": "groups", - "kind": 32768, - "kindString": "Parameter", + "comment": { + "summary": [ + { + "kind": "text", + "text": "Proxy groups which the Apify Proxy should use, if provided." + } + ] + }, + "defaultValue": "None", "flags": { - "keyword-only": "true" + "isOptional": true, + "keyword-only": true }, + "id": 163, + "kind": 32768, + "kindString": "Parameter", + "name": "groups", "type": { + "name": "Optional", "type": "reference", - "name": "Optional[List[str]]" - }, - "defaultValue": "None" + "typeArguments": [ + { + "type": "reference", + "name": "List", + "typeArguments": [ + { + "type": "reference", + "name": "str" + } + ] + } + ] + } }, { - "id": 138, - "name": "country_code", - "kind": 32768, - "kindString": "Parameter", + "comment": { + "summary": [ + { + "kind": "text", + "text": "Country which the Apify Proxy should use, if provided." + } + ] + }, + "defaultValue": "None", "flags": { - "keyword-only": "true" + "isOptional": true, + "keyword-only": true }, + "id": 164, + "kind": 32768, + "kindString": "Parameter", + "name": "country_code", "type": { + "name": "Optional", "type": "reference", - "name": "Optional[str]" - }, - "defaultValue": "None" + "typeArguments": [ + { + "type": "reference", + "name": "str" + } + ] + } }, { - "id": 139, - "name": "proxy_urls", - "kind": 32768, - "kindString": "Parameter", + "comment": { + "summary": [ + { + "kind": "text", + "text": "Custom proxy server URLs which should be rotated through." + } + ] + }, + "defaultValue": "None", "flags": { - "keyword-only": "true" + "isOptional": true, + "keyword-only": true }, + "id": 165, + "kind": 32768, + "kindString": "Parameter", + "name": "proxy_urls", "type": { + "name": "Optional", "type": "reference", - "name": "Optional[List[str]]" - }, - "defaultValue": "None" + "typeArguments": [ + { + "type": "reference", + "name": "List", + "typeArguments": [ + { + "type": "reference", + "name": "str" + } + ] + } + ] + } }, { - "id": 140, - "name": "new_url_function", - "kind": 32768, - "kindString": "Parameter", + "comment": { + "summary": [ + { + "kind": "text", + "text": "Function which returns a custom proxy URL to be used." + } + ] + }, + "defaultValue": "None", "flags": { - "keyword-only": "true" + "isOptional": true, + "keyword-only": true }, + "id": 166, + "kind": 32768, + "kindString": "Parameter", + "name": "new_url_function", "type": { + "name": "Optional", "type": "reference", - "name": "Optional[Union[Callable[[Optional[str]], str], Callable[[Optional[str]], Awaitable[str]]]]" - }, - "defaultValue": "None" + "typeArguments": [ + { + "type": "reference", + "name": "Union", + "typeArguments": [ + { + "type": "reference", + "name": "Callable", + "typeArguments": [ + { + "type": "reference", + "name": "[Optional[str]]" + }, + { + "type": "reference", + "name": "str" + } + ] + }, + { + "type": "reference", + "name": "Callable", + "typeArguments": [ + { + "type": "reference", + "name": "[Optional[str]]" + }, + { + "type": "reference", + "name": "Awaitable", + "typeArguments": [ + { + "type": "reference", + "name": "str" + } + ] + } + ] + } + ] + } + ] + } }, { - "id": 141, - "name": "actor_proxy_input", - "kind": 32768, - "kindString": "Parameter", + "comment": { + "summary": [ + { + "kind": "text", + "text": "Proxy configuration field from the actor input, if actor has such input field.\n" + } + ] + }, + "defaultValue": "None", "flags": { - "keyword-only": "true" + "isOptional": true, + "keyword-only": true }, + "id": 167, + "kind": 32768, + "kindString": "Parameter", + "name": "actor_proxy_input", "type": { + "name": "Optional", "type": "reference", - "name": "Optional[Dict]" - }, - "defaultValue": "None" + "typeArguments": [ + { + "type": "reference", + "name": "Dict" + } + ] + } } ], - "modifiers": [ - "async" - ] + "type": { + "name": "Optional", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "ProxyConfiguration", + "target": "286" + } + ] + } } ] } ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "The main class of the SDK, through which all the actor operations should be done." + } + ] + }, + "flags": {}, "groups": [ { - "title": "Constructors", - "children": [ - 2 - ] - }, - { - "title": "Methods", "children": [ - 5, - 8, - 11, - 14, - 17, - 19, - 23, - 27, - 30, - 37, + 35, + 113, + 149, + 118, + 129, + 160, + 44, + 48, + 100, + 80, + 82, 42, - 47, + 98, 52, + 139, 55, - 57, - 60, - 65, - 69, - 73, - 75, + 94, + 90, + 62, + 67, + 72, 77, - 88, - 93, - 104, - 114, - 121, - 124, - 132, - 135 - ] + 146, + 157, + 85, + 102 + ], + "title": "Methods" + }, + { + "children": [ + 38, + 39, + 40, + 41 + ], + "title": "Properties" } ], + "id": 34, + "module": "actor", + "name": "Actor", + "parsedDocstring": { + "text": "The main class of the SDK, through which all the actor operations should be done." + }, "sources": [ { - "filename": "src/apify/actor.py", - "line": 63, "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v0.2.0/src/apify/actor.py#L63" + "fileName": "/src/apify/actor.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 63 } - ] + ], + "type": { + "name": "Undefined", + "type": "reference" + } }, { - "id": 143, - "name": "Configuration", - "module": "config", - "kind": 128, - "kindString": "Class", + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, "flags": {}, + "groups": [], + "id": 168, + "module": "_crypto", + "name": "ENCRYPTION_KEY_LENGTH", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_crypto.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 13 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"A class for specifying the configuration of an actor.\\n\\nCan be used either globally via `Configuration.get_global_configuration()`,\\nor it can be specific to each `Actor` instance on the `actor.config` property.\"]}" + "text": "" } ] }, - "children": [ + "flags": {}, + "groups": [], + "id": 169, + "module": "_crypto", + "name": "ENCRYPTION_IV_LENGTH", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_crypto.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 14 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 170, + "module": "_crypto", + "name": "ENCRYPTION_AUTH_TAG_LENGTH", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_crypto.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 15 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Crate an instance of EventManager.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 171, + "module": "event_manager", + "name": "__init__", + "parsedDocstring": { + "text": "Crate an instance of EventManager.\n", + "args": { + "config": "The actor configuration to be used in this event manager." + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/event_manager.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 32 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ { - "id": 144, - "name": "__init__", - "module": "config", - "kind": 512, - "kindString": "Constructor", - "flags": {}, "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Create a `Configuration` instance.\\n\\nAll the parameters are loaded by default from environment variables when running on the Apify platform.\\nYou can override them here in the Configuration constructor, which might be useful for local testing of your actors.\\n\\nArgs:\\n api_base_url (str, optional): The URL of the Apify API.\\n This is the URL actually used for connecting to the API, so it can contain an IP address when running in a container on the platform.\\n api_public_base_url (str, optional): The public URL of the Apify API.\\n This will always contain the public URL of the API, even when running in a container on the platform.\\n Useful for generating shareable URLs to key-value store records or datasets.\\n container_port (int, optional): The port on which the container can listen for HTTP requests.\\n container_url (str, optional): The URL on which the container can listen for HTTP requests.\\n default_dataset_id (str, optional): The ID of the default dataset for the actor.\\n default_key_value_store_id (str, optional): The ID of the default key-value store for the actor.\\n default_request_queue_id (str, optional): The ID of the default request queue for the actor.\\n input_key (str, optional): The key of the input record in the actor's default key-value store\\n max_used_cpu_ratio (float, optional): The CPU usage above which the SYSTEM_INFO event will report the CPU is overloaded.\\n metamorph_after_sleep_millis (int, optional): How long should the actor sleep after calling metamorph.\\n persist_state_interval_millis (int, optional): How often should the actor emit the PERSIST_STATE event.\\n persist_storage (bool, optional): Whether the actor should persist its used storages to the filesystem when running locally.\\n proxy_hostname (str, optional): The hostname of Apify Proxy.\\n proxy_password (str, optional): The password for Apify Proxy.\\n proxy_port (str, optional): The port of Apify Proxy.\\n proxy_status_url (str, optional): The URL on which the Apify Proxy status page is available.\\n purge_on_start (str, optional): Whether the actor should purge its default storages on startup, when running locally.\\n token (str, optional): The API token for the Apify API this actor should use.\\n system_info_interval_millis (str, optional): How often should the actor emit the SYSTEM_INFO event when running locally.\"]}" + "text": "Crate an instance of EventManager.\n" } ] }, - "children": [], - "groups": [], - "sources": [ - { - "filename": "src/apify/config.py", - "line": 16, - "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v0.2.0/src/apify/config.py#L16" - } - ], - "signatures": [ + "flags": {}, + "id": 172, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "__init__", + "parameters": [ { - "id": 164, - "name": "__init__", - "kind": 4096, - "kindString": "Call signature", - "flags": {}, "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Create a `Configuration` instance.\\n\\nAll the parameters are loaded by default from environment variables when running on the Apify platform.\\nYou can override them here in the Configuration constructor, which might be useful for local testing of your actors.\\n\\nArgs:\\n api_base_url (str, optional): The URL of the Apify API.\\n This is the URL actually used for connecting to the API, so it can contain an IP address when running in a container on the platform.\\n api_public_base_url (str, optional): The public URL of the Apify API.\\n This will always contain the public URL of the API, even when running in a container on the platform.\\n Useful for generating shareable URLs to key-value store records or datasets.\\n container_port (int, optional): The port on which the container can listen for HTTP requests.\\n container_url (str, optional): The URL on which the container can listen for HTTP requests.\\n default_dataset_id (str, optional): The ID of the default dataset for the actor.\\n default_key_value_store_id (str, optional): The ID of the default key-value store for the actor.\\n default_request_queue_id (str, optional): The ID of the default request queue for the actor.\\n input_key (str, optional): The key of the input record in the actor's default key-value store\\n max_used_cpu_ratio (float, optional): The CPU usage above which the SYSTEM_INFO event will report the CPU is overloaded.\\n metamorph_after_sleep_millis (int, optional): How long should the actor sleep after calling metamorph.\\n persist_state_interval_millis (int, optional): How often should the actor emit the PERSIST_STATE event.\\n persist_storage (bool, optional): Whether the actor should persist its used storages to the filesystem when running locally.\\n proxy_hostname (str, optional): The hostname of Apify Proxy.\\n proxy_password (str, optional): The password for Apify Proxy.\\n proxy_port (str, optional): The port of Apify Proxy.\\n proxy_status_url (str, optional): The URL on which the Apify Proxy status page is available.\\n purge_on_start (str, optional): Whether the actor should purge its default storages on startup, when running locally.\\n token (str, optional): The API token for the Apify API this actor should use.\\n system_info_interval_millis (str, optional): How often should the actor emit the SYSTEM_INFO event when running locally.\"]}" + "text": "The actor configuration to be used in this event manager." } ] }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 173, + "kind": 32768, + "kindString": "Parameter", + "name": "config", "type": { + "name": "Configuration", "type": "reference", - "name": "None" - }, - "parameters": [ - { - "id": 145, - "name": "api_base_url", - "kind": 32768, - "kindString": "Parameter", - "flags": { - "keyword-only": "true" - }, - "type": { - "type": "reference", - "name": "Optional[str]" - }, - "defaultValue": "None" - }, - { - "id": 146, - "name": "api_public_base_url", - "kind": 32768, - "kindString": "Parameter", - "flags": { - "keyword-only": "true" - }, - "type": { - "type": "reference", - "name": "Optional[str]" - }, - "defaultValue": "None" - }, - { - "id": 147, - "name": "container_port", - "kind": 32768, - "kindString": "Parameter", - "flags": { - "keyword-only": "true" - }, - "type": { - "type": "reference", - "name": "Optional[int]" - }, - "defaultValue": "None" - }, - { - "id": 148, - "name": "container_url", - "kind": 32768, - "kindString": "Parameter", - "flags": { - "keyword-only": "true" - }, - "type": { - "type": "reference", - "name": "Optional[str]" - }, - "defaultValue": "None" - }, - { - "id": 149, - "name": "default_dataset_id", - "kind": 32768, - "kindString": "Parameter", - "flags": { - "keyword-only": "true" - }, - "type": { - "type": "reference", - "name": "Optional[str]" - }, - "defaultValue": "None" - }, - { - "id": 150, - "name": "default_key_value_store_id", - "kind": 32768, - "kindString": "Parameter", - "flags": { - "keyword-only": "true" - }, - "type": { - "type": "reference", - "name": "Optional[str]" - }, - "defaultValue": "None" - }, - { - "id": 151, - "name": "default_request_queue_id", - "kind": 32768, - "kindString": "Parameter", - "flags": { - "keyword-only": "true" - }, - "type": { - "type": "reference", - "name": "Optional[str]" - }, - "defaultValue": "None" - }, - { - "id": 152, - "name": "input_key", - "kind": 32768, - "kindString": "Parameter", - "flags": { - "keyword-only": "true" - }, - "type": { - "type": "reference", - "name": "Optional[str]" - }, - "defaultValue": "None" - }, - { - "id": 153, - "name": "max_used_cpu_ratio", - "kind": 32768, - "kindString": "Parameter", - "flags": { - "keyword-only": "true" - }, - "type": { - "type": "reference", - "name": "Optional[float]" - }, - "defaultValue": "None" - }, - { - "id": 154, - "name": "metamorph_after_sleep_millis", - "kind": 32768, - "kindString": "Parameter", - "flags": { - "keyword-only": "true" - }, - "type": { - "type": "reference", - "name": "Optional[int]" - }, - "defaultValue": "None" - }, - { - "id": 155, - "name": "persist_state_interval_millis", - "kind": 32768, - "kindString": "Parameter", - "flags": { - "keyword-only": "true" - }, - "type": { - "type": "reference", - "name": "Optional[int]" - }, - "defaultValue": "None" - }, - { - "id": 156, - "name": "persist_storage", - "kind": 32768, - "kindString": "Parameter", - "flags": { - "keyword-only": "true" - }, - "type": { - "type": "reference", - "name": "Optional[bool]" - }, - "defaultValue": "None" - }, - { - "id": 157, - "name": "proxy_hostname", - "kind": 32768, - "kindString": "Parameter", - "flags": { - "keyword-only": "true" - }, - "type": { - "type": "reference", - "name": "Optional[str]" - }, - "defaultValue": "None" - }, - { - "id": 158, - "name": "proxy_password", - "kind": 32768, - "kindString": "Parameter", - "flags": { - "keyword-only": "true" - }, - "type": { - "type": "reference", - "name": "Optional[str]" - }, - "defaultValue": "None" - }, - { - "id": 159, - "name": "proxy_port", - "kind": 32768, - "kindString": "Parameter", - "flags": { - "keyword-only": "true" - }, - "type": { - "type": "reference", - "name": "Optional[int]" - }, - "defaultValue": "None" - }, - { - "id": 160, - "name": "proxy_status_url", - "kind": 32768, - "kindString": "Parameter", - "flags": { - "keyword-only": "true" - }, - "type": { - "type": "reference", - "name": "Optional[str]" - }, - "defaultValue": "None" - }, - { - "id": 161, - "name": "purge_on_start", - "kind": 32768, - "kindString": "Parameter", - "flags": { - "keyword-only": "true" - }, - "type": { - "type": "reference", - "name": "Optional[bool]" - }, - "defaultValue": "None" - }, - { - "id": 162, - "name": "token", - "kind": 32768, - "kindString": "Parameter", - "flags": { - "keyword-only": "true" - }, - "type": { - "type": "reference", - "name": "Optional[str]" - }, - "defaultValue": "None" - }, - { - "id": 163, - "name": "system_info_interval_millis", - "kind": 32768, - "kindString": "Parameter", - "flags": { - "keyword-only": "true" - }, - "type": { - "type": "reference", - "name": "Optional[int]" - }, - "defaultValue": "None" - } - ] + "target": "1" + } } - ] - }, + ], + "type": { + "name": "None", + "type": "literal", + "value": null + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Initialize the event manager.\n\nWhen running this on the Apify Platform, this will start processing events\nsend by the platform to the events websocket and emitting them as events\nthat can be listened to by the `Actor.on()` method." + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 174, + "module": "event_manager", + "name": "init", + "parsedDocstring": { + "text": "Initialize the event manager.\n\nWhen running this on the Apify Platform, this will start processing events\nsend by the platform to the events websocket and emitting them as events\nthat can be listened to by the `Actor.on()` method." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/event_manager.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 44 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ { - "id": 165, - "name": "get_global_configuration", - "module": "config", - "kind": 2048, - "kindString": "Method", - "flags": {}, "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Retrieve the global configuration.\\n\\nThe global configuration applies when you call actor methods via their static versions, e.g. `Actor.init()`.\\nAlso accessible via `Actor.config`.\"]}" + "text": "Initialize the event manager.\n\nWhen running this on the Apify Platform, this will start processing events\nsend by the platform to the events websocket and emitting them as events\nthat can be listened to by the `Actor.on()` method." } ] }, - "children": [], - "groups": [], - "sources": [ - { - "filename": "src/apify/config.py", - "line": 117, - "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v0.2.0/src/apify/config.py#L117" - } + "flags": {}, + "id": 175, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" ], - "signatures": [ - { - "id": 166, - "name": "get_global_configuration", - "kind": 4096, - "kindString": "Call signature", - "flags": {}, - "comment": { - "summary": [ - { - "kind": "text", - "text": "{\"content\": [\"Retrieve the global configuration.\\n\\nThe global configuration applies when you call actor methods via their static versions, e.g. `Actor.init()`.\\nAlso accessible via `Actor.config`.\"]}" - } - ] - }, - "type": { - "type": "reference", - "name": "'Configuration'" - }, - "parameters": [] - } - ] - } - ], - "groups": [ - { - "title": "Constructors", - "children": [ - 144 - ] - }, - { - "title": "Methods", - "children": [ - 165 - ] - } - ], - "sources": [ - { - "filename": "src/apify/config.py", - "line": 7, - "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v0.2.0/src/apify/config.py#L7" + "name": "init", + "parameters": [], + "type": { + "name": "None", + "type": "literal", + "value": null + } } ] }, { - "id": 167, - "name": "ProxyInfo", - "module": "proxy_configuration", - "kind": 128, - "kindString": "Class", - "flags": {}, + "kind": 2048, + "kindString": "Method", + "children": [], "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Provides information about a proxy connection that is used for requests.\"]}" + "text": "Initialize the event manager.\n\nThis will stop listening for the platform events,\nand it will wait for all the event listeners to finish.\n" } ] }, - "children": [], + "decorations": [], + "flags": {}, "groups": [], + "id": 176, + "module": "event_manager", + "name": "close", + "parsedDocstring": { + "text": "Initialize the event manager.\n\nThis will stop listening for the platform events,\nand it will wait for all the event listeners to finish.\n", + "args": { + "event_listeners_timeout_secs": "Optional timeout after which the pending event listeners are canceled." + } + }, "sources": [ { - "filename": "src/apify/proxy_configuration.py", - "line": 61, "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v0.2.0/src/apify/proxy_configuration.py#L61" + "fileName": "/src/apify/event_manager.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 66 } - ] - }, - { - "id": 168, - "name": "ProxyConfiguration", - "module": "proxy_configuration", - "kind": 128, - "kindString": "Class", - "flags": {}, - "comment": { - "summary": [ - { - "kind": "text", - "text": "{\"content\": [\"Configures a connection to a proxy server with the provided options.\\n\\nProxy servers are used to prevent target websites from blocking your crawlers based on IP address rate limits or blacklists.\\nThe default servers used by this class are managed by [Apify Proxy](https://docs.apify.com/proxy).\\nTo be able to use Apify Proxy, you need an Apify account and access to the selected proxies. If you provide no configuration option,\\nthe proxies will be managed automatically using a smart algorithm.\\n\\nIf you want to use your own proxies, use the `proxy_urls` or `new_url_function` constructor options.\\nYour list of proxy URLs will be rotated by the configuration, if this option is provided.\"]}" - } - ] + ], + "type": { + "name": "Undefined", + "type": "reference" }, - "children": [ + "signatures": [ { - "id": 169, - "name": "__init__", - "module": "proxy_configuration", - "kind": 512, - "kindString": "Constructor", - "flags": {}, "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Create a ProxyConfiguration instance. It is highly recommended to use `Actor.create_proxy_configuration()` instead of this.\\n\\nArgs:\\n password (str, optional): Password for the Apify Proxy. If not provided, will use os.environ['APIFY_PROXY_PASSWORD'], if available.\\n groups (list of str, optional): Proxy groups which the Apify Proxy should use, if provided.\\n country_code (str, optional): Country which the Apify Proxy should use, if provided.\\n proxy_urls (list of str, optional): Custom proxy server URLs which should be rotated through.\\n new_url_function (Callable, optional): Function which returns a custom proxy URL to be used.\"]}" + "text": "Initialize the event manager.\n\nThis will stop listening for the platform events,\nand it will wait for all the event listeners to finish.\n" } ] }, - "children": [], - "groups": [], - "sources": [ - { - "filename": "src/apify/proxy_configuration.py", - "line": 127, - "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v0.2.0/src/apify/proxy_configuration.py#L127" - } + "flags": {}, + "id": 177, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" ], - "signatures": [ + "name": "close", + "parameters": [ { - "id": 177, - "name": "__init__", - "kind": 4096, - "kindString": "Call signature", - "flags": {}, "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Create a ProxyConfiguration instance. It is highly recommended to use `Actor.create_proxy_configuration()` instead of this.\\n\\nArgs:\\n password (str, optional): Password for the Apify Proxy. If not provided, will use os.environ['APIFY_PROXY_PASSWORD'], if available.\\n groups (list of str, optional): Proxy groups which the Apify Proxy should use, if provided.\\n country_code (str, optional): Country which the Apify Proxy should use, if provided.\\n proxy_urls (list of str, optional): Custom proxy server URLs which should be rotated through.\\n new_url_function (Callable, optional): Function which returns a custom proxy URL to be used.\"]}" + "text": "Optional timeout after which the pending event listeners are canceled." } ] }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": false + }, + "id": 178, + "kind": 32768, + "kindString": "Parameter", + "name": "event_listeners_timeout_secs", "type": { + "name": "Optional", "type": "reference", - "name": "None" - }, - "parameters": [ - { - "id": 170, - "name": "password", - "kind": 32768, - "kindString": "Parameter", - "flags": { - "keyword-only": "true" - }, - "type": { - "type": "reference", - "name": "Optional[str]" - }, - "defaultValue": "None" - }, - { - "id": 171, - "name": "groups", - "kind": 32768, - "kindString": "Parameter", - "flags": { - "keyword-only": "true" - }, - "type": { - "type": "reference", - "name": "Optional[List[str]]" - }, - "defaultValue": "None" - }, - { - "id": 172, - "name": "country_code", - "kind": 32768, - "kindString": "Parameter", - "flags": { - "keyword-only": "true" - }, - "type": { - "type": "reference", - "name": "Optional[str]" - }, - "defaultValue": "None" - }, - { - "id": 173, - "name": "proxy_urls", - "kind": 32768, - "kindString": "Parameter", - "flags": { - "keyword-only": "true" - }, - "type": { - "type": "reference", - "name": "Optional[List[str]]" - }, - "defaultValue": "None" - }, - { - "id": 174, - "name": "new_url_function", - "kind": 32768, - "kindString": "Parameter", - "flags": { - "keyword-only": "true" - }, - "type": { - "type": "reference", - "name": "Optional[Union[Callable[[Optional[str]], str], Callable[[Optional[str]], Awaitable[str]]]]" - }, - "defaultValue": "None" - }, - { - "id": 175, - "name": "_actor_config", - "kind": 32768, - "kindString": "Parameter", - "flags": { - "keyword-only": "true" - }, - "type": { - "type": "reference", - "name": "Optional[Configuration]" - }, - "defaultValue": "None" - }, - { - "id": 176, - "name": "_apify_client", - "kind": 32768, - "kindString": "Parameter", - "flags": { - "keyword-only": "true" - }, - "type": { + "typeArguments": [ + { "type": "reference", - "name": "Optional[ApifyClientAsync]" - }, - "defaultValue": "None" - } - ] + "name": "float" + } + ] + } } - ] - }, + ], + "type": { + "name": "None", + "type": "literal", + "value": null + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Add an event listener to the event manager.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 179, + "module": "event_manager", + "name": "on", + "parsedDocstring": { + "text": "Add an event listener to the event manager.\n", + "args": { + "event_name": "The actor event for which to listen to.", + "listener": "The function which is to be called when the event is emitted (can be async)." + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/event_manager.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 90 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ { - "id": 178, - "name": "initialize", - "module": "proxy_configuration", - "kind": 2048, - "kindString": "Method", - "flags": {}, "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Load the Apify Proxy password if the API token is provided and check access to Apify Proxy and provided proxy groups.\\n\\nOnly called if Apify Proxy configuration is used.\\nAlso checks if country has access to Apify Proxy groups if the country code is provided.\\n\\nYou should use the Actor.create_proxy_configuration function\\nto create a pre-initialized `ProxyConfiguration` instance instead of calling this manually.\"]}" + "text": "Add an event listener to the event manager.\n" } ] }, - "children": [], - "groups": [], - "sources": [ - { - "filename": "src/apify/proxy_configuration.py", - "line": 187, - "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v0.2.0/src/apify/proxy_configuration.py#L187" - } - ], - "signatures": [ + "flags": {}, + "id": 180, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "on", + "parameters": [ { - "id": 179, - "name": "initialize", - "kind": 4096, - "kindString": "Call signature", - "flags": {}, "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Load the Apify Proxy password if the API token is provided and check access to Apify Proxy and provided proxy groups.\\n\\nOnly called if Apify Proxy configuration is used.\\nAlso checks if country has access to Apify Proxy groups if the country code is provided.\\n\\nYou should use the Actor.create_proxy_configuration function\\nto create a pre-initialized `ProxyConfiguration` instance instead of calling this manually.\"]}" + "text": "The actor event for which to listen to." } ] }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 181, + "kind": 32768, + "kindString": "Parameter", + "name": "event_name", "type": { + "name": "ActorEventTypes", "type": "reference", - "name": "None" + "target": "195" + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The function which is to be called when the event is emitted (can be async)." + } + ] }, - "parameters": [], - "modifiers": [ - "async" - ] + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 182, + "kind": 32768, + "kindString": "Parameter", + "name": "listener", + "type": { + "name": "Callable", + "type": "reference" + } } - ] - }, + ], + "type": { + "name": "Callable", + "type": "reference" + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Remove a listener, or all listeners, from an actor event.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 183, + "module": "event_manager", + "name": "off", + "parsedDocstring": { + "text": "Remove a listener, or all listeners, from an actor event.\n", + "args": { + "event_name": "The actor event for which to remove listeners.", + "listener": "The listener which is supposed to be removed. If not passed, all listeners of this event are removed." + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/event_manager.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 125 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ { - "id": 180, - "name": "new_url", - "module": "proxy_configuration", - "kind": 2048, - "kindString": "Method", - "flags": {}, "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Return a new proxy URL based on provided configuration options and the `sessionId` parameter.\\n\\nArgs:\\n session_id (int or str, optional): Represents the identifier of a proxy session (https://docs.apify.com/proxy#sessions).\\n All the HTTP requests going through the proxy with the same session identifier\\n will use the same target proxy server (i.e. the same IP address).\\n The identifier must not be longer than 50 characters and include only the following: `0-9`, `a-z`, `A-Z`, `\\\".\\\"`, `\\\"_\\\"` and `\\\"~\\\"`.\\n\\nReturns:\\n str: A string with a proxy URL, including authentication credentials and port number.\\n For example, `http://bob:password123@proxy.example.com:8000`\"]}" + "text": "Remove a listener, or all listeners, from an actor event.\n" } ] }, - "children": [], - "groups": [], - "sources": [ - { - "filename": "src/apify/proxy_configuration.py", - "line": 200, - "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v0.2.0/src/apify/proxy_configuration.py#L200" - } - ], - "signatures": [ + "flags": {}, + "id": 184, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "off", + "parameters": [ { - "id": 182, - "name": "new_url", - "kind": 4096, - "kindString": "Call signature", - "flags": {}, "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Return a new proxy URL based on provided configuration options and the `sessionId` parameter.\\n\\nArgs:\\n session_id (int or str, optional): Represents the identifier of a proxy session (https://docs.apify.com/proxy#sessions).\\n All the HTTP requests going through the proxy with the same session identifier\\n will use the same target proxy server (i.e. the same IP address).\\n The identifier must not be longer than 50 characters and include only the following: `0-9`, `a-z`, `A-Z`, `\\\".\\\"`, `\\\"_\\\"` and `\\\"~\\\"`.\\n\\nReturns:\\n str: A string with a proxy URL, including authentication credentials and port number.\\n For example, `http://bob:password123@proxy.example.com:8000`\"]}" + "text": "The actor event for which to remove listeners." } ] }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 185, + "kind": 32768, + "kindString": "Parameter", + "name": "event_name", "type": { + "name": "ActorEventTypes", "type": "reference", - "name": "str" + "target": "195" + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The listener which is supposed to be removed. If not passed, all listeners of this event are removed." + } + ] }, - "parameters": [ - { - "id": 181, - "name": "session_id", - "kind": 32768, - "kindString": "Parameter", - "flags": {}, - "type": { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": false + }, + "id": 186, + "kind": 32768, + "kindString": "Parameter", + "name": "listener", + "type": { + "name": "Optional", + "type": "reference", + "typeArguments": [ + { "type": "reference", - "name": "Optional[Union[int, str]]" - }, - "defaultValue": "None" - } - ], - "modifiers": [ - "async" - ] + "name": "Callable" + } + ] + } } - ] - }, + ], + "type": { + "name": "None", + "type": "literal", + "value": null + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Emit an actor event manually.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 187, + "module": "event_manager", + "name": "emit", + "parsedDocstring": { + "text": "Emit an actor event manually.\n", + "args": { + "event_name": "The actor event which should be emitted.", + "data": "The data that should be emitted with the event." + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/event_manager.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 145 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ { - "id": 183, - "name": "new_proxy_info", - "module": "proxy_configuration", - "kind": 2048, - "kindString": "Method", - "flags": {}, "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Create a new ProxyInfo object.\\n\\nUse it if you want to work with a rich representation of a proxy URL.\\nIf you need the URL string only, use `ProxyConfiguration.new_url`.\\n\\nArgs:\\n session_id (int or str, optional): Represents the identifier of a proxy session (https://docs.apify.com/proxy#sessions).\\n All the HTTP requests going through the proxy with the same session identifier\\n will use the same target proxy server (i.e. the same IP address).\\n The identifier must not be longer than 50 characters and include only the following: `0-9`, `a-z`, `A-Z`, `\\\".\\\"`, `\\\"_\\\"` and `\\\"~\\\"`.\\n\\nReturns:\\n ProxyInfo: Dictionary that represents information about the proxy and its configuration.\"]}" + "text": "Emit an actor event manually.\n" } ] }, - "children": [], - "groups": [], - "sources": [ - { - "filename": "src/apify/proxy_configuration.py", - "line": 243, - "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v0.2.0/src/apify/proxy_configuration.py#L243" - } - ], - "signatures": [ + "flags": {}, + "id": 188, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "emit", + "parameters": [ { - "id": 185, - "name": "new_proxy_info", - "kind": 4096, - "kindString": "Call signature", - "flags": {}, "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Create a new ProxyInfo object.\\n\\nUse it if you want to work with a rich representation of a proxy URL.\\nIf you need the URL string only, use `ProxyConfiguration.new_url`.\\n\\nArgs:\\n session_id (int or str, optional): Represents the identifier of a proxy session (https://docs.apify.com/proxy#sessions).\\n All the HTTP requests going through the proxy with the same session identifier\\n will use the same target proxy server (i.e. the same IP address).\\n The identifier must not be longer than 50 characters and include only the following: `0-9`, `a-z`, `A-Z`, `\\\".\\\"`, `\\\"_\\\"` and `\\\"~\\\"`.\\n\\nReturns:\\n ProxyInfo: Dictionary that represents information about the proxy and its configuration.\"]}" + "text": "The actor event which should be emitted." } ] }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 189, + "kind": 32768, + "kindString": "Parameter", + "name": "event_name", "type": { + "name": "ActorEventTypes", "type": "reference", - "name": "ProxyInfo" + "target": "195" + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The data that should be emitted with the event." + } + ] }, - "parameters": [ - { - "id": 184, - "name": "session_id", - "kind": 32768, - "kindString": "Parameter", - "flags": {}, - "type": { - "type": "reference", - "name": "Optional[Union[int, str]]" - }, - "defaultValue": "None" - } - ], - "modifiers": [ - "async" - ] + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 190, + "kind": 32768, + "kindString": "Parameter", + "name": "data", + "type": { + "name": "Any", + "type": "reference" + } } - ] - } - ], - "groups": [ - { - "title": "Constructors", - "children": [ - 169 - ] - }, - { - "title": "Methods", - "children": [ - 178, - 180, - 183 - ] - } - ], - "sources": [ - { - "filename": "src/apify/proxy_configuration.py", - "line": 99, - "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v0.2.0/src/apify/proxy_configuration.py#L99" + ], + "type": { + "name": "None", + "type": "literal", + "value": null + } } ] }, { - "id": 186, - "name": "ActorLogFormatter", - "module": "log", - "kind": 128, - "kindString": "Class", - "flags": {}, + "kind": 2048, + "kindString": "Method", + "children": [], "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Log formatter that prints out the log message nicely formatted, with colored level and stringified extra fields.\\n\\nIt formats the log records so that they:\\n- start with the level (colorized, and padded to 5 chars so that it is nicely aligned)\\n- then have the actual log message, if it's multiline then it's nicely indented\\n- then have the stringified extra log fields\\n- then, if an exception is a part of the log record, prints the formatted exception.\"]}" + "text": "Wait for all event listeners which are currently being executed to complete.\n" } ] }, - "children": [ + "decorations": [], + "flags": {}, + "groups": [], + "id": 191, + "module": "event_manager", + "name": "wait_for_all_listeners_to_complete", + "parsedDocstring": { + "text": "Wait for all event listeners which are currently being executed to complete.\n", + "args": { + "timeout_secs": "Timeout for the wait. If the event listeners don't finish until the timeout, they will be canceled." + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/event_manager.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 156 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ { - "id": 187, - "name": "format", - "module": "log", - "kind": 2048, - "kindString": "Method", - "flags": {}, "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Format the log record nicely.\\n\\nThis formats the log record so that it:\\n- starts with the level (colorized, and padded to 5 chars so that it is nicely aligned)\\n- then has the actual log message, if it's multiline then it's nicely indented\\n- then has the stringified extra log fields\\n- then, if an exception is a part of the log record, prints the formatted exception.\"]}" + "text": "Wait for all event listeners which are currently being executed to complete.\n" } ] }, - "children": [], - "groups": [], - "sources": [ - { - "filename": "src/apify/log.py", - "line": 66, - "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v0.2.0/src/apify/log.py#L66" - } + "flags": {}, + "id": 192, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" ], - "signatures": [ + "name": "wait_for_all_listeners_to_complete", + "parameters": [ { - "id": 189, - "name": "format", - "kind": 4096, - "kindString": "Call signature", - "flags": {}, "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Format the log record nicely.\\n\\nThis formats the log record so that it:\\n- starts with the level (colorized, and padded to 5 chars so that it is nicely aligned)\\n- then has the actual log message, if it's multiline then it's nicely indented\\n- then has the stringified extra log fields\\n- then, if an exception is a part of the log record, prints the formatted exception.\"]}" + "text": "Timeout for the wait. If the event listeners don't finish until the timeout, they will be canceled." } ] }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 193, + "kind": 32768, + "kindString": "Parameter", + "name": "timeout_secs", "type": { + "name": "Optional", "type": "reference", - "name": "str" - }, - "parameters": [ - { - "id": 188, - "name": "record", - "kind": 32768, - "kindString": "Parameter", - "flags": {}, - "type": { + "typeArguments": [ + { "type": "reference", - "name": "logging.LogRecord" + "name": "float" } - } - ] + ] + } } - ] - } - ], - "groups": [ - { - "title": "Methods", - "children": [ - 187 - ] - } - ], - "sources": [ - { - "filename": "src/apify/log.py", - "line": 40, - "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v0.2.0/src/apify/log.py#L40" - } - ] - }, - { - "id": 190, - "name": "ActorEventTypes", - "module": "consts", - "kind": 128, - "kindString": "Class", - "flags": {}, - "comment": { - "summary": [ - { - "kind": "text", - "text": "{\"content\": [\"Possible values of actor event type.\"]}" + ], + "type": { + "name": "None", + "type": "literal", + "value": null } - ] - }, - "children": [], - "groups": [], - "sources": [ - { - "filename": "src/apify/consts.py", - "line": 6, - "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v0.2.0/src/apify/consts.py#L6" } ] }, { - "id": 191, - "name": "ApifyEnvVars", - "module": "consts", - "kind": 128, - "kindString": "Class", - "flags": {}, - "comment": { - "summary": [ - { - "kind": "text", - "text": "{\"content\": [\"Possible Apify-specific environment variables.\"]}" - } - ] - }, + "kind": 1024, + "kindString": "Property", "children": [], - "groups": [], - "sources": [ - { - "filename": "src/apify/consts.py", - "line": 19, - "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v0.2.0/src/apify/consts.py#L19" - } - ] - }, - { - "id": 192, - "name": "ActorExitCodes", - "module": "consts", - "kind": 128, - "kindString": "Class", - "flags": {}, "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Usual actor exit codes.\"]}" + "text": "" } ] }, - "children": [], + "flags": {}, "groups": [], + "id": 194, + "module": "_types", + "name": "JSONSerializable", + "parsedDocstring": { + "text": "" + }, "sources": [ { - "filename": "src/apify/consts.py", - "line": 198, "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v0.2.0/src/apify/consts.py#L198" + "fileName": "/src/apify/_types.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 6 } - ] + ], + "type": { + "name": "Undefined", + "type": "reference" + } }, { - "id": 193, - "name": "Dataset", - "module": "storages.dataset", - "kind": 128, - "kindString": "Class", - "flags": {}, - "comment": { - "summary": [ - { - "kind": "text", - "text": "{\"content\": [\"The `Dataset` class represents a store for structured data where each object stored has the same attributes.\\n\\nYou can imagine it as a table, where each object is a row and its attributes are columns.\\nDataset is an append-only storage - you can only add new records to it but you cannot modify or remove existing records.\\nTypically it is used to store crawling results.\\n\\nDo not instantiate this class directly, use the `Actor.open_dataset()` function instead.\\n\\n`Dataset` stores its data either on local disk or in the Apify cloud,\\ndepending on whether the `APIFY_LOCAL_STORAGE_DIR` or `APIFY_TOKEN` environment variables are set.\\n\\nIf the `APIFY_LOCAL_STORAGE_DIR` environment variable is set, the data is stored in\\nthe local directory in the following files:\\n```\\n{APIFY_LOCAL_STORAGE_DIR}/datasets/{DATASET_ID}/{INDEX}.json\\n```\\nNote that `{DATASET_ID}` is the name or ID of the dataset. The default dataset has ID: `default`,\\nunless you override it by setting the `APIFY_DEFAULT_DATASET_ID` environment variable.\\nEach dataset item is stored as a separate JSON file, where `{INDEX}` is a zero-based index of the item in the dataset.\\n\\nIf the `APIFY_TOKEN` environment variable is set but `APIFY_LOCAL_STORAGE_DIR` is not, the data is stored in the\\n[Apify Dataset](https://docs.apify.com/storage/dataset) cloud storage.\"]}" - } - ] - }, + "kind": 8, + "kindString": "Enumeration", "children": [ { - "id": 194, - "name": "__init__", - "module": "storages.dataset", - "kind": 512, - "kindString": "Constructor", - "flags": {}, + "kind": 16, + "kindString": "Enumeration Member", + "children": [], "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Create a `Dataset` instance.\\n\\nDo not use the constructor directly, use the `Actor.open_dataset()` function instead.\\n\\nArgs:\\n id (str): ID of the dataset.\\n name (str, optional): Name of the dataset.\\n client (ApifyClientAsync or MemoryStorageClient): The storage client which should be used.\\n config (Configuration): The configuration which should be used.\"]}" + "text": "Info about resource usage of the actor" } ] }, - "children": [], + "flags": {}, "groups": [], + "id": 196, + "module": "consts", + "name": "SYSTEM_INFO", + "parsedDocstring": { + "text": "Info about resource usage of the actor" + }, "sources": [ { - "filename": "src/apify/storages/dataset.py", - "line": 96, "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v0.2.0/src/apify/storages/dataset.py#L96" + "fileName": "/src/apify/consts.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 10 } ], - "signatures": [ + "type": { + "type": "literal", + "value": "'systemInfo'" + } + }, + { + "kind": 16, + "kindString": "Enumeration Member", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Sent when the actor is about to migrate" + } + ] + }, + "flags": {}, + "groups": [], + "id": 197, + "module": "consts", + "name": "MIGRATING", + "parsedDocstring": { + "text": "Sent when the actor is about to migrate" + }, + "sources": [ { - "id": 199, - "name": "__init__", - "kind": 4096, - "kindString": "Call signature", - "flags": {}, - "comment": { - "summary": [ - { - "kind": "text", - "text": "{\"content\": [\"Create a `Dataset` instance.\\n\\nDo not use the constructor directly, use the `Actor.open_dataset()` function instead.\\n\\nArgs:\\n id (str): ID of the dataset.\\n name (str, optional): Name of the dataset.\\n client (ApifyClientAsync or MemoryStorageClient): The storage client which should be used.\\n config (Configuration): The configuration which should be used.\"]}" - } - ] - }, - "type": { - "type": "reference", - "name": "None" - }, - "parameters": [ - { - "id": 195, - "name": "id", - "kind": 32768, - "kindString": "Parameter", - "flags": {}, - "type": { - "type": "reference", - "name": "str" - } - }, - { - "id": 196, - "name": "name", - "kind": 32768, - "kindString": "Parameter", - "flags": {}, - "type": { - "type": "reference", - "name": "Optional[str]" - } - }, - { - "id": 197, - "name": "client", - "kind": 32768, - "kindString": "Parameter", - "flags": {}, - "type": { - "type": "reference", - "name": "Union[ApifyClientAsync, MemoryStorageClient]" - } - }, - { - "id": 198, - "name": "config", - "kind": 32768, - "kindString": "Parameter", - "flags": {}, - "type": { - "type": "reference", - "name": "Configuration" - } - } - ] + "character": 1, + "fileName": "/src/apify/consts.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 12 } - ] + ], + "type": { + "type": "literal", + "value": "'migrating'" + } }, { - "id": 200, - "name": "push_data", - "module": "storages.dataset", - "kind": 2048, - "kindString": "Method", + "kind": 16, + "kindString": "Enumeration Member", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Sent when the actor should persist its state (every minute or when migrating)" + } + ] + }, "flags": {}, + "groups": [], + "id": 198, + "module": "consts", + "name": "PERSIST_STATE", + "parsedDocstring": { + "text": "Sent when the actor should persist its state (every minute or when migrating)" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/consts.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 14 + } + ], + "type": { + "type": "literal", + "value": "'persistState'" + } + }, + { + "kind": 16, + "kindString": "Enumeration Member", + "children": [], "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Store an object or an array of objects to the dataset.\\n\\nThe size of the data is limited by the receiving API and therefore `push_data()` will only\\nallow objects whose JSON representation is smaller than 9MB. When an array is passed,\\nnone of the included objects may be larger than 9MB, but the array itself may be of any size.\\n\\nArgs:\\n data (JSONSerializable): dict or array of dicts containing data to be stored in the default dataset.\\n The JSON representation of each item must be smaller than 9MB.\"]}" + "text": "Sent when the actor is aborting" } ] }, + "flags": {}, + "groups": [], + "id": 199, + "module": "consts", + "name": "ABORTING", + "parsedDocstring": { + "text": "Sent when the actor is aborting" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/consts.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 16 + } + ], + "type": { + "type": "literal", + "value": "'aborting'" + } + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Possible values of actor event type." + } + ] + }, + "flags": {}, + "groups": [ + { + "children": [ + 199, + 197, + 198, + 196 + ], + "title": "Enumeration members" + } + ], + "id": 195, + "module": "consts", + "name": "ActorEventTypes", + "parsedDocstring": { + "text": "Possible values of actor event type." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/consts.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 6 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 8, + "kindString": "Enumeration", + "children": [ + { + "kind": 16, + "kindString": "Enumeration Member", "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "ACT_ID" + } + ] + }, + "flags": {}, "groups": [], + "id": 201, + "module": "consts", + "name": "ACT_ID", + "parsedDocstring": { + "text": "ACT_ID" + }, "sources": [ { - "filename": "src/apify/storages/dataset.py", - "line": 136, "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v0.2.0/src/apify/storages/dataset.py#L136" + "fileName": "/src/apify/consts.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 25 } ], - "signatures": [ + "type": { + "type": "literal", + "value": "'APIFY_ACT_ID'" + } + }, + { + "kind": 16, + "kindString": "Enumeration Member", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "ACT_RUN_ID" + } + ] + }, + "flags": {}, + "groups": [], + "id": 202, + "module": "consts", + "name": "ACT_RUN_ID", + "parsedDocstring": { + "text": "ACT_RUN_ID" + }, + "sources": [ { - "id": 202, - "name": "push_data", - "kind": 4096, - "kindString": "Call signature", - "flags": {}, - "comment": { - "summary": [ - { - "kind": "text", - "text": "{\"content\": [\"Store an object or an array of objects to the dataset.\\n\\nThe size of the data is limited by the receiving API and therefore `push_data()` will only\\nallow objects whose JSON representation is smaller than 9MB. When an array is passed,\\nnone of the included objects may be larger than 9MB, but the array itself may be of any size.\\n\\nArgs:\\n data (JSONSerializable): dict or array of dicts containing data to be stored in the default dataset.\\n The JSON representation of each item must be smaller than 9MB.\"]}" - } - ] - }, - "type": { - "type": "reference", - "name": "None" - }, - "parameters": [ - { - "id": 201, - "name": "data", - "kind": 32768, - "kindString": "Parameter", - "flags": {}, - "type": { - "type": "reference", - "name": "JSONSerializable" - } - } - ], - "modifiers": [ - "async" - ] + "character": 1, + "fileName": "/src/apify/consts.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 27 } - ] + ], + "type": { + "type": "literal", + "value": "'APIFY_ACT_RUN_ID'" + } }, { + "kind": 16, + "kindString": "Enumeration Member", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "ACTOR_BUILD_ID" + } + ] + }, + "flags": {}, + "groups": [], "id": 203, - "name": "get_data", - "module": "storages.dataset", - "kind": 2048, - "kindString": "Method", + "module": "consts", + "name": "ACTOR_BUILD_ID", + "parsedDocstring": { + "text": "ACTOR_BUILD_ID" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/consts.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 29 + } + ], + "type": { + "type": "literal", + "value": "'APIFY_ACTOR_BUILD_ID'" + } + }, + { + "kind": 16, + "kindString": "Enumeration Member", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "ACTOR_BUILD_NUMBER" + } + ] + }, "flags": {}, + "groups": [], + "id": 204, + "module": "consts", + "name": "ACTOR_BUILD_NUMBER", + "parsedDocstring": { + "text": "ACTOR_BUILD_NUMBER" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/consts.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 31 + } + ], + "type": { + "type": "literal", + "value": "'APIFY_ACTOR_BUILD_NUMBER'" + } + }, + { + "kind": 16, + "kindString": "Enumeration Member", + "children": [], "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Get items from the dataset.\\n\\nArgs:\\n offset (int, optional): Number of items that should be skipped at the start. The default value is 0\\n limit (int, optional): Maximum number of items to return. By default there is no limit.\\n desc (bool, optional): By default, results are returned in the same order as they were stored.\\n To reverse the order, set this parameter to True.\\n clean (bool, optional): If True, returns only non-empty items and skips hidden fields (i.e. fields starting with the # character).\\n The clean parameter is just a shortcut for skip_hidden=True and skip_empty=True parameters.\\n Note that since some objects might be skipped from the output, that the result might contain less items than the limit value.\\n fields (list of str, optional): A list of fields which should be picked from the items,\\n only these fields will remain in the resulting record objects.\\n Note that the fields in the outputted items are sorted the same way as they are specified in the fields parameter.\\n You can use this feature to effectively fix the output format.\\n omit (list of str, optional): A list of fields which should be omitted from the items.\\n unwind (str, optional): Name of a field which should be unwound.\\n If the field is an array then every element of the array will become a separate record and merged with parent object.\\n If the unwound field is an object then it is merged with the parent object.\\n If the unwound field is missing or its value is neither an array nor an object and therefore cannot be merged with a parent object,\\n then the item gets preserved as it is. Note that the unwound items ignore the desc parameter.\\n skip_empty (bool, optional): If True, then empty items are skipped from the output.\\n Note that if used, the results might contain less items than the limit value.\\n skip_hidden (bool, optional): If True, then hidden fields are skipped from the output, i.e. fields starting with the # character.\\n flatten (list of str, optional): A list of fields that should be flattened\\n view (str, optional): Name of the dataset view to be used\\n\\nReturns:\\n ListPage: A page of the list of dataset items according to the specified filters.\"]}" + "text": "ACTOR_EVENTS_WS_URL" } ] }, + "flags": {}, + "groups": [], + "id": 205, + "module": "consts", + "name": "ACTOR_EVENTS_WS_URL", + "parsedDocstring": { + "text": "ACTOR_EVENTS_WS_URL" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/consts.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 33 + } + ], + "type": { + "type": "literal", + "value": "'APIFY_ACTOR_EVENTS_WS_URL'" + } + }, + { + "kind": 16, + "kindString": "Enumeration Member", "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "ACTOR_ID" + } + ] + }, + "flags": {}, "groups": [], + "id": 206, + "module": "consts", + "name": "ACTOR_ID", + "parsedDocstring": { + "text": "ACTOR_ID" + }, "sources": [ { - "filename": "src/apify/storages/dataset.py", - "line": 164, "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v0.2.0/src/apify/storages/dataset.py#L164" + "fileName": "/src/apify/consts.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 35 } ], - "signatures": [ + "type": { + "type": "literal", + "value": "'APIFY_ACTOR_ID'" + } + }, + { + "kind": 16, + "kindString": "Enumeration Member", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "ACTOR_RUN_ID" + } + ] + }, + "flags": {}, + "groups": [], + "id": 207, + "module": "consts", + "name": "ACTOR_RUN_ID", + "parsedDocstring": { + "text": "ACTOR_RUN_ID" + }, + "sources": [ { - "id": 215, - "name": "get_data", - "kind": 4096, - "kindString": "Call signature", - "flags": {}, - "comment": { - "summary": [ - { - "kind": "text", - "text": "{\"content\": [\"Get items from the dataset.\\n\\nArgs:\\n offset (int, optional): Number of items that should be skipped at the start. The default value is 0\\n limit (int, optional): Maximum number of items to return. By default there is no limit.\\n desc (bool, optional): By default, results are returned in the same order as they were stored.\\n To reverse the order, set this parameter to True.\\n clean (bool, optional): If True, returns only non-empty items and skips hidden fields (i.e. fields starting with the # character).\\n The clean parameter is just a shortcut for skip_hidden=True and skip_empty=True parameters.\\n Note that since some objects might be skipped from the output, that the result might contain less items than the limit value.\\n fields (list of str, optional): A list of fields which should be picked from the items,\\n only these fields will remain in the resulting record objects.\\n Note that the fields in the outputted items are sorted the same way as they are specified in the fields parameter.\\n You can use this feature to effectively fix the output format.\\n omit (list of str, optional): A list of fields which should be omitted from the items.\\n unwind (str, optional): Name of a field which should be unwound.\\n If the field is an array then every element of the array will become a separate record and merged with parent object.\\n If the unwound field is an object then it is merged with the parent object.\\n If the unwound field is missing or its value is neither an array nor an object and therefore cannot be merged with a parent object,\\n then the item gets preserved as it is. Note that the unwound items ignore the desc parameter.\\n skip_empty (bool, optional): If True, then empty items are skipped from the output.\\n Note that if used, the results might contain less items than the limit value.\\n skip_hidden (bool, optional): If True, then hidden fields are skipped from the output, i.e. fields starting with the # character.\\n flatten (list of str, optional): A list of fields that should be flattened\\n view (str, optional): Name of the dataset view to be used\\n\\nReturns:\\n ListPage: A page of the list of dataset items according to the specified filters.\"]}" - } - ] - }, - "type": { - "type": "reference", - "name": "ListPage" - }, - "parameters": [ - { - "id": 204, - "name": "offset", - "kind": 32768, - "kindString": "Parameter", - "flags": { - "keyword-only": "true" - }, - "type": { - "type": "reference", - "name": "Optional[int]" - }, - "defaultValue": "None" - }, - { - "id": 205, - "name": "limit", - "kind": 32768, - "kindString": "Parameter", - "flags": { - "keyword-only": "true" - }, - "type": { - "type": "reference", - "name": "Optional[int]" - }, - "defaultValue": "None" - }, - { - "id": 206, - "name": "clean", - "kind": 32768, - "kindString": "Parameter", - "flags": { - "keyword-only": "true" - }, - "type": { - "type": "reference", - "name": "Optional[bool]" - }, - "defaultValue": "None" - }, - { - "id": 207, - "name": "desc", - "kind": 32768, - "kindString": "Parameter", - "flags": { - "keyword-only": "true" - }, - "type": { - "type": "reference", - "name": "Optional[bool]" - }, - "defaultValue": "None" - }, - { - "id": 208, - "name": "fields", - "kind": 32768, - "kindString": "Parameter", - "flags": { - "keyword-only": "true" - }, - "type": { - "type": "reference", - "name": "Optional[List[str]]" - }, - "defaultValue": "None" - }, - { - "id": 209, - "name": "omit", - "kind": 32768, - "kindString": "Parameter", - "flags": { - "keyword-only": "true" - }, - "type": { - "type": "reference", - "name": "Optional[List[str]]" - }, - "defaultValue": "None" - }, - { - "id": 210, - "name": "unwind", - "kind": 32768, - "kindString": "Parameter", - "flags": { - "keyword-only": "true" - }, - "type": { - "type": "reference", - "name": "Optional[str]" - }, - "defaultValue": "None" - }, - { - "id": 211, - "name": "skip_empty", - "kind": 32768, - "kindString": "Parameter", - "flags": { - "keyword-only": "true" - }, - "type": { - "type": "reference", - "name": "Optional[bool]" - }, - "defaultValue": "None" - }, - { - "id": 212, - "name": "skip_hidden", - "kind": 32768, - "kindString": "Parameter", - "flags": { - "keyword-only": "true" - }, - "type": { - "type": "reference", - "name": "Optional[bool]" - }, - "defaultValue": "None" - }, - { - "id": 213, - "name": "flatten", - "kind": 32768, - "kindString": "Parameter", - "flags": { - "keyword-only": "true" - }, - "type": { - "type": "reference", - "name": "Optional[List[str]]" - }, - "defaultValue": "None" - }, - { - "id": 214, - "name": "view", - "kind": 32768, - "kindString": "Parameter", - "flags": { - "keyword-only": "true" - }, - "type": { - "type": "reference", - "name": "Optional[str]" - }, - "defaultValue": "None" - } - ], - "modifiers": [ - "async" - ] + "character": 1, + "fileName": "/src/apify/consts.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 37 } - ] + ], + "type": { + "type": "literal", + "value": "'APIFY_ACTOR_RUN_ID'" + } }, { - "id": 216, - "name": "export_to", - "module": "storages.dataset", - "kind": 2048, - "kindString": "Method", - "flags": {}, + "kind": 16, + "kindString": "Enumeration Member", + "children": [], "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Save the entirety of the dataset's contents into one file within a key-value store.\\n\\nArgs:\\n key (str): The key to save the data under.\\n to_key_value_store_id (str, optional): The id of the key-value store in which the result will be saved.\\n to_key_value_store_name (str, optional): The name of the key-value store in which the result will be saved.\\n You must specify only one of `to_key_value_store_id` and `to_key_value_store_name` arguments.\\n If you omit both, it uses the default key-value store.\\n content_type (str, optional): Either 'text/csv' or 'application/json'. Defaults to JSON.\"]}" + "text": "ACTOR_TASK_ID" } ] }, - "children": [], + "flags": {}, "groups": [], + "id": 208, + "module": "consts", + "name": "ACTOR_TASK_ID", + "parsedDocstring": { + "text": "ACTOR_TASK_ID" + }, "sources": [ { - "filename": "src/apify/storages/dataset.py", - "line": 262, "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v0.2.0/src/apify/storages/dataset.py#L262" + "fileName": "/src/apify/consts.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 39 } ], - "signatures": [ + "type": { + "type": "literal", + "value": "'APIFY_ACTOR_TASK_ID'" + } + }, + { + "kind": 16, + "kindString": "Enumeration Member", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "API_BASE_URL" + } + ] + }, + "flags": {}, + "groups": [], + "id": 209, + "module": "consts", + "name": "API_BASE_URL", + "parsedDocstring": { + "text": "API_BASE_URL" + }, + "sources": [ { - "id": 221, - "name": "export_to", - "kind": 4096, - "kindString": "Call signature", - "flags": {}, - "comment": { - "summary": [ - { - "kind": "text", - "text": "{\"content\": [\"Save the entirety of the dataset's contents into one file within a key-value store.\\n\\nArgs:\\n key (str): The key to save the data under.\\n to_key_value_store_id (str, optional): The id of the key-value store in which the result will be saved.\\n to_key_value_store_name (str, optional): The name of the key-value store in which the result will be saved.\\n You must specify only one of `to_key_value_store_id` and `to_key_value_store_name` arguments.\\n If you omit both, it uses the default key-value store.\\n content_type (str, optional): Either 'text/csv' or 'application/json'. Defaults to JSON.\"]}" - } - ] - }, - "type": { - "type": "reference", - "name": "None" - }, - "parameters": [ - { - "id": 217, - "name": "key", - "kind": 32768, - "kindString": "Parameter", - "flags": {}, - "type": { - "type": "reference", - "name": "str" - } - }, - { - "id": 218, - "name": "to_key_value_store_id", - "kind": 32768, - "kindString": "Parameter", - "flags": { - "keyword-only": "true" - }, - "type": { - "type": "reference", - "name": "Optional[str]" - }, - "defaultValue": "None" - }, - { - "id": 219, - "name": "to_key_value_store_name", - "kind": 32768, - "kindString": "Parameter", - "flags": { - "keyword-only": "true" - }, - "type": { - "type": "reference", - "name": "Optional[str]" - }, - "defaultValue": "None" - }, - { - "id": 220, - "name": "content_type", - "kind": 32768, - "kindString": "Parameter", - "flags": { - "keyword-only": "true" - }, - "type": { - "type": "reference", - "name": "Optional[str]" - }, - "defaultValue": "None" - } - ], - "modifiers": [ - "async" - ] + "character": 1, + "fileName": "/src/apify/consts.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 41 } - ] + ], + "type": { + "type": "literal", + "value": "'APIFY_API_BASE_URL'" + } }, { - "id": 222, - "name": "export_to_json", - "module": "storages.dataset", - "kind": 2048, - "kindString": "Method", - "flags": {}, + "kind": 16, + "kindString": "Enumeration Member", + "children": [], "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Save the entirety of the dataset's contents into one JSON file within a key-value store.\\n\\nArgs:\\n key (str): The key to save the data under.\\n from_dataset_id (str, optional): The ID of the dataset in case of calling the class method. Uses default dataset if omitted.\\n from_dataset_name (str, optional): The name of the dataset in case of calling the class method. Uses default dataset if omitted.\\n You must specify only one of `from_dataset_id` and `from_dataset_name` arguments.\\n If you omit both, it uses the default dataset.\\n to_key_value_store_id (str, optional): The id of the key-value store in which the result will be saved.\\n to_key_value_store_name (str, optional): The name of the key-value store in which the result will be saved.\\n You must specify only one of `to_key_value_store_id` and `to_key_value_store_name` arguments.\\n If you omit both, it uses the default key-value store.\"]}" + "text": "API_PUBLIC_BASE_URL" } ] }, + "flags": {}, + "groups": [], + "id": 210, + "module": "consts", + "name": "API_PUBLIC_BASE_URL", + "parsedDocstring": { + "text": "API_PUBLIC_BASE_URL" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/consts.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 43 + } + ], + "type": { + "type": "literal", + "value": "'APIFY_API_PUBLIC_BASE_URL'" + } + }, + { + "kind": 16, + "kindString": "Enumeration Member", "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "CHROME_EXECUTABLE_PATH" + } + ] + }, + "flags": {}, "groups": [], + "id": 211, + "module": "consts", + "name": "CHROME_EXECUTABLE_PATH", + "parsedDocstring": { + "text": "CHROME_EXECUTABLE_PATH" + }, "sources": [ { - "filename": "src/apify/storages/dataset.py", - "line": 307, "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v0.2.0/src/apify/storages/dataset.py#L307" + "fileName": "/src/apify/consts.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 45 } ], - "signatures": [ + "type": { + "type": "literal", + "value": "'APIFY_CHROME_EXECUTABLE_PATH'" + } + }, + { + "kind": 16, + "kindString": "Enumeration Member", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "CONTAINER_PORT" + } + ] + }, + "flags": {}, + "groups": [], + "id": 212, + "module": "consts", + "name": "CONTAINER_PORT", + "parsedDocstring": { + "text": "CONTAINER_PORT" + }, + "sources": [ { - "id": 228, - "name": "export_to_json", - "kind": 4096, - "kindString": "Call signature", - "flags": {}, - "comment": { - "summary": [ - { - "kind": "text", - "text": "{\"content\": [\"Save the entirety of the dataset's contents into one JSON file within a key-value store.\\n\\nArgs:\\n key (str): The key to save the data under.\\n from_dataset_id (str, optional): The ID of the dataset in case of calling the class method. Uses default dataset if omitted.\\n from_dataset_name (str, optional): The name of the dataset in case of calling the class method. Uses default dataset if omitted.\\n You must specify only one of `from_dataset_id` and `from_dataset_name` arguments.\\n If you omit both, it uses the default dataset.\\n to_key_value_store_id (str, optional): The id of the key-value store in which the result will be saved.\\n to_key_value_store_name (str, optional): The name of the key-value store in which the result will be saved.\\n You must specify only one of `to_key_value_store_id` and `to_key_value_store_name` arguments.\\n If you omit both, it uses the default key-value store.\"]}" - } - ] - }, - "type": { - "type": "reference", - "name": "None" - }, - "parameters": [ - { - "id": 223, - "name": "key", - "kind": 32768, - "kindString": "Parameter", - "flags": {}, - "type": { - "type": "reference", - "name": "str" - } - }, - { - "id": 224, - "name": "from_dataset_id", - "kind": 32768, - "kindString": "Parameter", - "flags": { - "keyword-only": "true" - }, - "type": { - "type": "reference", - "name": "Optional[str]" - }, - "defaultValue": "None" - }, - { - "id": 225, - "name": "from_dataset_name", - "kind": 32768, - "kindString": "Parameter", - "flags": { - "keyword-only": "true" - }, - "type": { - "type": "reference", - "name": "Optional[str]" - }, - "defaultValue": "None" - }, - { - "id": 226, - "name": "to_key_value_store_id", - "kind": 32768, - "kindString": "Parameter", - "flags": { - "keyword-only": "true" - }, - "type": { - "type": "reference", - "name": "Optional[str]" - }, - "defaultValue": "None" - }, - { - "id": 227, - "name": "to_key_value_store_name", - "kind": 32768, - "kindString": "Parameter", - "flags": { - "keyword-only": "true" - }, - "type": { - "type": "reference", - "name": "Optional[str]" - }, - "defaultValue": "None" - } - ], - "modifiers": [ - "async" - ] + "character": 1, + "fileName": "/src/apify/consts.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 47 } - ] + ], + "type": { + "type": "literal", + "value": "'APIFY_CONTAINER_PORT'" + } }, { - "id": 229, - "name": "export_to_csv", - "module": "storages.dataset", - "kind": 2048, - "kindString": "Method", - "flags": {}, + "kind": 16, + "kindString": "Enumeration Member", + "children": [], "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Save the entirety of the dataset's contents into one CSV file within a key-value store.\\n\\nArgs:\\n key (str): The key to save the data under.\\n from_dataset_id (str, optional): The ID of the dataset in case of calling the class method. Uses default dataset if omitted.\\n from_dataset_name (str, optional): The name of the dataset in case of calling the class method. Uses default dataset if omitted.\\n You must specify only one of `from_dataset_id` and `from_dataset_name` arguments.\\n If you omit both, it uses the default dataset.\\n to_key_value_store_id (str, optional): The id of the key-value store in which the result will be saved.\\n to_key_value_store_name (str, optional): The name of the key-value store in which the result will be saved.\\n You must specify only one of `to_key_value_store_id` and `to_key_value_store_name` arguments.\\n If you omit both, it uses the default key-value store.\"]}" + "text": "CONTAINER_URL" } ] }, - "children": [], + "flags": {}, "groups": [], + "id": 213, + "module": "consts", + "name": "CONTAINER_URL", + "parsedDocstring": { + "text": "CONTAINER_URL" + }, "sources": [ { - "filename": "src/apify/storages/dataset.py", - "line": 349, "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v0.2.0/src/apify/storages/dataset.py#L349" + "fileName": "/src/apify/consts.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 49 } ], - "signatures": [ + "type": { + "type": "literal", + "value": "'APIFY_CONTAINER_URL'" + } + }, + { + "kind": 16, + "kindString": "Enumeration Member", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "DEDICATED_CPUS" + } + ] + }, + "flags": {}, + "groups": [], + "id": 214, + "module": "consts", + "name": "DEDICATED_CPUS", + "parsedDocstring": { + "text": "DEDICATED_CPUS" + }, + "sources": [ { - "id": 235, - "name": "export_to_csv", - "kind": 4096, - "kindString": "Call signature", - "flags": {}, - "comment": { - "summary": [ - { - "kind": "text", - "text": "{\"content\": [\"Save the entirety of the dataset's contents into one CSV file within a key-value store.\\n\\nArgs:\\n key (str): The key to save the data under.\\n from_dataset_id (str, optional): The ID of the dataset in case of calling the class method. Uses default dataset if omitted.\\n from_dataset_name (str, optional): The name of the dataset in case of calling the class method. Uses default dataset if omitted.\\n You must specify only one of `from_dataset_id` and `from_dataset_name` arguments.\\n If you omit both, it uses the default dataset.\\n to_key_value_store_id (str, optional): The id of the key-value store in which the result will be saved.\\n to_key_value_store_name (str, optional): The name of the key-value store in which the result will be saved.\\n You must specify only one of `to_key_value_store_id` and `to_key_value_store_name` arguments.\\n If you omit both, it uses the default key-value store.\"]}" - } - ] - }, - "type": { - "type": "reference", - "name": "None" - }, - "parameters": [ - { - "id": 230, - "name": "key", - "kind": 32768, - "kindString": "Parameter", - "flags": {}, - "type": { - "type": "reference", - "name": "str" - } - }, - { - "id": 231, - "name": "from_dataset_id", - "kind": 32768, - "kindString": "Parameter", - "flags": { - "keyword-only": "true" - }, - "type": { - "type": "reference", - "name": "Optional[str]" - }, - "defaultValue": "None" - }, - { - "id": 232, - "name": "from_dataset_name", - "kind": 32768, - "kindString": "Parameter", - "flags": { - "keyword-only": "true" - }, - "type": { - "type": "reference", - "name": "Optional[str]" - }, - "defaultValue": "None" - }, - { - "id": 233, - "name": "to_key_value_store_id", - "kind": 32768, - "kindString": "Parameter", - "flags": { - "keyword-only": "true" - }, - "type": { - "type": "reference", - "name": "Optional[str]" - }, - "defaultValue": "None" - }, - { - "id": 234, - "name": "to_key_value_store_name", - "kind": 32768, - "kindString": "Parameter", - "flags": { - "keyword-only": "true" - }, - "type": { - "type": "reference", - "name": "Optional[str]" - }, - "defaultValue": "None" - } - ], - "modifiers": [ - "async" - ] + "character": 1, + "fileName": "/src/apify/consts.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 51 } - ] + ], + "type": { + "type": "literal", + "value": "'APIFY_DEDICATED_CPUS'" + } }, { - "id": 236, - "name": "get_info", - "module": "storages.dataset", - "kind": 2048, - "kindString": "Method", - "flags": {}, + "kind": 16, + "kindString": "Enumeration Member", + "children": [], "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Get an object containing general information about the dataset.\\n\\nReturns:\\n dict: Object returned by calling the GET dataset API endpoint.\"]}" + "text": "DEFAULT_BROWSER_PATH" } ] }, - "children": [], + "flags": {}, "groups": [], + "id": 215, + "module": "consts", + "name": "DEFAULT_BROWSER_PATH", + "parsedDocstring": { + "text": "DEFAULT_BROWSER_PATH" + }, "sources": [ { - "filename": "src/apify/storages/dataset.py", - "line": 390, "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v0.2.0/src/apify/storages/dataset.py#L390" + "fileName": "/src/apify/consts.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 53 } ], - "signatures": [ + "type": { + "type": "literal", + "value": "'APIFY_DEFAULT_BROWSER_PATH'" + } + }, + { + "kind": 16, + "kindString": "Enumeration Member", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "DEFAULT_DATASET_ID" + } + ] + }, + "flags": {}, + "groups": [], + "id": 216, + "module": "consts", + "name": "DEFAULT_DATASET_ID", + "parsedDocstring": { + "text": "DEFAULT_DATASET_ID" + }, + "sources": [ { - "id": 237, - "name": "get_info", - "kind": 4096, - "kindString": "Call signature", - "flags": {}, - "comment": { - "summary": [ - { - "kind": "text", - "text": "{\"content\": [\"Get an object containing general information about the dataset.\\n\\nReturns:\\n dict: Object returned by calling the GET dataset API endpoint.\"]}" - } - ] - }, - "type": { - "type": "reference", - "name": "Optional[Dict]" - }, - "parameters": [], - "modifiers": [ - "async" - ] + "character": 1, + "fileName": "/src/apify/consts.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 55 } - ] + ], + "type": { + "type": "literal", + "value": "'APIFY_DEFAULT_DATASET_ID'" + } }, { - "id": 238, - "name": "iterate_items", - "module": "storages.dataset", - "kind": 2048, - "kindString": "Method", - "flags": {}, + "kind": 16, + "kindString": "Enumeration Member", + "children": [], "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Iterate over the items in the dataset.\\n\\nArgs:\\n offset (int, optional): Number of items that should be skipped at the start. The default value is 0\\n limit (int, optional): Maximum number of items to return. By default there is no limit.\\n desc (bool, optional): By default, results are returned in the same order as they were stored.\\n To reverse the order, set this parameter to True.\\n clean (bool, optional): If True, returns only non-empty items and skips hidden fields (i.e. fields starting with the # character).\\n The clean parameter is just a shortcut for skip_hidden=True and skip_empty=True parameters.\\n Note that since some objects might be skipped from the output, that the result might contain less items than the limit value.\\n fields (list of str, optional): A list of fields which should be picked from the items,\\n only these fields will remain in the resulting record objects.\\n Note that the fields in the outputted items are sorted the same way as they are specified in the fields parameter.\\n You can use this feature to effectively fix the output format.\\n omit (list of str, optional): A list of fields which should be omitted from the items.\\n unwind (str, optional): Name of a field which should be unwound.\\n If the field is an array then every element of the array will become a separate record and merged with parent object.\\n If the unwound field is an object then it is merged with the parent object.\\n If the unwound field is missing or its value is neither an array nor an object and therefore cannot be merged with a parent object,\\n then the item gets preserved as it is. Note that the unwound items ignore the desc parameter.\\n skip_empty (bool, optional): If True, then empty items are skipped from the output.\\n Note that if used, the results might contain less items than the limit value.\\n skip_hidden (bool, optional): If True, then hidden fields are skipped from the output, i.e. fields starting with the # character.\\n\\nYields:\\n dict: An item from the dataset\"]}" + "text": "DEFAULT_KEY_VALUE_STORE_ID" } ] }, + "flags": {}, + "groups": [], + "id": 217, + "module": "consts", + "name": "DEFAULT_KEY_VALUE_STORE_ID", + "parsedDocstring": { + "text": "DEFAULT_KEY_VALUE_STORE_ID" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/consts.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 57 + } + ], + "type": { + "type": "literal", + "value": "'APIFY_DEFAULT_KEY_VALUE_STORE_ID'" + } + }, + { + "kind": 16, + "kindString": "Enumeration Member", "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "DEFAULT_REQUEST_QUEUE_ID" + } + ] + }, + "flags": {}, "groups": [], + "id": 218, + "module": "consts", + "name": "DEFAULT_REQUEST_QUEUE_ID", + "parsedDocstring": { + "text": "DEFAULT_REQUEST_QUEUE_ID" + }, "sources": [ { - "filename": "src/apify/storages/dataset.py", - "line": 398, "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v0.2.0/src/apify/storages/dataset.py#L398" + "fileName": "/src/apify/consts.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 59 } ], - "signatures": [ + "type": { + "type": "literal", + "value": "'APIFY_DEFAULT_REQUEST_QUEUE_ID'" + } + }, + { + "kind": 16, + "kindString": "Enumeration Member", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "DISABLE_BROWSER_SANDBOX" + } + ] + }, + "flags": {}, + "groups": [], + "id": 219, + "module": "consts", + "name": "DISABLE_BROWSER_SANDBOX", + "parsedDocstring": { + "text": "DISABLE_BROWSER_SANDBOX" + }, + "sources": [ { - "id": 248, - "name": "iterate_items", - "kind": 4096, - "kindString": "Call signature", - "flags": {}, - "comment": { - "summary": [ - { - "kind": "text", - "text": "{\"content\": [\"Iterate over the items in the dataset.\\n\\nArgs:\\n offset (int, optional): Number of items that should be skipped at the start. The default value is 0\\n limit (int, optional): Maximum number of items to return. By default there is no limit.\\n desc (bool, optional): By default, results are returned in the same order as they were stored.\\n To reverse the order, set this parameter to True.\\n clean (bool, optional): If True, returns only non-empty items and skips hidden fields (i.e. fields starting with the # character).\\n The clean parameter is just a shortcut for skip_hidden=True and skip_empty=True parameters.\\n Note that since some objects might be skipped from the output, that the result might contain less items than the limit value.\\n fields (list of str, optional): A list of fields which should be picked from the items,\\n only these fields will remain in the resulting record objects.\\n Note that the fields in the outputted items are sorted the same way as they are specified in the fields parameter.\\n You can use this feature to effectively fix the output format.\\n omit (list of str, optional): A list of fields which should be omitted from the items.\\n unwind (str, optional): Name of a field which should be unwound.\\n If the field is an array then every element of the array will become a separate record and merged with parent object.\\n If the unwound field is an object then it is merged with the parent object.\\n If the unwound field is missing or its value is neither an array nor an object and therefore cannot be merged with a parent object,\\n then the item gets preserved as it is. Note that the unwound items ignore the desc parameter.\\n skip_empty (bool, optional): If True, then empty items are skipped from the output.\\n Note that if used, the results might contain less items than the limit value.\\n skip_hidden (bool, optional): If True, then hidden fields are skipped from the output, i.e. fields starting with the # character.\\n\\nYields:\\n dict: An item from the dataset\"]}" - } - ] - }, - "type": { - "type": "reference", - "name": "AsyncIterator[Dict]" - }, - "parameters": [ - { - "id": 239, - "name": "offset", - "kind": 32768, - "kindString": "Parameter", - "flags": { - "keyword-only": "true" - }, - "type": { - "type": "reference", - "name": "int" - }, - "defaultValue": "0" - }, - { - "id": 240, - "name": "limit", - "kind": 32768, - "kindString": "Parameter", - "flags": { - "keyword-only": "true" - }, - "type": { - "type": "reference", - "name": "Optional[int]" - }, - "defaultValue": "None" - }, - { - "id": 241, - "name": "clean", - "kind": 32768, - "kindString": "Parameter", - "flags": { - "keyword-only": "true" - }, - "type": { - "type": "reference", - "name": "Optional[bool]" - }, - "defaultValue": "None" - }, - { - "id": 242, - "name": "desc", - "kind": 32768, - "kindString": "Parameter", - "flags": { - "keyword-only": "true" - }, - "type": { - "type": "reference", - "name": "Optional[bool]" - }, - "defaultValue": "None" - }, - { - "id": 243, - "name": "fields", - "kind": 32768, - "kindString": "Parameter", - "flags": { - "keyword-only": "true" - }, - "type": { - "type": "reference", - "name": "Optional[List[str]]" - }, - "defaultValue": "None" - }, - { - "id": 244, - "name": "omit", - "kind": 32768, - "kindString": "Parameter", - "flags": { - "keyword-only": "true" - }, - "type": { - "type": "reference", - "name": "Optional[List[str]]" - }, - "defaultValue": "None" - }, - { - "id": 245, - "name": "unwind", - "kind": 32768, - "kindString": "Parameter", - "flags": { - "keyword-only": "true" - }, - "type": { - "type": "reference", - "name": "Optional[str]" - }, - "defaultValue": "None" - }, - { - "id": 246, - "name": "skip_empty", - "kind": 32768, - "kindString": "Parameter", - "flags": { - "keyword-only": "true" - }, - "type": { - "type": "reference", - "name": "Optional[bool]" - }, - "defaultValue": "None" - }, - { - "id": 247, - "name": "skip_hidden", - "kind": 32768, - "kindString": "Parameter", - "flags": { - "keyword-only": "true" - }, - "type": { - "type": "reference", - "name": "Optional[bool]" - }, - "defaultValue": "None" - } - ] + "character": 1, + "fileName": "/src/apify/consts.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 61 } - ] + ], + "type": { + "type": "literal", + "value": "'APIFY_DISABLE_BROWSER_SANDBOX'" + } }, { - "id": 249, - "name": "drop", - "module": "storages.dataset", - "kind": 2048, - "kindString": "Method", - "flags": {}, + "kind": 16, + "kindString": "Enumeration Member", + "children": [], "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Remove the dataset either from the Apify cloud storage or from the local directory.\"]}" + "text": "DISABLE_OUTDATED_WARNING" } ] }, - "children": [], + "flags": {}, "groups": [], + "id": 220, + "module": "consts", + "name": "DISABLE_OUTDATED_WARNING", + "parsedDocstring": { + "text": "DISABLE_OUTDATED_WARNING" + }, "sources": [ { - "filename": "src/apify/storages/dataset.py", - "line": 450, "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v0.2.0/src/apify/storages/dataset.py#L450" + "fileName": "/src/apify/consts.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 63 } ], - "signatures": [ - { - "id": 250, - "name": "drop", - "kind": 4096, - "kindString": "Call signature", - "flags": {}, - "comment": { - "summary": [ - { - "kind": "text", - "text": "{\"content\": [\"Remove the dataset either from the Apify cloud storage or from the local directory.\"]}" - } - ] - }, - "type": { - "type": "reference", - "name": "None" - }, - "parameters": [], - "modifiers": [ - "async" - ] - } - ] + "type": { + "type": "literal", + "value": "'APIFY_DISABLE_OUTDATED_WARNING'" + } }, { - "id": 251, - "name": "open", - "module": "storages.dataset", - "kind": 2048, - "kindString": "Method", - "flags": {}, + "kind": 16, + "kindString": "Enumeration Member", + "children": [], "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Open a dataset.\\n\\nDatasets are used to store structured data where each object stored has the same attributes,\\nsuch as online store products or real estate offers.\\nThe actual data is stored either on the local filesystem or in the Apify cloud.\\n\\nArgs:\\n id (str, optional): ID of the dataset to be opened.\\n If neither `id` nor `name` are provided, the method returns the default dataset associated with the actor run.\\n If the dataset with the given ID does not exist, it raises an error.\\n name (str, optional): Name of the dataset to be opened.\\n If neither `id` nor `name` are provided, the method returns the default dataset associated with the actor run.\\n If the dataset with the given name does not exist, it is created.\\n force_cloud (bool, optional): If set to True, it will open a dataset on the Apify Platform even when running the actor locally.\\n Defaults to False.\\n config (Configuration, optional): A `Configuration` instance, uses global configuration if omitted.\\n\\nReturns:\\n Dataset: An instance of the `Dataset` class for the given ID or name.\"]}" + "text": "FACT" } ] }, - "children": [], + "flags": {}, "groups": [], + "id": 221, + "module": "consts", + "name": "FACT", + "parsedDocstring": { + "text": "FACT" + }, "sources": [ { - "filename": "src/apify/storages/dataset.py", - "line": 456, "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v0.2.0/src/apify/storages/dataset.py#L456" + "fileName": "/src/apify/consts.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 65 } ], - "signatures": [ - { - "id": 256, - "name": "open", - "kind": 4096, - "kindString": "Call signature", - "flags": {}, - "comment": { - "summary": [ - { - "kind": "text", - "text": "{\"content\": [\"Open a dataset.\\n\\nDatasets are used to store structured data where each object stored has the same attributes,\\nsuch as online store products or real estate offers.\\nThe actual data is stored either on the local filesystem or in the Apify cloud.\\n\\nArgs:\\n id (str, optional): ID of the dataset to be opened.\\n If neither `id` nor `name` are provided, the method returns the default dataset associated with the actor run.\\n If the dataset with the given ID does not exist, it raises an error.\\n name (str, optional): Name of the dataset to be opened.\\n If neither `id` nor `name` are provided, the method returns the default dataset associated with the actor run.\\n If the dataset with the given name does not exist, it is created.\\n force_cloud (bool, optional): If set to True, it will open a dataset on the Apify Platform even when running the actor locally.\\n Defaults to False.\\n config (Configuration, optional): A `Configuration` instance, uses global configuration if omitted.\\n\\nReturns:\\n Dataset: An instance of the `Dataset` class for the given ID or name.\"]}" - } - ] - }, - "type": { - "type": "reference", - "name": "'Dataset'" - }, - "parameters": [ - { - "id": 252, - "name": "id", - "kind": 32768, - "kindString": "Parameter", - "flags": { - "keyword-only": "true" - }, - "type": { - "type": "reference", - "name": "Optional[str]" - }, - "defaultValue": "None" - }, - { - "id": 253, - "name": "name", - "kind": 32768, - "kindString": "Parameter", - "flags": { - "keyword-only": "true" - }, - "type": { - "type": "reference", - "name": "Optional[str]" - }, - "defaultValue": "None" - }, - { - "id": 254, - "name": "force_cloud", - "kind": 32768, - "kindString": "Parameter", - "flags": { - "keyword-only": "true" - }, - "type": { - "type": "reference", - "name": "bool" - }, - "defaultValue": "False" - }, - { - "id": 255, - "name": "config", - "kind": 32768, - "kindString": "Parameter", - "flags": { - "keyword-only": "true" - }, - "type": { - "type": "reference", - "name": "Optional[Configuration]" - }, - "defaultValue": "None" - } - ], - "modifiers": [ - "async" - ] - } - ] - } - ], - "groups": [ - { - "title": "Constructors", - "children": [ - 194 - ] - }, - { - "title": "Methods", - "children": [ - 200, - 203, - 216, - 222, - 229, - 236, - 238, - 249, - 251 - ] - } - ], - "sources": [ - { - "filename": "src/apify/storages/dataset.py", - "line": 66, - "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v0.2.0/src/apify/storages/dataset.py#L66" - } - ] - }, - { - "id": 257, - "name": "KeyValueStore", - "module": "storages.key_value_store", - "kind": 128, - "kindString": "Class", - "flags": {}, - "comment": { - "summary": [ - { - "kind": "text", - "text": "{\"content\": [\"The `KeyValueStore` class represents a key-value store.\\n\\nYou can imagine it as a simple data storage that is used\\nfor saving and reading data records or files. Each data record is\\nrepresented by a unique key and associated with a MIME content type.\\n\\nDo not instantiate this class directly, use the `Actor.open_key_value_store()` function instead.\\n\\nEach crawler run is associated with a default key-value store, which is created exclusively\\nfor the run. By convention, the crawler input and output are stored into the\\ndefault key-value store under the `INPUT` and `OUTPUT` key, respectively.\\nTypically, input and output are JSON files, although it can be any other format.\\nTo access the default key-value store directly, you can use the\\n`KeyValueStore.get_value` and `KeyValueStore.set_value` convenience functions.\\n\\n`KeyValueStore` stores its data either on local disk or in the Apify cloud,\\ndepending on whether the `APIFY_LOCAL_STORAGE_DIR` or `APIFY_TOKEN` environment variables are set.\\n\\nIf the `APIFY_LOCAL_STORAGE_DIR` environment variable is set, the data is stored in\\nthe local directory in the following files:\\n```\\n{APIFY_LOCAL_STORAGE_DIR}/key_value_stores/{STORE_ID}/{INDEX}.{EXT}\\n```\\nNote that `{STORE_ID}` is the name or ID of the key-value store. The default key-value store has ID: `default`,\\nunless you override it by setting the `APIFY_DEFAULT_KEY_VALUE_STORE_ID` environment variable.\\nThe `{KEY}` is the key of the record and `{EXT}` corresponds to the MIME content type of the data value.\\n\\nIf the `APIFY_TOKEN` environment variable is set but `APIFY_LOCAL_STORAGE_DIR` is not, the data is stored in the\\n[Apify Key-value store](https://docs.apify.com/storage/key-value-store) cloud storage.\"]}" + "type": { + "type": "literal", + "value": "'APIFY_FACT'" } - ] - }, - "children": [ + }, { - "id": 258, - "name": "__init__", - "module": "storages.key_value_store", - "kind": 512, - "kindString": "Constructor", - "flags": {}, + "kind": 16, + "kindString": "Enumeration Member", + "children": [], "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Create a `KeyValueStore` instance.\\n\\nDo not use the constructor directly, use the `Actor.open_key_value_store()` function instead.\\n\\nArgs:\\n id (str): ID of the key-value store.\\n name (str, optional): Name of the key-value store.\\n client (ApifyClientAsync or MemoryStorageClient): The storage client which should be used.\\n config (Configuration): The configuration which should be used.\"]}" + "text": "HEADLESS" } ] }, - "children": [], + "flags": {}, "groups": [], + "id": 222, + "module": "consts", + "name": "HEADLESS", + "parsedDocstring": { + "text": "HEADLESS" + }, "sources": [ { - "filename": "src/apify/storages/key_value_store.py", - "line": 54, "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v0.2.0/src/apify/storages/key_value_store.py#L54" + "fileName": "/src/apify/consts.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 67 } ], - "signatures": [ - { - "id": 263, - "name": "__init__", - "kind": 4096, - "kindString": "Call signature", - "flags": {}, - "comment": { - "summary": [ - { - "kind": "text", - "text": "{\"content\": [\"Create a `KeyValueStore` instance.\\n\\nDo not use the constructor directly, use the `Actor.open_key_value_store()` function instead.\\n\\nArgs:\\n id (str): ID of the key-value store.\\n name (str, optional): Name of the key-value store.\\n client (ApifyClientAsync or MemoryStorageClient): The storage client which should be used.\\n config (Configuration): The configuration which should be used.\"]}" - } - ] - }, - "type": { - "type": "reference", - "name": "None" - }, - "parameters": [ - { - "id": 259, - "name": "id", - "kind": 32768, - "kindString": "Parameter", - "flags": {}, - "type": { - "type": "reference", - "name": "str" - } - }, - { - "id": 260, - "name": "name", - "kind": 32768, - "kindString": "Parameter", - "flags": {}, - "type": { - "type": "reference", - "name": "Optional[str]" - } - }, - { - "id": 261, - "name": "client", - "kind": 32768, - "kindString": "Parameter", - "flags": {}, - "type": { - "type": "reference", - "name": "Union[ApifyClientAsync, MemoryStorageClient]" - } - }, - { - "id": 262, - "name": "config", - "kind": 32768, - "kindString": "Parameter", - "flags": {}, - "type": { - "type": "reference", - "name": "Configuration" - } - } - ] - } - ] + "type": { + "type": "literal", + "value": "'APIFY_HEADLESS'" + } }, { - "id": 264, - "name": "get_value", - "module": "storages.key_value_store", - "kind": 2048, - "kindString": "Method", - "flags": {}, + "kind": 16, + "kindString": "Enumeration Member", + "children": [], "comment": { - "summary": [] + "summary": [ + { + "kind": "text", + "text": "INPUT_KEY" + } + ] }, - "children": [], + "flags": {}, "groups": [], + "id": 223, + "module": "consts", + "name": "INPUT_KEY", + "parsedDocstring": { + "text": "INPUT_KEY" + }, "sources": [ { - "filename": "src/apify/storages/key_value_store.py", - "line": 99, "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v0.2.0/src/apify/storages/key_value_store.py#L99" + "fileName": "/src/apify/consts.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 69 } ], - "signatures": [ - { - "id": 266, - "name": "get_value", - "kind": 4096, - "kindString": "Call signature", - "flags": {}, - "comment": { - "summary": [] - }, - "type": { - "type": "reference", - "name": "Any" - }, - "parameters": [ - { - "id": 265, - "name": "key", - "kind": 32768, - "kindString": "Parameter", - "flags": {}, - "type": { - "type": "reference", - "name": "str" - } - } - ], - "modifiers": [ - "async" - ] - } - ] + "type": { + "type": "literal", + "value": "'APIFY_INPUT_KEY'" + } }, { - "id": 267, - "name": "get_value", - "module": "storages.key_value_store", - "kind": 2048, - "kindString": "Method", - "flags": {}, + "kind": 16, + "kindString": "Enumeration Member", + "children": [], "comment": { - "summary": [] + "summary": [ + { + "kind": "text", + "text": "INPUT_SECRETS_PRIVATE_KEY_FILE" + } + ] }, - "children": [], + "flags": {}, "groups": [], + "id": 224, + "module": "consts", + "name": "INPUT_SECRETS_PRIVATE_KEY_FILE", + "parsedDocstring": { + "text": "INPUT_SECRETS_PRIVATE_KEY_FILE" + }, "sources": [ { - "filename": "src/apify/storages/key_value_store.py", - "line": 104, "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v0.2.0/src/apify/storages/key_value_store.py#L104" + "fileName": "/src/apify/consts.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 71 } ], - "signatures": [ - { - "id": 270, - "name": "get_value", - "kind": 4096, - "kindString": "Call signature", - "flags": {}, - "comment": { - "summary": [] - }, - "type": { - "type": "reference", - "name": "T" - }, - "parameters": [ - { - "id": 268, - "name": "key", - "kind": 32768, - "kindString": "Parameter", - "flags": {}, - "type": { - "type": "reference", - "name": "str" - } - }, - { - "id": 269, - "name": "default_value", - "kind": 32768, - "kindString": "Parameter", - "flags": {}, - "type": { - "type": "reference", - "name": "T" - } - } - ], - "modifiers": [ - "async" - ] - } - ] + "type": { + "type": "literal", + "value": "'APIFY_INPUT_SECRETS_PRIVATE_KEY_FILE'" + } }, { - "id": 271, - "name": "get_value", - "module": "storages.key_value_store", - "kind": 2048, - "kindString": "Method", - "flags": {}, + "kind": 16, + "kindString": "Enumeration Member", + "children": [], "comment": { - "summary": [] + "summary": [ + { + "kind": "text", + "text": "INPUT_SECRETS_PRIVATE_KEY_PASSPHRASE" + } + ] }, - "children": [], + "flags": {}, "groups": [], + "id": 225, + "module": "consts", + "name": "INPUT_SECRETS_PRIVATE_KEY_PASSPHRASE", + "parsedDocstring": { + "text": "INPUT_SECRETS_PRIVATE_KEY_PASSPHRASE" + }, "sources": [ { - "filename": "src/apify/storages/key_value_store.py", - "line": 109, "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v0.2.0/src/apify/storages/key_value_store.py#L109" + "fileName": "/src/apify/consts.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 73 } ], - "signatures": [ - { - "id": 274, - "name": "get_value", - "kind": 4096, - "kindString": "Call signature", - "flags": {}, - "comment": { - "summary": [] - }, - "type": { - "type": "reference", - "name": "Optional[T]" - }, - "parameters": [ - { - "id": 272, - "name": "key", - "kind": 32768, - "kindString": "Parameter", - "flags": {}, - "type": { - "type": "reference", - "name": "str" - } - }, - { - "id": 273, - "name": "default_value", - "kind": 32768, - "kindString": "Parameter", - "flags": {}, - "type": { - "type": "reference", - "name": "Optional[T]" - }, - "defaultValue": "None" - } - ], - "modifiers": [ - "async" - ] - } - ] + "type": { + "type": "literal", + "value": "'APIFY_INPUT_SECRETS_PRIVATE_KEY_PASSPHRASE'" + } }, { - "id": 275, - "name": "get_value", - "module": "storages.key_value_store", - "kind": 2048, - "kindString": "Method", - "flags": {}, + "kind": 16, + "kindString": "Enumeration Member", + "children": [], "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Get a value from the key-value store.\\n\\nArgs:\\n key (str): Key of the record to retrieve.\\n default_value (Any, optional): Default value returned in case the record does not exist.\\n\\nReturns:\\n Any: The value associated with the given key. `default_value` is used in case the record does not exist.\"]}" + "text": "IS_AT_HOME" } ] }, - "children": [], + "flags": {}, "groups": [], + "id": 226, + "module": "consts", + "name": "IS_AT_HOME", + "parsedDocstring": { + "text": "IS_AT_HOME" + }, "sources": [ { - "filename": "src/apify/storages/key_value_store.py", - "line": 113, "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v0.2.0/src/apify/storages/key_value_store.py#L113" + "fileName": "/src/apify/consts.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 75 } ], - "signatures": [ - { - "id": 278, - "name": "get_value", - "kind": 4096, - "kindString": "Call signature", - "flags": {}, - "comment": { - "summary": [ - { - "kind": "text", - "text": "{\"content\": [\"Get a value from the key-value store.\\n\\nArgs:\\n key (str): Key of the record to retrieve.\\n default_value (Any, optional): Default value returned in case the record does not exist.\\n\\nReturns:\\n Any: The value associated with the given key. `default_value` is used in case the record does not exist.\"]}" - } - ] - }, - "type": { - "type": "reference", - "name": "Optional[T]" - }, - "parameters": [ - { - "id": 276, - "name": "key", - "kind": 32768, - "kindString": "Parameter", - "flags": {}, - "type": { - "type": "reference", - "name": "str" - } - }, - { - "id": 277, - "name": "default_value", - "kind": 32768, - "kindString": "Parameter", - "flags": {}, - "type": { - "type": "reference", - "name": "Optional[T]" - }, - "defaultValue": "None" - } - ], - "modifiers": [ - "async" - ] - } - ] + "type": { + "type": "literal", + "value": "'APIFY_IS_AT_HOME'" + } }, { - "id": 279, - "name": "iterate_keys", - "module": "storages.key_value_store", - "kind": 2048, - "kindString": "Method", - "flags": {}, + "kind": 16, + "kindString": "Enumeration Member", + "children": [], "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Iterate over the keys in the key-value store.\\n\\nArgs:\\n exclusive_start_key (str, optional): All keys up to this one (including) are skipped from the result.\\n\\nYields:\\n IterateKeysTuple: A tuple `(key, info)`,\\n where `key` is the record key, and `info` is an object that contains a single property `size`\\n indicating size of the record in bytes.\"]}" + "text": "LOCAL_STORAGE_DIR" } ] }, - "children": [], + "flags": {}, "groups": [], + "id": 227, + "module": "consts", + "name": "LOCAL_STORAGE_DIR", + "parsedDocstring": { + "text": "LOCAL_STORAGE_DIR" + }, "sources": [ { - "filename": "src/apify/storages/key_value_store.py", - "line": 130, "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v0.2.0/src/apify/storages/key_value_store.py#L130" + "fileName": "/src/apify/consts.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 77 } ], - "signatures": [ + "type": { + "type": "literal", + "value": "'APIFY_LOCAL_STORAGE_DIR'" + } + }, + { + "kind": 16, + "kindString": "Enumeration Member", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "LOG_FORMAT" + } + ] + }, + "flags": {}, + "groups": [], + "id": 228, + "module": "consts", + "name": "LOG_FORMAT", + "parsedDocstring": { + "text": "LOG_FORMAT" + }, + "sources": [ { - "id": 281, - "name": "iterate_keys", - "kind": 4096, - "kindString": "Call signature", - "flags": {}, - "comment": { - "summary": [ - { - "kind": "text", - "text": "{\"content\": [\"Iterate over the keys in the key-value store.\\n\\nArgs:\\n exclusive_start_key (str, optional): All keys up to this one (including) are skipped from the result.\\n\\nYields:\\n IterateKeysTuple: A tuple `(key, info)`,\\n where `key` is the record key, and `info` is an object that contains a single property `size`\\n indicating size of the record in bytes.\"]}" - } - ] - }, - "type": { - "type": "reference", - "name": "AsyncIterator[IterateKeysTuple]" - }, - "parameters": [ - { - "id": 280, - "name": "exclusive_start_key", - "kind": 32768, - "kindString": "Parameter", - "flags": {}, - "type": { - "type": "reference", - "name": "Optional[str]" - }, - "defaultValue": "None" - } - ], - "modifiers": [ - "async" - ] + "character": 1, + "fileName": "/src/apify/consts.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 79 } - ] + ], + "type": { + "type": "literal", + "value": "'APIFY_LOG_FORMAT'" + } }, { - "id": 282, - "name": "set_value", - "module": "storages.key_value_store", - "kind": 2048, - "kindString": "Method", - "flags": {}, + "kind": 16, + "kindString": "Enumeration Member", + "children": [], "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Set or delete a value in the key-value store.\\n\\nArgs:\\n key (str): The key under which the value should be saved.\\n value (Any, optional): The value to save. If the value is `None`, the corresponding key-value pair will be deleted.\\n content_type (str, optional): The content type of the saved value.\"]}" + "text": "LOG_LEVEL" } ] }, - "children": [], + "flags": {}, "groups": [], + "id": 229, + "module": "consts", + "name": "LOG_LEVEL", + "parsedDocstring": { + "text": "LOG_LEVEL" + }, "sources": [ { - "filename": "src/apify/storages/key_value_store.py", - "line": 151, "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v0.2.0/src/apify/storages/key_value_store.py#L151" + "fileName": "/src/apify/consts.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 81 } ], - "signatures": [ + "type": { + "type": "literal", + "value": "'APIFY_LOG_LEVEL'" + } + }, + { + "kind": 16, + "kindString": "Enumeration Member", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "MAX_USED_CPU_RATIO" + } + ] + }, + "flags": {}, + "groups": [], + "id": 230, + "module": "consts", + "name": "MAX_USED_CPU_RATIO", + "parsedDocstring": { + "text": "MAX_USED_CPU_RATIO" + }, + "sources": [ { - "id": 286, - "name": "set_value", - "kind": 4096, - "kindString": "Call signature", - "flags": {}, - "comment": { - "summary": [ - { - "kind": "text", - "text": "{\"content\": [\"Set or delete a value in the key-value store.\\n\\nArgs:\\n key (str): The key under which the value should be saved.\\n value (Any, optional): The value to save. If the value is `None`, the corresponding key-value pair will be deleted.\\n content_type (str, optional): The content type of the saved value.\"]}" - } - ] - }, - "type": { - "type": "reference", - "name": "None" - }, - "parameters": [ - { - "id": 283, - "name": "key", - "kind": 32768, - "kindString": "Parameter", - "flags": {}, - "type": { - "type": "reference", - "name": "str" - } - }, - { - "id": 284, - "name": "value", - "kind": 32768, - "kindString": "Parameter", - "flags": {}, - "type": { - "type": "reference", - "name": "Optional[T]" - } - }, - { - "id": 285, - "name": "content_type", - "kind": 32768, - "kindString": "Parameter", - "flags": {}, - "type": { - "type": "reference", - "name": "Optional[str]" - }, - "defaultValue": "None" - } - ], - "modifiers": [ - "async" - ] + "character": 1, + "fileName": "/src/apify/consts.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 83 } - ] + ], + "type": { + "type": "literal", + "value": "'APIFY_MAX_USED_CPU_RATIO'" + } }, { - "id": 287, - "name": "get_public_url", - "module": "storages.key_value_store", - "kind": 2048, - "kindString": "Method", - "flags": {}, + "kind": 16, + "kindString": "Enumeration Member", + "children": [], "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Get a URL for the given key that may be used to publicly access the value in the remote key-value store.\\n\\nArgs:\\n key (str): The key for which the URL should be generated.\"]}" + "text": "MEMORY_MBYTES" } ] }, - "children": [], + "flags": {}, "groups": [], + "id": 231, + "module": "consts", + "name": "MEMORY_MBYTES", + "parsedDocstring": { + "text": "MEMORY_MBYTES" + }, "sources": [ { - "filename": "src/apify/storages/key_value_store.py", - "line": 169, "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v0.2.0/src/apify/storages/key_value_store.py#L169" + "fileName": "/src/apify/consts.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 85 } ], - "signatures": [ + "type": { + "type": "literal", + "value": "'APIFY_MEMORY_MBYTES'" + } + }, + { + "kind": 16, + "kindString": "Enumeration Member", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "META_ORIGIN" + } + ] + }, + "flags": {}, + "groups": [], + "id": 232, + "module": "consts", + "name": "META_ORIGIN", + "parsedDocstring": { + "text": "META_ORIGIN" + }, + "sources": [ { - "id": 289, - "name": "get_public_url", - "kind": 4096, - "kindString": "Call signature", - "flags": {}, - "comment": { - "summary": [ - { - "kind": "text", - "text": "{\"content\": [\"Get a URL for the given key that may be used to publicly access the value in the remote key-value store.\\n\\nArgs:\\n key (str): The key for which the URL should be generated.\"]}" - } - ] - }, - "type": { - "type": "reference", - "name": "str" - }, - "parameters": [ - { - "id": 288, - "name": "key", - "kind": 32768, - "kindString": "Parameter", - "flags": {}, - "type": { - "type": "reference", - "name": "str" - } - } - ], - "modifiers": [ - "async" - ] + "character": 1, + "fileName": "/src/apify/consts.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 87 } - ] + ], + "type": { + "type": "literal", + "value": "'APIFY_META_ORIGIN'" + } }, { - "id": 290, - "name": "drop", - "module": "storages.key_value_store", - "kind": 2048, - "kindString": "Method", - "flags": {}, + "kind": 16, + "kindString": "Enumeration Member", + "children": [], "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Remove the key-value store either from the Apify cloud storage or from the local directory.\"]}" + "text": "PERSIST_STORAGE" } ] }, - "children": [], + "flags": {}, "groups": [], + "id": 233, + "module": "consts", + "name": "PERSIST_STORAGE", + "parsedDocstring": { + "text": "PERSIST_STORAGE" + }, "sources": [ { - "filename": "src/apify/storages/key_value_store.py", - "line": 186, "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v0.2.0/src/apify/storages/key_value_store.py#L186" + "fileName": "/src/apify/consts.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 89 } ], - "signatures": [ + "type": { + "type": "literal", + "value": "'APIFY_PERSIST_STORAGE'" + } + }, + { + "kind": 16, + "kindString": "Enumeration Member", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "PROXY_HOSTNAME" + } + ] + }, + "flags": {}, + "groups": [], + "id": 234, + "module": "consts", + "name": "PROXY_HOSTNAME", + "parsedDocstring": { + "text": "PROXY_HOSTNAME" + }, + "sources": [ { - "id": 291, - "name": "drop", - "kind": 4096, - "kindString": "Call signature", - "flags": {}, - "comment": { - "summary": [ - { - "kind": "text", - "text": "{\"content\": [\"Remove the key-value store either from the Apify cloud storage or from the local directory.\"]}" - } - ] - }, - "type": { - "type": "reference", - "name": "None" - }, - "parameters": [], - "modifiers": [ - "async" - ] + "character": 1, + "fileName": "/src/apify/consts.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 91 } - ] + ], + "type": { + "type": "literal", + "value": "'APIFY_PROXY_HOSTNAME'" + } }, { - "id": 292, - "name": "open", - "module": "storages.key_value_store", - "kind": 2048, - "kindString": "Method", - "flags": {}, + "kind": 16, + "kindString": "Enumeration Member", + "children": [], "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Open a key-value store.\\n\\nKey-value stores are used to store records or files, along with their MIME content type.\\nThe records are stored and retrieved using a unique key.\\nThe actual data is stored either on a local filesystem or in the Apify cloud.\\n\\nArgs:\\n id (str, optional): ID of the key-value store to be opened.\\n If neither `id` nor `name` are provided, the method returns the default key-value store associated with the actor run.\\n If the key-value store with the given ID does not exist, it raises an error.\\n name (str, optional): Name of the key-value store to be opened.\\n If neither `id` nor `name` are provided, the method returns the default key-value store associated with the actor run.\\n If the key-value store with the given name does not exist, it is created.\\n force_cloud (bool, optional): If set to True, it will open a key-value store on the Apify Platform even when running the actor locally.\\n Defaults to False.\\n config (Configuration, optional): A `Configuration` instance, uses global configuration if omitted.\\n\\nReturns:\\n KeyValueStore: An instance of the `KeyValueStore` class for the given ID or name.\"]}" + "text": "PROXY_PASSWORD" } ] }, - "children": [], + "flags": {}, "groups": [], + "id": 235, + "module": "consts", + "name": "PROXY_PASSWORD", + "parsedDocstring": { + "text": "PROXY_PASSWORD" + }, "sources": [ { - "filename": "src/apify/storages/key_value_store.py", - "line": 192, "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v0.2.0/src/apify/storages/key_value_store.py#L192" + "fileName": "/src/apify/consts.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 93 } ], - "signatures": [ - { - "id": 297, - "name": "open", - "kind": 4096, - "kindString": "Call signature", - "flags": {}, - "comment": { - "summary": [ - { - "kind": "text", - "text": "{\"content\": [\"Open a key-value store.\\n\\nKey-value stores are used to store records or files, along with their MIME content type.\\nThe records are stored and retrieved using a unique key.\\nThe actual data is stored either on a local filesystem or in the Apify cloud.\\n\\nArgs:\\n id (str, optional): ID of the key-value store to be opened.\\n If neither `id` nor `name` are provided, the method returns the default key-value store associated with the actor run.\\n If the key-value store with the given ID does not exist, it raises an error.\\n name (str, optional): Name of the key-value store to be opened.\\n If neither `id` nor `name` are provided, the method returns the default key-value store associated with the actor run.\\n If the key-value store with the given name does not exist, it is created.\\n force_cloud (bool, optional): If set to True, it will open a key-value store on the Apify Platform even when running the actor locally.\\n Defaults to False.\\n config (Configuration, optional): A `Configuration` instance, uses global configuration if omitted.\\n\\nReturns:\\n KeyValueStore: An instance of the `KeyValueStore` class for the given ID or name.\"]}" - } - ] - }, - "type": { - "type": "reference", - "name": "'KeyValueStore'" - }, - "parameters": [ - { - "id": 293, - "name": "id", - "kind": 32768, - "kindString": "Parameter", - "flags": { - "keyword-only": "true" - }, - "type": { - "type": "reference", - "name": "Optional[str]" - }, - "defaultValue": "None" - }, - { - "id": 294, - "name": "name", - "kind": 32768, - "kindString": "Parameter", - "flags": { - "keyword-only": "true" - }, - "type": { - "type": "reference", - "name": "Optional[str]" - }, - "defaultValue": "None" - }, - { - "id": 295, - "name": "force_cloud", - "kind": 32768, - "kindString": "Parameter", - "flags": { - "keyword-only": "true" - }, - "type": { - "type": "reference", - "name": "bool" - }, - "defaultValue": "False" - }, - { - "id": 296, - "name": "config", - "kind": 32768, - "kindString": "Parameter", - "flags": { - "keyword-only": "true" - }, - "type": { - "type": "reference", - "name": "Optional[Configuration]" - }, - "defaultValue": "None" - } - ], - "modifiers": [ - "async" - ] - } - ] - } - ], - "groups": [ - { - "title": "Constructors", - "children": [ - 258 - ] - }, - { - "title": "Methods", - "children": [ - 264, - 267, - 271, - 275, - 279, - 282, - 287, - 290, - 292 - ] - } - ], - "sources": [ - { - "filename": "src/apify/storages/key_value_store.py", - "line": 17, - "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v0.2.0/src/apify/storages/key_value_store.py#L17" - } - ] - }, - { - "id": 298, - "name": "RequestQueue", - "module": "storages.request_queue", - "kind": 128, - "kindString": "Class", - "flags": {}, - "comment": { - "summary": [ - { - "kind": "text", - "text": "{\"content\": [\"Represents a queue of URLs to crawl.\\n\\nCan be used for deep crawling of websites where you start with several URLs and then recursively\\nfollow links to other pages. The data structure supports both breadth-first and depth-first crawling orders.\\n\\nEach URL is represented using an instance of the {@apilink Request} class.\\nThe queue can only contain unique URLs. More precisely, it can only contain request dictionaries\\nwith distinct `uniqueKey` properties. By default, `uniqueKey` is generated from the URL, but it can also be overridden.\\nTo add a single URL multiple times to the queue,\\ncorresponding request dictionary will need to have different `uniqueKey` properties.\\n\\nDo not instantiate this class directly, use the `Actor.open_request_queue()` function instead.\\n\\n`RequestQueue` stores its data either on local disk or in the Apify cloud,\\ndepending on whether the `APIFY_LOCAL_STORAGE_DIR` or `APIFY_TOKEN` environment variables are set.\\n\\nIf the `APIFY_LOCAL_STORAGE_DIR` environment variable is set, the data is stored in\\nthe local directory in the following files:\\n```\\n{APIFY_LOCAL_STORAGE_DIR}/request_queues/{QUEUE_ID}/{REQUEST_ID}.json\\n```\\nNote that `{QUEUE_ID}` is the name or ID of the request queue. The default request queue has ID: `default`,\\nunless you override it by setting the `APIFY_DEFAULT_REQUEST_QUEUE_ID` environment variable.\\nThe `{REQUEST_ID}` is the id of the request.\\n\\nIf the `APIFY_TOKEN` environment variable is set but `APIFY_LOCAL_STORAGE_DIR` is not, the data is stored in the\\n[Apify Request Queue](https://docs.apify.com/storage/request-queue)\\ncloud storage.\"]}" + "type": { + "type": "literal", + "value": "'APIFY_PROXY_PASSWORD'" } - ] - }, - "children": [ + }, { - "id": 299, - "name": "__init__", - "module": "storages.request_queue", - "kind": 512, - "kindString": "Constructor", - "flags": {}, + "kind": 16, + "kindString": "Enumeration Member", + "children": [], "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Create a `RequestQueue` instance.\\n\\nDo not use the constructor directly, use the `Actor.open_request_queue()` function instead.\\n\\nArgs:\\n id (str): ID of the request queue.\\n name (str, optional): Name of the request queue.\\n client (ApifyClientAsync or MemoryStorageClient): The storage client which should be used.\\n config (Configuration): The configuration which should be used.\"]}" + "text": "PROXY_PORT" } ] }, - "children": [], + "flags": {}, "groups": [], + "id": 236, + "module": "consts", + "name": "PROXY_PORT", + "parsedDocstring": { + "text": "PROXY_PORT" + }, "sources": [ { - "filename": "src/apify/storages/request_queue.py", - "line": 87, "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v0.2.0/src/apify/storages/request_queue.py#L87" + "fileName": "/src/apify/consts.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 95 } ], - "signatures": [ - { - "id": 304, - "name": "__init__", - "kind": 4096, - "kindString": "Call signature", - "flags": {}, - "comment": { - "summary": [ - { - "kind": "text", - "text": "{\"content\": [\"Create a `RequestQueue` instance.\\n\\nDo not use the constructor directly, use the `Actor.open_request_queue()` function instead.\\n\\nArgs:\\n id (str): ID of the request queue.\\n name (str, optional): Name of the request queue.\\n client (ApifyClientAsync or MemoryStorageClient): The storage client which should be used.\\n config (Configuration): The configuration which should be used.\"]}" - } - ] - }, - "type": { - "type": "reference", - "name": "None" - }, - "parameters": [ - { - "id": 300, - "name": "id", - "kind": 32768, - "kindString": "Parameter", - "flags": {}, - "type": { - "type": "reference", - "name": "str" - } - }, - { - "id": 301, - "name": "name", - "kind": 32768, - "kindString": "Parameter", - "flags": {}, - "type": { - "type": "reference", - "name": "Optional[str]" - } - }, - { - "id": 302, - "name": "client", - "kind": 32768, - "kindString": "Parameter", - "flags": {}, - "type": { - "type": "reference", - "name": "Union[ApifyClientAsync, MemoryStorageClient]" - } - }, - { - "id": 303, - "name": "config", - "kind": 32768, - "kindString": "Parameter", - "flags": {}, - "type": { - "type": "reference", - "name": "Configuration" - } - } - ] - } - ] + "type": { + "type": "literal", + "value": "'APIFY_PROXY_PORT'" + } }, { - "id": 305, - "name": "add_request", - "module": "storages.request_queue", - "kind": 2048, - "kindString": "Method", - "flags": {}, + "kind": 16, + "kindString": "Enumeration Member", + "children": [], "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Add a request to the queue.\\n\\nArgs:\\n request (dict): The request to add to the queue\\n forefront (bool, optional): Whether to add the request to the head or the end of the queue\\n\\nReturns:\\n dict: Information about the queue operation with keys `requestId`, `uniqueKey`, `wasAlreadyPresent`, `wasAlreadyHandled`.\"]}" + "text": "PROXY_STATUS_URL" } ] }, - "children": [], + "flags": {}, "groups": [], + "id": 237, + "module": "consts", + "name": "PROXY_STATUS_URL", + "parsedDocstring": { + "text": "PROXY_STATUS_URL" + }, "sources": [ { - "filename": "src/apify/storages/request_queue.py", - "line": 131, "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v0.2.0/src/apify/storages/request_queue.py#L131" + "fileName": "/src/apify/consts.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 97 } ], - "signatures": [ - { - "id": 308, - "name": "add_request", - "kind": 4096, - "kindString": "Call signature", - "flags": {}, - "comment": { - "summary": [ - { - "kind": "text", - "text": "{\"content\": [\"Add a request to the queue.\\n\\nArgs:\\n request (dict): The request to add to the queue\\n forefront (bool, optional): Whether to add the request to the head or the end of the queue\\n\\nReturns:\\n dict: Information about the queue operation with keys `requestId`, `uniqueKey`, `wasAlreadyPresent`, `wasAlreadyHandled`.\"]}" - } - ] - }, - "type": { - "type": "reference", - "name": "Dict" - }, - "parameters": [ - { - "id": 306, - "name": "request", - "kind": 32768, - "kindString": "Parameter", - "flags": {}, - "type": { - "type": "reference", - "name": "Dict" - } - }, - { - "id": 307, - "name": "forefront", - "kind": 32768, - "kindString": "Parameter", - "flags": { - "keyword-only": "true" - }, - "type": { - "type": "reference", - "name": "bool" - }, - "defaultValue": "False" - } - ], - "modifiers": [ - "async" - ] - } - ] + "type": { + "type": "literal", + "value": "'APIFY_PROXY_STATUS_URL'" + } }, { - "id": 309, - "name": "get_request", - "module": "storages.request_queue", - "kind": 2048, - "kindString": "Method", - "flags": {}, + "kind": 16, + "kindString": "Enumeration Member", + "children": [], "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Retrieve a request from the queue.\\n\\nArgs:\\n request_id (str): ID of the request to retrieve.\\n\\nReturns:\\n dict, optional: The retrieved request, or `None`, if it does not exist.\"]}" + "text": "SDK_LATEST_VERSION" } ] }, - "children": [], + "flags": {}, "groups": [], + "id": 238, + "module": "consts", + "name": "SDK_LATEST_VERSION", + "parsedDocstring": { + "text": "SDK_LATEST_VERSION" + }, "sources": [ { - "filename": "src/apify/storages/request_queue.py", - "line": 176, "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v0.2.0/src/apify/storages/request_queue.py#L176" + "fileName": "/src/apify/consts.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 99 } ], - "signatures": [ - { - "id": 311, - "name": "get_request", - "kind": 4096, - "kindString": "Call signature", - "flags": {}, - "comment": { - "summary": [ - { - "kind": "text", - "text": "{\"content\": [\"Retrieve a request from the queue.\\n\\nArgs:\\n request_id (str): ID of the request to retrieve.\\n\\nReturns:\\n dict, optional: The retrieved request, or `None`, if it does not exist.\"]}" - } - ] - }, - "type": { - "type": "reference", - "name": "Optional[Dict]" - }, - "parameters": [ - { - "id": 310, - "name": "request_id", - "kind": 32768, - "kindString": "Parameter", - "flags": {}, - "type": { - "type": "reference", - "name": "str" - } - } - ], - "modifiers": [ - "async" - ] - } - ] + "type": { + "type": "literal", + "value": "'APIFY_SDK_LATEST_VERSION'" + } }, { - "id": 312, - "name": "fetch_next_request", - "module": "storages.request_queue", - "kind": 2048, - "kindString": "Method", - "flags": {}, + "kind": 16, + "kindString": "Enumeration Member", + "children": [], "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Return the next request in the queue to be processed.\\n\\nOnce you successfully finish processing of the request, you need to call\\n`RequestQueue.mark_request_as_handled` to mark the request as handled in the queue.\\nIf there was some error in processing the request, call `RequestQueue.reclaim_request` instead,\\nso that the queue will give the request to some other consumer in another call to the `fetch_next_request` method.\\n\\nNote that the `None` return value does not mean the queue processing finished, it means there are currently no pending requests.\\nTo check whether all requests in queue were finished, use `RequestQueue.is_finished` instead.\\n\\nReturns:\\n dict, optional: The request or `None` if there are no more pending requests.\"]}" + "text": "STARTED_AT" } ] }, - "children": [], + "flags": {}, "groups": [], + "id": 239, + "module": "consts", + "name": "STARTED_AT", + "parsedDocstring": { + "text": "STARTED_AT" + }, "sources": [ { - "filename": "src/apify/storages/request_queue.py", - "line": 188, "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v0.2.0/src/apify/storages/request_queue.py#L188" + "fileName": "/src/apify/consts.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 101 } ], - "signatures": [ - { - "id": 313, - "name": "fetch_next_request", - "kind": 4096, - "kindString": "Call signature", - "flags": {}, - "comment": { - "summary": [ - { - "kind": "text", - "text": "{\"content\": [\"Return the next request in the queue to be processed.\\n\\nOnce you successfully finish processing of the request, you need to call\\n`RequestQueue.mark_request_as_handled` to mark the request as handled in the queue.\\nIf there was some error in processing the request, call `RequestQueue.reclaim_request` instead,\\nso that the queue will give the request to some other consumer in another call to the `fetch_next_request` method.\\n\\nNote that the `None` return value does not mean the queue processing finished, it means there are currently no pending requests.\\nTo check whether all requests in queue were finished, use `RequestQueue.is_finished` instead.\\n\\nReturns:\\n dict, optional: The request or `None` if there are no more pending requests.\"]}" - } - ] - }, - "type": { - "type": "reference", - "name": "Optional[Dict]" - }, - "parameters": [], - "modifiers": [ - "async" - ] - } - ] + "type": { + "type": "literal", + "value": "'APIFY_STARTED_AT'" + } }, { - "id": 314, - "name": "mark_request_as_handled", - "module": "storages.request_queue", - "kind": 2048, - "kindString": "Method", - "flags": {}, + "kind": 16, + "kindString": "Enumeration Member", + "children": [], "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Mark a request as handled after successful processing.\\n\\nHandled requests will never again be returned by the `RequestQueue.fetch_next_request` method.\\n\\nArgs:\\n request (dict): The request to mark as handled.\\n\\nReturns:\\n dict, optional: Information about the queue operation with keys `requestId`, `uniqueKey`, `wasAlreadyPresent`, `wasAlreadyHandled`.\\n `None` if the given request was not in progress.\"]}" + "text": "TIMEOUT_AT" } ] }, - "children": [], + "flags": {}, "groups": [], + "id": 240, + "module": "consts", + "name": "TIMEOUT_AT", + "parsedDocstring": { + "text": "TIMEOUT_AT" + }, "sources": [ { - "filename": "src/apify/storages/request_queue.py", - "line": 253, "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v0.2.0/src/apify/storages/request_queue.py#L253" + "fileName": "/src/apify/consts.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 103 } ], - "signatures": [ - { - "id": 316, - "name": "mark_request_as_handled", - "kind": 4096, - "kindString": "Call signature", - "flags": {}, - "comment": { - "summary": [ - { - "kind": "text", - "text": "{\"content\": [\"Mark a request as handled after successful processing.\\n\\nHandled requests will never again be returned by the `RequestQueue.fetch_next_request` method.\\n\\nArgs:\\n request (dict): The request to mark as handled.\\n\\nReturns:\\n dict, optional: Information about the queue operation with keys `requestId`, `uniqueKey`, `wasAlreadyPresent`, `wasAlreadyHandled`.\\n `None` if the given request was not in progress.\"]}" - } - ] - }, - "type": { - "type": "reference", - "name": "Optional[Dict]" - }, - "parameters": [ - { - "id": 315, - "name": "request", - "kind": 32768, - "kindString": "Parameter", - "flags": {}, - "type": { - "type": "reference", - "name": "Dict" - } - } - ], - "modifiers": [ - "async" - ] - } - ] + "type": { + "type": "literal", + "value": "'APIFY_TIMEOUT_AT'" + } }, { - "id": 317, - "name": "reclaim_request", - "module": "storages.request_queue", - "kind": 2048, - "kindString": "Method", - "flags": {}, + "kind": 16, + "kindString": "Enumeration Member", + "children": [], "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Reclaim a failed request back to the queue.\\n\\nThe request will be returned for processing later again\\nby another call to `RequestQueue.fetchNextRequest`.\\n\\nArgs:\\n request (dict): The request to return to the queue.\\n forefront (bool, optional): Whether to add the request to the head or the end of the queue\\nReturns:\\n dict, optional: Information about the queue operation with keys `requestId`, `uniqueKey`, `wasAlreadyPresent`, `wasAlreadyHandled`.\\n `None` if the given request was not in progress.\"]}" + "text": "TOKEN" } ] }, - "children": [], + "flags": {}, "groups": [], + "id": 241, + "module": "consts", + "name": "TOKEN", + "parsedDocstring": { + "text": "TOKEN" + }, "sources": [ { - "filename": "src/apify/storages/request_queue.py", - "line": 289, "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v0.2.0/src/apify/storages/request_queue.py#L289" + "fileName": "/src/apify/consts.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 105 } ], - "signatures": [ - { - "id": 320, - "name": "reclaim_request", - "kind": 4096, - "kindString": "Call signature", - "flags": {}, - "comment": { - "summary": [ - { - "kind": "text", - "text": "{\"content\": [\"Reclaim a failed request back to the queue.\\n\\nThe request will be returned for processing later again\\nby another call to `RequestQueue.fetchNextRequest`.\\n\\nArgs:\\n request (dict): The request to return to the queue.\\n forefront (bool, optional): Whether to add the request to the head or the end of the queue\\nReturns:\\n dict, optional: Information about the queue operation with keys `requestId`, `uniqueKey`, `wasAlreadyPresent`, `wasAlreadyHandled`.\\n `None` if the given request was not in progress.\"]}" - } - ] - }, - "type": { - "type": "reference", - "name": "Optional[Dict]" - }, - "parameters": [ - { - "id": 318, - "name": "request", - "kind": 32768, - "kindString": "Parameter", - "flags": {}, - "type": { - "type": "reference", - "name": "Dict" - } - }, - { - "id": 319, - "name": "forefront", - "kind": 32768, - "kindString": "Parameter", - "flags": {}, - "type": { - "type": "reference", - "name": "bool" - }, - "defaultValue": "False" - } - ], - "modifiers": [ - "async" - ] - } - ] + "type": { + "type": "literal", + "value": "'APIFY_TOKEN'" + } }, { - "id": 321, - "name": "is_empty", - "module": "storages.request_queue", - "kind": 2048, - "kindString": "Method", - "flags": {}, + "kind": 16, + "kindString": "Enumeration Member", + "children": [], "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Check whether the queue is empty.\\n\\nReturns:\\n bool: `True` if the next call to `RequestQueue.fetchNextRequest` would return `None`, otherwise `False`.\"]}" + "text": "USER_ID" } ] }, - "children": [], + "flags": {}, "groups": [], + "id": 242, + "module": "consts", + "name": "USER_ID", + "parsedDocstring": { + "text": "USER_ID" + }, "sources": [ { - "filename": "src/apify/storages/request_queue.py", - "line": 337, "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v0.2.0/src/apify/storages/request_queue.py#L337" + "fileName": "/src/apify/consts.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 107 } ], - "signatures": [ - { - "id": 322, - "name": "is_empty", - "kind": 4096, - "kindString": "Call signature", - "flags": {}, - "comment": { - "summary": [ - { - "kind": "text", - "text": "{\"content\": [\"Check whether the queue is empty.\\n\\nReturns:\\n bool: `True` if the next call to `RequestQueue.fetchNextRequest` would return `None`, otherwise `False`.\"]}" - } - ] - }, - "type": { - "type": "reference", - "name": "bool" - }, - "parameters": [], - "modifiers": [ - "async" - ] - } - ] + "type": { + "type": "literal", + "value": "'APIFY_USER_ID'" + } }, { - "id": 323, - "name": "is_finished", - "module": "storages.request_queue", - "kind": 2048, - "kindString": "Method", - "flags": {}, + "kind": 16, + "kindString": "Enumeration Member", + "children": [], "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Check whether the queue is finished.\\n\\nDue to the nature of distributed storage used by the queue,\\nthe function might occasionally return a false negative,\\nbut it will never return a false positive.\\n\\nReturns:\\n bool: `True` if all requests were already handled and there are no more left. `False` otherwise.\"]}" + "text": "WORKFLOW_KEY" } ] }, - "children": [], + "flags": {}, "groups": [], + "id": 243, + "module": "consts", + "name": "WORKFLOW_KEY", + "parsedDocstring": { + "text": "WORKFLOW_KEY" + }, "sources": [ { - "filename": "src/apify/storages/request_queue.py", - "line": 346, "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v0.2.0/src/apify/storages/request_queue.py#L346" + "fileName": "/src/apify/consts.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 109 } ], - "signatures": [ - { - "id": 324, - "name": "is_finished", - "kind": 4096, - "kindString": "Call signature", - "flags": {}, - "comment": { - "summary": [ - { - "kind": "text", - "text": "{\"content\": [\"Check whether the queue is finished.\\n\\nDue to the nature of distributed storage used by the queue,\\nthe function might occasionally return a false negative,\\nbut it will never return a false positive.\\n\\nReturns:\\n bool: `True` if all requests were already handled and there are no more left. `False` otherwise.\"]}" - } - ] - }, - "type": { - "type": "reference", - "name": "bool" - }, - "parameters": [], - "modifiers": [ - "async" - ] - } - ] + "type": { + "type": "literal", + "value": "'APIFY_WORKFLOW_KEY'" + } }, { - "id": 325, - "name": "drop", - "module": "storages.request_queue", - "kind": 2048, - "kindString": "Method", - "flags": {}, + "kind": 16, + "kindString": "Enumeration Member", + "children": [], "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Remove the request queue either from the Apify cloud storage or from the local directory.\"]}" + "text": "XVFB" } ] }, - "children": [], + "flags": {}, "groups": [], + "id": 244, + "module": "consts", + "name": "XVFB", + "parsedDocstring": { + "text": "XVFB" + }, "sources": [ { - "filename": "src/apify/storages/request_queue.py", - "line": 480, "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v0.2.0/src/apify/storages/request_queue.py#L480" + "fileName": "/src/apify/consts.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 111 } ], - "signatures": [ - { - "id": 326, - "name": "drop", - "kind": 4096, - "kindString": "Call signature", - "flags": {}, - "comment": { - "summary": [ - { - "kind": "text", - "text": "{\"content\": [\"Remove the request queue either from the Apify cloud storage or from the local directory.\"]}" - } - ] - }, - "type": { - "type": "reference", - "name": "None" - }, - "parameters": [], - "modifiers": [ - "async" - ] - } - ] + "type": { + "type": "literal", + "value": "'APIFY_XVFB'" + } }, { - "id": 327, - "name": "get_info", - "module": "storages.request_queue", - "kind": 2048, - "kindString": "Method", - "flags": {}, + "kind": 16, + "kindString": "Enumeration Member", + "children": [], "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Get an object containing general information about the request queue.\\n\\nReturns:\\n dict: Object returned by calling the GET request queue API endpoint.\"]}" + "text": "Extra ones not in @apify/consts:\nMETAMORPH_AFTER_SLEEP_MILLIS" } ] }, - "children": [], + "flags": {}, "groups": [], + "id": 245, + "module": "consts", + "name": "METAMORPH_AFTER_SLEEP_MILLIS", + "parsedDocstring": { + "text": "Extra ones not in @apify/consts:\nMETAMORPH_AFTER_SLEEP_MILLIS" + }, "sources": [ { - "filename": "src/apify/storages/request_queue.py", - "line": 485, "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v0.2.0/src/apify/storages/request_queue.py#L485" + "fileName": "/src/apify/consts.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 115 } ], - "signatures": [ - { - "id": 328, - "name": "get_info", - "kind": 4096, - "kindString": "Call signature", - "flags": {}, - "comment": { - "summary": [ - { - "kind": "text", - "text": "{\"content\": [\"Get an object containing general information about the request queue.\\n\\nReturns:\\n dict: Object returned by calling the GET request queue API endpoint.\"]}" - } - ] - }, - "type": { - "type": "reference", - "name": "Optional[Dict]" - }, - "parameters": [], - "modifiers": [ - "async" - ] - } - ] + "type": { + "type": "literal", + "value": "'APIFY_METAMORPH_AFTER_SLEEP_MILLIS'" + } }, { - "id": 329, - "name": "open", - "module": "storages.request_queue", - "kind": 2048, - "kindString": "Method", - "flags": {}, + "kind": 16, + "kindString": "Enumeration Member", + "children": [], "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Open a request queue.\\n\\nRequest queue represents a queue of URLs to crawl, which is stored either on local filesystem or in the Apify cloud.\\nThe queue is used for deep crawling of websites, where you start with several URLs and then\\nrecursively follow links to other pages. The data structure supports both breadth-first\\nand depth-first crawling orders.\\n\\nArgs:\\n id (str, optional): ID of the request queue to be opened.\\n If neither `id` nor `name` are provided, the method returns the default request queue associated with the actor run.\\n If the request queue with the given ID does not exist, it raises an error.\\n name (str, optional): Name of the request queue to be opened.\\n If neither `id` nor `name` are provided, the method returns the default request queue associated with the actor run.\\n If the request queue with the given name does not exist, it is created.\\n force_cloud (bool, optional): If set to True, it will open a request queue on the Apify Platform even when running the actor locally.\\n Defaults to False.\\n config (Configuration, optional): A `Configuration` instance, uses global configuration if omitted.\\n\\nReturns:\\n RequestQueue: An instance of the `RequestQueue` class for the given ID or name.\"]}" + "text": "PERSIST_STATE_INTERVAL_MILLIS" } ] }, - "children": [], + "flags": {}, "groups": [], + "id": 246, + "module": "consts", + "name": "PERSIST_STATE_INTERVAL_MILLIS", + "parsedDocstring": { + "text": "PERSIST_STATE_INTERVAL_MILLIS" + }, "sources": [ { - "filename": "src/apify/storages/request_queue.py", - "line": 494, "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v0.2.0/src/apify/storages/request_queue.py#L494" + "fileName": "/src/apify/consts.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 117 } ], - "signatures": [ - { - "id": 334, - "name": "open", - "kind": 4096, - "kindString": "Call signature", - "flags": {}, - "comment": { - "summary": [ - { - "kind": "text", - "text": "{\"content\": [\"Open a request queue.\\n\\nRequest queue represents a queue of URLs to crawl, which is stored either on local filesystem or in the Apify cloud.\\nThe queue is used for deep crawling of websites, where you start with several URLs and then\\nrecursively follow links to other pages. The data structure supports both breadth-first\\nand depth-first crawling orders.\\n\\nArgs:\\n id (str, optional): ID of the request queue to be opened.\\n If neither `id` nor `name` are provided, the method returns the default request queue associated with the actor run.\\n If the request queue with the given ID does not exist, it raises an error.\\n name (str, optional): Name of the request queue to be opened.\\n If neither `id` nor `name` are provided, the method returns the default request queue associated with the actor run.\\n If the request queue with the given name does not exist, it is created.\\n force_cloud (bool, optional): If set to True, it will open a request queue on the Apify Platform even when running the actor locally.\\n Defaults to False.\\n config (Configuration, optional): A `Configuration` instance, uses global configuration if omitted.\\n\\nReturns:\\n RequestQueue: An instance of the `RequestQueue` class for the given ID or name.\"]}" - } - ] - }, - "type": { - "type": "reference", - "name": "'RequestQueue'" - }, - "parameters": [ - { - "id": 330, - "name": "id", - "kind": 32768, - "kindString": "Parameter", - "flags": { - "keyword-only": "true" - }, - "type": { - "type": "reference", - "name": "Optional[str]" - }, - "defaultValue": "None" - }, - { - "id": 331, - "name": "name", - "kind": 32768, - "kindString": "Parameter", - "flags": { - "keyword-only": "true" - }, - "type": { - "type": "reference", - "name": "Optional[str]" - }, - "defaultValue": "None" - }, - { - "id": 332, - "name": "force_cloud", - "kind": 32768, - "kindString": "Parameter", - "flags": { - "keyword-only": "true" - }, - "type": { - "type": "reference", - "name": "bool" - }, - "defaultValue": "False" - }, - { - "id": 333, - "name": "config", - "kind": 32768, - "kindString": "Parameter", - "flags": { - "keyword-only": "true" - }, - "type": { - "type": "reference", - "name": "Optional[Configuration]" - }, - "defaultValue": "None" - } - ], - "modifiers": [ - "async" - ] + "type": { + "type": "literal", + "value": "'APIFY_PERSIST_STATE_INTERVAL_MILLIS'" + } + }, + { + "kind": 16, + "kindString": "Enumeration Member", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "PURGE_ON_START" + } + ] + }, + "flags": {}, + "groups": [], + "id": 247, + "module": "consts", + "name": "PURGE_ON_START", + "parsedDocstring": { + "text": "PURGE_ON_START" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/consts.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 119 } - ] + ], + "type": { + "type": "literal", + "value": "'APIFY_PURGE_ON_START'" + } + }, + { + "kind": 16, + "kindString": "Enumeration Member", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "SYSTEM_INFO_INTERVAL_MILLIS" + } + ] + }, + "flags": {}, + "groups": [], + "id": 248, + "module": "consts", + "name": "SYSTEM_INFO_INTERVAL_MILLIS", + "parsedDocstring": { + "text": "SYSTEM_INFO_INTERVAL_MILLIS" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/consts.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 121 + } + ], + "type": { + "type": "literal", + "value": "'APIFY_SYSTEM_INFO_INTERVAL_MILLIS'" + } } ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Possible Apify-specific environment variables." + } + ] + }, + "flags": {}, "groups": [ { - "title": "Constructors", "children": [ - 299 - ] - }, + 201, + 202, + 203, + 204, + 205, + 206, + 207, + 208, + 209, + 210, + 211, + 212, + 213, + 214, + 215, + 216, + 217, + 218, + 219, + 220, + 221, + 222, + 223, + 224, + 225, + 226, + 227, + 228, + 229, + 230, + 231, + 232, + 245, + 246, + 233, + 234, + 235, + 236, + 237, + 247, + 238, + 239, + 248, + 240, + 241, + 242, + 243, + 244 + ], + "title": "Enumeration members" + } + ], + "id": 200, + "module": "consts", + "name": "ApifyEnvVars", + "parsedDocstring": { + "text": "Possible Apify-specific environment variables." + }, + "sources": [ { - "title": "Methods", - "children": [ - 305, - 309, - 312, - 314, - 317, - 321, - 323, - 325, - 327, - 329 - ] + "character": 1, + "fileName": "/src/apify/consts.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 19 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 249, + "module": "consts", + "name": "INTEGER_ENV_VARS", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/consts.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 135 } ], + "type": { + "name": "List", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "_INTEGER_ENV_VARS_TYPE" + } + ] + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 250, + "module": "consts", + "name": "FLOAT_ENV_VARS", + "parsedDocstring": { + "text": "" + }, "sources": [ { - "filename": "src/apify/storages/request_queue.py", - "line": 43, "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v0.2.0/src/apify/storages/request_queue.py#L43" + "fileName": "/src/apify/consts.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 141 } - ] - } - ], - "groups": [ + ], + "type": { + "name": "List", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "_FLOAT_ENV_VARS_TYPE" + } + ] + } + }, { - "title": "Other", - "children": [ - 1, - 143, - 167, - 168, - 186, - 190, - 191, - 192, - 193, - 257, - 298 - ] - } - ], - "sources": [ + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 251, + "module": "consts", + "name": "BOOL_ENV_VARS", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/consts.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 153 + } + ], + "type": { + "name": "List", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "_BOOL_ENV_VARS_TYPE" + } + ] + } + }, { - "fileName": "src/index.ts", - "line": 1, - "character": 0, - "url": "http://example.com/blob/v0.2.0/src/dummy.py" + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 252, + "module": "consts", + "name": "DATETIME_ENV_VARS", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/consts.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 160 + } + ], + "type": { + "name": "List", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "_DATETIME_ENV_VARS_TYPE" + } + ] + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 253, + "module": "consts", + "name": "STRING_ENV_VARS", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/consts.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 195 + } + ], + "type": { + "name": "List", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "_STRING_ENV_VARS_TYPE" + } + ] + } + }, + { + "kind": 8, + "kindString": "Enumeration", + "children": [ + { + "kind": 16, + "kindString": "Enumeration Member", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "The actor finished successfully" + } + ] + }, + "flags": {}, + "groups": [], + "id": 255, + "module": "consts", + "name": "SUCCESS", + "parsedDocstring": { + "text": "The actor finished successfully" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/consts.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 202 + } + ], + "type": { + "type": "literal", + "value": "0" + } + }, + { + "kind": 16, + "kindString": "Enumeration Member", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "The main function of the actor threw an Exception" + } + ] + }, + "flags": {}, + "groups": [], + "id": 256, + "module": "consts", + "name": "ERROR_USER_FUNCTION_THREW", + "parsedDocstring": { + "text": "The main function of the actor threw an Exception" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/consts.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 205 + } + ], + "type": { + "type": "literal", + "value": "91" + } + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Usual actor exit codes." + } + ] + }, + "flags": {}, + "groups": [ + { + "children": [ + 256, + 255 + ], + "title": "Enumeration members" + } + ], + "id": 254, + "module": "consts", + "name": "ActorExitCodes", + "parsedDocstring": { + "text": "Usual actor exit codes." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/consts.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 198 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 8, + "kindString": "Enumeration", + "children": [ + { + "kind": 16, + "kindString": "Enumeration Member", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 258, + "module": "consts", + "name": "DATASET", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/consts.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 211 + } + ], + "type": { + "type": "literal", + "value": "'Dataset'" + } + }, + { + "kind": 16, + "kindString": "Enumeration Member", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 259, + "module": "consts", + "name": "KEY_VALUE_STORE", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/consts.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 212 + } + ], + "type": { + "type": "literal", + "value": "'Key-value store'" + } + }, + { + "kind": 16, + "kindString": "Enumeration Member", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 260, + "module": "consts", + "name": "REQUEST_QUEUE", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/consts.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 213 + } + ], + "type": { + "type": "literal", + "value": "'Request queue'" + } + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Possible Apify storage types." + } + ] + }, + "flags": {}, + "groups": [ + { + "children": [ + 258, + 259, + 260 + ], + "title": "Enumeration members" + } + ], + "id": 257, + "module": "consts", + "name": "_StorageTypes", + "parsedDocstring": { + "text": "Possible Apify storage types." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/consts.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 208 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 261, + "module": "consts", + "name": "DEFAULT_API_PARAM_LIMIT", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/consts.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 216 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 262, + "module": "consts", + "name": "REQUEST_ID_LENGTH", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/consts.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 218 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 263, + "module": "consts", + "name": "REQUEST_QUEUE_HEAD_MAX_LIMIT", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/consts.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 220 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 264, + "module": "consts", + "name": "EVENT_LISTENERS_TIMEOUT_SECS", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/consts.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 222 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 265, + "module": "consts", + "name": "BASE64_REGEXP", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/consts.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 224 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 266, + "module": "consts", + "name": "ENCRYPTED_INPUT_VALUE_PREFIX", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/consts.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 225 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 267, + "module": "consts", + "name": "ENCRYPTED_INPUT_VALUE_REGEXP", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/consts.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 226 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "9MB" + } + ] + }, + "flags": {}, + "groups": [], + "id": 268, + "module": "consts", + "name": "MAX_PAYLOAD_SIZE_BYTES", + "parsedDocstring": { + "text": "9MB" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/consts.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 228 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 269, + "module": "log", + "name": "logger_name", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/log.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 15 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 270, + "module": "log", + "name": "logger", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/log.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 18 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 272, + "module": "log", + "name": "empty_record", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/log.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 55 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Log formatter that prints out the log message nicely formatted, with colored level and stringified extra fields.\n\nIt formats the log records so that they:\n- start with the level (colorized, and padded to 5 chars so that it is nicely aligned)\n- then have the actual log message, if it's multiline then it's nicely indented\n- then have the stringified extra log fields\n- then, if an exception is a part of the log record, prints the formatted exception." + } + ] + }, + "flags": {}, + "groups": [ + { + "children": [ + 272 + ], + "title": "Properties" + } + ], + "id": 271, + "module": "log", + "name": "ActorLogFormatter", + "parsedDocstring": { + "text": "Log formatter that prints out the log message nicely formatted, with colored level and stringified extra fields.\n\nIt formats the log records so that they:\n- start with the level (colorized, and padded to 5 chars so that it is nicely aligned)\n- then have the actual log message, if it's multiline then it's nicely indented\n- then have the stringified extra log fields\n- then, if an exception is a part of the log record, prints the formatted exception." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/log.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 40 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 273, + "module": "_version", + "name": "__version__", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_version.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 1 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 274, + "module": "proxy_configuration", + "name": "APIFY_PROXY_VALUE_REGEX", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/proxy_configuration.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 17 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 275, + "module": "proxy_configuration", + "name": "COUNTRY_CODE_REGEX", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/proxy_configuration.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 18 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 276, + "module": "proxy_configuration", + "name": "SESSION_ID_MAX_LENGTH", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/proxy_configuration.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 19 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "The URL of the proxy." + } + ] + }, + "flags": {}, + "groups": [], + "id": 278, + "module": "proxy_configuration", + "name": "url", + "parsedDocstring": { + "text": "The URL of the proxy." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/proxy_configuration.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 64 + } + ], + "type": { + "name": "str", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "The hostname of the proxy." + } + ] + }, + "flags": {}, + "groups": [], + "id": 279, + "module": "proxy_configuration", + "name": "hostname", + "parsedDocstring": { + "text": "The hostname of the proxy." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/proxy_configuration.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 67 + } + ], + "type": { + "name": "str", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "The proxy port." + } + ] + }, + "flags": {}, + "groups": [], + "id": 280, + "module": "proxy_configuration", + "name": "port", + "parsedDocstring": { + "text": "The proxy port." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/proxy_configuration.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 70 + } + ], + "type": { + "name": "int", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "The username for the proxy." + } + ] + }, + "flags": {}, + "groups": [], + "id": 281, + "module": "proxy_configuration", + "name": "username", + "parsedDocstring": { + "text": "The username for the proxy." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/proxy_configuration.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 73 + } + ], + "type": { + "name": "NotRequired", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "str" + } + ] + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "The password for the proxy." + } + ] + }, + "flags": {}, + "groups": [], + "id": 282, + "module": "proxy_configuration", + "name": "password", + "parsedDocstring": { + "text": "The password for the proxy." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/proxy_configuration.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 76 + } + ], + "type": { + "name": "str", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "An array of proxy groups to be used by the [Apify Proxy](https://docs.apify.com/proxy).\nIf not provided, the proxy will select the groups automatically." + } + ] + }, + "flags": {}, + "groups": [], + "id": 283, + "module": "proxy_configuration", + "name": "groups", + "parsedDocstring": { + "text": "An array of proxy groups to be used by the [Apify Proxy](https://docs.apify.com/proxy).\nIf not provided, the proxy will select the groups automatically." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/proxy_configuration.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 79 + } + ], + "type": { + "name": "NotRequired", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "List", + "typeArguments": [ + { + "type": "reference", + "name": "str" + } + ] + } + ] + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "If set and relevant proxies are available in your Apify account, all proxied requests will\nuse IP addresses that are geolocated to the specified country. For example `GB` for IPs\nfrom Great Britain. Note that online services often have their own rules for handling\ngeolocation and thus the country selection is a best attempt at geolocation, rather than\na guaranteed hit. This parameter is optional, by default, each proxied request is assigned\nan IP address from a random country. The country code needs to be a two letter ISO country code.\nSee the [full list of available country codes](https://en.wikipedia.org/wiki/ISO_3166-1_alpha-2#Officially_assigned_code_elements).\nThis parameter is optional, by default, the proxy uses all available proxy servers from all countries." + } + ] + }, + "flags": {}, + "groups": [], + "id": 284, + "module": "proxy_configuration", + "name": "country_code", + "parsedDocstring": { + "text": "If set and relevant proxies are available in your Apify account, all proxied requests will\nuse IP addresses that are geolocated to the specified country. For example `GB` for IPs\nfrom Great Britain. Note that online services often have their own rules for handling\ngeolocation and thus the country selection is a best attempt at geolocation, rather than\na guaranteed hit. This parameter is optional, by default, each proxied request is assigned\nan IP address from a random country. The country code needs to be a two letter ISO country code.\nSee the [full list of available country codes](https://en.wikipedia.org/wiki/ISO_3166-1_alpha-2#Officially_assigned_code_elements).\nThis parameter is optional, by default, the proxy uses all available proxy servers from all countries." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/proxy_configuration.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 84 + } + ], + "type": { + "name": "NotRequired", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "str" + } + ] + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "The identifier of the used proxy session, if used. Using the same session ID guarantees getting the same proxy URL." + } + ] + }, + "flags": {}, + "groups": [], + "id": 285, + "module": "proxy_configuration", + "name": "session_id", + "parsedDocstring": { + "text": "The identifier of the used proxy session, if used. Using the same session ID guarantees getting the same proxy URL." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/proxy_configuration.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 95 + } + ], + "type": { + "name": "NotRequired", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "str" + } + ] + } + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Provides information about a proxy connection that is used for requests." + } + ] + }, + "flags": {}, + "groups": [ + { + "children": [ + 284, + 283, + 279, + 282, + 280, + 285, + 278, + 281 + ], + "title": "Properties" + } + ], + "id": 277, + "module": "proxy_configuration", + "name": "ProxyInfo", + "parsedDocstring": { + "text": "Provides information about a proxy connection that is used for requests." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/proxy_configuration.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 61 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 287, + "module": "proxy_configuration", + "name": "is_man_in_the_middle", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/proxy_configuration.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 111 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Load the Apify Proxy password if the API token is provided and check access to Apify Proxy and provided proxy groups.\n\nOnly called if Apify Proxy configuration is used.\nAlso checks if country has access to Apify Proxy groups if the country code is provided.\n\nYou should use the Actor.create_proxy_configuration function\nto create a pre-initialized `ProxyConfiguration` instance instead of calling this manually." + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 288, + "module": "proxy_configuration", + "name": "initialize", + "parsedDocstring": { + "text": "Load the Apify Proxy password if the API token is provided and check access to Apify Proxy and provided proxy groups.\n\nOnly called if Apify Proxy configuration is used.\nAlso checks if country has access to Apify Proxy groups if the country code is provided.\n\nYou should use the Actor.create_proxy_configuration function\nto create a pre-initialized `ProxyConfiguration` instance instead of calling this manually." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/proxy_configuration.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 187 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Load the Apify Proxy password if the API token is provided and check access to Apify Proxy and provided proxy groups.\n\nOnly called if Apify Proxy configuration is used.\nAlso checks if country has access to Apify Proxy groups if the country code is provided.\n\nYou should use the Actor.create_proxy_configuration function\nto create a pre-initialized `ProxyConfiguration` instance instead of calling this manually." + } + ] + }, + "flags": {}, + "id": 289, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "initialize", + "parameters": [], + "type": { + "name": "None", + "type": "literal", + "value": null + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Return a new proxy URL based on provided configuration options and the `sessionId` parameter.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 290, + "module": "proxy_configuration", + "name": "new_url", + "parsedDocstring": { + "text": "Return a new proxy URL based on provided configuration options and the `sessionId` parameter.\n", + "args": { + "session_id": "Represents the identifier of a proxy session (https://docs.apify.com/proxy#sessions).\nAll the HTTP requests going through the proxy with the same session identifier\nwill use the same target proxy server (i.e. the same IP address).\nThe identifier must not be longer than 50 characters and include only the following: `0-9`, `a-z`, `A-Z`, `\".\"`, `\"_\"` and `\"~\"`.\n" + }, + "returns": "str: A string with a proxy URL, including authentication credentials and port number.\nFor example, `http://bob:password123@proxy.example.com:8000`" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/proxy_configuration.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 200 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "str: A string with a proxy URL, including authentication credentials and port number.\nFor example, `http://bob:password123@proxy.example.com:8000`" + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Return a new proxy URL based on provided configuration options and the `sessionId` parameter.\n" + } + ] + }, + "flags": {}, + "id": 291, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "new_url", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Represents the identifier of a proxy session (https://docs.apify.com/proxy#sessions).\nAll the HTTP requests going through the proxy with the same session identifier\nwill use the same target proxy server (i.e. the same IP address).\nThe identifier must not be longer than 50 characters and include only the following: `0-9`, `a-z`, `A-Z`, `\".\"`, `\"_\"` and `\"~\"`.\n" + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": false + }, + "id": 292, + "kind": 32768, + "kindString": "Parameter", + "name": "session_id", + "type": { + "name": "Optional", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "Union", + "typeArguments": [ + { + "type": "reference", + "name": "int" + }, + { + "type": "reference", + "name": "str" + } + ] + } + ] + } + } + ], + "type": { + "name": "str", + "type": "reference" + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Create a new ProxyInfo object.\n\nUse it if you want to work with a rich representation of a proxy URL.\nIf you need the URL string only, use `ProxyConfiguration.new_url`.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 293, + "module": "proxy_configuration", + "name": "new_proxy_info", + "parsedDocstring": { + "text": "Create a new ProxyInfo object.\n\nUse it if you want to work with a rich representation of a proxy URL.\nIf you need the URL string only, use `ProxyConfiguration.new_url`.\n", + "args": { + "session_id": "Represents the identifier of a proxy session (https://docs.apify.com/proxy#sessions).\nAll the HTTP requests going through the proxy with the same session identifier\nwill use the same target proxy server (i.e. the same IP address).\nThe identifier must not be longer than 50 characters and include only the following: `0-9`, `a-z`, `A-Z`, `\".\"`, `\"_\"` and `\"~\"`.\n" + }, + "returns": "ProxyInfo: Dictionary that represents information about the proxy and its configuration." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/proxy_configuration.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 243 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "ProxyInfo: Dictionary that represents information about the proxy and its configuration." + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Create a new ProxyInfo object.\n\nUse it if you want to work with a rich representation of a proxy URL.\nIf you need the URL string only, use `ProxyConfiguration.new_url`.\n" + } + ] + }, + "flags": {}, + "id": 294, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "new_proxy_info", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Represents the identifier of a proxy session (https://docs.apify.com/proxy#sessions).\nAll the HTTP requests going through the proxy with the same session identifier\nwill use the same target proxy server (i.e. the same IP address).\nThe identifier must not be longer than 50 characters and include only the following: `0-9`, `a-z`, `A-Z`, `\".\"`, `\"_\"` and `\"~\"`.\n" + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": false + }, + "id": 295, + "kind": 32768, + "kindString": "Parameter", + "name": "session_id", + "type": { + "name": "Optional", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "Union", + "typeArguments": [ + { + "type": "reference", + "name": "int" + }, + { + "type": "reference", + "name": "str" + } + ] + } + ] + } + } + ], + "type": { + "name": "ProxyInfo", + "type": "reference", + "target": "277" + } + } + ] + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Configures a connection to a proxy server with the provided options.\n\nProxy servers are used to prevent target websites from blocking your crawlers based on IP address rate limits or blacklists.\nThe default servers used by this class are managed by [Apify Proxy](https://docs.apify.com/proxy).\nTo be able to use Apify Proxy, you need an Apify account and access to the selected proxies. If you provide no configuration option,\nthe proxies will be managed automatically using a smart algorithm.\n\nIf you want to use your own proxies, use the `proxy_urls` or `new_url_function` constructor options.\nYour list of proxy URLs will be rotated by the configuration, if this option is provided." + } + ] + }, + "flags": {}, + "groups": [ + { + "children": [ + 288, + 293, + 290 + ], + "title": "Methods" + }, + { + "children": [ + 287 + ], + "title": "Properties" + } + ], + "id": 286, + "module": "proxy_configuration", + "name": "ProxyConfiguration", + "parsedDocstring": { + "text": "Configures a connection to a proxy server with the provided options.\n\nProxy servers are used to prevent target websites from blocking your crawlers based on IP address rate limits or blacklists.\nThe default servers used by this class are managed by [Apify Proxy](https://docs.apify.com/proxy).\nTo be able to use Apify Proxy, you need an Apify account and access to the selected proxies. If you provide no configuration option,\nthe proxies will be managed automatically using a smart algorithm.\n\nIf you want to use your own proxies, use the `proxy_urls` or `new_url_function` constructor options.\nYour list of proxy URLs will be rotated by the configuration, if this option is provided." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/proxy_configuration.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 99 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 296, + "module": "_utils", + "name": "T", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_utils.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 46 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 297, + "module": "_utils", + "name": "DualPropertyType", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_utils.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 70 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 298, + "module": "_utils", + "name": "DualPropertyOwner", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_utils.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 71 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Initialize the dualproperty.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 299, + "module": "_utils", + "name": "__init__", + "parsedDocstring": { + "text": "Initialize the dualproperty.\n", + "args": { + "getter": "The getter of the property.\nIt should accept either an instance or a class as its first argument." + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_utils.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 82 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Initialize the dualproperty.\n" + } + ] + }, + "flags": {}, + "id": 300, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "__init__", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The getter of the property.\nIt should accept either an instance or a class as its first argument." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 301, + "kind": 32768, + "kindString": "Parameter", + "name": "getter", + "type": { + "name": "Callable", + "type": "reference", + "typeArguments": [ + { + "type": "literal", + "value": "..." + }, + { + "type": "reference", + "name": "DualPropertyType", + "target": "297" + } + ] + } + } + ], + "type": { + "name": "None", + "type": "literal", + "value": null + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Call the getter with the right object.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 302, + "module": "_utils", + "name": "__get__", + "parsedDocstring": { + "text": "Call the getter with the right object.\n", + "args": { + "obj": "The instance of class T on which the getter will be called", + "owner": "The class object of class T on which the getter will be called, if obj is None\n" + }, + "returns": "The result of the getter." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_utils.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 91 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "The result of the getter." + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Call the getter with the right object.\n" + } + ] + }, + "flags": {}, + "id": 303, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "__get__", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The instance of class T on which the getter will be called" + } + ] + }, + "flags": { + "isOptional": true, + "keyword-only": false + }, + "id": 304, + "kind": 32768, + "kindString": "Parameter", + "name": "obj", + "type": { + "name": "Optional", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "DualPropertyOwner", + "target": "298" + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The class object of class T on which the getter will be called, if obj is None\n" + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 305, + "kind": 32768, + "kindString": "Parameter", + "name": "owner", + "type": { + "name": "Type", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "DualPropertyOwner", + "target": "298" + } + ] + } + } + ], + "type": { + "name": "DualPropertyType", + "type": "reference", + "target": "297" + } + } + ] + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 306, + "module": "_utils", + "name": "ImplementationType", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_utils.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 348 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 307, + "module": "_utils", + "name": "MetadataType", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_utils.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 349 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Create a LRUCache with a specific max_length." + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 308, + "module": "_utils", + "name": "__init__", + "parsedDocstring": { + "text": "Create a LRUCache with a specific max_length." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_utils.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 368 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Create a LRUCache with a specific max_length." + } + ] + }, + "flags": {}, + "id": 309, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "__init__", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 310, + "kind": 32768, + "kindString": "Parameter", + "name": "max_length", + "type": { + "name": "int", + "type": "reference" + } + } + ], + "type": { + "name": "None", + "type": "literal", + "value": null + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Get an item from the cache. Move it to the end if present." + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 311, + "module": "_utils", + "name": "__getitem__", + "parsedDocstring": { + "text": "Get an item from the cache. Move it to the end if present." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_utils.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 373 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Get an item from the cache. Move it to the end if present." + } + ] + }, + "flags": {}, + "id": 312, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "__getitem__", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 313, + "kind": 32768, + "kindString": "Parameter", + "name": "key", + "type": { + "name": "str", + "type": "reference" + } + } + ], + "type": { + "name": "T", + "type": "reference", + "target": "296" + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Add an item to the cache. Remove least used item if max_length exceeded." + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 314, + "module": "_utils", + "name": "__setitem__", + "parsedDocstring": { + "text": "Add an item to the cache. Remove least used item if max_length exceeded." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_utils.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 381 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Add an item to the cache. Remove least used item if max_length exceeded." + } + ] + }, + "flags": {}, + "id": 315, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "__setitem__", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 316, + "kind": 32768, + "kindString": "Parameter", + "name": "key", + "type": { + "name": "str", + "type": "reference" + } + }, + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 317, + "kind": 32768, + "kindString": "Parameter", + "name": "value", + "type": { + "name": "T", + "type": "reference", + "target": "296" + } + } + ], + "type": { + "name": "None", + "type": "literal", + "value": null + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Remove an item from the cache." + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 318, + "module": "_utils", + "name": "__delitem__", + "parsedDocstring": { + "text": "Remove an item from the cache." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_utils.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 387 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Remove an item from the cache." + } + ] + }, + "flags": {}, + "id": 319, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "__delitem__", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 320, + "kind": 32768, + "kindString": "Parameter", + "name": "key", + "type": { + "name": "str", + "type": "reference" + } + } + ], + "type": { + "name": "None", + "type": "literal", + "value": null + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Iterate over the keys of the cache in order of insertion." + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 321, + "module": "_utils", + "name": "__iter__", + "parsedDocstring": { + "text": "Iterate over the keys of the cache in order of insertion." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_utils.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 391 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Iterate over the keys of the cache in order of insertion." + } + ] + }, + "flags": {}, + "id": 322, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "__iter__", + "parameters": [], + "type": { + "name": "Iterator", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "str" + } + ] + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Get the number of items in the cache." + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 323, + "module": "_utils", + "name": "__len__", + "parsedDocstring": { + "text": "Get the number of items in the cache." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_utils.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 395 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Get the number of items in the cache." + } + ] + }, + "flags": {}, + "id": 324, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "__len__", + "parameters": [], + "type": { + "name": "int", + "type": "reference" + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Iterate over the values in the cache in order of insertion." + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 325, + "module": "_utils", + "name": "values", + "parsedDocstring": { + "text": "Iterate over the values in the cache in order of insertion." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_utils.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 399 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Iterate over the values in the cache in order of insertion." + } + ] + }, + "flags": {}, + "id": 326, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "values", + "parameters": [], + "type": { + "name": "ValuesView", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "T", + "target": "296" + } + ] + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Iterate over the pairs of (key, value) in the cache in order of insertion." + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 327, + "module": "_utils", + "name": "items", + "parsedDocstring": { + "text": "Iterate over the pairs of (key, value) in the cache in order of insertion." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_utils.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 403 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Iterate over the pairs of (key, value) in the cache in order of insertion." + } + ] + }, + "flags": {}, + "id": 328, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "items", + "parameters": [], + "type": { + "name": "ItemsView", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "reference", + "name": "T", + "target": "296" + } + ] + } + } + ] + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 329, + "module": "_utils", + "name": "PARSE_DATE_FIELDS_MAX_DEPTH", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_utils.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 447 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 330, + "module": "_utils", + "name": "PARSE_DATE_FIELDS_KEY_SUFFIX", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_utils.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 448 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 331, + "module": "_utils", + "name": "ListOrDictOrAny", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_utils.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 449 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Initialize the MemoryStorageClient.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 332, + "module": "_memory_storage.memory_storage_client", + "name": "__init__", + "parsedDocstring": { + "text": "Initialize the MemoryStorageClient.\n", + "args": { + "local_data_directory": "A local directory where all data will be persisted", + "persist_storage": "Whether to persist the data to the `local_data_directory` or just keep them in memory", + "write_metadata": "Whether to persist metadata of the storages as well" + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_memory_storage/memory_storage_client.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 46 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Initialize the MemoryStorageClient.\n" + } + ] + }, + "flags": {}, + "id": 333, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "__init__", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "A local directory where all data will be persisted" + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 334, + "kind": 32768, + "kindString": "Parameter", + "name": "local_data_directory", + "type": { + "name": "Optional", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "str" + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Whether to persist metadata of the storages as well" + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 335, + "kind": 32768, + "kindString": "Parameter", + "name": "write_metadata", + "type": { + "name": "Optional", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "bool" + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Whether to persist the data to the `local_data_directory` or just keep them in memory" + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 336, + "kind": 32768, + "kindString": "Parameter", + "name": "persist_storage", + "type": { + "name": "Optional", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "bool" + } + ] + } + } + ], + "type": { + "name": "None", + "type": "literal", + "value": null + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Retrieve the sub-client for manipulating datasets." + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 337, + "module": "_memory_storage.memory_storage_client", + "name": "datasets", + "parsedDocstring": { + "text": "Retrieve the sub-client for manipulating datasets." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_memory_storage/memory_storage_client.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 67 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Retrieve the sub-client for manipulating datasets." + } + ] + }, + "flags": {}, + "id": 338, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "datasets", + "parameters": [], + "type": { + "name": "DatasetCollectionClient", + "type": "reference" + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Retrieve the sub-client for manipulating a single dataset.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 339, + "module": "_memory_storage.memory_storage_client", + "name": "dataset", + "parsedDocstring": { + "text": "Retrieve the sub-client for manipulating a single dataset.\n", + "args": { + "dataset_id": "ID of the dataset to be manipulated" + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_memory_storage/memory_storage_client.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 71 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Retrieve the sub-client for manipulating a single dataset.\n" + } + ] + }, + "flags": {}, + "id": 340, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "dataset", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "ID of the dataset to be manipulated" + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 341, + "kind": 32768, + "kindString": "Parameter", + "name": "dataset_id", + "type": { + "name": "str", + "type": "reference" + } + } + ], + "type": { + "name": "DatasetClient", + "type": "reference" + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Retrieve the sub-client for manipulating key-value stores." + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 342, + "module": "_memory_storage.memory_storage_client", + "name": "key_value_stores", + "parsedDocstring": { + "text": "Retrieve the sub-client for manipulating key-value stores." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_memory_storage/memory_storage_client.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 79 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Retrieve the sub-client for manipulating key-value stores." + } + ] + }, + "flags": {}, + "id": 343, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "key_value_stores", + "parameters": [], + "type": { + "name": "KeyValueStoreCollectionClient", + "type": "reference" + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Retrieve the sub-client for manipulating a single key-value store.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 344, + "module": "_memory_storage.memory_storage_client", + "name": "key_value_store", + "parsedDocstring": { + "text": "Retrieve the sub-client for manipulating a single key-value store.\n", + "args": { + "key_value_store_id": "ID of the key-value store to be manipulated" + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_memory_storage/memory_storage_client.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 83 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Retrieve the sub-client for manipulating a single key-value store.\n" + } + ] + }, + "flags": {}, + "id": 345, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "key_value_store", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "ID of the key-value store to be manipulated" + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 346, + "kind": 32768, + "kindString": "Parameter", + "name": "key_value_store_id", + "type": { + "name": "str", + "type": "reference" + } + } + ], + "type": { + "name": "KeyValueStoreClient", + "type": "reference" + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Retrieve the sub-client for manipulating request queues." + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 347, + "module": "_memory_storage.memory_storage_client", + "name": "request_queues", + "parsedDocstring": { + "text": "Retrieve the sub-client for manipulating request queues." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_memory_storage/memory_storage_client.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 91 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Retrieve the sub-client for manipulating request queues." + } + ] + }, + "flags": {}, + "id": 348, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "request_queues", + "parameters": [], + "type": { + "name": "RequestQueueCollectionClient", + "type": "reference" + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Retrieve the sub-client for manipulating a single request queue.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 349, + "module": "_memory_storage.memory_storage_client", + "name": "request_queue", + "parsedDocstring": { + "text": "Retrieve the sub-client for manipulating a single request queue.\n", + "args": { + "request_queue_id": "ID of the request queue to be manipulated", + "client_key": "A unique identifier of the client accessing the request queue" + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_memory_storage/memory_storage_client.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 95 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Retrieve the sub-client for manipulating a single request queue.\n" + } + ] + }, + "flags": {}, + "id": 350, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "request_queue", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "ID of the request queue to be manipulated" + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 351, + "kind": 32768, + "kindString": "Parameter", + "name": "request_queue_id", + "type": { + "name": "str", + "type": "reference" + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "A unique identifier of the client accessing the request queue" + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 352, + "kind": 32768, + "kindString": "Parameter", + "name": "client_key", + "type": { + "name": "Optional", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "str" + } + ] + } + } + ], + "type": { + "name": "RequestQueueClient", + "type": "reference" + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "List the available key-value stores.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 353, + "module": "_memory_storage.resource_clients.key_value_store_collection", + "name": "list", + "parsedDocstring": { + "text": "List the available key-value stores.\n", + "returns": "ListPage: The list of available key-value stores matching the specified filters." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_memory_storage/resource_clients/key_value_store_collection.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 20 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "ListPage: The list of available key-value stores matching the specified filters." + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "List the available key-value stores.\n" + } + ] + }, + "flags": {}, + "id": 354, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "list", + "parameters": [], + "type": { + "name": "ListPage", + "type": "reference" + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Retrieve a named key-value store, or create a new one when it doesn't exist.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 355, + "module": "_memory_storage.resource_clients.key_value_store_collection", + "name": "get_or_create", + "parsedDocstring": { + "text": "Retrieve a named key-value store, or create a new one when it doesn't exist.\n", + "args": { + "name": "The name of the key-value store to retrieve or create.", + "schema": "The schema of the key-value store\n" + }, + "returns": "dict: The retrieved or newly-created key-value store." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_memory_storage/resource_clients/key_value_store_collection.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 28 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "dict: The retrieved or newly-created key-value store." + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Retrieve a named key-value store, or create a new one when it doesn't exist.\n" + } + ] + }, + "flags": {}, + "id": 356, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "get_or_create", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The name of the key-value store to retrieve or create." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 357, + "kind": 32768, + "kindString": "Parameter", + "name": "name", + "type": { + "name": "Optional", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "str" + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The schema of the key-value store\n" + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 358, + "kind": 32768, + "kindString": "Parameter", + "name": "schema", + "type": { + "name": "Optional", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "Dict" + } + ] + } + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 359, + "kind": 32768, + "kindString": "Parameter", + "name": "_id", + "type": { + "name": "Optional", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "str" + } + ] + } + } + ], + "type": { + "name": "Dict", + "type": "reference" + } + } + ] + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 360, + "module": "_memory_storage.resource_clients.base_resource_collection_client", + "name": "ResourceClientType", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_memory_storage/resource_clients/base_resource_collection_client.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 15 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Initialize the DatasetCollectionClient with the passed arguments." + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 361, + "module": "_memory_storage.resource_clients.base_resource_collection_client", + "name": "__init__", + "parsedDocstring": { + "text": "Initialize the DatasetCollectionClient with the passed arguments." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_memory_storage/resource_clients/base_resource_collection_client.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 25 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Initialize the DatasetCollectionClient with the passed arguments." + } + ] + }, + "flags": {}, + "id": 362, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "__init__", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 363, + "kind": 32768, + "kindString": "Parameter", + "name": "base_storage_directory", + "type": { + "name": "str", + "type": "reference" + } + }, + { + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 364, + "kind": 32768, + "kindString": "Parameter", + "name": "memory_storage_client", + "type": { + "name": "'MemoryStorageClient'", + "type": "literal", + "value": "MemoryStorageClient" + } + } + ], + "type": { + "name": "None", + "type": "literal", + "value": null + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "List the available storages.\n" + } + ] + }, + "decorations": [ + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 365, + "module": "_memory_storage.resource_clients.base_resource_collection_client", + "name": "list", + "parsedDocstring": { + "text": "List the available storages.\n", + "returns": "ListPage: The list of available storages matching the specified filters." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_memory_storage/resource_clients/base_resource_collection_client.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 39 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "ListPage: The list of available storages matching the specified filters." + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "List the available storages.\n" + } + ] + }, + "flags": {}, + "id": 366, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "list", + "parameters": [], + "type": { + "name": "ListPage", + "type": "reference" + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Retrieve a named storage, or create a new one when it doesn't exist.\n" + } + ] + }, + "decorations": [ + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 367, + "module": "_memory_storage.resource_clients.base_resource_collection_client", + "name": "get_or_create", + "parsedDocstring": { + "text": "Retrieve a named storage, or create a new one when it doesn't exist.\n", + "args": { + "name": "The name of the storage to retrieve or create.", + "schema": "The schema of the storage\n" + }, + "returns": "dict: The retrieved or newly-created storage." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_memory_storage/resource_clients/base_resource_collection_client.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 59 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "dict: The retrieved or newly-created storage." + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Retrieve a named storage, or create a new one when it doesn't exist.\n" + } + ] + }, + "flags": {}, + "id": 368, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "get_or_create", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The name of the storage to retrieve or create." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 369, + "kind": 32768, + "kindString": "Parameter", + "name": "name", + "type": { + "name": "Optional", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "str" + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The schema of the storage\n" + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 370, + "kind": 32768, + "kindString": "Parameter", + "name": "schema", + "type": { + "name": "Optional", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "Dict" + } + ] + } + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 371, + "kind": 32768, + "kindString": "Parameter", + "name": "_id", + "type": { + "name": "Optional", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "str" + } + ] + } + } + ], + "type": { + "name": "Dict", + "type": "reference" + } + } + ] + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 372, + "module": "_memory_storage.resource_clients.dataset", + "name": "LIST_ITEMS_LIMIT", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_memory_storage/resource_clients/dataset.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 23 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 373, + "module": "_memory_storage.resource_clients.dataset", + "name": "LOCAL_ENTRY_NAME_DIGITS", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_memory_storage/resource_clients/dataset.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 27 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Initialize the DatasetClient." + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 374, + "module": "_memory_storage.resource_clients.dataset", + "name": "__init__", + "parsedDocstring": { + "text": "Initialize the DatasetClient." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_memory_storage/resource_clients/dataset.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 45 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Initialize the DatasetClient." + } + ] + }, + "flags": {}, + "id": 375, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "__init__", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 376, + "kind": 32768, + "kindString": "Parameter", + "name": "base_storage_directory", + "type": { + "name": "str", + "type": "reference" + } + }, + { + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 377, + "kind": 32768, + "kindString": "Parameter", + "name": "memory_storage_client", + "type": { + "name": "'MemoryStorageClient'", + "type": "literal", + "value": "MemoryStorageClient" + } + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 378, + "kind": 32768, + "kindString": "Parameter", + "name": "id", + "type": { + "name": "Optional", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "str" + } + ] + } + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 379, + "kind": 32768, + "kindString": "Parameter", + "name": "name", + "type": { + "name": "Optional", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "str" + } + ] + } + } + ], + "type": { + "name": "None", + "type": "literal", + "value": null + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Retrieve the dataset.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 380, + "module": "_memory_storage.resource_clients.dataset", + "name": "get", + "parsedDocstring": { + "text": "Retrieve the dataset.\n", + "returns": "dict, optional: The retrieved dataset, or None, if it does not exist" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_memory_storage/resource_clients/dataset.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 64 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "dict, optional: The retrieved dataset, or None, if it does not exist" + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Retrieve the dataset.\n" + } + ] + }, + "flags": {}, + "id": 381, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "get", + "parameters": [], + "type": { + "name": "Optional", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "Dict" + } + ] + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Update the dataset with specified fields.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 382, + "module": "_memory_storage.resource_clients.dataset", + "name": "update", + "parsedDocstring": { + "text": "Update the dataset with specified fields.\n", + "args": { + "name": "The new name for the dataset\n" + }, + "returns": "dict: The updated dataset" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_memory_storage/resource_clients/dataset.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 79 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "dict: The updated dataset" + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Update the dataset with specified fields.\n" + } + ] + }, + "flags": {}, + "id": 383, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "update", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The new name for the dataset\n" + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 384, + "kind": 32768, + "kindString": "Parameter", + "name": "name", + "type": { + "name": "Optional", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "str" + } + ] + } + } + ], + "type": { + "name": "Dict", + "type": "reference" + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Delete the dataset." + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 385, + "module": "_memory_storage.resource_clients.dataset", + "name": "delete", + "parsedDocstring": { + "text": "Delete the dataset." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_memory_storage/resource_clients/dataset.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 125 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Delete the dataset." + } + ] + }, + "flags": {}, + "id": 386, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "delete", + "parameters": [], + "type": { + "name": "None", + "type": "literal", + "value": null + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "List the items of the dataset.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 387, + "module": "_memory_storage.resource_clients.dataset", + "name": "list_items", + "parsedDocstring": { + "text": "List the items of the dataset.\n", + "args": { + "offset": "Number of items that should be skipped at the start. The default value is 0", + "limit": "Maximum number of items to return. By default there is no limit.", + "desc": "By default, results are returned in the same order as they were stored.\nTo reverse the order, set this parameter to True.", + "clean": "If True, returns only non-empty items and skips hidden fields (i.e. fields starting with the # character).\nThe clean parameter is just a shortcut for skip_hidden=True and skip_empty=True parameters.\nNote that since some objects might be skipped from the output, that the result might contain less items than the limit value.", + "fields": "A list of fields which should be picked from the items,\nonly these fields will remain in the resulting record objects.\nNote that the fields in the outputted items are sorted the same way as they are specified in the fields parameter.\nYou can use this feature to effectively fix the output format.", + "omit": "A list of fields which should be omitted from the items.", + "unwind": "Name of a field which should be unwound.\nIf the field is an array then every element of the array will become a separate record and merged with parent object.\nIf the unwound field is an object then it is merged with the parent object.\nIf the unwound field is missing or its value is neither an array nor an object and therefore cannot be merged with a parent object,\nthen the item gets preserved as it is. Note that the unwound items ignore the desc parameter.", + "skip_empty": "If True, then empty items are skipped from the output.\nNote that if used, the results might contain less items than the limit value.", + "skip_hidden": "If True, then hidden fields are skipped from the output, i.e. fields starting with the # character.", + "flatten": "A list of fields that should be flattened", + "view": "Name of the dataset view to be used\n" + }, + "returns": "ListPage: A page of the list of dataset items according to the specified filters." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_memory_storage/resource_clients/dataset.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 138 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "ListPage: A page of the list of dataset items according to the specified filters." + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "List the items of the dataset.\n" + } + ] + }, + "flags": {}, + "id": 388, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "list_items", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Number of items that should be skipped at the start. The default value is 0" + } + ] + }, + "defaultValue": "0", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 389, + "kind": 32768, + "kindString": "Parameter", + "name": "offset", + "type": { + "name": "Optional", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "int" + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Maximum number of items to return. By default there is no limit." + } + ] + }, + "defaultValue": "LIST_ITEMS_LIMIT", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 390, + "kind": 32768, + "kindString": "Parameter", + "name": "limit", + "type": { + "name": "Optional", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "int" + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "If True, returns only non-empty items and skips hidden fields (i.e. fields starting with the # character).\nThe clean parameter is just a shortcut for skip_hidden=True and skip_empty=True parameters.\nNote that since some objects might be skipped from the output, that the result might contain less items than the limit value." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 391, + "kind": 32768, + "kindString": "Parameter", + "name": "clean", + "type": { + "name": "Optional", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "bool" + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "By default, results are returned in the same order as they were stored.\nTo reverse the order, set this parameter to True." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 392, + "kind": 32768, + "kindString": "Parameter", + "name": "desc", + "type": { + "name": "Optional", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "bool" + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "A list of fields which should be picked from the items,\nonly these fields will remain in the resulting record objects.\nNote that the fields in the outputted items are sorted the same way as they are specified in the fields parameter.\nYou can use this feature to effectively fix the output format." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 393, + "kind": 32768, + "kindString": "Parameter", + "name": "fields", + "type": { + "name": "Optional", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "List", + "typeArguments": [ + { + "type": "reference", + "name": "str" + } + ] + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "A list of fields which should be omitted from the items." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 394, + "kind": 32768, + "kindString": "Parameter", + "name": "omit", + "type": { + "name": "Optional", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "List", + "typeArguments": [ + { + "type": "reference", + "name": "str" + } + ] + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Name of a field which should be unwound.\nIf the field is an array then every element of the array will become a separate record and merged with parent object.\nIf the unwound field is an object then it is merged with the parent object.\nIf the unwound field is missing or its value is neither an array nor an object and therefore cannot be merged with a parent object,\nthen the item gets preserved as it is. Note that the unwound items ignore the desc parameter." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 395, + "kind": 32768, + "kindString": "Parameter", + "name": "unwind", + "type": { + "name": "Optional", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "str" + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "If True, then empty items are skipped from the output.\nNote that if used, the results might contain less items than the limit value." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 396, + "kind": 32768, + "kindString": "Parameter", + "name": "skip_empty", + "type": { + "name": "Optional", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "bool" + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "If True, then hidden fields are skipped from the output, i.e. fields starting with the # character." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 397, + "kind": 32768, + "kindString": "Parameter", + "name": "skip_hidden", + "type": { + "name": "Optional", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "bool" + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "A list of fields that should be flattened" + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 398, + "kind": 32768, + "kindString": "Parameter", + "name": "flatten", + "type": { + "name": "Optional", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "List", + "typeArguments": [ + { + "type": "reference", + "name": "str" + } + ] + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Name of the dataset view to be used\n" + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 399, + "kind": 32768, + "kindString": "Parameter", + "name": "view", + "type": { + "name": "Optional", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "str" + } + ] + } + } + ], + "type": { + "name": "ListPage", + "type": "reference" + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Iterate over the items in the dataset.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 400, + "module": "_memory_storage.resource_clients.dataset", + "name": "iterate_items", + "parsedDocstring": { + "text": "Iterate over the items in the dataset.\n", + "args": { + "offset": "Number of items that should be skipped at the start. The default value is 0", + "limit": "Maximum number of items to return. By default there is no limit.", + "desc": "By default, results are returned in the same order as they were stored.\nTo reverse the order, set this parameter to True.", + "clean": "If True, returns only non-empty items and skips hidden fields (i.e. fields starting with the # character).\nThe clean parameter is just a shortcut for skip_hidden=True and skip_empty=True parameters.\nNote that since some objects might be skipped from the output, that the result might contain less items than the limit value.", + "fields": "A list of fields which should be picked from the items,\nonly these fields will remain in the resulting record objects.\nNote that the fields in the outputted items are sorted the same way as they are specified in the fields parameter.\nYou can use this feature to effectively fix the output format.", + "omit": "A list of fields which should be omitted from the items.", + "unwind": "Name of a field which should be unwound.\nIf the field is an array then every element of the array will become a separate record and merged with parent object.\nIf the unwound field is an object then it is merged with the parent object.\nIf the unwound field is missing or its value is neither an array nor an object and therefore cannot be merged with a parent object,\nthen the item gets preserved as it is. Note that the unwound items ignore the desc parameter.", + "skip_empty": "If True, then empty items are skipped from the output.\nNote that if used, the results might contain less items than the limit value.", + "skip_hidden": "If True, then hidden fields are skipped from the output, i.e. fields starting with the # character.\n" + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_memory_storage/resource_clients/dataset.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 218 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Iterate over the items in the dataset.\n" + } + ] + }, + "flags": {}, + "id": 401, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "iterate_items", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Number of items that should be skipped at the start. The default value is 0" + } + ] + }, + "defaultValue": "0", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 402, + "kind": 32768, + "kindString": "Parameter", + "name": "offset", + "type": { + "name": "int", + "type": "reference" + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Maximum number of items to return. By default there is no limit." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 403, + "kind": 32768, + "kindString": "Parameter", + "name": "limit", + "type": { + "name": "Optional", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "int" + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "If True, returns only non-empty items and skips hidden fields (i.e. fields starting with the # character).\nThe clean parameter is just a shortcut for skip_hidden=True and skip_empty=True parameters.\nNote that since some objects might be skipped from the output, that the result might contain less items than the limit value." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 404, + "kind": 32768, + "kindString": "Parameter", + "name": "clean", + "type": { + "name": "Optional", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "bool" + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "By default, results are returned in the same order as they were stored.\nTo reverse the order, set this parameter to True." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 405, + "kind": 32768, + "kindString": "Parameter", + "name": "desc", + "type": { + "name": "Optional", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "bool" + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "A list of fields which should be picked from the items,\nonly these fields will remain in the resulting record objects.\nNote that the fields in the outputted items are sorted the same way as they are specified in the fields parameter.\nYou can use this feature to effectively fix the output format." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 406, + "kind": 32768, + "kindString": "Parameter", + "name": "fields", + "type": { + "name": "Optional", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "List", + "typeArguments": [ + { + "type": "reference", + "name": "str" + } + ] + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "A list of fields which should be omitted from the items." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 407, + "kind": 32768, + "kindString": "Parameter", + "name": "omit", + "type": { + "name": "Optional", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "List", + "typeArguments": [ + { + "type": "reference", + "name": "str" + } + ] + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Name of a field which should be unwound.\nIf the field is an array then every element of the array will become a separate record and merged with parent object.\nIf the unwound field is an object then it is merged with the parent object.\nIf the unwound field is missing or its value is neither an array nor an object and therefore cannot be merged with a parent object,\nthen the item gets preserved as it is. Note that the unwound items ignore the desc parameter." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 408, + "kind": 32768, + "kindString": "Parameter", + "name": "unwind", + "type": { + "name": "Optional", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "str" + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "If True, then empty items are skipped from the output.\nNote that if used, the results might contain less items than the limit value." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 409, + "kind": 32768, + "kindString": "Parameter", + "name": "skip_empty", + "type": { + "name": "Optional", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "bool" + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "If True, then hidden fields are skipped from the output, i.e. fields starting with the # character.\n" + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 410, + "kind": 32768, + "kindString": "Parameter", + "name": "skip_hidden", + "type": { + "name": "Optional", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "bool" + } + ] + } + } + ], + "type": { + "name": "AsyncIterator", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "Dict" + } + ] + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 411, + "module": "_memory_storage.resource_clients.dataset", + "name": "get_items_as_bytes", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_memory_storage/resource_clients/dataset.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 287 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "flags": {}, + "id": 412, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "get_items_as_bytes", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 413, + "kind": 32768, + "kindString": "Parameter", + "name": "_args", + "type": { + "name": "Any", + "type": "reference" + } + }, + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 414, + "kind": 32768, + "kindString": "Parameter", + "name": "_kwargs", + "type": { + "name": "Any", + "type": "reference" + } + } + ], + "type": { + "name": "bytes", + "type": "reference" + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 415, + "module": "_memory_storage.resource_clients.dataset", + "name": "stream_items", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_memory_storage/resource_clients/dataset.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 290 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "flags": {}, + "id": 416, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "stream_items", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 417, + "kind": 32768, + "kindString": "Parameter", + "name": "_args", + "type": { + "name": "Any", + "type": "reference" + } + }, + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 418, + "kind": 32768, + "kindString": "Parameter", + "name": "_kwargs", + "type": { + "name": "Any", + "type": "reference" + } + } + ], + "type": { + "name": "AsyncIterator", + "type": "reference" + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Push items to the dataset.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 419, + "module": "_memory_storage.resource_clients.dataset", + "name": "push_items", + "parsedDocstring": { + "text": "Push items to the dataset.\n", + "args": { + "items": "The items which to push in the dataset. Either a stringified JSON, a dictionary, or a list of strings or dictionaries." + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_memory_storage/resource_clients/dataset.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 293 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Push items to the dataset.\n" + } + ] + }, + "flags": {}, + "id": 420, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "push_items", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The items which to push in the dataset. Either a stringified JSON, a dictionary, or a list of strings or dictionaries." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 421, + "kind": 32768, + "kindString": "Parameter", + "name": "items", + "type": { + "name": "JSONSerializable", + "type": "reference", + "target": "194" + } + } + ], + "type": { + "name": "None", + "type": "literal", + "value": null + } + } + ] + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 422, + "module": "_memory_storage.resource_clients.key_value_store", + "name": "DEFAULT_LOCAL_FILE_EXTENSION", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_memory_storage/resource_clients/key_value_store.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 33 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Initialize the KeyValueStoreClient." + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 423, + "module": "_memory_storage.resource_clients.key_value_store", + "name": "__init__", + "parsedDocstring": { + "text": "Initialize the KeyValueStoreClient." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_memory_storage/resource_clients/key_value_store.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 50 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Initialize the KeyValueStoreClient." + } + ] + }, + "flags": {}, + "id": 424, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "__init__", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 425, + "kind": 32768, + "kindString": "Parameter", + "name": "base_storage_directory", + "type": { + "name": "str", + "type": "reference" + } + }, + { + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 426, + "kind": 32768, + "kindString": "Parameter", + "name": "memory_storage_client", + "type": { + "name": "'MemoryStorageClient'", + "type": "literal", + "value": "MemoryStorageClient" + } + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 427, + "kind": 32768, + "kindString": "Parameter", + "name": "id", + "type": { + "name": "Optional", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "str" + } + ] + } + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 428, + "kind": 32768, + "kindString": "Parameter", + "name": "name", + "type": { + "name": "Optional", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "str" + } + ] + } + } + ], + "type": { + "name": "None", + "type": "literal", + "value": null + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Retrieve the key-value store.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 429, + "module": "_memory_storage.resource_clients.key_value_store", + "name": "get", + "parsedDocstring": { + "text": "Retrieve the key-value store.\n", + "returns": "dict, optional: The retrieved key-value store, or None if it does not exist" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_memory_storage/resource_clients/key_value_store.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 69 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "dict, optional: The retrieved key-value store, or None if it does not exist" + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Retrieve the key-value store.\n" + } + ] + }, + "flags": {}, + "id": 430, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "get", + "parameters": [], + "type": { + "name": "Optional", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "Dict" + } + ] + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Update the key-value store with specified fields.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 431, + "module": "_memory_storage.resource_clients.key_value_store", + "name": "update", + "parsedDocstring": { + "text": "Update the key-value store with specified fields.\n", + "args": { + "name": "The new name for key-value store\n" + }, + "returns": "dict: The updated key-value store" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_memory_storage/resource_clients/key_value_store.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 84 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "dict: The updated key-value store" + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Update the key-value store with specified fields.\n" + } + ] + }, + "flags": {}, + "id": 432, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "update", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The new name for key-value store\n" + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 433, + "kind": 32768, + "kindString": "Parameter", + "name": "name", + "type": { + "name": "Optional", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "str" + } + ] + } + } + ], + "type": { + "name": "Dict", + "type": "reference" + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Delete the key-value store." + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 434, + "module": "_memory_storage.resource_clients.key_value_store", + "name": "delete", + "parsedDocstring": { + "text": "Delete the key-value store." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_memory_storage/resource_clients/key_value_store.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 127 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Delete the key-value store." + } + ] + }, + "flags": {}, + "id": 435, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "delete", + "parameters": [], + "type": { + "name": "None", + "type": "literal", + "value": null + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "List the keys in the key-value store.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 436, + "module": "_memory_storage.resource_clients.key_value_store", + "name": "list_keys", + "parsedDocstring": { + "text": "List the keys in the key-value store.\n", + "args": { + "limit": "Number of keys to be returned. Maximum value is 1000", + "exclusive_start_key": "All keys up to this one (including) are skipped from the result\n" + }, + "returns": "dict: The list of keys in the key-value store matching the given arguments" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_memory_storage/resource_clients/key_value_store.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 139 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "dict: The list of keys in the key-value store matching the given arguments" + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "List the keys in the key-value store.\n" + } + ] + }, + "flags": {}, + "id": 437, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "list_keys", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Number of keys to be returned. Maximum value is 1000" + } + ] + }, + "defaultValue": "DEFAULT_API_PARAM_LIMIT", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 438, + "kind": 32768, + "kindString": "Parameter", + "name": "limit", + "type": { + "name": "int", + "type": "reference" + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "All keys up to this one (including) are skipped from the result\n" + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 439, + "kind": 32768, + "kindString": "Parameter", + "name": "exclusive_start_key", + "type": { + "name": "Optional", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "str" + } + ] + } + } + ], + "type": { + "name": "Dict", + "type": "reference" + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Retrieve the given record from the key-value store.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 440, + "module": "_memory_storage.resource_clients.key_value_store", + "name": "get_record", + "parsedDocstring": { + "text": "Retrieve the given record from the key-value store.\n", + "args": { + "key": "Key of the record to retrieve\n" + }, + "returns": "dict, optional: The requested record, or None, if the record does not exist" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_memory_storage/resource_clients/key_value_store.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 234 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "dict, optional: The requested record, or None, if the record does not exist" + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Retrieve the given record from the key-value store.\n" + } + ] + }, + "flags": {}, + "id": 441, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "get_record", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Key of the record to retrieve\n" + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 442, + "kind": 32768, + "kindString": "Parameter", + "name": "key", + "type": { + "name": "str", + "type": "reference" + } + } + ], + "type": { + "name": "Optional", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "Dict" + } + ] + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Retrieve the given record from the key-value store, without parsing it.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 443, + "module": "_memory_storage.resource_clients.key_value_store", + "name": "get_record_as_bytes", + "parsedDocstring": { + "text": "Retrieve the given record from the key-value store, without parsing it.\n", + "args": { + "key": "Key of the record to retrieve\n" + }, + "returns": "dict, optional: The requested record, or None, if the record does not exist" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_memory_storage/resource_clients/key_value_store.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 245 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "dict, optional: The requested record, or None, if the record does not exist" + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Retrieve the given record from the key-value store, without parsing it.\n" + } + ] + }, + "flags": {}, + "id": 444, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "get_record_as_bytes", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Key of the record to retrieve\n" + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 445, + "kind": 32768, + "kindString": "Parameter", + "name": "key", + "type": { + "name": "str", + "type": "reference" + } + } + ], + "type": { + "name": "Optional", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "Dict" + } + ] + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 446, + "module": "_memory_storage.resource_clients.key_value_store", + "name": "stream_record", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_memory_storage/resource_clients/key_value_store.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 256 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "flags": {}, + "id": 447, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "stream_record", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 448, + "kind": 32768, + "kindString": "Parameter", + "name": "_key", + "type": { + "name": "str", + "type": "reference" + } + } + ], + "type": { + "name": "AsyncIterator", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "Optional", + "typeArguments": [ + { + "type": "reference", + "name": "Dict" + } + ] + } + ] + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Set a value to the given record in the key-value store.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 449, + "module": "_memory_storage.resource_clients.key_value_store", + "name": "set_record", + "parsedDocstring": { + "text": "Set a value to the given record in the key-value store.\n", + "args": { + "key": "The key of the record to save the value to", + "value": "The value to save into the record", + "content_type": "The content type of the saved value" + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_memory_storage/resource_clients/key_value_store.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 259 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Set a value to the given record in the key-value store.\n" + } + ] + }, + "flags": {}, + "id": 450, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "set_record", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The key of the record to save the value to" + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 451, + "kind": 32768, + "kindString": "Parameter", + "name": "key", + "type": { + "name": "str", + "type": "reference" + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The value to save into the record" + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 452, + "kind": 32768, + "kindString": "Parameter", + "name": "value", + "type": { + "name": "Any", + "type": "reference" + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The content type of the saved value" + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": false + }, + "id": 453, + "kind": 32768, + "kindString": "Parameter", + "name": "content_type", + "type": { + "name": "Optional", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "str" + } + ] + } + } + ], + "type": { + "name": "None", + "type": "literal", + "value": null + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Delete the specified record from the key-value store.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 454, + "module": "_memory_storage.resource_clients.key_value_store", + "name": "delete_record", + "parsedDocstring": { + "text": "Delete the specified record from the key-value store.\n", + "args": { + "key": "The key of the record which to delete" + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_memory_storage/resource_clients/key_value_store.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 309 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Delete the specified record from the key-value store.\n" + } + ] + }, + "flags": {}, + "id": 455, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "delete_record", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The key of the record which to delete" + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 456, + "kind": 32768, + "kindString": "Parameter", + "name": "key", + "type": { + "name": "str", + "type": "reference" + } + } + ], + "type": { + "name": "None", + "type": "literal", + "value": null + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Initialize the RequestQueueClient." + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 457, + "module": "_memory_storage.resource_clients.request_queue", + "name": "__init__", + "parsedDocstring": { + "text": "Initialize the RequestQueueClient." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_memory_storage/resource_clients/request_queue.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 46 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Initialize the RequestQueueClient." + } + ] + }, + "flags": {}, + "id": 458, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "__init__", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 459, + "kind": 32768, + "kindString": "Parameter", + "name": "base_storage_directory", + "type": { + "name": "str", + "type": "reference" + } + }, + { + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 460, + "kind": 32768, + "kindString": "Parameter", + "name": "memory_storage_client", + "type": { + "name": "'MemoryStorageClient'", + "type": "literal", + "value": "MemoryStorageClient" + } + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 461, + "kind": 32768, + "kindString": "Parameter", + "name": "id", + "type": { + "name": "Optional", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "str" + } + ] + } + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 462, + "kind": 32768, + "kindString": "Parameter", + "name": "name", + "type": { + "name": "Optional", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "str" + } + ] + } + } + ], + "type": { + "name": "None", + "type": "literal", + "value": null + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Retrieve the request queue.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 463, + "module": "_memory_storage.resource_clients.request_queue", + "name": "get", + "parsedDocstring": { + "text": "Retrieve the request queue.\n", + "returns": "dict, optional: The retrieved request queue, or None, if it does not exist" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_memory_storage/resource_clients/request_queue.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 65 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "dict, optional: The retrieved request queue, or None, if it does not exist" + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Retrieve the request queue.\n" + } + ] + }, + "flags": {}, + "id": 464, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "get", + "parameters": [], + "type": { + "name": "Optional", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "Dict" + } + ] + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Update the request queue with specified fields.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 465, + "module": "_memory_storage.resource_clients.request_queue", + "name": "update", + "parsedDocstring": { + "text": "Update the request queue with specified fields.\n", + "args": { + "name": "The new name for the request queue\n" + }, + "returns": "dict: The updated request queue" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_memory_storage/resource_clients/request_queue.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 80 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "dict: The updated request queue" + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Update the request queue with specified fields.\n" + } + ] + }, + "flags": {}, + "id": 466, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "update", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The new name for the request queue\n" + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 467, + "kind": 32768, + "kindString": "Parameter", + "name": "name", + "type": { + "name": "Optional", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "str" + } + ] + } + } + ], + "type": { + "name": "Dict", + "type": "reference" + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Delete the request queue." + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 468, + "module": "_memory_storage.resource_clients.request_queue", + "name": "delete", + "parsedDocstring": { + "text": "Delete the request queue." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_memory_storage/resource_clients/request_queue.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 121 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Delete the request queue." + } + ] + }, + "flags": {}, + "id": 469, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "delete", + "parameters": [], + "type": { + "name": "None", + "type": "literal", + "value": null + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Retrieve a given number of requests from the beginning of the queue.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 470, + "module": "_memory_storage.resource_clients.request_queue", + "name": "list_head", + "parsedDocstring": { + "text": "Retrieve a given number of requests from the beginning of the queue.\n", + "args": { + "limit": "How many requests to retrieve\n" + }, + "returns": "dict: The desired number of requests from the beginning of the queue." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_memory_storage/resource_clients/request_queue.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 135 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "dict: The desired number of requests from the beginning of the queue." + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Retrieve a given number of requests from the beginning of the queue.\n" + } + ] + }, + "flags": {}, + "id": 471, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "list_head", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "How many requests to retrieve\n" + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 472, + "kind": 32768, + "kindString": "Parameter", + "name": "limit", + "type": { + "name": "Optional", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "int" + } + ] + } + } + ], + "type": { + "name": "Dict", + "type": "reference" + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Add a request to the queue.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 473, + "module": "_memory_storage.resource_clients.request_queue", + "name": "add_request", + "parsedDocstring": { + "text": "Add a request to the queue.\n", + "args": { + "request": "The request to add to the queue", + "forefront": "Whether to add the request to the head or the end of the queue\n" + }, + "returns": "dict: The added request." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_memory_storage/resource_clients/request_queue.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 175 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "dict: The added request." + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Add a request to the queue.\n" + } + ] + }, + "flags": {}, + "id": 474, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "add_request", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The request to add to the queue" + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 475, + "kind": 32768, + "kindString": "Parameter", + "name": "request", + "type": { + "name": "Dict", + "type": "reference" + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Whether to add the request to the head or the end of the queue\n" + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 476, + "kind": 32768, + "kindString": "Parameter", + "name": "forefront", + "type": { + "name": "Optional", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "bool" + } + ] + } + } + ], + "type": { + "name": "Dict", + "type": "reference" + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Retrieve a request from the queue.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 477, + "module": "_memory_storage.resource_clients.request_queue", + "name": "get_request", + "parsedDocstring": { + "text": "Retrieve a request from the queue.\n", + "args": { + "request_id": "ID of the request to retrieve\n" + }, + "returns": "dict, optional: The retrieved request, or None, if it did not exist." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_memory_storage/resource_clients/request_queue.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 227 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "dict, optional: The retrieved request, or None, if it did not exist." + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Retrieve a request from the queue.\n" + } + ] + }, + "flags": {}, + "id": 478, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "get_request", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "ID of the request to retrieve\n" + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 479, + "kind": 32768, + "kindString": "Parameter", + "name": "request_id", + "type": { + "name": "str", + "type": "reference" + } + } + ], + "type": { + "name": "Optional", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "Dict" + } + ] + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Update a request in the queue.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 480, + "module": "_memory_storage.resource_clients.request_queue", + "name": "update_request", + "parsedDocstring": { + "text": "Update a request in the queue.\n", + "args": { + "request": "The updated request", + "forefront": "Whether to put the updated request in the beginning or the end of the queue\n" + }, + "returns": "dict: The updated request" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_memory_storage/resource_clients/request_queue.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 248 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "dict: The updated request" + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Update a request in the queue.\n" + } + ] + }, + "flags": {}, + "id": 481, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "update_request", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The updated request" + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 482, + "kind": 32768, + "kindString": "Parameter", + "name": "request", + "type": { + "name": "Dict", + "type": "reference" + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Whether to put the updated request in the beginning or the end of the queue\n" + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 483, + "kind": 32768, + "kindString": "Parameter", + "name": "forefront", + "type": { + "name": "Optional", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "bool" + } + ] + } + } + ], + "type": { + "name": "Dict", + "type": "reference" + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Delete a request from the queue.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 484, + "module": "_memory_storage.resource_clients.request_queue", + "name": "delete_request", + "parsedDocstring": { + "text": "Delete a request from the queue.\n", + "args": { + "request_id": "ID of the request to delete." + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_memory_storage/resource_clients/request_queue.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 305 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Delete a request from the queue.\n" + } + ] + }, + "flags": {}, + "id": 485, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "delete_request", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "ID of the request to delete." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 486, + "kind": 32768, + "kindString": "Parameter", + "name": "request_id", + "type": { + "name": "str", + "type": "reference" + } + } + ], + "type": { + "name": "None", + "type": "literal", + "value": null + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Initialize the BaseResourceClient." + } + ] + }, + "decorations": [ + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 487, + "module": "_memory_storage.resource_clients.base_resource_client", + "name": "__init__", + "parsedDocstring": { + "text": "Initialize the BaseResourceClient." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_memory_storage/resource_clients/base_resource_client.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 23 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Initialize the BaseResourceClient." + } + ] + }, + "flags": {}, + "id": 488, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "__init__", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 489, + "kind": 32768, + "kindString": "Parameter", + "name": "base_storage_directory", + "type": { + "name": "str", + "type": "reference" + } + }, + { + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 490, + "kind": 32768, + "kindString": "Parameter", + "name": "memory_storage_client", + "type": { + "name": "'MemoryStorageClient'", + "type": "literal", + "value": "MemoryStorageClient" + } + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 491, + "kind": 32768, + "kindString": "Parameter", + "name": "id", + "type": { + "name": "Optional", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "str" + } + ] + } + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 492, + "kind": 32768, + "kindString": "Parameter", + "name": "name", + "type": { + "name": "Optional", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "str" + } + ] + } + } + ], + "type": { + "name": "None", + "type": "literal", + "value": null + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Retrieve the storage.\n" + } + ] + }, + "decorations": [ + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 493, + "module": "_memory_storage.resource_clients.base_resource_client", + "name": "get", + "parsedDocstring": { + "text": "Retrieve the storage.\n", + "returns": "dict, optional: The retrieved storage, or None, if it does not exist" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_memory_storage/resource_clients/base_resource_client.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 35 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "dict, optional: The retrieved storage, or None, if it does not exist" + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Retrieve the storage.\n" + } + ] + }, + "flags": {}, + "id": 494, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "get", + "parameters": [], + "type": { + "name": "Optional", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "Dict" + } + ] + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "List the available request queues.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 495, + "module": "_memory_storage.resource_clients.request_queue_collection", + "name": "list", + "parsedDocstring": { + "text": "List the available request queues.\n", + "returns": "ListPage: The list of available request queues matching the specified filters." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_memory_storage/resource_clients/request_queue_collection.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 20 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "ListPage: The list of available request queues matching the specified filters." + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "List the available request queues.\n" + } + ] + }, + "flags": {}, + "id": 496, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "list", + "parameters": [], + "type": { + "name": "ListPage", + "type": "reference" + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Retrieve a named request queue, or create a new one when it doesn't exist.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 497, + "module": "_memory_storage.resource_clients.request_queue_collection", + "name": "get_or_create", + "parsedDocstring": { + "text": "Retrieve a named request queue, or create a new one when it doesn't exist.\n", + "args": { + "name": "The name of the request queue to retrieve or create.", + "schema": "The schema of the request queue\n" + }, + "returns": "dict: The retrieved or newly-created request queue." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_memory_storage/resource_clients/request_queue_collection.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 28 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "dict: The retrieved or newly-created request queue." + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Retrieve a named request queue, or create a new one when it doesn't exist.\n" + } + ] + }, + "flags": {}, + "id": 498, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "get_or_create", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The name of the request queue to retrieve or create." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 499, + "kind": 32768, + "kindString": "Parameter", + "name": "name", + "type": { + "name": "Optional", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "str" + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The schema of the request queue\n" + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 500, + "kind": 32768, + "kindString": "Parameter", + "name": "schema", + "type": { + "name": "Optional", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "Dict" + } + ] + } + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 501, + "kind": 32768, + "kindString": "Parameter", + "name": "_id", + "type": { + "name": "Optional", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "str" + } + ] + } + } + ], + "type": { + "name": "Dict", + "type": "reference" + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "List the available datasets.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 502, + "module": "_memory_storage.resource_clients.dataset_collection", + "name": "list", + "parsedDocstring": { + "text": "List the available datasets.\n", + "returns": "ListPage: The list of available datasets matching the specified filters." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_memory_storage/resource_clients/dataset_collection.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 20 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "ListPage: The list of available datasets matching the specified filters." + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "List the available datasets.\n" + } + ] + }, + "flags": {}, + "id": 503, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "list", + "parameters": [], + "type": { + "name": "ListPage", + "type": "reference" + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Retrieve a named dataset, or create a new one when it doesn't exist.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 504, + "module": "_memory_storage.resource_clients.dataset_collection", + "name": "get_or_create", + "parsedDocstring": { + "text": "Retrieve a named dataset, or create a new one when it doesn't exist.\n", + "args": { + "name": "The name of the dataset to retrieve or create.", + "schema": "The schema of the dataset\n" + }, + "returns": "dict: The retrieved or newly-created dataset." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_memory_storage/resource_clients/dataset_collection.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 28 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "dict: The retrieved or newly-created dataset." + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Retrieve a named dataset, or create a new one when it doesn't exist.\n" + } + ] + }, + "flags": {}, + "id": 505, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "get_or_create", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The name of the dataset to retrieve or create." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 506, + "kind": 32768, + "kindString": "Parameter", + "name": "name", + "type": { + "name": "Optional", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "str" + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The schema of the dataset\n" + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 507, + "kind": 32768, + "kindString": "Parameter", + "name": "schema", + "type": { + "name": "Optional", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "Dict" + } + ] + } + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 508, + "kind": 32768, + "kindString": "Parameter", + "name": "_id", + "type": { + "name": "Optional", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "str" + } + ] + } + } + ], + "type": { + "name": "Dict", + "type": "reference" + } + } + ] + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "0.01%" + } + ] + }, + "flags": {}, + "groups": [], + "id": 509, + "module": "storages.dataset", + "name": "SAFETY_BUFFER_PERCENT", + "parsedDocstring": { + "text": "0.01%" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storages/dataset.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 19 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 510, + "module": "storages.dataset", + "name": "EFFECTIVE_LIMIT_BYTES", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storages/dataset.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 20 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Store an object or an array of objects to the dataset.\n\nThe size of the data is limited by the receiving API and therefore `push_data()` will only\nallow objects whose JSON representation is smaller than 9MB. When an array is passed,\nnone of the included objects may be larger than 9MB, but the array itself may be of any size.\n" + } + ] + }, + "decorations": [ + { + "name": "classmethod" + } + ], + "flags": {}, + "groups": [], + "id": 512, + "module": "storages.dataset", + "name": "push_data", + "parsedDocstring": { + "text": "Store an object or an array of objects to the dataset.\n\nThe size of the data is limited by the receiving API and therefore `push_data()` will only\nallow objects whose JSON representation is smaller than 9MB. When an array is passed,\nnone of the included objects may be larger than 9MB, but the array itself may be of any size.\n", + "args": { + "data": "dict or array of dicts containing data to be stored in the default dataset.\nThe JSON representation of each item must be smaller than 9MB." + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storages/dataset.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 136 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Store an object or an array of objects to the dataset.\n\nThe size of the data is limited by the receiving API and therefore `push_data()` will only\nallow objects whose JSON representation is smaller than 9MB. When an array is passed,\nnone of the included objects may be larger than 9MB, but the array itself may be of any size.\n" + } + ] + }, + "flags": {}, + "id": 513, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "push_data", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "dict or array of dicts containing data to be stored in the default dataset.\nThe JSON representation of each item must be smaller than 9MB." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 514, + "kind": 32768, + "kindString": "Parameter", + "name": "data", + "type": { + "name": "JSONSerializable", + "type": "reference", + "target": "194" + } + } + ], + "type": { + "name": "None", + "type": "literal", + "value": null + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Get items from the dataset.\n" + } + ] + }, + "decorations": [ + { + "name": "classmethod" + } + ], + "flags": {}, + "groups": [], + "id": 515, + "module": "storages.dataset", + "name": "get_data", + "parsedDocstring": { + "text": "Get items from the dataset.\n", + "args": { + "offset": "Number of items that should be skipped at the start. The default value is 0", + "limit": "Maximum number of items to return. By default there is no limit.", + "desc": "By default, results are returned in the same order as they were stored.\nTo reverse the order, set this parameter to True.", + "clean": "If True, returns only non-empty items and skips hidden fields (i.e. fields starting with the # character).\nThe clean parameter is just a shortcut for skip_hidden=True and skip_empty=True parameters.\nNote that since some objects might be skipped from the output, that the result might contain less items than the limit value.", + "fields": "A list of fields which should be picked from the items,\nonly these fields will remain in the resulting record objects.\nNote that the fields in the outputted items are sorted the same way as they are specified in the fields parameter.\nYou can use this feature to effectively fix the output format.", + "omit": "A list of fields which should be omitted from the items.", + "unwind": "Name of a field which should be unwound.\nIf the field is an array then every element of the array will become a separate record and merged with parent object.\nIf the unwound field is an object then it is merged with the parent object.\nIf the unwound field is missing or its value is neither an array nor an object and therefore cannot be merged with a parent object,\nthen the item gets preserved as it is. Note that the unwound items ignore the desc parameter.", + "skip_empty": "If True, then empty items are skipped from the output.\nNote that if used, the results might contain less items than the limit value.", + "skip_hidden": "If True, then hidden fields are skipped from the output, i.e. fields starting with the # character.", + "flatten": "A list of fields that should be flattened", + "view": "Name of the dataset view to be used\n" + }, + "returns": "ListPage: A page of the list of dataset items according to the specified filters." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storages/dataset.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 164 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "ListPage: A page of the list of dataset items according to the specified filters." + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Get items from the dataset.\n" + } + ] + }, + "flags": {}, + "id": 516, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "get_data", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Number of items that should be skipped at the start. The default value is 0" + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 517, + "kind": 32768, + "kindString": "Parameter", + "name": "offset", + "type": { + "name": "Optional", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "int" + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Maximum number of items to return. By default there is no limit." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 518, + "kind": 32768, + "kindString": "Parameter", + "name": "limit", + "type": { + "name": "Optional", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "int" + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "If True, returns only non-empty items and skips hidden fields (i.e. fields starting with the # character).\nThe clean parameter is just a shortcut for skip_hidden=True and skip_empty=True parameters.\nNote that since some objects might be skipped from the output, that the result might contain less items than the limit value." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 519, + "kind": 32768, + "kindString": "Parameter", + "name": "clean", + "type": { + "name": "Optional", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "bool" + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "By default, results are returned in the same order as they were stored.\nTo reverse the order, set this parameter to True." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 520, + "kind": 32768, + "kindString": "Parameter", + "name": "desc", + "type": { + "name": "Optional", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "bool" + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "A list of fields which should be picked from the items,\nonly these fields will remain in the resulting record objects.\nNote that the fields in the outputted items are sorted the same way as they are specified in the fields parameter.\nYou can use this feature to effectively fix the output format." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 521, + "kind": 32768, + "kindString": "Parameter", + "name": "fields", + "type": { + "name": "Optional", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "List", + "typeArguments": [ + { + "type": "reference", + "name": "str" + } + ] + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "A list of fields which should be omitted from the items." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 522, + "kind": 32768, + "kindString": "Parameter", + "name": "omit", + "type": { + "name": "Optional", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "List", + "typeArguments": [ + { + "type": "reference", + "name": "str" + } + ] + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Name of a field which should be unwound.\nIf the field is an array then every element of the array will become a separate record and merged with parent object.\nIf the unwound field is an object then it is merged with the parent object.\nIf the unwound field is missing or its value is neither an array nor an object and therefore cannot be merged with a parent object,\nthen the item gets preserved as it is. Note that the unwound items ignore the desc parameter." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 523, + "kind": 32768, + "kindString": "Parameter", + "name": "unwind", + "type": { + "name": "Optional", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "str" + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "If True, then empty items are skipped from the output.\nNote that if used, the results might contain less items than the limit value." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 524, + "kind": 32768, + "kindString": "Parameter", + "name": "skip_empty", + "type": { + "name": "Optional", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "bool" + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "If True, then hidden fields are skipped from the output, i.e. fields starting with the # character." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 525, + "kind": 32768, + "kindString": "Parameter", + "name": "skip_hidden", + "type": { + "name": "Optional", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "bool" + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "A list of fields that should be flattened" + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 526, + "kind": 32768, + "kindString": "Parameter", + "name": "flatten", + "type": { + "name": "Optional", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "List", + "typeArguments": [ + { + "type": "reference", + "name": "str" + } + ] + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Name of the dataset view to be used\n" + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 527, + "kind": 32768, + "kindString": "Parameter", + "name": "view", + "type": { + "name": "Optional", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "str" + } + ] + } + } + ], + "type": { + "name": "ListPage", + "type": "reference" + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Save the entirety of the dataset's contents into one file within a key-value store.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 528, + "module": "storages.dataset", + "name": "export_to", + "parsedDocstring": { + "text": "Save the entirety of the dataset's contents into one file within a key-value store.\n", + "args": { + "key": "The key to save the data under.", + "to_key_value_store_id": "The id of the key-value store in which the result will be saved.", + "to_key_value_store_name": "The name of the key-value store in which the result will be saved.\nYou must specify only one of `to_key_value_store_id` and `to_key_value_store_name` arguments.\nIf you omit both, it uses the default key-value store.", + "content_type": "Either 'text/csv' or 'application/json'. Defaults to JSON." + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storages/dataset.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 262 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Save the entirety of the dataset's contents into one file within a key-value store.\n" + } + ] + }, + "flags": {}, + "id": 529, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "export_to", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The key to save the data under." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 530, + "kind": 32768, + "kindString": "Parameter", + "name": "key", + "type": { + "name": "str", + "type": "reference" + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The id of the key-value store in which the result will be saved." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 531, + "kind": 32768, + "kindString": "Parameter", + "name": "to_key_value_store_id", + "type": { + "name": "Optional", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "str" + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The name of the key-value store in which the result will be saved.\nYou must specify only one of `to_key_value_store_id` and `to_key_value_store_name` arguments.\nIf you omit both, it uses the default key-value store." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 532, + "kind": 32768, + "kindString": "Parameter", + "name": "to_key_value_store_name", + "type": { + "name": "Optional", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "str" + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Either 'text/csv' or 'application/json'. Defaults to JSON." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 533, + "kind": 32768, + "kindString": "Parameter", + "name": "content_type", + "type": { + "name": "Optional", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "str" + } + ] + } + } + ], + "type": { + "name": "None", + "type": "literal", + "value": null + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Save the entirety of the dataset's contents into one JSON file within a key-value store.\n" + } + ] + }, + "decorations": [ + { + "name": "classmethod" + } + ], + "flags": {}, + "groups": [], + "id": 534, + "module": "storages.dataset", + "name": "export_to_json", + "parsedDocstring": { + "text": "Save the entirety of the dataset's contents into one JSON file within a key-value store.\n", + "args": { + "key": "The key to save the data under.", + "from_dataset_id": "The ID of the dataset in case of calling the class method. Uses default dataset if omitted.", + "from_dataset_name": "The name of the dataset in case of calling the class method. Uses default dataset if omitted.\nYou must specify only one of `from_dataset_id` and `from_dataset_name` arguments.\nIf you omit both, it uses the default dataset.", + "to_key_value_store_id": "The id of the key-value store in which the result will be saved.", + "to_key_value_store_name": "The name of the key-value store in which the result will be saved.\nYou must specify only one of `to_key_value_store_id` and `to_key_value_store_name` arguments.\nIf you omit both, it uses the default key-value store." + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storages/dataset.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 307 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Save the entirety of the dataset's contents into one JSON file within a key-value store.\n" + } + ] + }, + "flags": {}, + "id": 535, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "export_to_json", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The key to save the data under." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 536, + "kind": 32768, + "kindString": "Parameter", + "name": "key", + "type": { + "name": "str", + "type": "reference" + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The ID of the dataset in case of calling the class method. Uses default dataset if omitted." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 537, + "kind": 32768, + "kindString": "Parameter", + "name": "from_dataset_id", + "type": { + "name": "Optional", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "str" + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The name of the dataset in case of calling the class method. Uses default dataset if omitted.\nYou must specify only one of `from_dataset_id` and `from_dataset_name` arguments.\nIf you omit both, it uses the default dataset." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 538, + "kind": 32768, + "kindString": "Parameter", + "name": "from_dataset_name", + "type": { + "name": "Optional", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "str" + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The id of the key-value store in which the result will be saved." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 539, + "kind": 32768, + "kindString": "Parameter", + "name": "to_key_value_store_id", + "type": { + "name": "Optional", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "str" + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The name of the key-value store in which the result will be saved.\nYou must specify only one of `to_key_value_store_id` and `to_key_value_store_name` arguments.\nIf you omit both, it uses the default key-value store." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 540, + "kind": 32768, + "kindString": "Parameter", + "name": "to_key_value_store_name", + "type": { + "name": "Optional", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "str" + } + ] + } + } + ], + "type": { + "name": "None", + "type": "literal", + "value": null + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Save the entirety of the dataset's contents into one CSV file within a key-value store.\n" + } + ] + }, + "decorations": [ + { + "name": "classmethod" + } + ], + "flags": {}, + "groups": [], + "id": 541, + "module": "storages.dataset", + "name": "export_to_csv", + "parsedDocstring": { + "text": "Save the entirety of the dataset's contents into one CSV file within a key-value store.\n", + "args": { + "key": "The key to save the data under.", + "from_dataset_id": "The ID of the dataset in case of calling the class method. Uses default dataset if omitted.", + "from_dataset_name": "The name of the dataset in case of calling the class method. Uses default dataset if omitted.\nYou must specify only one of `from_dataset_id` and `from_dataset_name` arguments.\nIf you omit both, it uses the default dataset.", + "to_key_value_store_id": "The id of the key-value store in which the result will be saved.", + "to_key_value_store_name": "The name of the key-value store in which the result will be saved.\nYou must specify only one of `to_key_value_store_id` and `to_key_value_store_name` arguments.\nIf you omit both, it uses the default key-value store." + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storages/dataset.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 349 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Save the entirety of the dataset's contents into one CSV file within a key-value store.\n" + } + ] + }, + "flags": {}, + "id": 542, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "export_to_csv", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The key to save the data under." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 543, + "kind": 32768, + "kindString": "Parameter", + "name": "key", + "type": { + "name": "str", + "type": "reference" + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The ID of the dataset in case of calling the class method. Uses default dataset if omitted." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 544, + "kind": 32768, + "kindString": "Parameter", + "name": "from_dataset_id", + "type": { + "name": "Optional", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "str" + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The name of the dataset in case of calling the class method. Uses default dataset if omitted.\nYou must specify only one of `from_dataset_id` and `from_dataset_name` arguments.\nIf you omit both, it uses the default dataset." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 545, + "kind": 32768, + "kindString": "Parameter", + "name": "from_dataset_name", + "type": { + "name": "Optional", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "str" + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The id of the key-value store in which the result will be saved." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 546, + "kind": 32768, + "kindString": "Parameter", + "name": "to_key_value_store_id", + "type": { + "name": "Optional", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "str" + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The name of the key-value store in which the result will be saved.\nYou must specify only one of `to_key_value_store_id` and `to_key_value_store_name` arguments.\nIf you omit both, it uses the default key-value store." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 547, + "kind": 32768, + "kindString": "Parameter", + "name": "to_key_value_store_name", + "type": { + "name": "Optional", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "str" + } + ] + } + } + ], + "type": { + "name": "None", + "type": "literal", + "value": null + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Get an object containing general information about the dataset.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 548, + "module": "storages.dataset", + "name": "get_info", + "parsedDocstring": { + "text": "Get an object containing general information about the dataset.\n", + "returns": "dict: Object returned by calling the GET dataset API endpoint." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storages/dataset.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 390 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "dict: Object returned by calling the GET dataset API endpoint." + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Get an object containing general information about the dataset.\n" + } + ] + }, + "flags": {}, + "id": 549, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "get_info", + "parameters": [], + "type": { + "name": "Optional", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "Dict" + } + ] + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Iterate over the items in the dataset.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 550, + "module": "storages.dataset", + "name": "iterate_items", + "parsedDocstring": { + "text": "Iterate over the items in the dataset.\n", + "args": { + "offset": "Number of items that should be skipped at the start. The default value is 0", + "limit": "Maximum number of items to return. By default there is no limit.", + "desc": "By default, results are returned in the same order as they were stored.\nTo reverse the order, set this parameter to True.", + "clean": "If True, returns only non-empty items and skips hidden fields (i.e. fields starting with the # character).\nThe clean parameter is just a shortcut for skip_hidden=True and skip_empty=True parameters.\nNote that since some objects might be skipped from the output, that the result might contain less items than the limit value.", + "fields": "A list of fields which should be picked from the items,\nonly these fields will remain in the resulting record objects.\nNote that the fields in the outputted items are sorted the same way as they are specified in the fields parameter.\nYou can use this feature to effectively fix the output format.", + "omit": "A list of fields which should be omitted from the items.", + "unwind": "Name of a field which should be unwound.\nIf the field is an array then every element of the array will become a separate record and merged with parent object.\nIf the unwound field is an object then it is merged with the parent object.\nIf the unwound field is missing or its value is neither an array nor an object and therefore cannot be merged with a parent object,\nthen the item gets preserved as it is. Note that the unwound items ignore the desc parameter.", + "skip_empty": "If True, then empty items are skipped from the output.\nNote that if used, the results might contain less items than the limit value.", + "skip_hidden": "If True, then hidden fields are skipped from the output, i.e. fields starting with the # character.\n" + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storages/dataset.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 398 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Iterate over the items in the dataset.\n" + } + ] + }, + "flags": {}, + "id": 551, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "iterate_items", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Number of items that should be skipped at the start. The default value is 0" + } + ] + }, + "defaultValue": "0", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 552, + "kind": 32768, + "kindString": "Parameter", + "name": "offset", + "type": { + "name": "int", + "type": "reference" + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Maximum number of items to return. By default there is no limit." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 553, + "kind": 32768, + "kindString": "Parameter", + "name": "limit", + "type": { + "name": "Optional", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "int" + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "If True, returns only non-empty items and skips hidden fields (i.e. fields starting with the # character).\nThe clean parameter is just a shortcut for skip_hidden=True and skip_empty=True parameters.\nNote that since some objects might be skipped from the output, that the result might contain less items than the limit value." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 554, + "kind": 32768, + "kindString": "Parameter", + "name": "clean", + "type": { + "name": "Optional", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "bool" + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "By default, results are returned in the same order as they were stored.\nTo reverse the order, set this parameter to True." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 555, + "kind": 32768, + "kindString": "Parameter", + "name": "desc", + "type": { + "name": "Optional", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "bool" + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "A list of fields which should be picked from the items,\nonly these fields will remain in the resulting record objects.\nNote that the fields in the outputted items are sorted the same way as they are specified in the fields parameter.\nYou can use this feature to effectively fix the output format." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 556, + "kind": 32768, + "kindString": "Parameter", + "name": "fields", + "type": { + "name": "Optional", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "List", + "typeArguments": [ + { + "type": "reference", + "name": "str" + } + ] + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "A list of fields which should be omitted from the items." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 557, + "kind": 32768, + "kindString": "Parameter", + "name": "omit", + "type": { + "name": "Optional", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "List", + "typeArguments": [ + { + "type": "reference", + "name": "str" + } + ] + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Name of a field which should be unwound.\nIf the field is an array then every element of the array will become a separate record and merged with parent object.\nIf the unwound field is an object then it is merged with the parent object.\nIf the unwound field is missing or its value is neither an array nor an object and therefore cannot be merged with a parent object,\nthen the item gets preserved as it is. Note that the unwound items ignore the desc parameter." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 558, + "kind": 32768, + "kindString": "Parameter", + "name": "unwind", + "type": { + "name": "Optional", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "str" + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "If True, then empty items are skipped from the output.\nNote that if used, the results might contain less items than the limit value." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 559, + "kind": 32768, + "kindString": "Parameter", + "name": "skip_empty", + "type": { + "name": "Optional", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "bool" + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "If True, then hidden fields are skipped from the output, i.e. fields starting with the # character.\n" + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 560, + "kind": 32768, + "kindString": "Parameter", + "name": "skip_hidden", + "type": { + "name": "Optional", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "bool" + } + ] + } + } + ], + "type": { + "name": "AsyncIterator", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "Dict" + } + ] + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Remove the dataset either from the Apify cloud storage or from the local directory." + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 561, + "module": "storages.dataset", + "name": "drop", + "parsedDocstring": { + "text": "Remove the dataset either from the Apify cloud storage or from the local directory." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storages/dataset.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 450 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Remove the dataset either from the Apify cloud storage or from the local directory." + } + ] + }, + "flags": {}, + "id": 562, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "drop", + "parameters": [], + "type": { + "name": "None", + "type": "literal", + "value": null + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Open a dataset.\n\nDatasets are used to store structured data where each object stored has the same attributes,\nsuch as online store products or real estate offers.\nThe actual data is stored either on the local filesystem or in the Apify cloud.\n" + } + ] + }, + "decorations": [ + { + "name": "classmethod" + } + ], + "flags": {}, + "groups": [], + "id": 563, + "module": "storages.dataset", + "name": "open", + "parsedDocstring": { + "text": "Open a dataset.\n\nDatasets are used to store structured data where each object stored has the same attributes,\nsuch as online store products or real estate offers.\nThe actual data is stored either on the local filesystem or in the Apify cloud.\n", + "args": { + "id": "ID of the dataset to be opened.\nIf neither `id` nor `name` are provided, the method returns the default dataset associated with the actor run.\nIf the dataset with the given ID does not exist, it raises an error.", + "name": "Name of the dataset to be opened.\nIf neither `id` nor `name` are provided, the method returns the default dataset associated with the actor run.\nIf the dataset with the given name does not exist, it is created.", + "force_cloud": "If set to True, it will open a dataset on the Apify Platform even when running the actor locally.\nDefaults to False.", + "config": "A `Configuration` instance, uses global configuration if omitted.\n" + }, + "returns": "Dataset: An instance of the `Dataset` class for the given ID or name." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storages/dataset.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 456 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "Dataset: An instance of the `Dataset` class for the given ID or name." + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Open a dataset.\n\nDatasets are used to store structured data where each object stored has the same attributes,\nsuch as online store products or real estate offers.\nThe actual data is stored either on the local filesystem or in the Apify cloud.\n" + } + ] + }, + "flags": {}, + "id": 564, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "open", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "ID of the dataset to be opened.\nIf neither `id` nor `name` are provided, the method returns the default dataset associated with the actor run.\nIf the dataset with the given ID does not exist, it raises an error." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 565, + "kind": 32768, + "kindString": "Parameter", + "name": "id", + "type": { + "name": "Optional", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "str" + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Name of the dataset to be opened.\nIf neither `id` nor `name` are provided, the method returns the default dataset associated with the actor run.\nIf the dataset with the given name does not exist, it is created." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 566, + "kind": 32768, + "kindString": "Parameter", + "name": "name", + "type": { + "name": "Optional", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "str" + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "If set to True, it will open a dataset on the Apify Platform even when running the actor locally.\nDefaults to False." + } + ] + }, + "defaultValue": "False", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 567, + "kind": 32768, + "kindString": "Parameter", + "name": "force_cloud", + "type": { + "name": "bool", + "type": "reference" + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "A `Configuration` instance, uses global configuration if omitted.\n" + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 568, + "kind": 32768, + "kindString": "Parameter", + "name": "config", + "type": { + "name": "Optional", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "Configuration", + "target": "1" + } + ] + } + } + ], + "type": { + "name": "'Dataset'", + "type": "literal", + "value": "Dataset" + } + } + ] + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "The `Dataset` class represents a store for structured data where each object stored has the same attributes.\n\nYou can imagine it as a table, where each object is a row and its attributes are columns.\nDataset is an append-only storage - you can only add new records to it but you cannot modify or remove existing records.\nTypically it is used to store crawling results.\n\nDo not instantiate this class directly, use the `Actor.open_dataset()` function instead.\n\n`Dataset` stores its data either on local disk or in the Apify cloud,\ndepending on whether the `APIFY_LOCAL_STORAGE_DIR` or `APIFY_TOKEN` environment variables are set.\n\nIf the `APIFY_LOCAL_STORAGE_DIR` environment variable is set, the data is stored in\nthe local directory in the following files:\n```\n{APIFY_LOCAL_STORAGE_DIR}/datasets/{DATASET_ID}/{INDEX}.json\n```\n\nNote that `{DATASET_ID}` is the name or ID of the dataset. The default dataset has ID: `default`,\nunless you override it by setting the `APIFY_DEFAULT_DATASET_ID` environment variable.\nEach dataset item is stored as a separate JSON file, where `{INDEX}` is a zero-based index of the item in the dataset.\n\nIf the `APIFY_TOKEN` environment variable is set but `APIFY_LOCAL_STORAGE_DIR` is not, the data is stored in the\n[Apify Dataset](https://docs.apify.com/storage/dataset) cloud storage." + } + ] + }, + "flags": {}, + "groups": [ + { + "children": [ + 561, + 528, + 541, + 534, + 515, + 548, + 550, + 563, + 512 + ], + "title": "Methods" + } + ], + "id": 511, + "module": "storages.dataset", + "name": "Dataset", + "parsedDocstring": { + "text": "The `Dataset` class represents a store for structured data where each object stored has the same attributes.\n\nYou can imagine it as a table, where each object is a row and its attributes are columns.\nDataset is an append-only storage - you can only add new records to it but you cannot modify or remove existing records.\nTypically it is used to store crawling results.\n\nDo not instantiate this class directly, use the `Actor.open_dataset()` function instead.\n\n`Dataset` stores its data either on local disk or in the Apify cloud,\ndepending on whether the `APIFY_LOCAL_STORAGE_DIR` or `APIFY_TOKEN` environment variables are set.\n\nIf the `APIFY_LOCAL_STORAGE_DIR` environment variable is set, the data is stored in\nthe local directory in the following files:\n```\n{APIFY_LOCAL_STORAGE_DIR}/datasets/{DATASET_ID}/{INDEX}.json\n```\n\nNote that `{DATASET_ID}` is the name or ID of the dataset. The default dataset has ID: `default`,\nunless you override it by setting the `APIFY_DEFAULT_DATASET_ID` environment variable.\nEach dataset item is stored as a separate JSON file, where `{INDEX}` is a zero-based index of the item in the dataset.\n\nIf the `APIFY_TOKEN` environment variable is set but `APIFY_LOCAL_STORAGE_DIR` is not, the data is stored in the\n[Apify Dataset](https://docs.apify.com/storage/dataset) cloud storage." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storages/dataset.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 66 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 569, + "module": "storages.key_value_store", + "name": "T", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storages/key_value_store.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 12 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 570, + "module": "storages.key_value_store", + "name": "IterateKeysInfo", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storages/key_value_store.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 13 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 571, + "module": "storages.key_value_store", + "name": "IterateKeysTuple", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storages/key_value_store.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 14 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Get a value from the key-value store.\n" + } + ] + }, + "decorations": [ + { + "name": "classmethod" + } + ], + "flags": {}, + "groups": [], + "id": 573, + "module": "storages.key_value_store", + "name": "get_value", + "parsedDocstring": { + "text": "Get a value from the key-value store.\n", + "args": { + "key": "Key of the record to retrieve.", + "default_value": "Default value returned in case the record does not exist.\n" + }, + "returns": "Any: The value associated with the given key. `default_value` is used in case the record does not exist." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storages/key_value_store.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 113 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "Any: The value associated with the given key. `default_value` is used in case the record does not exist." + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Get a value from the key-value store.\n" + } + ] + }, + "flags": {}, + "id": 574, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "get_value", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Key of the record to retrieve." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 575, + "kind": 32768, + "kindString": "Parameter", + "name": "key", + "type": { + "name": "str", + "type": "reference" + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Default value returned in case the record does not exist.\n" + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": false + }, + "id": 576, + "kind": 32768, + "kindString": "Parameter", + "name": "default_value", + "type": { + "name": "Optional", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "T", + "target": "296" + } + ] + } + } + ], + "type": { + "name": "Optional", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "T", + "target": "296" + } + ] + } + }, + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "Any: The value associated with the given key. `default_value` is used in case the record does not exist." + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Get a value from the key-value store.\n" + } + ] + }, + "flags": {}, + "id": 596, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "get_value", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Key of the record to retrieve." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 597, + "kind": 32768, + "kindString": "Parameter", + "name": "key", + "type": { + "name": "str", + "type": "reference" + } + } + ], + "type": { + "name": "Any", + "type": "reference" + } + }, + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "Any: The value associated with the given key. `default_value` is used in case the record does not exist." + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Get a value from the key-value store.\n" + } + ] + }, + "flags": {}, + "id": 598, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "get_value", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Key of the record to retrieve." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 599, + "kind": 32768, + "kindString": "Parameter", + "name": "key", + "type": { + "name": "str", + "type": "reference" + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Default value returned in case the record does not exist.\n" + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 600, + "kind": 32768, + "kindString": "Parameter", + "name": "default_value", + "type": { + "name": "T", + "type": "reference", + "target": "296" + } + } + ], + "type": { + "name": "T", + "type": "reference", + "target": "296" + } + }, + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "Any: The value associated with the given key. `default_value` is used in case the record does not exist." + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Get a value from the key-value store.\n" + } + ] + }, + "flags": {}, + "id": 601, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "get_value", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Key of the record to retrieve." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 602, + "kind": 32768, + "kindString": "Parameter", + "name": "key", + "type": { + "name": "str", + "type": "reference" + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Default value returned in case the record does not exist.\n" + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": false + }, + "id": 603, + "kind": 32768, + "kindString": "Parameter", + "name": "default_value", + "type": { + "name": "Optional", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "T", + "target": "296" + } + ] + } + } + ], + "type": { + "name": "Optional", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "T", + "target": "296" + } + ] + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Iterate over the keys in the key-value store.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 577, + "module": "storages.key_value_store", + "name": "iterate_keys", + "parsedDocstring": { + "text": "Iterate over the keys in the key-value store.\n", + "args": { + "exclusive_start_key": "All keys up to this one (including) are skipped from the result.\n" + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storages/key_value_store.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 130 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Iterate over the keys in the key-value store.\n" + } + ] + }, + "flags": {}, + "id": 578, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "iterate_keys", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "All keys up to this one (including) are skipped from the result.\n" + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": false + }, + "id": 579, + "kind": 32768, + "kindString": "Parameter", + "name": "exclusive_start_key", + "type": { + "name": "Optional", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "str" + } + ] + } + } + ], + "type": { + "name": "AsyncIterator", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "IterateKeysTuple", + "target": "571" + } + ] + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Set or delete a value in the key-value store.\n" + } + ] + }, + "decorations": [ + { + "name": "classmethod" + } + ], + "flags": {}, + "groups": [], + "id": 580, + "module": "storages.key_value_store", + "name": "set_value", + "parsedDocstring": { + "text": "Set or delete a value in the key-value store.\n", + "args": { + "key": "The key under which the value should be saved.", + "value": "The value to save. If the value is `None`, the corresponding key-value pair will be deleted.", + "content_type": "The content type of the saved value." + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storages/key_value_store.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 151 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Set or delete a value in the key-value store.\n" + } + ] + }, + "flags": {}, + "id": 581, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "set_value", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The key under which the value should be saved." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 582, + "kind": 32768, + "kindString": "Parameter", + "name": "key", + "type": { + "name": "str", + "type": "reference" + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The value to save. If the value is `None`, the corresponding key-value pair will be deleted." + } + ] + }, + "flags": { + "isOptional": true, + "keyword-only": false + }, + "id": 583, + "kind": 32768, + "kindString": "Parameter", + "name": "value", + "type": { + "name": "Optional", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "T", + "target": "296" + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The content type of the saved value." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": false + }, + "id": 584, + "kind": 32768, + "kindString": "Parameter", + "name": "content_type", + "type": { + "name": "Optional", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "str" + } + ] + } + } + ], + "type": { + "name": "None", + "type": "literal", + "value": null + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Get a URL for the given key that may be used to publicly access the value in the remote key-value store.\n" + } + ] + }, + "decorations": [ + { + "name": "classmethod" + } + ], + "flags": {}, + "groups": [], + "id": 585, + "module": "storages.key_value_store", + "name": "get_public_url", + "parsedDocstring": { + "text": "Get a URL for the given key that may be used to publicly access the value in the remote key-value store.\n", + "args": { + "key": "The key for which the URL should be generated." + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storages/key_value_store.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 169 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Get a URL for the given key that may be used to publicly access the value in the remote key-value store.\n" + } + ] + }, + "flags": {}, + "id": 586, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "get_public_url", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The key for which the URL should be generated." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 587, + "kind": 32768, + "kindString": "Parameter", + "name": "key", + "type": { + "name": "str", + "type": "reference" + } + } + ], + "type": { + "name": "str", + "type": "reference" + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Remove the key-value store either from the Apify cloud storage or from the local directory." + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 588, + "module": "storages.key_value_store", + "name": "drop", + "parsedDocstring": { + "text": "Remove the key-value store either from the Apify cloud storage or from the local directory." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storages/key_value_store.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 186 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Remove the key-value store either from the Apify cloud storage or from the local directory." + } + ] + }, + "flags": {}, + "id": 589, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "drop", + "parameters": [], + "type": { + "name": "None", + "type": "literal", + "value": null + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Open a key-value store.\n\nKey-value stores are used to store records or files, along with their MIME content type.\nThe records are stored and retrieved using a unique key.\nThe actual data is stored either on a local filesystem or in the Apify cloud.\n" + } + ] + }, + "decorations": [ + { + "name": "classmethod" + } + ], + "flags": {}, + "groups": [], + "id": 590, + "module": "storages.key_value_store", + "name": "open", + "parsedDocstring": { + "text": "Open a key-value store.\n\nKey-value stores are used to store records or files, along with their MIME content type.\nThe records are stored and retrieved using a unique key.\nThe actual data is stored either on a local filesystem or in the Apify cloud.\n", + "args": { + "id": "ID of the key-value store to be opened.\nIf neither `id` nor `name` are provided, the method returns the default key-value store associated with the actor run.\nIf the key-value store with the given ID does not exist, it raises an error.", + "name": "Name of the key-value store to be opened.\nIf neither `id` nor `name` are provided, the method returns the default key-value store associated with the actor run.\nIf the key-value store with the given name does not exist, it is created.", + "force_cloud": "If set to True, it will open a key-value store on the Apify Platform even when running the actor locally.\nDefaults to False.", + "config": "A `Configuration` instance, uses global configuration if omitted.\n" + }, + "returns": "KeyValueStore: An instance of the `KeyValueStore` class for the given ID or name." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storages/key_value_store.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 192 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "KeyValueStore: An instance of the `KeyValueStore` class for the given ID or name." + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Open a key-value store.\n\nKey-value stores are used to store records or files, along with their MIME content type.\nThe records are stored and retrieved using a unique key.\nThe actual data is stored either on a local filesystem or in the Apify cloud.\n" + } + ] + }, + "flags": {}, + "id": 591, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "open", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "ID of the key-value store to be opened.\nIf neither `id` nor `name` are provided, the method returns the default key-value store associated with the actor run.\nIf the key-value store with the given ID does not exist, it raises an error." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 592, + "kind": 32768, + "kindString": "Parameter", + "name": "id", + "type": { + "name": "Optional", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "str" + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Name of the key-value store to be opened.\nIf neither `id` nor `name` are provided, the method returns the default key-value store associated with the actor run.\nIf the key-value store with the given name does not exist, it is created." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 593, + "kind": 32768, + "kindString": "Parameter", + "name": "name", + "type": { + "name": "Optional", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "str" + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "If set to True, it will open a key-value store on the Apify Platform even when running the actor locally.\nDefaults to False." + } + ] + }, + "defaultValue": "False", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 594, + "kind": 32768, + "kindString": "Parameter", + "name": "force_cloud", + "type": { + "name": "bool", + "type": "reference" + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "A `Configuration` instance, uses global configuration if omitted.\n" + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 595, + "kind": 32768, + "kindString": "Parameter", + "name": "config", + "type": { + "name": "Optional", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "Configuration", + "target": "1" + } + ] + } + } + ], + "type": { + "name": "'KeyValueStore'", + "type": "literal", + "value": "KeyValueStore" + } + } + ] + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "The `KeyValueStore` class represents a key-value store.\n\nYou can imagine it as a simple data storage that is used\nfor saving and reading data records or files. Each data record is\nrepresented by a unique key and associated with a MIME content type.\n\nDo not instantiate this class directly, use the `Actor.open_key_value_store()` function instead.\n\nEach crawler run is associated with a default key-value store, which is created exclusively\nfor the run. By convention, the crawler input and output are stored into the\ndefault key-value store under the `INPUT` and `OUTPUT` key, respectively.\nTypically, input and output are JSON files, although it can be any other format.\nTo access the default key-value store directly, you can use the\n`KeyValueStore.get_value` and `KeyValueStore.set_value` convenience functions.\n\n`KeyValueStore` stores its data either on local disk or in the Apify cloud,\ndepending on whether the `APIFY_LOCAL_STORAGE_DIR` or `APIFY_TOKEN` environment variables are set.\n\nIf the `APIFY_LOCAL_STORAGE_DIR` environment variable is set, the data is stored in\nthe local directory in the following files:\n```\n{APIFY_LOCAL_STORAGE_DIR}/key_value_stores/{STORE_ID}/{INDEX}.{EXT}\n```\n\nNote that `{STORE_ID}` is the name or ID of the key-value store. The default key-value store has ID: `default`,\nunless you override it by setting the `APIFY_DEFAULT_KEY_VALUE_STORE_ID` environment variable.\nThe `{KEY}` is the key of the record and `{EXT}` corresponds to the MIME content type of the data value.\n\nIf the `APIFY_TOKEN` environment variable is set but `APIFY_LOCAL_STORAGE_DIR` is not, the data is stored in the\n[Apify Key-value store](https://docs.apify.com/storage/key-value-store) cloud storage." + } + ] + }, + "flags": {}, + "groups": [ + { + "children": [ + 588, + 585, + 573, + 577, + 590, + 580 + ], + "title": "Methods" + } + ], + "id": 572, + "module": "storages.key_value_store", + "name": "KeyValueStore", + "parsedDocstring": { + "text": "The `KeyValueStore` class represents a key-value store.\n\nYou can imagine it as a simple data storage that is used\nfor saving and reading data records or files. Each data record is\nrepresented by a unique key and associated with a MIME content type.\n\nDo not instantiate this class directly, use the `Actor.open_key_value_store()` function instead.\n\nEach crawler run is associated with a default key-value store, which is created exclusively\nfor the run. By convention, the crawler input and output are stored into the\ndefault key-value store under the `INPUT` and `OUTPUT` key, respectively.\nTypically, input and output are JSON files, although it can be any other format.\nTo access the default key-value store directly, you can use the\n`KeyValueStore.get_value` and `KeyValueStore.set_value` convenience functions.\n\n`KeyValueStore` stores its data either on local disk or in the Apify cloud,\ndepending on whether the `APIFY_LOCAL_STORAGE_DIR` or `APIFY_TOKEN` environment variables are set.\n\nIf the `APIFY_LOCAL_STORAGE_DIR` environment variable is set, the data is stored in\nthe local directory in the following files:\n```\n{APIFY_LOCAL_STORAGE_DIR}/key_value_stores/{STORE_ID}/{INDEX}.{EXT}\n```\n\nNote that `{STORE_ID}` is the name or ID of the key-value store. The default key-value store has ID: `default`,\nunless you override it by setting the `APIFY_DEFAULT_KEY_VALUE_STORE_ID` environment variable.\nThe `{KEY}` is the key of the record and `{EXT}` corresponds to the MIME content type of the data value.\n\nIf the `APIFY_TOKEN` environment variable is set but `APIFY_LOCAL_STORAGE_DIR` is not, the data is stored in the\n[Apify Key-value store](https://docs.apify.com/storage/key-value-store) cloud storage." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storages/key_value_store.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 17 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Create a `StorageClientManager` instance." + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 604, + "module": "storages.storage_client_manager", + "name": "__init__", + "parsedDocstring": { + "text": "Create a `StorageClientManager` instance." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storages/storage_client_manager.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 21 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Create a `StorageClientManager` instance." + } + ] + }, + "flags": {}, + "id": 605, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "__init__", + "parameters": [], + "type": { + "name": "None", + "type": "literal", + "value": null + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Set the config for the StorageClientManager.\n" + } + ] + }, + "decorations": [ + { + "name": "classmethod" + } + ], + "flags": {}, + "groups": [], + "id": 606, + "module": "storages.storage_client_manager", + "name": "set_config", + "parsedDocstring": { + "text": "Set the config for the StorageClientManager.\n", + "args": { + "config": "The configuration this StorageClientManager should use." + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storages/storage_client_manager.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 26 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Set the config for the StorageClientManager.\n" + } + ] + }, + "flags": {}, + "id": 607, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "set_config", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The configuration this StorageClientManager should use." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 608, + "kind": 32768, + "kindString": "Parameter", + "name": "config", + "type": { + "name": "Configuration", + "type": "reference", + "target": "1" + } + } + ], + "type": { + "name": "None", + "type": "literal", + "value": null + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Get the current storage client instance.\n" + } + ] + }, + "decorations": [ + { + "name": "classmethod" + } + ], + "flags": {}, + "groups": [], + "id": 609, + "module": "storages.storage_client_manager", + "name": "get_storage_client", + "parsedDocstring": { + "text": "Get the current storage client instance.\n", + "returns": "ApifyClientAsync or MemoryStorageClient: The current storage client instance." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storages/storage_client_manager.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 35 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "ApifyClientAsync or MemoryStorageClient: The current storage client instance." + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Get the current storage client instance.\n" + } + ] + }, + "flags": {}, + "id": 610, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "get_storage_client", + "parameters": [ + { + "defaultValue": "False", + "flags": { + "isOptional": true, + "keyword-only": false + }, + "id": 611, + "kind": 32768, + "kindString": "Parameter", + "name": "force_cloud", + "type": { + "name": "bool", + "type": "reference" + } + } + ], + "type": { + "name": "Union", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "ApifyClientAsync" + }, + { + "type": "reference", + "name": "MemoryStorageClient" + } + ] + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Set the storage client.\n" + } + ] + }, + "decorations": [ + { + "name": "classmethod" + } + ], + "flags": {}, + "groups": [], + "id": 612, + "module": "storages.storage_client_manager", + "name": "set_cloud_client", + "parsedDocstring": { + "text": "Set the storage client.\n", + "args": { + "client": "The instance of a storage client." + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storages/storage_client_manager.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 52 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Set the storage client.\n" + } + ] + }, + "flags": {}, + "id": 613, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "set_cloud_client", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The instance of a storage client." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 614, + "kind": 32768, + "kindString": "Parameter", + "name": "client", + "type": { + "name": "ApifyClientAsync", + "type": "reference" + } + } + ], + "type": { + "name": "None", + "type": "literal", + "value": null + } + } + ] + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 615, + "module": "storages.base_storage", + "name": "BaseResourceClientType", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storages/base_storage.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 15 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 616, + "module": "storages.base_storage", + "name": "BaseResourceCollectionClientType", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storages/base_storage.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 16 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Initialize the storage.\n\nDo not use this method directly, but use `Actor.open_()` instead.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 617, + "module": "storages.base_storage", + "name": "__init__", + "parsedDocstring": { + "text": "Initialize the storage.\n\nDo not use this method directly, but use `Actor.open_()` instead.\n", + "args": { + "id": "The storage id", + "name": "The storage name", + "client": "The storage client", + "config": "The configuration" + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storages/base_storage.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 32 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Initialize the storage.\n\nDo not use this method directly, but use `Actor.open_()` instead.\n" + } + ] + }, + "flags": {}, + "id": 618, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "__init__", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The storage id" + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 619, + "kind": 32768, + "kindString": "Parameter", + "name": "id", + "type": { + "name": "str", + "type": "reference" + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The storage name" + } + ] + }, + "flags": { + "isOptional": true, + "keyword-only": false + }, + "id": 620, + "kind": 32768, + "kindString": "Parameter", + "name": "name", + "type": { + "name": "Optional", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "str" + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The storage client" + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 621, + "kind": 32768, + "kindString": "Parameter", + "name": "client", + "type": { + "name": "Union", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "ApifyClientAsync" + }, + { + "type": "reference", + "name": "MemoryStorageClient" + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The configuration" + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 622, + "kind": 32768, + "kindString": "Parameter", + "name": "config", + "type": { + "name": "Configuration", + "type": "reference", + "target": "1" + } + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Open a storage, or return a cached storage object if it was opened before.\n\nOpens a storage with the given ID or name.\nReturns the cached storage object if the storage was opened before.\n" + } + ] + }, + "decorations": [ + { + "name": "classmethod" + }, + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 623, + "module": "storages.base_storage", + "name": "open", + "parsedDocstring": { + "text": "Open a storage, or return a cached storage object if it was opened before.\n\nOpens a storage with the given ID or name.\nReturns the cached storage object if the storage was opened before.\n", + "args": { + "id": "ID of the storage to be opened.\nIf neither `id` nor `name` are provided, the method returns the default storage associated with the actor run.\nIf the storage with the given ID does not exist, it raises an error.", + "name": "Name of the storage to be opened.\nIf neither `id` nor `name` are provided, the method returns the default storage associated with the actor run.\nIf the storage with the given name does not exist, it is created.", + "force_cloud": "If set to True, it will open a storage on the Apify Platform even when running the actor locally.\nDefaults to False.", + "config": "A `Configuration` instance, uses global configuration if omitted.\n" + }, + "returns": "An instance of the storage." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storages/base_storage.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 79 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "An instance of the storage." + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Open a storage, or return a cached storage object if it was opened before.\n\nOpens a storage with the given ID or name.\nReturns the cached storage object if the storage was opened before.\n" + } + ] + }, + "flags": {}, + "id": 624, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "open", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "ID of the storage to be opened.\nIf neither `id` nor `name` are provided, the method returns the default storage associated with the actor run.\nIf the storage with the given ID does not exist, it raises an error." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 625, + "kind": 32768, + "kindString": "Parameter", + "name": "id", + "type": { + "name": "Optional", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "str" + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Name of the storage to be opened.\nIf neither `id` nor `name` are provided, the method returns the default storage associated with the actor run.\nIf the storage with the given name does not exist, it is created." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 626, + "kind": 32768, + "kindString": "Parameter", + "name": "name", + "type": { + "name": "Optional", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "str" + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "If set to True, it will open a storage on the Apify Platform even when running the actor locally.\nDefaults to False." + } + ] + }, + "defaultValue": "False", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 627, + "kind": 32768, + "kindString": "Parameter", + "name": "force_cloud", + "type": { + "name": "bool", + "type": "reference" + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "A `Configuration` instance, uses global configuration if omitted.\n" + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 628, + "kind": 32768, + "kindString": "Parameter", + "name": "config", + "type": { + "name": "Optional", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "Configuration", + "target": "1" + } + ] + } + } + ], + "type": { + "name": "Self", + "type": "reference" + } + } + ] + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 629, + "module": "storages.request_queue", + "name": "MAX_CACHED_REQUESTS", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storages/request_queue.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 20 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 630, + "module": "storages.request_queue", + "name": "QUERY_HEAD_MIN_LENGTH", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storages/request_queue.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 23 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 631, + "module": "storages.request_queue", + "name": "QUERY_HEAD_BUFFER", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storages/request_queue.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 25 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 632, + "module": "storages.request_queue", + "name": "API_PROCESSED_REQUESTS_DELAY_MILLIS", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storages/request_queue.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 29 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 633, + "module": "storages.request_queue", + "name": "MAX_QUERIES_FOR_CONSISTENCY", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storages/request_queue.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 32 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 634, + "module": "storages.request_queue", + "name": "RECENTLY_HANDLED_CACHE_SIZE", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storages/request_queue.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 36 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 635, + "module": "storages.request_queue", + "name": "STORAGE_CONSISTENCY_DELAY_MILLIS", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storages/request_queue.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 40 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Add a request to the queue.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 637, + "module": "storages.request_queue", + "name": "add_request", + "parsedDocstring": { + "text": "Add a request to the queue.\n", + "args": { + "request": "The request to add to the queue", + "forefront": "Whether to add the request to the head or the end of the queue\n" + }, + "returns": "dict: Information about the queue operation with keys `requestId`, `uniqueKey`, `wasAlreadyPresent`, `wasAlreadyHandled`." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storages/request_queue.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 131 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "dict: Information about the queue operation with keys `requestId`, `uniqueKey`, `wasAlreadyPresent`, `wasAlreadyHandled`." + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Add a request to the queue.\n" + } + ] + }, + "flags": {}, + "id": 638, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "add_request", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The request to add to the queue" + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 639, + "kind": 32768, + "kindString": "Parameter", + "name": "request", + "type": { + "name": "Dict", + "type": "reference" + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Whether to add the request to the head or the end of the queue\n" + } + ] + }, + "defaultValue": "False", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 640, + "kind": 32768, + "kindString": "Parameter", + "name": "forefront", + "type": { + "name": "bool", + "type": "reference" + } + } + ], + "type": { + "name": "Dict", + "type": "reference" + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Retrieve a request from the queue.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 641, + "module": "storages.request_queue", + "name": "get_request", + "parsedDocstring": { + "text": "Retrieve a request from the queue.\n", + "args": { + "request_id": "ID of the request to retrieve.\n" + }, + "returns": "dict, optional: The retrieved request, or `None`, if it does not exist." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storages/request_queue.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 176 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "dict, optional: The retrieved request, or `None`, if it does not exist." + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Retrieve a request from the queue.\n" + } + ] + }, + "flags": {}, + "id": 642, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "get_request", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "ID of the request to retrieve.\n" + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 643, + "kind": 32768, + "kindString": "Parameter", + "name": "request_id", + "type": { + "name": "str", + "type": "reference" + } + } + ], + "type": { + "name": "Optional", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "Dict" + } + ] + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Return the next request in the queue to be processed.\n\nOnce you successfully finish processing of the request, you need to call\n`RequestQueue.mark_request_as_handled` to mark the request as handled in the queue.\nIf there was some error in processing the request, call `RequestQueue.reclaim_request` instead,\nso that the queue will give the request to some other consumer in another call to the `fetch_next_request` method.\n\nNote that the `None` return value does not mean the queue processing finished, it means there are currently no pending requests.\nTo check whether all requests in queue were finished, use `RequestQueue.is_finished` instead.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 644, + "module": "storages.request_queue", + "name": "fetch_next_request", + "parsedDocstring": { + "text": "Return the next request in the queue to be processed.\n\nOnce you successfully finish processing of the request, you need to call\n`RequestQueue.mark_request_as_handled` to mark the request as handled in the queue.\nIf there was some error in processing the request, call `RequestQueue.reclaim_request` instead,\nso that the queue will give the request to some other consumer in another call to the `fetch_next_request` method.\n\nNote that the `None` return value does not mean the queue processing finished, it means there are currently no pending requests.\nTo check whether all requests in queue were finished, use `RequestQueue.is_finished` instead.\n", + "returns": "dict, optional: The request or `None` if there are no more pending requests." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storages/request_queue.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 188 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "dict, optional: The request or `None` if there are no more pending requests." + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Return the next request in the queue to be processed.\n\nOnce you successfully finish processing of the request, you need to call\n`RequestQueue.mark_request_as_handled` to mark the request as handled in the queue.\nIf there was some error in processing the request, call `RequestQueue.reclaim_request` instead,\nso that the queue will give the request to some other consumer in another call to the `fetch_next_request` method.\n\nNote that the `None` return value does not mean the queue processing finished, it means there are currently no pending requests.\nTo check whether all requests in queue were finished, use `RequestQueue.is_finished` instead.\n" + } + ] + }, + "flags": {}, + "id": 645, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "fetch_next_request", + "parameters": [], + "type": { + "name": "Optional", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "Dict" + } + ] + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Mark a request as handled after successful processing.\n\nHandled requests will never again be returned by the `RequestQueue.fetch_next_request` method.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 646, + "module": "storages.request_queue", + "name": "mark_request_as_handled", + "parsedDocstring": { + "text": "Mark a request as handled after successful processing.\n\nHandled requests will never again be returned by the `RequestQueue.fetch_next_request` method.\n", + "args": { + "request": "The request to mark as handled.\n" + }, + "returns": "dict, optional: Information about the queue operation with keys `requestId`, `uniqueKey`, `wasAlreadyPresent`, `wasAlreadyHandled`.\n`None` if the given request was not in progress." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storages/request_queue.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 253 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "dict, optional: Information about the queue operation with keys `requestId`, `uniqueKey`, `wasAlreadyPresent`, `wasAlreadyHandled`.\n`None` if the given request was not in progress." + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Mark a request as handled after successful processing.\n\nHandled requests will never again be returned by the `RequestQueue.fetch_next_request` method.\n" + } + ] + }, + "flags": {}, + "id": 647, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "mark_request_as_handled", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The request to mark as handled.\n" + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 648, + "kind": 32768, + "kindString": "Parameter", + "name": "request", + "type": { + "name": "Dict", + "type": "reference" + } + } + ], + "type": { + "name": "Optional", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "Dict" + } + ] + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Reclaim a failed request back to the queue.\n\nThe request will be returned for processing later again\nby another call to `RequestQueue.fetchNextRequest`.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 649, + "module": "storages.request_queue", + "name": "reclaim_request", + "parsedDocstring": { + "text": "Reclaim a failed request back to the queue.\n\nThe request will be returned for processing later again\nby another call to `RequestQueue.fetchNextRequest`.\n", + "args": { + "request": "The request to return to the queue.", + "forefront": "Whether to add the request to the head or the end of the queue" + }, + "returns": "dict, optional: Information about the queue operation with keys `requestId`, `uniqueKey`, `wasAlreadyPresent`, `wasAlreadyHandled`.\n`None` if the given request was not in progress." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storages/request_queue.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 289 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "dict, optional: Information about the queue operation with keys `requestId`, `uniqueKey`, `wasAlreadyPresent`, `wasAlreadyHandled`.\n`None` if the given request was not in progress." + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Reclaim a failed request back to the queue.\n\nThe request will be returned for processing later again\nby another call to `RequestQueue.fetchNextRequest`.\n" + } + ] + }, + "flags": {}, + "id": 650, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "reclaim_request", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The request to return to the queue." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 651, + "kind": 32768, + "kindString": "Parameter", + "name": "request", + "type": { + "name": "Dict", + "type": "reference" + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Whether to add the request to the head or the end of the queue" + } + ] + }, + "defaultValue": "False", + "flags": { + "isOptional": true, + "keyword-only": false + }, + "id": 652, + "kind": 32768, + "kindString": "Parameter", + "name": "forefront", + "type": { + "name": "bool", + "type": "reference" + } + } + ], + "type": { + "name": "Optional", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "Dict" + } + ] + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Check whether the queue is empty.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 653, + "module": "storages.request_queue", + "name": "is_empty", + "parsedDocstring": { + "text": "Check whether the queue is empty.\n", + "returns": "bool: `True` if the next call to `RequestQueue.fetchNextRequest` would return `None`, otherwise `False`." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storages/request_queue.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 337 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "bool: `True` if the next call to `RequestQueue.fetchNextRequest` would return `None`, otherwise `False`." + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Check whether the queue is empty.\n" + } + ] + }, + "flags": {}, + "id": 654, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "is_empty", + "parameters": [], + "type": { + "name": "bool", + "type": "reference" + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Check whether the queue is finished.\n\nDue to the nature of distributed storage used by the queue,\nthe function might occasionally return a false negative,\nbut it will never return a false positive.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 655, + "module": "storages.request_queue", + "name": "is_finished", + "parsedDocstring": { + "text": "Check whether the queue is finished.\n\nDue to the nature of distributed storage used by the queue,\nthe function might occasionally return a false negative,\nbut it will never return a false positive.\n", + "returns": "bool: `True` if all requests were already handled and there are no more left. `False` otherwise." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storages/request_queue.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 346 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "bool: `True` if all requests were already handled and there are no more left. `False` otherwise." + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Check whether the queue is finished.\n\nDue to the nature of distributed storage used by the queue,\nthe function might occasionally return a false negative,\nbut it will never return a false positive.\n" + } + ] + }, + "flags": {}, + "id": 656, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "is_finished", + "parameters": [], + "type": { + "name": "bool", + "type": "reference" + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Remove the request queue either from the Apify cloud storage or from the local directory." + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 657, + "module": "storages.request_queue", + "name": "drop", + "parsedDocstring": { + "text": "Remove the request queue either from the Apify cloud storage or from the local directory." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storages/request_queue.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 480 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Remove the request queue either from the Apify cloud storage or from the local directory." + } + ] + }, + "flags": {}, + "id": 658, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "drop", + "parameters": [], + "type": { + "name": "None", + "type": "literal", + "value": null + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Get an object containing general information about the request queue.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 659, + "module": "storages.request_queue", + "name": "get_info", + "parsedDocstring": { + "text": "Get an object containing general information about the request queue.\n", + "returns": "dict: Object returned by calling the GET request queue API endpoint." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storages/request_queue.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 485 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "dict: Object returned by calling the GET request queue API endpoint." + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Get an object containing general information about the request queue.\n" + } + ] + }, + "flags": {}, + "id": 660, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "get_info", + "parameters": [], + "type": { + "name": "Optional", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "Dict" + } + ] + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Open a request queue.\n\nRequest queue represents a queue of URLs to crawl, which is stored either on local filesystem or in the Apify cloud.\nThe queue is used for deep crawling of websites, where you start with several URLs and then\nrecursively follow links to other pages. The data structure supports both breadth-first\nand depth-first crawling orders.\n" + } + ] + }, + "decorations": [ + { + "name": "classmethod" + } + ], + "flags": {}, + "groups": [], + "id": 661, + "module": "storages.request_queue", + "name": "open", + "parsedDocstring": { + "text": "Open a request queue.\n\nRequest queue represents a queue of URLs to crawl, which is stored either on local filesystem or in the Apify cloud.\nThe queue is used for deep crawling of websites, where you start with several URLs and then\nrecursively follow links to other pages. The data structure supports both breadth-first\nand depth-first crawling orders.\n", + "args": { + "id": "ID of the request queue to be opened.\nIf neither `id` nor `name` are provided, the method returns the default request queue associated with the actor run.\nIf the request queue with the given ID does not exist, it raises an error.", + "name": "Name of the request queue to be opened.\nIf neither `id` nor `name` are provided, the method returns the default request queue associated with the actor run.\nIf the request queue with the given name does not exist, it is created.", + "force_cloud": "If set to True, it will open a request queue on the Apify Platform even when running the actor locally.\nDefaults to False.", + "config": "A `Configuration` instance, uses global configuration if omitted.\n" + }, + "returns": "RequestQueue: An instance of the `RequestQueue` class for the given ID or name." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storages/request_queue.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 494 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "RequestQueue: An instance of the `RequestQueue` class for the given ID or name." + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Open a request queue.\n\nRequest queue represents a queue of URLs to crawl, which is stored either on local filesystem or in the Apify cloud.\nThe queue is used for deep crawling of websites, where you start with several URLs and then\nrecursively follow links to other pages. The data structure supports both breadth-first\nand depth-first crawling orders.\n" + } + ] + }, + "flags": {}, + "id": 662, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "open", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "ID of the request queue to be opened.\nIf neither `id` nor `name` are provided, the method returns the default request queue associated with the actor run.\nIf the request queue with the given ID does not exist, it raises an error." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 663, + "kind": 32768, + "kindString": "Parameter", + "name": "id", + "type": { + "name": "Optional", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "str" + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Name of the request queue to be opened.\nIf neither `id` nor `name` are provided, the method returns the default request queue associated with the actor run.\nIf the request queue with the given name does not exist, it is created." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 664, + "kind": 32768, + "kindString": "Parameter", + "name": "name", + "type": { + "name": "Optional", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "str" + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "If set to True, it will open a request queue on the Apify Platform even when running the actor locally.\nDefaults to False." + } + ] + }, + "defaultValue": "False", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 665, + "kind": 32768, + "kindString": "Parameter", + "name": "force_cloud", + "type": { + "name": "bool", + "type": "reference" + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "A `Configuration` instance, uses global configuration if omitted.\n" + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 666, + "kind": 32768, + "kindString": "Parameter", + "name": "config", + "type": { + "name": "Optional", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "Configuration", + "target": "1" + } + ] + } + } + ], + "type": { + "name": "'RequestQueue'", + "type": "literal", + "value": "RequestQueue" + } + } + ] + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Represents a queue of URLs to crawl.\n\nCan be used for deep crawling of websites where you start with several URLs and then recursively\nfollow links to other pages. The data structure supports both breadth-first and depth-first crawling orders.\n\nEach URL is represented using an instance of the {@apilink Request} class.\nThe queue can only contain unique URLs. More precisely, it can only contain request dictionaries\nwith distinct `uniqueKey` properties. By default, `uniqueKey` is generated from the URL, but it can also be overridden.\nTo add a single URL multiple times to the queue,\ncorresponding request dictionary will need to have different `uniqueKey` properties.\n\nDo not instantiate this class directly, use the `Actor.open_request_queue()` function instead.\n\n`RequestQueue` stores its data either on local disk or in the Apify cloud,\ndepending on whether the `APIFY_LOCAL_STORAGE_DIR` or `APIFY_TOKEN` environment variables are set.\n\nIf the `APIFY_LOCAL_STORAGE_DIR` environment variable is set, the data is stored in\nthe local directory in the following files:\n```\n{APIFY_LOCAL_STORAGE_DIR}/request_queues/{QUEUE_ID}/{REQUEST_ID}.json\n```\n\nNote that `{QUEUE_ID}` is the name or ID of the request queue. The default request queue has ID: `default`,\nunless you override it by setting the `APIFY_DEFAULT_REQUEST_QUEUE_ID` environment variable.\nThe `{REQUEST_ID}` is the id of the request.\n\nIf the `APIFY_TOKEN` environment variable is set but `APIFY_LOCAL_STORAGE_DIR` is not, the data is stored in the\n[Apify Request Queue](https://docs.apify.com/storage/request-queue)\ncloud storage." + } + ] + }, + "flags": {}, + "groups": [ + { + "children": [ + 637, + 657, + 644, + 659, + 641, + 653, + 655, + 646, + 661, + 649 + ], + "title": "Methods" + } + ], + "id": 636, + "module": "storages.request_queue", + "name": "RequestQueue", + "parsedDocstring": { + "text": "Represents a queue of URLs to crawl.\n\nCan be used for deep crawling of websites where you start with several URLs and then recursively\nfollow links to other pages. The data structure supports both breadth-first and depth-first crawling orders.\n\nEach URL is represented using an instance of the {@apilink Request} class.\nThe queue can only contain unique URLs. More precisely, it can only contain request dictionaries\nwith distinct `uniqueKey` properties. By default, `uniqueKey` is generated from the URL, but it can also be overridden.\nTo add a single URL multiple times to the queue,\ncorresponding request dictionary will need to have different `uniqueKey` properties.\n\nDo not instantiate this class directly, use the `Actor.open_request_queue()` function instead.\n\n`RequestQueue` stores its data either on local disk or in the Apify cloud,\ndepending on whether the `APIFY_LOCAL_STORAGE_DIR` or `APIFY_TOKEN` environment variables are set.\n\nIf the `APIFY_LOCAL_STORAGE_DIR` environment variable is set, the data is stored in\nthe local directory in the following files:\n```\n{APIFY_LOCAL_STORAGE_DIR}/request_queues/{QUEUE_ID}/{REQUEST_ID}.json\n```\n\nNote that `{QUEUE_ID}` is the name or ID of the request queue. The default request queue has ID: `default`,\nunless you override it by setting the `APIFY_DEFAULT_REQUEST_QUEUE_ID` environment variable.\nThe `{REQUEST_ID}` is the id of the request.\n\nIf the `APIFY_TOKEN` environment variable is set but `APIFY_LOCAL_STORAGE_DIR` is not, the data is stored in the\n[Apify Request Queue](https://docs.apify.com/storage/request-queue)\ncloud storage." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storages/request_queue.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 43 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Get the storage ID." + } + ] + }, + "decorations": [ + { + "name": "property" + }, + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 2997, + "module": "storages._base", + "name": "id", + "parsedDocstring": { + "text": "Get the storage ID." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_base.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 20, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storages/_base.py#L20" + } + ], + "type": { + "name": "str", + "type": "reference", + "target": 0, + "ref": {} + }, + "permalink": "/python/api/class/Storage#id", + "parentId": 2330 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Get the storage name." + } + ] + }, + "decorations": [ + { + "name": "property" + }, + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 2998, + "module": "storages._base", + "name": "name", + "parsedDocstring": { + "text": "Get the storage name." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_base.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 25, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storages/_base.py#L25" + } + ], + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "permalink": "/python/api/class/Storage#name", + "parentId": 2330 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Get the storage metadata." + } + ] + }, + "decorations": [ + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 2999, + "module": "storages._base", + "name": "get_metadata", + "parsedDocstring": { + "text": "Get the storage metadata." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_base.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 29, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storages/_base.py#L29" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Get the storage metadata." + } + ] + }, + "flags": {}, + "id": 3000, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "get_metadata", + "parameters": [], + "type": { + "name": "DatasetMetadata | KeyValueStoreMetadata | RequestQueueMetadata", + "type": "union", + "types": [ + { + "type": "union", + "types": [ + { + "type": "reference", + "name": "DatasetMetadata", + "target": "1314", + "ref": { + "id": 1980, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 42 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/DatasetMetadata" + } + }, + { + "type": "reference", + "name": "KeyValueStoreMetadata", + "target": "1317", + "ref": { + "id": 1983, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 52 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/KeyValueStoreMetadata" + } + } + ] + }, + { + "type": "reference", + "name": "RequestQueueMetadata", + "target": "1319", + "ref": { + "id": 1985, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 59 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/RequestQueueMetadata" + } + } + ] + }, + "overwrites": { + "name": "Storage.get_metadata", + "target": 2333, + "type": "reference", + "ref": { + "id": 2999, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_base.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 29 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/Storage#get_metadata" + } + }, + "parentId": 2333 + } + ], + "permalink": "/python/api/class/Storage#get_metadata", + "parentId": 2330 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Open a storage, either restore existing or create a new one.\n" + } + ] + }, + "decorations": [ + { + "name": "classmethod" + }, + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 3001, + "module": "storages._base", + "name": "open", + "parsedDocstring": { + "text": "Open a storage, either restore existing or create a new one.\n", + "args": { + "id": "The storage ID.", + "name": "The storage name (global scope, persists across runs). Name can only contain letters \"a\" through \"z\",\nthe digits \"0\" through \"9\", and the hyphen (\"-\") but only in the middle of the string\n(e.g. \"my-value-1\").", + "alias": "The storage alias (run scope, creates unnamed storage).", + "configuration": "Configuration object used during the storage creation or restoration process.", + "storage_client": "Underlying storage client to use. If not provided, the default global storage client\nfrom the service locator will be used." + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_base.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 34, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storages/_base.py#L34" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Open a storage, either restore existing or create a new one.\n" + } + ] + }, + "flags": {}, + "id": 3002, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "open", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The storage ID." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 3003, + "kind": 32768, + "kindString": "Parameter", + "name": "id", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2336 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The storage name (global scope, persists across runs). Name can only contain letters \"a\" through \"z\",\nthe digits \"0\" through \"9\", and the hyphen (\"-\") but only in the middle of the string\n(e.g. \"my-value-1\")." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 3004, + "kind": 32768, + "kindString": "Parameter", + "name": "name", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2336 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The storage alias (run scope, creates unnamed storage)." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 3005, + "kind": 32768, + "kindString": "Parameter", + "name": "alias", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2336 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Configuration object used during the storage creation or restoration process." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 3006, + "kind": 32768, + "kindString": "Parameter", + "name": "configuration", + "type": { + "name": "Configuration | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "Configuration", + "target": "2", + "ref": { + "id": 668, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/configuration.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 20 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/Configuration" + } + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2336 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Underlying storage client to use. If not provided, the default global storage client\nfrom the service locator will be used." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 3007, + "kind": 32768, + "kindString": "Parameter", + "name": "storage_client", + "type": { + "name": "StorageClient | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "StorageClient", + "target": "1282", + "ref": { + "id": 1948, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 19 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/StorageClient" + } + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2336 + } + ], + "type": { + "name": "Storage", + "type": "reference", + "target": "2330", + "ref": { + "id": 2996, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_base.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 15 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/Storage" + } + }, + "overwrites": { + "name": "Storage.open", + "target": 2335, + "type": "reference", + "ref": { + "id": 3001, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_base.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 34 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/Storage#open" + } + }, + "parentId": 2335 + } + ], + "permalink": "/python/api/class/Storage#open", + "parentId": 2330 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Drop the storage, removing it from the underlying storage client and clearing the cache." + } + ] + }, + "decorations": [ + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 3008, + "module": "storages._base", + "name": "drop", + "parsedDocstring": { + "text": "Drop the storage, removing it from the underlying storage client and clearing the cache." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_base.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 57, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storages/_base.py#L57" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Drop the storage, removing it from the underlying storage client and clearing the cache." + } + ] + }, + "flags": {}, + "id": 3009, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "drop", + "parameters": [], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "overwrites": { + "name": "Storage.drop", + "target": 2342, + "type": "reference", + "ref": { + "id": 3008, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_base.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 57 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/Storage#drop" + } + }, + "parentId": 2342 + } + ], + "permalink": "/python/api/class/Storage#drop", + "parentId": 2330 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Purge the storage, removing all items from the underlying storage client.\n\nThis method does not remove the storage itself, e.g. don't remove the metadata,\nbut clears all items within it." + } + ] + }, + "decorations": [ + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 3010, + "module": "storages._base", + "name": "purge", + "parsedDocstring": { + "text": "Purge the storage, removing all items from the underlying storage client.\n\nThis method does not remove the storage itself, e.g. don't remove the metadata,\nbut clears all items within it." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_base.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 61, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storages/_base.py#L61" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Purge the storage, removing all items from the underlying storage client.\n\nThis method does not remove the storage itself, e.g. don't remove the metadata,\nbut clears all items within it." + } + ] + }, + "flags": {}, + "id": 3011, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "purge", + "parameters": [], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "overwrites": { + "name": "Storage.purge", + "target": 2344, + "type": "reference", + "ref": { + "id": 3010, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_base.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 61 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/Storage#purge" + } + }, + "parentId": 2344 + } + ], + "permalink": "/python/api/class/Storage#purge", + "parentId": 2330 + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Base class for storages." + } + ] + }, + "decorations": [ + { + "args": "('Storages')", + "name": "docs_group" + } + ], + "flags": {}, + "groups": [ + { + "children": [ + 3008, + 2999, + 3001, + 3010 + ], + "title": "Methods" + }, + { + "children": [ + 2997, + 2998 + ], + "title": "Properties" + } + ], + "id": 2996, + "module": "storages._base", + "name": "Storage", + "parsedDocstring": { + "text": "Base class for storages." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_base.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 15, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storages/_base.py#L15" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "extendedBy": [ + { + "name": "KeyValueStore", + "target": "2269", + "type": "reference", + "ref": { + "id": 2935, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_key_value_store.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 40 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/KeyValueStore" + } + }, + { + "name": "RequestQueue", + "target": "2348", + "type": "reference", + "ref": { + "id": 3014, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_request_queue.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 33 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/RequestQueue" + } + }, + { + "name": "Dataset", + "target": "2402", + "type": "reference", + "ref": { + "id": 3068, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_dataset.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 33 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/Dataset" + } + } + ], + "permalink": "/python/api/class/Storage", + "categories": [] + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Initialize a new instance.\n\nPreferably use the `Dataset.open` constructor to create a new instance.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 5471, + "module": "storages._dataset", + "name": "__init__", + "parsedDocstring": { + "text": "Initialize a new instance.\n\nPreferably use the `Dataset.open` constructor to create a new instance.\n", + "args": { + "client": "An instance of a storage client.", + "id": "The unique identifier of the storage.", + "name": "The name of the storage, if available." + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_dataset.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 69, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storages/_dataset.py#L69" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Initialize a new instance.\n\nPreferably use the `Dataset.open` constructor to create a new instance.\n" + } + ] + }, + "flags": {}, + "id": 5472, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "__init__", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "An instance of a storage client." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 5473, + "kind": 32768, + "kindString": "Parameter", + "name": "client", + "type": { + "name": "DatasetClient", + "type": "reference", + "target": "1220", + "ref": { + "id": 4288, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_dataset_client.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 13 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/DatasetClient" + } + }, + "parentId": 2404 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The unique identifier of the storage." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 5474, + "kind": 32768, + "kindString": "Parameter", + "name": "id", + "type": { + "name": "str", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2404 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The name of the storage, if available." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 5475, + "kind": 32768, + "kindString": "Parameter", + "name": "name", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2404 + } + ], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "parentId": 2403 + } + ], + "permalink": "/python/api/class/Dataset#__init__", + "parentId": 2402 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Get the storage ID." + } + ] + }, + "decorations": [ + { + "name": "property" + }, + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 5476, + "module": "storages._base", + "name": "id", + "parsedDocstring": { + "text": "Get the storage ID." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_dataset.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 87, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storages/_dataset.py#L87" + } + ], + "type": { + "name": "str", + "type": "reference", + "target": 0, + "ref": {} + }, + "overwrites": { + "name": "Storage.id", + "target": 2331, + "type": "reference", + "ref": { + "id": 5399, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_base.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 20 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api/class/Storage#id" + } + }, + "permalink": "/python/api/class/Dataset#id", + "parentId": 2402 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Get the storage name." + } + ] + }, + "decorations": [ + { + "name": "property" + }, + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 5477, + "module": "storages._base", + "name": "name", + "parsedDocstring": { + "text": "Get the storage name." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_dataset.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 92, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storages/_dataset.py#L92" + } + ], + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "overwrites": { + "name": "Storage.name", + "target": 2332, + "type": "reference", + "ref": { + "id": 5400, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_base.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 25 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api/class/Storage#name" + } + }, + "permalink": "/python/api/class/Dataset#name", + "parentId": 2402 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Get the storage metadata." + } + ] + }, + "decorations": [ + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 5478, + "module": "storages._base", + "name": "get_metadata", + "parsedDocstring": { + "text": "Get the storage metadata." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_dataset.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 96, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storages/_dataset.py#L96" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Get the storage metadata." + } + ] + }, + "flags": {}, + "id": 5402, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "get_metadata", + "parameters": [], + "type": { + "name": "DatasetMetadata | KeyValueStoreMetadata | RequestQueueMetadata", + "type": "union", + "types": [ + { + "type": "union", + "types": [ + { + "type": "reference", + "name": "DatasetMetadata", + "target": "1314", + "ref": { + "id": 4382, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 42 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/DatasetMetadata" + } + }, + { + "type": "reference", + "name": "KeyValueStoreMetadata", + "target": "1317", + "ref": { + "id": 4385, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 52 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/KeyValueStoreMetadata" + } + } + ] + }, + { + "type": "reference", + "name": "RequestQueueMetadata", + "target": "1319", + "ref": { + "id": 4387, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 59 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/RequestQueueMetadata" + } + } + ] + }, + "overwrites": { + "name": "Storage.get_metadata", + "target": 2333, + "type": "reference", + "ref": { + "id": 5401, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_base.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 29 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/Storage#get_metadata" + } + }, + "parentId": 2410 + } + ], + "overwrites": { + "name": "Storage.get_metadata", + "target": 2333, + "type": "reference", + "ref": { + "id": 5401, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_base.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 29 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/Storage#get_metadata" + } + }, + "permalink": "/python/api/class/Dataset#get_metadata", + "parentId": 2402 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Open a storage, either restore existing or create a new one.\n" + } + ] + }, + "decorations": [ + { + "name": "classmethod" + }, + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 5480, + "module": "storages._base", + "name": "open", + "parsedDocstring": { + "text": "Open a storage, either restore existing or create a new one.\n", + "args": { + "id": "The storage ID.", + "name": "The storage name (global scope, persists across runs). Name can only contain letters \"a\" through \"z\",\nthe digits \"0\" through \"9\", and the hyphen (\"-\") but only in the middle of the string\n(e.g. \"my-value-1\").", + "alias": "The storage alias (run scope, creates unnamed storage).", + "configuration": "Configuration object used during the storage creation or restoration process.", + "storage_client": "Underlying storage client to use. If not provided, the default global storage client\nfrom the service locator will be used." + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_dataset.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 101, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storages/_dataset.py#L101" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Open a storage, either restore existing or create a new one.\n" + } + ] + }, + "flags": {}, + "id": 5404, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "open", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The storage ID." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 5405, + "kind": 32768, + "kindString": "Parameter", + "name": "id", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2336 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The storage name (global scope, persists across runs). Name can only contain letters \"a\" through \"z\",\nthe digits \"0\" through \"9\", and the hyphen (\"-\") but only in the middle of the string\n(e.g. \"my-value-1\")." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 5406, + "kind": 32768, + "kindString": "Parameter", + "name": "name", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2336 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The storage alias (run scope, creates unnamed storage)." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 5407, + "kind": 32768, + "kindString": "Parameter", + "name": "alias", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2336 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Configuration object used during the storage creation or restoration process." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 5408, + "kind": 32768, + "kindString": "Parameter", + "name": "configuration", + "type": { + "name": "Configuration | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "Configuration", + "target": "2", + "ref": { + "id": 3070, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/configuration.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 20 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/Configuration" + } + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2336 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Underlying storage client to use. If not provided, the default global storage client\nfrom the service locator will be used." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 5409, + "kind": 32768, + "kindString": "Parameter", + "name": "storage_client", + "type": { + "name": "StorageClient | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "StorageClient", + "target": "1282", + "ref": { + "id": 4350, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 19 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/StorageClient" + } + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2336 + } + ], + "type": { + "name": "Storage", + "type": "reference", + "target": "2330", + "ref": { + "id": 5398, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_base.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 15 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/Storage" + } + }, + "overwrites": { + "name": "Storage.open", + "target": 2335, + "type": "reference", + "ref": { + "id": 5403, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_base.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 34 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/Storage#open" + } + }, + "parentId": 2412 + } + ], + "overwrites": { + "name": "Storage.open", + "target": 2335, + "type": "reference", + "ref": { + "id": 5403, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_base.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 34 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/Storage#open" + } + }, + "permalink": "/python/api/class/Dataset#open", + "parentId": 2402 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Drop the storage, removing it from the underlying storage client and clearing the cache." + } + ] + }, + "decorations": [ + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 5487, + "module": "storages._base", + "name": "drop", + "parsedDocstring": { + "text": "Drop the storage, removing it from the underlying storage client and clearing the cache." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_dataset.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 128, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storages/_dataset.py#L128" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Drop the storage, removing it from the underlying storage client and clearing the cache." + } + ] + }, + "flags": {}, + "id": 5411, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "drop", + "parameters": [], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "overwrites": { + "name": "Storage.drop", + "target": 2342, + "type": "reference", + "ref": { + "id": 5410, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_base.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 57 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/Storage#drop" + } + }, + "parentId": 2419 + } + ], + "overwrites": { + "name": "Storage.drop", + "target": 2342, + "type": "reference", + "ref": { + "id": 5410, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_base.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 57 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/Storage#drop" + } + }, + "permalink": "/python/api/class/Dataset#drop", + "parentId": 2402 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Purge the storage, removing all items from the underlying storage client.\n\nThis method does not remove the storage itself, e.g. don't remove the metadata,\nbut clears all items within it." + } + ] + }, + "decorations": [ + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 5489, + "module": "storages._base", + "name": "purge", + "parsedDocstring": { + "text": "Purge the storage, removing all items from the underlying storage client.\n\nThis method does not remove the storage itself, e.g. don't remove the metadata,\nbut clears all items within it." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_dataset.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 134, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storages/_dataset.py#L134" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Purge the storage, removing all items from the underlying storage client.\n\nThis method does not remove the storage itself, e.g. don't remove the metadata,\nbut clears all items within it." + } + ] + }, + "flags": {}, + "id": 5413, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "purge", + "parameters": [], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "overwrites": { + "name": "Storage.purge", + "target": 2344, + "type": "reference", + "ref": { + "id": 5412, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_base.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 61 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/Storage#purge" + } + }, + "parentId": 2421 + } + ], + "overwrites": { + "name": "Storage.purge", + "target": 2344, + "type": "reference", + "ref": { + "id": 5412, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_base.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 61 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/Storage#purge" + } + }, + "permalink": "/python/api/class/Dataset#purge", + "parentId": 2402 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Store an object or an array of objects to the dataset.\n\nThe size of the data is limited by the receiving API and therefore `push_data()` will only\nallow objects whose JSON representation is smaller than 9MB. When an array is passed,\nnone of the included objects may be larger than 9MB, but the array itself may be of any size.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 5491, + "module": "storages._dataset", + "name": "push_data", + "parsedDocstring": { + "text": "Store an object or an array of objects to the dataset.\n\nThe size of the data is limited by the receiving API and therefore `push_data()` will only\nallow objects whose JSON representation is smaller than 9MB. When an array is passed,\nnone of the included objects may be larger than 9MB, but the array itself may be of any size.\n", + "args": { + "data": "A JSON serializable data structure to be stored in the dataset. The JSON representation\nof each item must be smaller than 9MB." + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_dataset.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 137, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storages/_dataset.py#L137" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Store an object or an array of objects to the dataset.\n\nThe size of the data is limited by the receiving API and therefore `push_data()` will only\nallow objects whose JSON representation is smaller than 9MB. When an array is passed,\nnone of the included objects may be larger than 9MB, but the array itself may be of any size.\n" + } + ] + }, + "flags": {}, + "id": 5492, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "push_data", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "A JSON serializable data structure to be stored in the dataset. The JSON representation\nof each item must be smaller than 9MB." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 5493, + "kind": 32768, + "kindString": "Parameter", + "name": "data", + "type": { + "name": "list[dict[str, Any]] | dict[str, Any]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "list", + "typeArguments": [ + { + "type": "reference", + "name": "dict", + "typeArguments": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "reference", + "name": "Any", + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + }, + { + "type": "reference", + "name": "dict", + "typeArguments": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "reference", + "name": "Any", + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + } + ] + }, + "parentId": 2424 + } + ], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "parentId": 2423 + } + ], + "permalink": "/python/api/class/Dataset#push_data", + "parentId": 2402 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Retrieve a paginated list of items from a dataset based on various filtering parameters.\n\nThis method provides the flexibility to filter, sort, and modify the appearance of dataset items\nwhen listed. Each parameter modifies the result set according to its purpose. The method also\nsupports pagination through 'offset' and 'limit' parameters.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 5494, + "module": "storages._dataset", + "name": "get_data", + "parsedDocstring": { + "text": "Retrieve a paginated list of items from a dataset based on various filtering parameters.\n\nThis method provides the flexibility to filter, sort, and modify the appearance of dataset items\nwhen listed. Each parameter modifies the result set according to its purpose. The method also\nsupports pagination through 'offset' and 'limit' parameters.\n", + "args": { + "offset": "Skips the specified number of items at the start.", + "limit": "The maximum number of items to retrieve. Unlimited if None.", + "clean": "Return only non-empty items and excludes hidden fields. Shortcut for skip_hidden and skip_empty.", + "desc": "Set to True to sort results in descending order.", + "fields": "Fields to include in each item. Sorts fields as specified if provided.", + "omit": "Fields to exclude from each item.", + "unwind": "Unwinds items by a specified array field, turning each element into a separate item.", + "skip_empty": "Excludes empty items from the results if True.", + "skip_hidden": "Excludes fields starting with '#' if True.", + "flatten": "Fields to be flattened in returned items.", + "view": "Specifies the dataset view to be used.\n" + }, + "returns": "An object with filtered, sorted, and paginated dataset items plus pagination details." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_dataset.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 150, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storages/_dataset.py#L150" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "An object with filtered, sorted, and paginated dataset items plus pagination details." + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Retrieve a paginated list of items from a dataset based on various filtering parameters.\n\nThis method provides the flexibility to filter, sort, and modify the appearance of dataset items\nwhen listed. Each parameter modifies the result set according to its purpose. The method also\nsupports pagination through 'offset' and 'limit' parameters.\n" + } + ] + }, + "flags": {}, + "id": 5495, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "get_data", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Skips the specified number of items at the start." + } + ] + }, + "defaultValue": "0", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 5496, + "kind": 32768, + "kindString": "Parameter", + "name": "offset", + "type": { + "name": "int", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2427 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The maximum number of items to retrieve. Unlimited if None." + } + ] + }, + "defaultValue": "999_999_999_999", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 5497, + "kind": 32768, + "kindString": "Parameter", + "name": "limit", + "type": { + "name": "int | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "int", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2427 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Return only non-empty items and excludes hidden fields. Shortcut for skip_hidden and skip_empty." + } + ] + }, + "defaultValue": "False", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 5498, + "kind": 32768, + "kindString": "Parameter", + "name": "clean", + "type": { + "name": "bool", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2427 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Set to True to sort results in descending order." + } + ] + }, + "defaultValue": "False", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 5499, + "kind": 32768, + "kindString": "Parameter", + "name": "desc", + "type": { + "name": "bool", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2427 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Fields to include in each item. Sorts fields as specified if provided." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 5500, + "kind": 32768, + "kindString": "Parameter", + "name": "fields", + "type": { + "name": "list[str] | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "list", + "typeArguments": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2427 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Fields to exclude from each item." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 5501, + "kind": 32768, + "kindString": "Parameter", + "name": "omit", + "type": { + "name": "list[str] | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "list", + "typeArguments": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2427 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Unwinds items by a specified array field, turning each element into a separate item." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 5502, + "kind": 32768, + "kindString": "Parameter", + "name": "unwind", + "type": { + "name": "list[str] | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "list", + "typeArguments": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2427 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Excludes empty items from the results if True." + } + ] + }, + "defaultValue": "False", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 5503, + "kind": 32768, + "kindString": "Parameter", + "name": "skip_empty", + "type": { + "name": "bool", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2427 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Excludes fields starting with '#' if True." + } + ] + }, + "defaultValue": "False", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 5504, + "kind": 32768, + "kindString": "Parameter", + "name": "skip_hidden", + "type": { + "name": "bool", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2427 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Fields to be flattened in returned items." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 5505, + "kind": 32768, + "kindString": "Parameter", + "name": "flatten", + "type": { + "name": "list[str] | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "list", + "typeArguments": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2427 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Specifies the dataset view to be used.\n" + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 5506, + "kind": 32768, + "kindString": "Parameter", + "name": "view", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2427 + } + ], + "type": { + "name": "DatasetItemsListPage", + "type": "reference", + "target": "1333", + "ref": { + "id": 4401, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 110 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/DatasetItemsListPage" + } + }, + "parentId": 2426 + } + ], + "permalink": "/python/api/class/Dataset#get_data", + "parentId": 2402 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Iterate over items in the dataset according to specified filters and sorting.\n\nThis method allows for asynchronously iterating through dataset items while applying various filters such as\nskipping empty items, hiding specific fields, and sorting. It supports pagination via `offset` and `limit`\nparameters, and can modify the appearance of dataset items using `fields`, `omit`, `unwind`, `skip_empty`, and\n`skip_hidden` parameters.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 5507, + "module": "storages._dataset", + "name": "iterate_items", + "parsedDocstring": { + "text": "Iterate over items in the dataset according to specified filters and sorting.\n\nThis method allows for asynchronously iterating through dataset items while applying various filters such as\nskipping empty items, hiding specific fields, and sorting. It supports pagination via `offset` and `limit`\nparameters, and can modify the appearance of dataset items using `fields`, `omit`, `unwind`, `skip_empty`, and\n`skip_hidden` parameters.\n", + "args": { + "offset": "Skips the specified number of items at the start.", + "limit": "The maximum number of items to retrieve. Unlimited if None.", + "clean": "Return only non-empty items and excludes hidden fields. Shortcut for skip_hidden and skip_empty.", + "desc": "Set to True to sort results in descending order.", + "fields": "Fields to include in each item. Sorts fields as specified if provided.", + "omit": "Fields to exclude from each item.", + "unwind": "Unwinds items by a specified array field, turning each element into a separate item.", + "skip_empty": "Excludes empty items from the results if True.", + "skip_hidden": "Excludes fields starting with '#' if True.\n" + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_dataset.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 201, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storages/_dataset.py#L201" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Iterate over items in the dataset according to specified filters and sorting.\n\nThis method allows for asynchronously iterating through dataset items while applying various filters such as\nskipping empty items, hiding specific fields, and sorting. It supports pagination via `offset` and `limit`\nparameters, and can modify the appearance of dataset items using `fields`, `omit`, `unwind`, `skip_empty`, and\n`skip_hidden` parameters.\n" + } + ] + }, + "flags": {}, + "id": 5508, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "iterate_items", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Skips the specified number of items at the start." + } + ] + }, + "defaultValue": "0", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 5509, + "kind": 32768, + "kindString": "Parameter", + "name": "offset", + "type": { + "name": "int", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2440 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The maximum number of items to retrieve. Unlimited if None." + } + ] + }, + "defaultValue": "999_999_999_999", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 5510, + "kind": 32768, + "kindString": "Parameter", + "name": "limit", + "type": { + "name": "int | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "int", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2440 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Return only non-empty items and excludes hidden fields. Shortcut for skip_hidden and skip_empty." + } + ] + }, + "defaultValue": "False", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 5511, + "kind": 32768, + "kindString": "Parameter", + "name": "clean", + "type": { + "name": "bool", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2440 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Set to True to sort results in descending order." + } + ] + }, + "defaultValue": "False", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 5512, + "kind": 32768, + "kindString": "Parameter", + "name": "desc", + "type": { + "name": "bool", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2440 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Fields to include in each item. Sorts fields as specified if provided." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 5513, + "kind": 32768, + "kindString": "Parameter", + "name": "fields", + "type": { + "name": "list[str] | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "list", + "typeArguments": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2440 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Fields to exclude from each item." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 5514, + "kind": 32768, + "kindString": "Parameter", + "name": "omit", + "type": { + "name": "list[str] | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "list", + "typeArguments": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2440 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Unwinds items by a specified array field, turning each element into a separate item." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 5515, + "kind": 32768, + "kindString": "Parameter", + "name": "unwind", + "type": { + "name": "list[str] | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "list", + "typeArguments": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2440 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Excludes empty items from the results if True." + } + ] + }, + "defaultValue": "False", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 5516, + "kind": 32768, + "kindString": "Parameter", + "name": "skip_empty", + "type": { + "name": "bool", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2440 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Excludes fields starting with '#' if True.\n" + } + ] + }, + "defaultValue": "False", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 5517, + "kind": 32768, + "kindString": "Parameter", + "name": "skip_hidden", + "type": { + "name": "bool", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2440 + } + ], + "type": { + "name": "AsyncIterator", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "dict", + "typeArguments": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "reference", + "name": "Any", + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + }, + "parentId": 2439 + } + ], + "permalink": "/python/api/class/Dataset#iterate_items", + "parentId": 2402 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Retrieve a list of all items from the dataset according to specified filters and sorting.\n\nThis method collects all dataset items into a list while applying various filters such as\nskipping empty items, hiding specific fields, and sorting. It supports pagination via `offset` and `limit`\nparameters, and can modify the appearance of dataset items using `fields`, `omit`, `unwind`, `skip_empty`, and\n`skip_hidden` parameters.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 5518, + "module": "storages._dataset", + "name": "list_items", + "parsedDocstring": { + "text": "Retrieve a list of all items from the dataset according to specified filters and sorting.\n\nThis method collects all dataset items into a list while applying various filters such as\nskipping empty items, hiding specific fields, and sorting. It supports pagination via `offset` and `limit`\nparameters, and can modify the appearance of dataset items using `fields`, `omit`, `unwind`, `skip_empty`, and\n`skip_hidden` parameters.\n", + "args": { + "offset": "Skips the specified number of items at the start.", + "limit": "The maximum number of items to retrieve. Unlimited if None.", + "clean": "Return only non-empty items and excludes hidden fields. Shortcut for skip_hidden and skip_empty.", + "desc": "Set to True to sort results in descending order.", + "fields": "Fields to include in each item. Sorts fields as specified if provided.", + "omit": "Fields to exclude from each item.", + "unwind": "Unwinds items by a specified array field, turning each element into a separate item.", + "skip_empty": "Excludes empty items from the results if True.", + "skip_hidden": "Excludes fields starting with '#' if True.\n" + }, + "returns": "A list of dictionary objects, each representing a dataset item after applying\nthe specified filters and transformations." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_dataset.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 249, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storages/_dataset.py#L249" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "A list of dictionary objects, each representing a dataset item after applying\nthe specified filters and transformations." + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Retrieve a list of all items from the dataset according to specified filters and sorting.\n\nThis method collects all dataset items into a list while applying various filters such as\nskipping empty items, hiding specific fields, and sorting. It supports pagination via `offset` and `limit`\nparameters, and can modify the appearance of dataset items using `fields`, `omit`, `unwind`, `skip_empty`, and\n`skip_hidden` parameters.\n" + } + ] + }, + "flags": {}, + "id": 5519, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "list_items", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Skips the specified number of items at the start." + } + ] + }, + "defaultValue": "0", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 5520, + "kind": 32768, + "kindString": "Parameter", + "name": "offset", + "type": { + "name": "int", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2451 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The maximum number of items to retrieve. Unlimited if None." + } + ] + }, + "defaultValue": "999_999_999_999", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 5521, + "kind": 32768, + "kindString": "Parameter", + "name": "limit", + "type": { + "name": "int | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "int", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2451 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Return only non-empty items and excludes hidden fields. Shortcut for skip_hidden and skip_empty." + } + ] + }, + "defaultValue": "False", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 5522, + "kind": 32768, + "kindString": "Parameter", + "name": "clean", + "type": { + "name": "bool", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2451 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Set to True to sort results in descending order." + } + ] + }, + "defaultValue": "False", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 5523, + "kind": 32768, + "kindString": "Parameter", + "name": "desc", + "type": { + "name": "bool", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2451 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Fields to include in each item. Sorts fields as specified if provided." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 5524, + "kind": 32768, + "kindString": "Parameter", + "name": "fields", + "type": { + "name": "list[str] | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "list", + "typeArguments": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2451 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Fields to exclude from each item." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 5525, + "kind": 32768, + "kindString": "Parameter", + "name": "omit", + "type": { + "name": "list[str] | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "list", + "typeArguments": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2451 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Unwinds items by a specified array field, turning each element into a separate item." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 5526, + "kind": 32768, + "kindString": "Parameter", + "name": "unwind", + "type": { + "name": "list[str] | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "list", + "typeArguments": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2451 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Excludes empty items from the results if True." + } + ] + }, + "defaultValue": "False", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 5527, + "kind": 32768, + "kindString": "Parameter", + "name": "skip_empty", + "type": { + "name": "bool", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2451 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Excludes fields starting with '#' if True.\n" + } + ] + }, + "defaultValue": "False", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 5528, + "kind": 32768, + "kindString": "Parameter", + "name": "skip_hidden", + "type": { + "name": "bool", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2451 + } + ], + "type": { + "name": "list", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "dict", + "typeArguments": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "reference", + "name": "Any", + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + }, + "parentId": 2450 + } + ], + "permalink": "/python/api/class/Dataset#list_items", + "parentId": 2402 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Export the entire dataset into a specified file stored under a key in a key-value store.\n\nThis method consolidates all entries from a specified dataset into one file, which is then saved under a\ngiven key in a key-value store. The format of the exported file is determined by the `content_type` parameter.\nEither the dataset's ID or name should be specified, and similarly, either the target key-value store's ID or\nname should be used.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 5529, + "module": "storages._dataset", + "name": "export_to", + "parsedDocstring": { + "text": "Export the entire dataset into a specified file stored under a key in a key-value store.\n\nThis method consolidates all entries from a specified dataset into one file, which is then saved under a\ngiven key in a key-value store. The format of the exported file is determined by the `content_type` parameter.\nEither the dataset's ID or name should be specified, and similarly, either the target key-value store's ID or\nname should be used.\n", + "args": { + "key": "The key under which to save the data in the key-value store.", + "content_type": "The format in which to export the data.", + "to_kvs_id": "ID of the key-value store to save the exported file.\nSpecify only one of ID or name.", + "to_kvs_name": "Name of the key-value store to save the exported file.\nSpecify only one of ID or name.", + "to_kvs_storage_client": "Storage client to use for the key-value store.", + "to_kvs_configuration": "Configuration for the key-value store.", + "kwargs": "Additional parameters for the export operation, specific to the chosen content type." + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_dataset.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 323, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storages/_dataset.py#L323" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Export the entire dataset into a specified file stored under a key in a key-value store.\n\nThis method consolidates all entries from a specified dataset into one file, which is then saved under a\ngiven key in a key-value store. The format of the exported file is determined by the `content_type` parameter.\nEither the dataset's ID or name should be specified, and similarly, either the target key-value store's ID or\nname should be used.\n" + } + ] + }, + "flags": {}, + "id": 5530, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "export_to", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The key under which to save the data in the key-value store." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 5531, + "kind": 32768, + "kindString": "Parameter", + "name": "key", + "type": { + "name": "str", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2462 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The format in which to export the data." + } + ] + }, + "defaultValue": "'json'", + "flags": { + "isOptional": true, + "keyword-only": false + }, + "id": 5532, + "kind": 32768, + "kindString": "Parameter", + "name": "content_type", + "type": { + "name": "Literal", + "type": "reference", + "typeArguments": [ + { + "type": "literal", + "value": "json" + }, + { + "type": "literal", + "value": "csv" + } + ], + "target": 0, + "ref": {} + }, + "parentId": 2462 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "ID of the key-value store to save the exported file.\nSpecify only one of ID or name." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": false + }, + "id": 5533, + "kind": 32768, + "kindString": "Parameter", + "name": "to_kvs_id", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2462 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Name of the key-value store to save the exported file.\nSpecify only one of ID or name." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": false + }, + "id": 5534, + "kind": 32768, + "kindString": "Parameter", + "name": "to_kvs_name", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2462 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Storage client to use for the key-value store." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": false + }, + "id": 5535, + "kind": 32768, + "kindString": "Parameter", + "name": "to_kvs_storage_client", + "type": { + "name": "StorageClient | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "StorageClient", + "target": "1282", + "ref": { + "id": 4350, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 19 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/StorageClient" + } + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2462 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Configuration for the key-value store." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": false + }, + "id": 5536, + "kind": 32768, + "kindString": "Parameter", + "name": "to_kvs_configuration", + "type": { + "name": "Configuration | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "Configuration", + "target": "2", + "ref": { + "id": 3070, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/configuration.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 20 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/Configuration" + } + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2462 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Additional parameters for the export operation, specific to the chosen content type." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 5537, + "kind": 32768, + "kindString": "Parameter", + "name": "kwargs", + "type": { + "name": "Any", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2462 + } + ], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "parentId": 2461 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Export the entire dataset into a specified file stored under a key in a key-value store.\n\nThis method consolidates all entries from a specified dataset into one file, which is then saved under a\ngiven key in a key-value store. The format of the exported file is determined by the `content_type` parameter.\nEither the dataset's ID or name should be specified, and similarly, either the target key-value store's ID or\nname should be used.\n" + } + ] + }, + "flags": {}, + "id": 5538, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "export_to", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The key under which to save the data in the key-value store." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 5539, + "kind": 32768, + "kindString": "Parameter", + "name": "key", + "type": { + "name": "str", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2470 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The format in which to export the data." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 5540, + "kind": 32768, + "kindString": "Parameter", + "name": "content_type", + "type": { + "name": "Literal", + "type": "reference", + "typeArguments": [ + { + "type": "literal", + "value": "json" + } + ], + "target": 0, + "ref": {} + }, + "parentId": 2470 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "ID of the key-value store to save the exported file.\nSpecify only one of ID or name." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": false + }, + "id": 5541, + "kind": 32768, + "kindString": "Parameter", + "name": "to_kvs_id", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2470 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Name of the key-value store to save the exported file.\nSpecify only one of ID or name." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": false + }, + "id": 5542, + "kind": 32768, + "kindString": "Parameter", + "name": "to_kvs_name", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2470 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Storage client to use for the key-value store." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": false + }, + "id": 5543, + "kind": 32768, + "kindString": "Parameter", + "name": "to_kvs_storage_client", + "type": { + "name": "StorageClient | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "StorageClient", + "target": "1282", + "ref": { + "id": 4350, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 19 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/StorageClient" + } + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2470 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Configuration for the key-value store." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": false + }, + "id": 5544, + "kind": 32768, + "kindString": "Parameter", + "name": "to_kvs_configuration", + "type": { + "name": "Configuration | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "Configuration", + "target": "2", + "ref": { + "id": 3070, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/configuration.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 20 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/Configuration" + } + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2470 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "If True (default: False), dict keys that are not of a basic type (str, int, float, bool, None) will be skipped\ninstead of raising a `TypeError`." + } + ] + }, + "flags": { + "keyword-only": true, + "optional": true + }, + "groups": [], + "id": 3397, + "module": "_types", + "name": "skipkeys", + "parsedDocstring": { + "text": "If True (default: False), dict keys that are not of a basic type (str, int, float, bool, None) will be skipped\ninstead of raising a `TypeError`." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_types.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 759, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/_types.py#L759" + } + ], + "type": { + "name": "NotRequired", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "bool", + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + }, + "parentId": 2470 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Determines if non-ASCII characters should be escaped in the output JSON string." + } + ] + }, + "flags": { + "keyword-only": true, + "optional": true + }, + "groups": [], + "id": 3398, + "module": "_types", + "name": "ensure_ascii", + "parsedDocstring": { + "text": "Determines if non-ASCII characters should be escaped in the output JSON string." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_types.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 763, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/_types.py#L763" + } + ], + "type": { + "name": "NotRequired", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "bool", + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + }, + "parentId": 2470 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "If False (default: True), skips the circular reference check for container types. A circular reference will\nresult in a `RecursionError` or worse if unchecked." + } + ] + }, + "flags": { + "keyword-only": true, + "optional": true + }, + "groups": [], + "id": 3399, + "module": "_types", + "name": "check_circular", + "parsedDocstring": { + "text": "If False (default: True), skips the circular reference check for container types. A circular reference will\nresult in a `RecursionError` or worse if unchecked." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_types.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 766, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/_types.py#L766" + } + ], + "type": { + "name": "NotRequired", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "bool", + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + }, + "parentId": 2470 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "If False (default: True), raises a ValueError for out-of-range float values (nan, inf, -inf) to strictly comply\nwith the JSON specification. If True, uses their JavaScript equivalents (NaN, Infinity, -Infinity)." + } + ] + }, + "flags": { + "keyword-only": true, + "optional": true + }, + "groups": [], + "id": 3400, + "module": "_types", + "name": "allow_nan", + "parsedDocstring": { + "text": "If False (default: True), raises a ValueError for out-of-range float values (nan, inf, -inf) to strictly comply\nwith the JSON specification. If True, uses their JavaScript equivalents (NaN, Infinity, -Infinity)." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_types.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 770, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/_types.py#L770" + } + ], + "type": { + "name": "NotRequired", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "bool", + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + }, + "parentId": 2470 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Allows specifying a custom JSON encoder." + } + ] + }, + "flags": { + "keyword-only": true, + "optional": true + }, + "groups": [], + "id": 3401, + "module": "_types", + "name": "cls", + "parsedDocstring": { + "text": "Allows specifying a custom JSON encoder." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_types.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 774, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/_types.py#L774" + } + ], + "type": { + "name": "NotRequired", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "type", + "typeArguments": [ + { + "type": "reference", + "name": "json.JSONEncoder", + "target": 0, + "ref": {} + } + ], + "target": "3422", + "ref": { + "id": 6490, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_utils/sitemap.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 62 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api/class/SitemapSource#type" + } + } + ], + "target": 0, + "ref": {} + }, + "parentId": 2470 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Specifies the number of spaces for indentation in the pretty-printed JSON output." + } + ] + }, + "flags": { + "keyword-only": true, + "optional": true + }, + "groups": [], + "id": 3402, + "module": "_types", + "name": "indent", + "parsedDocstring": { + "text": "Specifies the number of spaces for indentation in the pretty-printed JSON output." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_types.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 777, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/_types.py#L777" + } + ], + "type": { + "name": "NotRequired", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "int", + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + }, + "parentId": 2470 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "A tuple of (item_separator, key_separator). The default is (', ', ': ') if indent is None and (',', ': ')\notherwise." + } + ] + }, + "flags": { + "keyword-only": true, + "optional": true + }, + "groups": [], + "id": 3403, + "module": "_types", + "name": "separators", + "parsedDocstring": { + "text": "A tuple of (item_separator, key_separator). The default is (', ', ': ') if indent is None and (',', ': ')\notherwise." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_types.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 780, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/_types.py#L780" + } + ], + "type": { + "name": "NotRequired", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "tuple", + "typeArguments": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + }, + "parentId": 2470 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "A function called for objects that can't be serialized otherwise. It should return a JSON-encodable version\nof the object or raise a `TypeError`." + } + ] + }, + "flags": { + "keyword-only": true, + "optional": true + }, + "groups": [], + "id": 3404, + "module": "_types", + "name": "default", + "parsedDocstring": { + "text": "A function called for objects that can't be serialized otherwise. It should return a JSON-encodable version\nof the object or raise a `TypeError`." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_types.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 784, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/_types.py#L784" + } + ], + "type": { + "name": "NotRequired", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "Callable", + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + }, + "parentId": 2470 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Specifies whether the output JSON object should have keys sorted alphabetically." + } + ] + }, + "flags": { + "keyword-only": true, + "optional": true + }, + "groups": [], + "id": 3405, + "module": "_types", + "name": "sort_keys", + "parsedDocstring": { + "text": "Specifies whether the output JSON object should have keys sorted alphabetically." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_types.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 788, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/_types.py#L788" + } + ], + "type": { + "name": "NotRequired", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "bool", + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + }, + "parentId": 2470 + } + ], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "parentId": 2461 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Export the entire dataset into a specified file stored under a key in a key-value store.\n\nThis method consolidates all entries from a specified dataset into one file, which is then saved under a\ngiven key in a key-value store. The format of the exported file is determined by the `content_type` parameter.\nEither the dataset's ID or name should be specified, and similarly, either the target key-value store's ID or\nname should be used.\n" + } + ] + }, + "flags": {}, + "id": 5546, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "export_to", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The key under which to save the data in the key-value store." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 5547, + "kind": 32768, + "kindString": "Parameter", + "name": "key", + "type": { + "name": "str", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2478 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The format in which to export the data." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 5548, + "kind": 32768, + "kindString": "Parameter", + "name": "content_type", + "type": { + "name": "Literal", + "type": "reference", + "typeArguments": [ + { + "type": "literal", + "value": "csv" + } + ], + "target": 0, + "ref": {} + }, + "parentId": 2478 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "ID of the key-value store to save the exported file.\nSpecify only one of ID or name." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": false + }, + "id": 5549, + "kind": 32768, + "kindString": "Parameter", + "name": "to_kvs_id", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2478 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Name of the key-value store to save the exported file.\nSpecify only one of ID or name." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": false + }, + "id": 5550, + "kind": 32768, + "kindString": "Parameter", + "name": "to_kvs_name", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2478 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Storage client to use for the key-value store." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": false + }, + "id": 5551, + "kind": 32768, + "kindString": "Parameter", + "name": "to_kvs_storage_client", + "type": { + "name": "StorageClient | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "StorageClient", + "target": "1282", + "ref": { + "id": 4350, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 19 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/StorageClient" + } + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2478 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Configuration for the key-value store." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": false + }, + "id": 5552, + "kind": 32768, + "kindString": "Parameter", + "name": "to_kvs_configuration", + "type": { + "name": "Configuration | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "Configuration", + "target": "2", + "ref": { + "id": 3070, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/configuration.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 20 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/Configuration" + } + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2478 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Specifies a dialect to be used in CSV parsing and writing." + } + ] + }, + "flags": { + "keyword-only": true, + "optional": true + }, + "groups": [], + "id": 3407, + "module": "_types", + "name": "dialect", + "parsedDocstring": { + "text": "Specifies a dialect to be used in CSV parsing and writing." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_types.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 795, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/_types.py#L795" + } + ], + "type": { + "name": "NotRequired", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + }, + "parentId": 2478 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "A one-character string used to separate fields. Defaults to ','." + } + ] + }, + "flags": { + "keyword-only": true, + "optional": true + }, + "groups": [], + "id": 3408, + "module": "_types", + "name": "delimiter", + "parsedDocstring": { + "text": "A one-character string used to separate fields. Defaults to ','." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_types.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 798, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/_types.py#L798" + } + ], + "type": { + "name": "NotRequired", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + }, + "parentId": 2478 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Controls how instances of `quotechar` inside a field should be quoted. When True, the character is doubled;\nwhen False, the `escapechar` is used as a prefix. Defaults to True." + } + ] + }, + "flags": { + "keyword-only": true, + "optional": true + }, + "groups": [], + "id": 3409, + "module": "_types", + "name": "doublequote", + "parsedDocstring": { + "text": "Controls how instances of `quotechar` inside a field should be quoted. When True, the character is doubled;\nwhen False, the `escapechar` is used as a prefix. Defaults to True." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_types.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 801, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/_types.py#L801" + } + ], + "type": { + "name": "NotRequired", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "bool", + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + }, + "parentId": 2478 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "A one-character string used to escape the delimiter if `quoting` is set to `QUOTE_NONE` and the `quotechar`\nif `doublequote` is False. Defaults to None, disabling escaping." + } + ] + }, + "flags": { + "keyword-only": true, + "optional": true + }, + "groups": [], + "id": 3410, + "module": "_types", + "name": "escapechar", + "parsedDocstring": { + "text": "A one-character string used to escape the delimiter if `quoting` is set to `QUOTE_NONE` and the `quotechar`\nif `doublequote` is False. Defaults to None, disabling escaping." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_types.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 805, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/_types.py#L805" + } + ], + "type": { + "name": "NotRequired", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + }, + "parentId": 2478 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "The string used to terminate lines produced by the writer. Defaults to '\\r\\n'." + } + ] + }, + "flags": { + "keyword-only": true, + "optional": true + }, + "groups": [], + "id": 3411, + "module": "_types", + "name": "lineterminator", + "parsedDocstring": { + "text": "The string used to terminate lines produced by the writer. Defaults to '\\r\\n'." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_types.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 809, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/_types.py#L809" + } + ], + "type": { + "name": "NotRequired", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + }, + "parentId": 2478 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "A one-character string used to quote fields containing special characters, like the delimiter or quotechar,\nor fields containing new-line characters. Defaults to '\"'." + } + ] + }, + "flags": { + "keyword-only": true, + "optional": true + }, + "groups": [], + "id": 3412, + "module": "_types", + "name": "quotechar", + "parsedDocstring": { + "text": "A one-character string used to quote fields containing special characters, like the delimiter or quotechar,\nor fields containing new-line characters. Defaults to '\"'." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_types.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 812, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/_types.py#L812" + } + ], + "type": { + "name": "NotRequired", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + }, + "parentId": 2478 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Controls when quotes should be generated by the writer and recognized by the reader. Can take any of\nthe `QUOTE_*` constants, with a default of `QUOTE_MINIMAL`." + } + ] + }, + "flags": { + "keyword-only": true, + "optional": true + }, + "groups": [], + "id": 3413, + "module": "_types", + "name": "quoting", + "parsedDocstring": { + "text": "Controls when quotes should be generated by the writer and recognized by the reader. Can take any of\nthe `QUOTE_*` constants, with a default of `QUOTE_MINIMAL`." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_types.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 816, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/_types.py#L816" + } + ], + "type": { + "name": "NotRequired", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "Literal", + "typeArguments": [ + { + "type": "literal", + "value": 0 + }, + { + "type": "literal", + "value": 1 + }, + { + "type": "literal", + "value": 2 + }, + { + "type": "literal", + "value": 3 + } + ], + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + }, + "parentId": 2478 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "When True, spaces immediately following the delimiter are ignored. Defaults to False." + } + ] + }, + "flags": { + "keyword-only": true, + "optional": true + }, + "groups": [], + "id": 3414, + "module": "_types", + "name": "skipinitialspace", + "parsedDocstring": { + "text": "When True, spaces immediately following the delimiter are ignored. Defaults to False." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_types.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 820, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/_types.py#L820" + } + ], + "type": { + "name": "NotRequired", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "bool", + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + }, + "parentId": 2478 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "When True, raises an exception on bad CSV input. Defaults to False." + } + ] + }, + "flags": { + "keyword-only": true, + "optional": true + }, + "groups": [], + "id": 3415, + "module": "_types", + "name": "strict", + "parsedDocstring": { + "text": "When True, raises an exception on bad CSV input. Defaults to False." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_types.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 823, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/_types.py#L823" + } + ], + "type": { + "name": "NotRequired", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "bool", + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + }, + "parentId": 2478 + } + ], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "parentId": 2461 + } + ], + "permalink": "/python/api/class/Dataset#export_to", + "parentId": 2402 + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Dataset is a storage for managing structured tabular data.\n\nThe dataset class provides a high-level interface for storing and retrieving structured data\nwith consistent schema, similar to database tables or spreadsheets. It abstracts the underlying\nstorage implementation details, offering a consistent API regardless of where the data is\nphysically stored.\n\nDataset operates in an append-only mode, allowing new records to be added but not modified\nor deleted after creation. This makes it particularly suitable for storing crawling results\nand other data that should be immutable once collected.\n\nThe class provides methods for adding data, retrieving data with various filtering options,\nand exporting data to different formats. You can create a dataset using the `open` class method,\nspecifying either a name or ID. The underlying storage implementation is determined by\nthe configured storage client.\n\n### Usage\n\n```python\nfrom crawlee.storages import Dataset\n\n# Open a dataset\ndataset = await Dataset.open(name='my-dataset')\n\n# Add data\nawait dataset.push_data({'title': 'Example Product', 'price': 99.99})\n\n# Retrieve filtered data\nresults = await dataset.get_data(limit=10, desc=True)\n\n# Export data\nawait dataset.export_to('results.json', content_type='json')\n```" + } + ] + }, + "decorations": [ + { + "args": "('Storages')", + "name": "docs_group" + } + ], + "flags": {}, + "groups": [ + { + "children": [ + 5471, + 5487, + 5529, + 5494, + 5478, + 5507, + 5518, + 5480, + 5489, + 5491 + ], + "title": "Methods" + }, + { + "children": [ + 5476, + 5477 + ], + "title": "Properties" + } + ], + "id": 5470, + "module": "storages._dataset", + "name": "Dataset", + "parsedDocstring": { + "text": "Dataset is a storage for managing structured tabular data.\n\nThe dataset class provides a high-level interface for storing and retrieving structured data\nwith consistent schema, similar to database tables or spreadsheets. It abstracts the underlying\nstorage implementation details, offering a consistent API regardless of where the data is\nphysically stored.\n\nDataset operates in an append-only mode, allowing new records to be added but not modified\nor deleted after creation. This makes it particularly suitable for storing crawling results\nand other data that should be immutable once collected.\n\nThe class provides methods for adding data, retrieving data with various filtering options,\nand exporting data to different formats. You can create a dataset using the `open` class method,\nspecifying either a name or ID. The underlying storage implementation is determined by\nthe configured storage client.\n\n### Usage\n\n```python\nfrom crawlee.storages import Dataset\n\n# Open a dataset\ndataset = await Dataset.open(name='my-dataset')\n\n# Add data\nawait dataset.push_data({'title': 'Example Product', 'price': 99.99})\n\n# Retrieve filtered data\nresults = await dataset.get_data(limit=10, desc=True)\n\n# Export data\nawait dataset.export_to('results.json', content_type='json')\n```" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_dataset.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 33, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storages/_dataset.py#L33" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "extendedTypes": [ + { + "name": "Storage", + "target": "2330", + "type": "reference", + "ref": { + "id": 5398, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_base.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 15 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/Storage" + } + } + ], + "permalink": "/python/api/class/Dataset", + "categories": [] + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Initialize a new instance.\n\nPreferably use the `KeyValueStore.open` constructor to create a new instance.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 8760, + "module": "storages._key_value_store", + "name": "__init__", + "parsedDocstring": { + "text": "Initialize a new instance.\n\nPreferably use the `KeyValueStore.open` constructor to create a new instance.\n", + "args": { + "client": "An instance of a storage client.", + "id": "The unique identifier of the storage.", + "name": "The name of the storage, if available." + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_key_value_store.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 78, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storages/_key_value_store.py#L78" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Initialize a new instance.\n\nPreferably use the `KeyValueStore.open` constructor to create a new instance.\n" + } + ] + }, + "flags": {}, + "id": 8761, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "__init__", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "An instance of a storage client." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 8762, + "kind": 32768, + "kindString": "Parameter", + "name": "client", + "type": { + "name": "KeyValueStoreClient", + "type": "reference", + "target": "1254", + "ref": { + "id": 7744, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_key_value_store_client.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 14 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/KeyValueStoreClient" + } + }, + "parentId": 2271 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The unique identifier of the storage." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 8763, + "kind": 32768, + "kindString": "Parameter", + "name": "id", + "type": { + "name": "str", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2271 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The name of the storage, if available." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 8764, + "kind": 32768, + "kindString": "Parameter", + "name": "name", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2271 + } + ], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "parentId": 2270 + } + ], + "permalink": "/python/api/class/KeyValueStore#__init__", + "parentId": 2269 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Get the storage ID." + } + ] + }, + "decorations": [ + { + "name": "property" + }, + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 8765, + "module": "storages._base", + "name": "id", + "parsedDocstring": { + "text": "Get the storage ID." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_key_value_store.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 99, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storages/_key_value_store.py#L99" + } + ], + "type": { + "name": "str", + "type": "reference", + "target": 0, + "ref": {} + }, + "overwrites": { + "name": "Storage.id", + "target": 2331, + "type": "reference", + "ref": { + "id": 8821, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_base.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 20 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api/class/Storage#id" + } + }, + "permalink": "/python/api/class/KeyValueStore#id", + "parentId": 2269 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Get the storage name." + } + ] + }, + "decorations": [ + { + "name": "property" + }, + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 8766, + "module": "storages._base", + "name": "name", + "parsedDocstring": { + "text": "Get the storage name." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_key_value_store.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 104, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storages/_key_value_store.py#L104" + } + ], + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "overwrites": { + "name": "Storage.name", + "target": 2332, + "type": "reference", + "ref": { + "id": 8822, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_base.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 25 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api/class/Storage#name" + } + }, + "permalink": "/python/api/class/KeyValueStore#name", + "parentId": 2269 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Get the storage metadata." + } + ] + }, + "decorations": [ + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 8767, + "module": "storages._base", + "name": "get_metadata", + "parsedDocstring": { + "text": "Get the storage metadata." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_key_value_store.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 108, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storages/_key_value_store.py#L108" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Get the storage metadata." + } + ] + }, + "flags": {}, + "id": 8824, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "get_metadata", + "parameters": [], + "type": { + "name": "DatasetMetadata | KeyValueStoreMetadata | RequestQueueMetadata", + "type": "union", + "types": [ + { + "type": "union", + "types": [ + { + "type": "reference", + "name": "DatasetMetadata", + "target": "1314", + "ref": { + "id": 7804, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 42 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/DatasetMetadata" + } + }, + { + "type": "reference", + "name": "KeyValueStoreMetadata", + "target": "1317", + "ref": { + "id": 7807, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 52 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/KeyValueStoreMetadata" + } + } + ] + }, + { + "type": "reference", + "name": "RequestQueueMetadata", + "target": "1319", + "ref": { + "id": 7809, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 59 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/RequestQueueMetadata" + } + } + ] + }, + "overwrites": { + "name": "Storage.get_metadata", + "target": 2333, + "type": "reference", + "ref": { + "id": 8823, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_base.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 29 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/Storage#get_metadata" + } + }, + "parentId": 2277 + } + ], + "overwrites": { + "name": "Storage.get_metadata", + "target": 2333, + "type": "reference", + "ref": { + "id": 8823, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_base.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 29 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/Storage#get_metadata" + } + }, + "permalink": "/python/api/class/KeyValueStore#get_metadata", + "parentId": 2269 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Open a storage, either restore existing or create a new one.\n" + } + ] + }, + "decorations": [ + { + "name": "classmethod" + }, + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 8769, + "module": "storages._base", + "name": "open", + "parsedDocstring": { + "text": "Open a storage, either restore existing or create a new one.\n", + "args": { + "id": "The storage ID.", + "name": "The storage name (global scope, persists across runs). Name can only contain letters \"a\" through \"z\",\nthe digits \"0\" through \"9\", and the hyphen (\"-\") but only in the middle of the string\n(e.g. \"my-value-1\").", + "alias": "The storage alias (run scope, creates unnamed storage).", + "configuration": "Configuration object used during the storage creation or restoration process.", + "storage_client": "Underlying storage client to use. If not provided, the default global storage client\nfrom the service locator will be used." + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_key_value_store.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 113, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storages/_key_value_store.py#L113" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Open a storage, either restore existing or create a new one.\n" + } + ] + }, + "flags": {}, + "id": 8826, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "open", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The storage ID." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 8827, + "kind": 32768, + "kindString": "Parameter", + "name": "id", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2336 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The storage name (global scope, persists across runs). Name can only contain letters \"a\" through \"z\",\nthe digits \"0\" through \"9\", and the hyphen (\"-\") but only in the middle of the string\n(e.g. \"my-value-1\")." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 8828, + "kind": 32768, + "kindString": "Parameter", + "name": "name", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2336 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The storage alias (run scope, creates unnamed storage)." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 8829, + "kind": 32768, + "kindString": "Parameter", + "name": "alias", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2336 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Configuration object used during the storage creation or restoration process." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 8830, + "kind": 32768, + "kindString": "Parameter", + "name": "configuration", + "type": { + "name": "Configuration | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "Configuration", + "target": "2", + "ref": { + "id": 6492, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/configuration.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 20 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/Configuration" + } + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2336 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Underlying storage client to use. If not provided, the default global storage client\nfrom the service locator will be used." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 8831, + "kind": 32768, + "kindString": "Parameter", + "name": "storage_client", + "type": { + "name": "StorageClient | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "StorageClient", + "target": "1282", + "ref": { + "id": 7772, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 19 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/StorageClient" + } + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2336 + } + ], + "type": { + "name": "Storage", + "type": "reference", + "target": "2330", + "ref": { + "id": 8820, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_base.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 15 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/Storage" + } + }, + "overwrites": { + "name": "Storage.open", + "target": 2335, + "type": "reference", + "ref": { + "id": 8825, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_base.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 34 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/Storage#open" + } + }, + "parentId": 2279 + } + ], + "overwrites": { + "name": "Storage.open", + "target": 2335, + "type": "reference", + "ref": { + "id": 8825, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_base.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 34 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/Storage#open" + } + }, + "permalink": "/python/api/class/KeyValueStore#open", + "parentId": 2269 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Drop the storage, removing it from the underlying storage client and clearing the cache." + } + ] + }, + "decorations": [ + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 8776, + "module": "storages._base", + "name": "drop", + "parsedDocstring": { + "text": "Drop the storage, removing it from the underlying storage client and clearing the cache." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_key_value_store.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 140, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storages/_key_value_store.py#L140" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Drop the storage, removing it from the underlying storage client and clearing the cache." + } + ] + }, + "flags": {}, + "id": 8833, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "drop", + "parameters": [], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "overwrites": { + "name": "Storage.drop", + "target": 2342, + "type": "reference", + "ref": { + "id": 8832, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_base.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 57 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/Storage#drop" + } + }, + "parentId": 2286 + } + ], + "overwrites": { + "name": "Storage.drop", + "target": 2342, + "type": "reference", + "ref": { + "id": 8832, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_base.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 57 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/Storage#drop" + } + }, + "permalink": "/python/api/class/KeyValueStore#drop", + "parentId": 2269 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Purge the storage, removing all items from the underlying storage client.\n\nThis method does not remove the storage itself, e.g. don't remove the metadata,\nbut clears all items within it." + } + ] + }, + "decorations": [ + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 8778, + "module": "storages._base", + "name": "purge", + "parsedDocstring": { + "text": "Purge the storage, removing all items from the underlying storage client.\n\nThis method does not remove the storage itself, e.g. don't remove the metadata,\nbut clears all items within it." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_key_value_store.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 148, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storages/_key_value_store.py#L148" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Purge the storage, removing all items from the underlying storage client.\n\nThis method does not remove the storage itself, e.g. don't remove the metadata,\nbut clears all items within it." + } + ] + }, + "flags": {}, + "id": 8835, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "purge", + "parameters": [], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "overwrites": { + "name": "Storage.purge", + "target": 2344, + "type": "reference", + "ref": { + "id": 8834, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_base.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 61 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/Storage#purge" + } + }, + "parentId": 2288 + } + ], + "overwrites": { + "name": "Storage.purge", + "target": 2344, + "type": "reference", + "ref": { + "id": 8834, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_base.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 61 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/Storage#purge" + } + }, + "permalink": "/python/api/class/KeyValueStore#purge", + "parentId": 2269 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Get a value from the KVS.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 8780, + "module": "storages._key_value_store", + "name": "get_value", + "parsedDocstring": { + "text": "Get a value from the KVS.\n", + "args": { + "key": "Key of the record to retrieve.", + "default_value": "Default value returned in case the record does not exist.\n" + }, + "returns": "The value associated with the given key. `default_value` is used in case the record does not exist." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_key_value_store.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 160, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storages/_key_value_store.py#L160" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "The value associated with the given key. `default_value` is used in case the record does not exist." + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Get a value from the KVS.\n" + } + ] + }, + "flags": {}, + "id": 8781, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "get_value", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Key of the record to retrieve." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 8782, + "kind": 32768, + "kindString": "Parameter", + "name": "key", + "type": { + "name": "str", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2291 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Default value returned in case the record does not exist.\n" + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": false + }, + "id": 8783, + "kind": 32768, + "kindString": "Parameter", + "name": "default_value", + "type": { + "name": "T | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "T", + "target": "83", + "ref": { + "id": 6573, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_types.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 36 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api#T" + } + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2291 + } + ], + "type": { + "name": "T | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "T", + "target": "83", + "ref": { + "id": 6573, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_types.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 36 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api#T" + } + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2290 + }, + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "The value associated with the given key. `default_value` is used in case the record does not exist." + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Get a value from the KVS.\n" + } + ] + }, + "flags": {}, + "id": 8812, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "get_value", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Key of the record to retrieve." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 8813, + "kind": 32768, + "kindString": "Parameter", + "name": "key", + "type": { + "name": "str", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2322 + } + ], + "type": { + "name": "Any", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2290 + }, + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "The value associated with the given key. `default_value` is used in case the record does not exist." + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Get a value from the KVS.\n" + } + ] + }, + "flags": {}, + "id": 8814, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "get_value", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Key of the record to retrieve." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 8815, + "kind": 32768, + "kindString": "Parameter", + "name": "key", + "type": { + "name": "str", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2324 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Default value returned in case the record does not exist.\n" + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 8816, + "kind": 32768, + "kindString": "Parameter", + "name": "default_value", + "type": { + "name": "T", + "type": "reference", + "target": "83", + "ref": { + "id": 6573, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_types.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 36 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api#T" + } + }, + "parentId": 2324 + } + ], + "type": { + "name": "T", + "type": "reference", + "target": "83", + "ref": { + "id": 6573, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_types.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 36 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api#T" + } + }, + "parentId": 2290 + }, + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "The value associated with the given key. `default_value` is used in case the record does not exist." + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Get a value from the KVS.\n" + } + ] + }, + "flags": {}, + "id": 8817, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "get_value", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Key of the record to retrieve." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 8818, + "kind": 32768, + "kindString": "Parameter", + "name": "key", + "type": { + "name": "str", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2327 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Default value returned in case the record does not exist.\n" + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": false + }, + "id": 8819, + "kind": 32768, + "kindString": "Parameter", + "name": "default_value", + "type": { + "name": "T | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "T", + "target": "83", + "ref": { + "id": 6573, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_types.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 36 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api#T" + } + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2327 + } + ], + "type": { + "name": "T | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "T", + "target": "83", + "ref": { + "id": 6573, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_types.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 36 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api#T" + } + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2290 + } + ], + "permalink": "/python/api/class/KeyValueStore#get_value", + "parentId": 2269 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Set a value in the KVS.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 8784, + "module": "storages._key_value_store", + "name": "set_value", + "parsedDocstring": { + "text": "Set a value in the KVS.\n", + "args": { + "key": "Key of the record to set.", + "value": "Value to set.", + "content_type": "The MIME content type string." + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_key_value_store.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 173, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storages/_key_value_store.py#L173" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Set a value in the KVS.\n" + } + ] + }, + "flags": {}, + "id": 8785, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "set_value", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Key of the record to set." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 8786, + "kind": 32768, + "kindString": "Parameter", + "name": "key", + "type": { + "name": "str", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2295 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Value to set." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 8787, + "kind": 32768, + "kindString": "Parameter", + "name": "value", + "type": { + "name": "Any", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2295 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The MIME content type string." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": false + }, + "id": 8788, + "kind": 32768, + "kindString": "Parameter", + "name": "content_type", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2295 + } + ], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "parentId": 2294 + } + ], + "permalink": "/python/api/class/KeyValueStore#set_value", + "parentId": 2269 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Delete a value from the KVS.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 8789, + "module": "storages._key_value_store", + "name": "delete_value", + "parsedDocstring": { + "text": "Delete a value from the KVS.\n", + "args": { + "key": "Key of the record to delete." + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_key_value_store.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 188, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storages/_key_value_store.py#L188" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Delete a value from the KVS.\n" + } + ] + }, + "flags": {}, + "id": 8790, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "delete_value", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Key of the record to delete." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 8791, + "kind": 32768, + "kindString": "Parameter", + "name": "key", + "type": { + "name": "str", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2300 + } + ], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "parentId": 2299 + } + ], + "permalink": "/python/api/class/KeyValueStore#delete_value", + "parentId": 2269 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Iterate over the existing keys in the KVS.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 8792, + "module": "storages._key_value_store", + "name": "iterate_keys", + "parsedDocstring": { + "text": "Iterate over the existing keys in the KVS.\n", + "args": { + "exclusive_start_key": "Key to start the iteration from.", + "limit": "Maximum number of keys to return. None means no limit.\n" + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_key_value_store.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 196, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storages/_key_value_store.py#L196" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Iterate over the existing keys in the KVS.\n" + } + ] + }, + "flags": {}, + "id": 8793, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "iterate_keys", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Key to start the iteration from." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": false + }, + "id": 8794, + "kind": 32768, + "kindString": "Parameter", + "name": "exclusive_start_key", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2303 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Maximum number of keys to return. None means no limit.\n" + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": false + }, + "id": 8795, + "kind": 32768, + "kindString": "Parameter", + "name": "limit", + "type": { + "name": "int | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "int", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2303 + } + ], + "type": { + "name": "AsyncIterator", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "KeyValueStoreRecordMetadata", + "target": "1325", + "ref": { + "id": 7815, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 78 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/KeyValueStoreRecordMetadata" + } + } + ], + "target": 0, + "ref": {} + }, + "parentId": 2302 + } + ], + "permalink": "/python/api/class/KeyValueStore#iterate_keys", + "parentId": 2269 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "List all the existing keys in the KVS.\n\nIt uses client's `iterate_keys` method to get the keys.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 8796, + "module": "storages._key_value_store", + "name": "list_keys", + "parsedDocstring": { + "text": "List all the existing keys in the KVS.\n\nIt uses client's `iterate_keys` method to get the keys.\n", + "args": { + "exclusive_start_key": "Key to start the iteration from.", + "limit": "Maximum number of keys to return.\n" + }, + "returns": "A list of keys in the KVS." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_key_value_store.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 216, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storages/_key_value_store.py#L216" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "A list of keys in the KVS." + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "List all the existing keys in the KVS.\n\nIt uses client's `iterate_keys` method to get the keys.\n" + } + ] + }, + "flags": {}, + "id": 8797, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "list_keys", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Key to start the iteration from." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": false + }, + "id": 8798, + "kind": 32768, + "kindString": "Parameter", + "name": "exclusive_start_key", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2307 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Maximum number of keys to return.\n" + } + ] + }, + "defaultValue": "1000", + "flags": { + "isOptional": true, + "keyword-only": false + }, + "id": 8799, + "kind": 32768, + "kindString": "Parameter", + "name": "limit", + "type": { + "name": "int", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2307 + } + ], + "type": { + "name": "list", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "KeyValueStoreRecordMetadata", + "target": "1325", + "ref": { + "id": 7815, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 78 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/KeyValueStoreRecordMetadata" + } + } + ], + "target": 0, + "ref": {} + }, + "parentId": 2306 + } + ], + "permalink": "/python/api/class/KeyValueStore#list_keys", + "parentId": 2269 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Check if a record with the given key exists in the key-value store.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 8800, + "module": "storages._key_value_store", + "name": "record_exists", + "parsedDocstring": { + "text": "Check if a record with the given key exists in the key-value store.\n", + "args": { + "key": "Key of the record to check for existence.\n" + }, + "returns": "True if a record with the given key exists, False otherwise." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_key_value_store.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 240, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storages/_key_value_store.py#L240" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "True if a record with the given key exists, False otherwise." + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Check if a record with the given key exists in the key-value store.\n" + } + ] + }, + "flags": {}, + "id": 8801, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "record_exists", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Key of the record to check for existence.\n" + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 8802, + "kind": 32768, + "kindString": "Parameter", + "name": "key", + "type": { + "name": "str", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2311 + } + ], + "type": { + "name": "bool", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2310 + } + ], + "permalink": "/python/api/class/KeyValueStore#record_exists", + "parentId": 2269 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Get the public URL for the given key.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 8803, + "module": "storages._key_value_store", + "name": "get_public_url", + "parsedDocstring": { + "text": "Get the public URL for the given key.\n", + "args": { + "key": "Key of the record for which URL is required.\n" + }, + "returns": "The public URL for the given key." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_key_value_store.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 251, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storages/_key_value_store.py#L251" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "The public URL for the given key." + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Get the public URL for the given key.\n" + } + ] + }, + "flags": {}, + "id": 8804, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "get_public_url", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Key of the record for which URL is required.\n" + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 8805, + "kind": 32768, + "kindString": "Parameter", + "name": "key", + "type": { + "name": "str", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2314 + } + ], + "type": { + "name": "str", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2313 + } + ], + "permalink": "/python/api/class/KeyValueStore#get_public_url", + "parentId": 2269 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Get a value from KVS that will be automatically saved on changes.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 8806, + "module": "storages._key_value_store", + "name": "get_auto_saved_value", + "parsedDocstring": { + "text": "Get a value from KVS that will be automatically saved on changes.\n", + "args": { + "key": "Key of the record, to store the value.", + "default_value": "Value to be used if the record does not exist yet. Should be a dictionary.\n" + }, + "returns": "Return the value of the key." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_key_value_store.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 262, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storages/_key_value_store.py#L262" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "Return the value of the key." + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Get a value from KVS that will be automatically saved on changes.\n" + } + ] + }, + "flags": {}, + "id": 8807, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "get_auto_saved_value", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Key of the record, to store the value." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 8808, + "kind": 32768, + "kindString": "Parameter", + "name": "key", + "type": { + "name": "str", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2317 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Value to be used if the record does not exist yet. Should be a dictionary.\n" + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": false + }, + "id": 8809, + "kind": 32768, + "kindString": "Parameter", + "name": "default_value", + "type": { + "name": "dict[str, JsonSerializable] | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "dict", + "typeArguments": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "reference", + "name": "JsonSerializable", + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2317 + } + ], + "type": { + "name": "dict", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "reference", + "name": "JsonSerializable", + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + }, + "parentId": 2316 + } + ], + "permalink": "/python/api/class/KeyValueStore#get_auto_saved_value", + "parentId": 2269 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Force autosaved values to be saved without waiting for an event in Event Manager." + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 8810, + "module": "storages._key_value_store", + "name": "persist_autosaved_values", + "parsedDocstring": { + "text": "Force autosaved values to be saved without waiting for an event in Event Manager." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_key_value_store.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 299, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storages/_key_value_store.py#L299" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Force autosaved values to be saved without waiting for an event in Event Manager." + } + ] + }, + "flags": {}, + "id": 8811, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "persist_autosaved_values", + "parameters": [], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "parentId": 2320 + } + ], + "permalink": "/python/api/class/KeyValueStore#persist_autosaved_values", + "parentId": 2269 + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Key-value store is a storage for reading and writing data records with unique key identifiers.\n\nThe key-value store class acts as a high-level interface for storing, retrieving, and managing data records\nidentified by unique string keys. It abstracts away the underlying storage implementation details,\nallowing you to work with the same API regardless of whether data is stored in memory, on disk,\nor in the cloud.\n\nEach data record is associated with a specific MIME content type, allowing storage of various\ndata formats such as JSON, text, images, HTML snapshots or any binary data. This class is\ncommonly used to store inputs, outputs, and other artifacts of crawler operations.\n\nYou can instantiate a key-value store using the `open` class method, which will create a store\nwith the specified name or id. The underlying storage implementation is determined by the configured\nstorage client.\n\n### Usage\n\n```python\nfrom crawlee.storages import KeyValueStore\n\n# Open a named key-value store\nkvs = await KeyValueStore.open(name='my-store')\n\n# Store and retrieve data\nawait kvs.set_value('product-1234.json', [{'name': 'Smartphone', 'price': 799.99}])\nproduct = await kvs.get_value('product-1234')\n```" + } + ] + }, + "decorations": [ + { + "args": "('Storages')", + "name": "docs_group" + } + ], + "flags": {}, + "groups": [ + { + "children": [ + 8760, + 8789, + 8776, + 8806, + 8767, + 8803, + 8780, + 8792, + 8796, + 8769, + 8810, + 8778, + 8800, + 8784 + ], + "title": "Methods" + }, + { + "children": [ + 8765, + 8766 + ], + "title": "Properties" + } + ], + "id": 8759, + "module": "storages._key_value_store", + "name": "KeyValueStore", + "parsedDocstring": { + "text": "Key-value store is a storage for reading and writing data records with unique key identifiers.\n\nThe key-value store class acts as a high-level interface for storing, retrieving, and managing data records\nidentified by unique string keys. It abstracts away the underlying storage implementation details,\nallowing you to work with the same API regardless of whether data is stored in memory, on disk,\nor in the cloud.\n\nEach data record is associated with a specific MIME content type, allowing storage of various\ndata formats such as JSON, text, images, HTML snapshots or any binary data. This class is\ncommonly used to store inputs, outputs, and other artifacts of crawler operations.\n\nYou can instantiate a key-value store using the `open` class method, which will create a store\nwith the specified name or id. The underlying storage implementation is determined by the configured\nstorage client.\n\n### Usage\n\n```python\nfrom crawlee.storages import KeyValueStore\n\n# Open a named key-value store\nkvs = await KeyValueStore.open(name='my-store')\n\n# Store and retrieve data\nawait kvs.set_value('product-1234.json', [{'name': 'Smartphone', 'price': 799.99}])\nproduct = await kvs.get_value('product-1234')\n```" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_key_value_store.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 40, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storages/_key_value_store.py#L40" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "extendedTypes": [ + { + "name": "Storage", + "target": "2330", + "type": "reference", + "ref": { + "id": 8820, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_base.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 15 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/Storage" + } + } + ], + "permalink": "/python/api/class/KeyValueStore", + "categories": [] + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Initialize a new instance.\n\nPreferably use the `RequestQueue.open` constructor to create a new instance.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 11184, + "module": "storages._request_queue", + "name": "__init__", + "parsedDocstring": { + "text": "Initialize a new instance.\n\nPreferably use the `RequestQueue.open` constructor to create a new instance.\n", + "args": { + "client": "An instance of a storage client.", + "id": "The unique identifier of the storage.", + "name": "The name of the storage, if available." + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_request_queue.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 74, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storages/_request_queue.py#L74" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Initialize a new instance.\n\nPreferably use the `RequestQueue.open` constructor to create a new instance.\n" + } + ] + }, + "flags": {}, + "id": 11185, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "__init__", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "An instance of a storage client." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 11186, + "kind": 32768, + "kindString": "Parameter", + "name": "client", + "type": { + "name": "RequestQueueClient", + "type": "reference", + "target": "1195", + "ref": { + "id": 10030, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_request_queue_client.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 13 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/RequestQueueClient" + } + }, + "parentId": 2350 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The unique identifier of the storage." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 11187, + "kind": 32768, + "kindString": "Parameter", + "name": "id", + "type": { + "name": "str", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2350 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The name of the storage, if available." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 11188, + "kind": 32768, + "kindString": "Parameter", + "name": "name", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2350 + } + ], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "parentId": 2349 + } + ], + "permalink": "/python/api/class/RequestQueue#__init__", + "parentId": 2348 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Get the storage ID." + } + ] + }, + "decorations": [ + { + "name": "property" + }, + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 11189, + "module": "storages._base", + "name": "id", + "parsedDocstring": { + "text": "Get the storage ID." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_request_queue.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 95, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storages/_request_queue.py#L95" + } + ], + "type": { + "name": "str", + "type": "reference", + "target": 0, + "ref": {} + }, + "overwrites": { + "name": "Storage.id", + "target": 2331, + "type": "reference", + "ref": { + "id": 11166, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_base.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 20 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api/class/Storage#id" + } + }, + "permalink": "/python/api/class/RequestQueue#id", + "parentId": 2348 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Get the storage name." + } + ] + }, + "decorations": [ + { + "name": "property" + }, + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 11190, + "module": "storages._base", + "name": "name", + "parsedDocstring": { + "text": "Get the storage name." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_request_queue.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 100, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storages/_request_queue.py#L100" + } + ], + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "overwrites": { + "name": "Storage.name", + "target": 2332, + "type": "reference", + "ref": { + "id": 11167, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_base.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 25 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api/class/Storage#name" + } + }, + "permalink": "/python/api/class/RequestQueue#name", + "parentId": 2348 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Get the storage metadata." + } + ] + }, + "decorations": [ + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 11191, + "module": "storages._base", + "name": "get_metadata", + "parsedDocstring": { + "text": "Get the storage metadata." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_request_queue.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 104, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storages/_request_queue.py#L104" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Get the storage metadata." + } + ] + }, + "flags": {}, + "id": 11169, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "get_metadata", + "parameters": [], + "type": { + "name": "DatasetMetadata | KeyValueStoreMetadata | RequestQueueMetadata", + "type": "union", + "types": [ + { + "type": "union", + "types": [ + { + "type": "reference", + "name": "DatasetMetadata", + "target": "1314", + "ref": { + "id": 10149, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 42 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/DatasetMetadata" + } + }, + { + "type": "reference", + "name": "KeyValueStoreMetadata", + "target": "1317", + "ref": { + "id": 10152, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 52 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/KeyValueStoreMetadata" + } + } + ] + }, + { + "type": "reference", + "name": "RequestQueueMetadata", + "target": "1319", + "ref": { + "id": 10154, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 59 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/RequestQueueMetadata" + } + } + ] + }, + "overwrites": { + "name": "Storage.get_metadata", + "target": 2333, + "type": "reference", + "ref": { + "id": 11168, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_base.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 29 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/Storage#get_metadata" + } + }, + "parentId": 2356 + } + ], + "overwrites": { + "name": "Storage.get_metadata", + "target": 2333, + "type": "reference", + "ref": { + "id": 11168, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_base.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 29 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/Storage#get_metadata" + } + }, + "permalink": "/python/api/class/RequestQueue#get_metadata", + "parentId": 2348 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Get the number of requests in the loader that have been handled." + } + ] + }, + "decorations": [ + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 11193, + "module": "request_loaders._request_loader", + "name": "get_handled_count", + "parsedDocstring": { + "text": "Get the number of requests in the loader that have been handled." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_request_queue.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 108, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storages/_request_queue.py#L108" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Get the number of requests in the loader that have been handled." + } + ] + }, + "flags": {}, + "id": 11771, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "get_handled_count", + "parameters": [], + "type": { + "name": "int", + "type": "reference", + "target": 0, + "ref": {} + }, + "overwrites": { + "name": "RequestManager.get_handled_count", + "target": 4143, + "type": "reference", + "ref": { + "id": 12978, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 29 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestManager#get_handled_count" + } + }, + "parentId": 2358 + } + ], + "overwrites": { + "name": "RequestManager.get_handled_count", + "target": 4143, + "type": "reference", + "ref": { + "id": 12978, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 29 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestManager#get_handled_count" + } + }, + "permalink": "/python/api/class/RequestQueue#get_handled_count", + "parentId": 2348 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Get an offline approximation of the total number of requests in the loader (i.e. pending + handled)." + } + ] + }, + "decorations": [ + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 11195, + "module": "request_loaders._request_loader", + "name": "get_total_count", + "parsedDocstring": { + "text": "Get an offline approximation of the total number of requests in the loader (i.e. pending + handled)." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_request_queue.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 113, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storages/_request_queue.py#L113" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Get an offline approximation of the total number of requests in the loader (i.e. pending + handled)." + } + ] + }, + "flags": {}, + "id": 11773, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "get_total_count", + "parameters": [], + "type": { + "name": "int", + "type": "reference", + "target": 0, + "ref": {} + }, + "overwrites": { + "name": "RequestManager.get_total_count", + "target": 4144, + "type": "reference", + "ref": { + "id": 12979, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 33 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestManager#get_total_count" + } + }, + "parentId": 2360 + } + ], + "overwrites": { + "name": "RequestManager.get_total_count", + "target": 4144, + "type": "reference", + "ref": { + "id": 12979, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 33 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestManager#get_total_count" + } + }, + "permalink": "/python/api/class/RequestQueue#get_total_count", + "parentId": 2348 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Open a storage, either restore existing or create a new one.\n" + } + ] + }, + "decorations": [ + { + "name": "classmethod" + }, + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 11197, + "module": "storages._base", + "name": "open", + "parsedDocstring": { + "text": "Open a storage, either restore existing or create a new one.\n", + "args": { + "id": "The storage ID.", + "name": "The storage name (global scope, persists across runs). Name can only contain letters \"a\" through \"z\",\nthe digits \"0\" through \"9\", and the hyphen (\"-\") but only in the middle of the string\n(e.g. \"my-value-1\").", + "alias": "The storage alias (run scope, creates unnamed storage).", + "configuration": "Configuration object used during the storage creation or restoration process.", + "storage_client": "Underlying storage client to use. If not provided, the default global storage client\nfrom the service locator will be used." + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_request_queue.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 119, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storages/_request_queue.py#L119" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Open a storage, either restore existing or create a new one.\n" + } + ] + }, + "flags": {}, + "id": 11171, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "open", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The storage ID." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 11172, + "kind": 32768, + "kindString": "Parameter", + "name": "id", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2336 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The storage name (global scope, persists across runs). Name can only contain letters \"a\" through \"z\",\nthe digits \"0\" through \"9\", and the hyphen (\"-\") but only in the middle of the string\n(e.g. \"my-value-1\")." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 11173, + "kind": 32768, + "kindString": "Parameter", + "name": "name", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2336 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The storage alias (run scope, creates unnamed storage)." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 11174, + "kind": 32768, + "kindString": "Parameter", + "name": "alias", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2336 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Configuration object used during the storage creation or restoration process." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 11175, + "kind": 32768, + "kindString": "Parameter", + "name": "configuration", + "type": { + "name": "Configuration | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "Configuration", + "target": "2", + "ref": { + "id": 8837, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/configuration.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 20 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/Configuration" + } + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2336 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Underlying storage client to use. If not provided, the default global storage client\nfrom the service locator will be used." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 11176, + "kind": 32768, + "kindString": "Parameter", + "name": "storage_client", + "type": { + "name": "StorageClient | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "StorageClient", + "target": "1282", + "ref": { + "id": 10117, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 19 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/StorageClient" + } + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2336 + } + ], + "type": { + "name": "Storage", + "type": "reference", + "target": "2330", + "ref": { + "id": 11165, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_base.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 15 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/Storage" + } + }, + "overwrites": { + "name": "Storage.open", + "target": 2335, + "type": "reference", + "ref": { + "id": 11170, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_base.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 34 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/Storage#open" + } + }, + "parentId": 2362 + } + ], + "overwrites": { + "name": "Storage.open", + "target": 2335, + "type": "reference", + "ref": { + "id": 11170, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_base.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 34 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/Storage#open" + } + }, + "permalink": "/python/api/class/RequestQueue#open", + "parentId": 2348 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Remove persistent state either from the Apify Cloud storage or from the local database." + } + ] + }, + "decorations": [ + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 11204, + "module": "request_loaders._request_manager", + "name": "drop", + "parsedDocstring": { + "text": "Remove persistent state either from the Apify Cloud storage or from the local database." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_request_queue.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 144, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storages/_request_queue.py#L144" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Remove persistent state either from the Apify Cloud storage or from the local database." + } + ] + }, + "flags": {}, + "id": 11838, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "drop", + "parameters": [], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "overwrites": { + "name": "Storage.drop", + "target": 2342, + "type": "reference", + "ref": { + "id": 11177, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_base.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 57 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/Storage#drop" + } + }, + "parentId": 2369 + } + ], + "overwrites": { + "name": "Storage.drop", + "target": 2342, + "type": "reference", + "ref": { + "id": 11177, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_base.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 57 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/Storage#drop" + } + }, + "permalink": "/python/api/class/RequestQueue#drop", + "parentId": 2348 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Remove all queued requests from this manager while keeping it usable.\n\nDefault implementation is a no-op. Subclasses with persistent state should override to clear it." + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 11206, + "module": "request_loaders._request_manager", + "name": "purge", + "parsedDocstring": { + "text": "Remove all queued requests from this manager while keeping it usable.\n\nDefault implementation is a no-op. Subclasses with persistent state should override to clear it." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_request_queue.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 152, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storages/_request_queue.py#L152" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Remove all queued requests from this manager while keeping it usable.\n\nDefault implementation is a no-op. Subclasses with persistent state should override to clear it." + } + ] + }, + "flags": {}, + "id": 11840, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "purge", + "parameters": [], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "overwrites": { + "name": "Storage.purge", + "target": 2344, + "type": "reference", + "ref": { + "id": 11179, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_base.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 61 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/Storage#purge" + } + }, + "parentId": 2371 + } + ], + "overwrites": { + "name": "Storage.purge", + "target": 2344, + "type": "reference", + "ref": { + "id": 11179, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_base.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 61 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/Storage#purge" + } + }, + "permalink": "/python/api/class/RequestQueue#purge", + "parentId": 2348 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Add a single request to the manager and store it in underlying resource client.\n" + } + ] + }, + "decorations": [ + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 11208, + "module": "request_loaders._request_manager", + "name": "add_request", + "parsedDocstring": { + "text": "Add a single request to the manager and store it in underlying resource client.\n", + "args": { + "request": "The request object (or its string representation) to be added to the manager.", + "forefront": "Determines whether the request should be added to the beginning (if True) or the end (if False)\nof the manager.\n" + }, + "returns": "Information about the request addition to the manager or None if the request was not added." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_request_queue.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 177, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storages/_request_queue.py#L177" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "Information about the request addition to the manager or None if the request was not added." + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Add a single request to the manager and store it in underlying resource client.\n" + } + ] + }, + "flags": {}, + "id": 11842, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "add_request", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The request object (or its string representation) to be added to the manager." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 11843, + "kind": 32768, + "kindString": "Parameter", + "name": "request", + "type": { + "name": "str | Request", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "reference", + "name": "Request", + "target": "409", + "ref": { + "id": 9244, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_request.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 145 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/Request" + } + } + ] + }, + "parentId": 3007 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Determines whether the request should be added to the beginning (if True) or the end (if False)\nof the manager.\n" + } + ] + }, + "defaultValue": "False", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 11844, + "kind": 32768, + "kindString": "Parameter", + "name": "forefront", + "type": { + "name": "bool", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 3007 + } + ], + "type": { + "name": "ProcessedRequest | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "ProcessedRequest", + "target": "1340", + "ref": { + "id": 10175, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 140 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/ProcessedRequest" + } + }, + { + "type": "literal", + "value": {} + } + ] + }, + "overwrites": { + "name": "RequestManager.add_request", + "target": 3006, + "type": "reference", + "ref": { + "id": 11841, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_manager.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 32 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestManager#add_request" + } + }, + "parentId": 2373 + } + ], + "overwrites": { + "name": "RequestManager.add_request", + "target": 3006, + "type": "reference", + "ref": { + "id": 11841, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_manager.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 32 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestManager#add_request" + } + }, + "permalink": "/python/api/class/RequestQueue#add_request", + "parentId": 2348 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Add requests to the manager in batches.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 11212, + "module": "request_loaders._request_manager", + "name": "add_requests", + "parsedDocstring": { + "text": "Add requests to the manager in batches.\n", + "args": { + "requests": "Requests to enqueue.", + "forefront": "If True, add requests to the beginning of the queue.", + "batch_size": "The number of requests to add in one batch.", + "wait_time_between_batches": "Time to wait between adding batches.", + "wait_for_all_requests_to_be_added": "If True, wait for all requests to be added before returning.", + "wait_for_all_requests_to_be_added_timeout": "Timeout for waiting for all requests to be added." + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_request_queue.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 201, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storages/_request_queue.py#L201" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Add requests to the manager in batches.\n" + } + ] + }, + "flags": {}, + "id": 11846, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "add_requests", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Requests to enqueue." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 11847, + "kind": 32768, + "kindString": "Parameter", + "name": "requests", + "type": { + "name": "Sequence", + "type": "reference", + "typeArguments": [ + { + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "reference", + "name": "Request", + "target": "409", + "ref": { + "id": 9244, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_request.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 145 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/Request" + } + } + ] + } + ], + "target": 0, + "ref": {} + }, + "parentId": 3011 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "If True, add requests to the beginning of the queue." + } + ] + }, + "defaultValue": "False", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 11848, + "kind": 32768, + "kindString": "Parameter", + "name": "forefront", + "type": { + "name": "bool", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 3011 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The number of requests to add in one batch." + } + ] + }, + "defaultValue": "1000", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 11849, + "kind": 32768, + "kindString": "Parameter", + "name": "batch_size", + "type": { + "name": "int", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 3011 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Time to wait between adding batches." + } + ] + }, + "defaultValue": "timedelta(seconds=1)", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 11850, + "kind": 32768, + "kindString": "Parameter", + "name": "wait_time_between_batches", + "type": { + "name": "timedelta", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 3011 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "If True, wait for all requests to be added before returning." + } + ] + }, + "defaultValue": "False", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 11851, + "kind": 32768, + "kindString": "Parameter", + "name": "wait_for_all_requests_to_be_added", + "type": { + "name": "bool", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 3011 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Timeout for waiting for all requests to be added." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 11852, + "kind": 32768, + "kindString": "Parameter", + "name": "wait_for_all_requests_to_be_added_timeout", + "type": { + "name": "timedelta | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "timedelta", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 3011 + } + ], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "overwrites": { + "name": "RequestManager.add_requests", + "target": 3010, + "type": "reference", + "ref": { + "id": 11845, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_manager.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 49 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestManager#add_requests" + } + }, + "parentId": 2377 + } + ], + "overwrites": { + "name": "RequestManager.add_requests", + "target": 3010, + "type": "reference", + "ref": { + "id": 11845, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_manager.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 49 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestManager#add_requests" + } + }, + "permalink": "/python/api/class/RequestQueue#add_requests", + "parentId": 2348 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Return the next request in the queue to be processed.\n\nOnce you successfully finish processing of the request, you need to call `RequestQueue.mark_request_as_handled`\nto mark the request as handled in the queue. If there was some error in processing the request, call\n`RequestQueue.reclaim_request` instead, so that the queue will give the request to some other consumer\nin another call to the `fetch_next_request` method.\n\nNote that the `None` return value does not mean the queue processing finished, it means there are currently\nno pending requests. To check whether all requests in queue were finished, use `RequestQueue.is_finished`\ninstead.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 11220, + "module": "storages._request_queue", + "name": "fetch_next_request", + "parsedDocstring": { + "text": "Return the next request in the queue to be processed.\n\nOnce you successfully finish processing of the request, you need to call `RequestQueue.mark_request_as_handled`\nto mark the request as handled in the queue. If there was some error in processing the request, call\n`RequestQueue.reclaim_request` instead, so that the queue will give the request to some other consumer\nin another call to the `fetch_next_request` method.\n\nNote that the `None` return value does not mean the queue processing finished, it means there are currently\nno pending requests. To check whether all requests in queue were finished, use `RequestQueue.is_finished`\ninstead.\n", + "returns": "The next request to process, or `None` if there are no more pending requests." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_request_queue.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 251, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storages/_request_queue.py#L251" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "The next request to process, or `None` if there are no more pending requests." + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Return the next request in the queue to be processed.\n\nOnce you successfully finish processing of the request, you need to call `RequestQueue.mark_request_as_handled`\nto mark the request as handled in the queue. If there was some error in processing the request, call\n`RequestQueue.reclaim_request` instead, so that the queue will give the request to some other consumer\nin another call to the `fetch_next_request` method.\n\nNote that the `None` return value does not mean the queue processing finished, it means there are currently\nno pending requests. To check whether all requests in queue were finished, use `RequestQueue.is_finished`\ninstead.\n" + } + ] + }, + "flags": {}, + "id": 11221, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "fetch_next_request", + "parameters": [], + "type": { + "name": "Request | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "Request", + "target": "409", + "ref": { + "id": 9244, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_request.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 145 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/Request" + } + }, + { + "type": "literal", + "value": {} + } + ] + }, + "overwrites": { + "name": "RequestManager.fetch_next_request", + "target": 4147, + "type": "reference", + "ref": { + "id": 12982, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 45 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestManager#fetch_next_request" + } + }, + "parentId": 2385 + } + ], + "overwrites": { + "name": "RequestManager.fetch_next_request", + "target": 4147, + "type": "reference", + "ref": { + "id": 12982, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 45 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestManager#fetch_next_request" + } + }, + "permalink": "/python/api/class/RequestQueue#fetch_next_request", + "parentId": 2348 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Retrieve a specific request from the queue by its ID.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 11222, + "module": "storages._request_queue", + "name": "get_request", + "parsedDocstring": { + "text": "Retrieve a specific request from the queue by its ID.\n", + "args": { + "unique_key": "Unique key of the request to retrieve.\n" + }, + "returns": "The request with the specified ID, or `None` if no such request exists." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_request_queue.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 268, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storages/_request_queue.py#L268" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "The request with the specified ID, or `None` if no such request exists." + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Retrieve a specific request from the queue by its ID.\n" + } + ] + }, + "flags": {}, + "id": 11223, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "get_request", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Unique key of the request to retrieve.\n" + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 11224, + "kind": 32768, + "kindString": "Parameter", + "name": "unique_key", + "type": { + "name": "str", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2388 + } + ], + "type": { + "name": "Request | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "Request", + "target": "409", + "ref": { + "id": 9244, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_request.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 145 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/Request" + } + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2387 + } + ], + "permalink": "/python/api/class/RequestQueue#get_request", + "parentId": 2348 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Mark a request as handled after successful processing.\n\nThis method should be called after a request has been successfully processed.\nOnce marked as handled, the request will be removed from the queue and will\nnot be returned in subsequent calls to `fetch_next_request` method.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 11225, + "module": "storages._request_queue", + "name": "mark_request_as_handled", + "parsedDocstring": { + "text": "Mark a request as handled after successful processing.\n\nThis method should be called after a request has been successfully processed.\nOnce marked as handled, the request will be removed from the queue and will\nnot be returned in subsequent calls to `fetch_next_request` method.\n", + "args": { + "request": "The request to mark as handled.\n" + }, + "returns": "Information about the queue operation." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_request_queue.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 279, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storages/_request_queue.py#L279" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "Information about the queue operation." + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Mark a request as handled after successful processing.\n\nThis method should be called after a request has been successfully processed.\nOnce marked as handled, the request will be removed from the queue and will\nnot be returned in subsequent calls to `fetch_next_request` method.\n" + } + ] + }, + "flags": {}, + "id": 11226, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "mark_request_as_handled", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The request to mark as handled.\n" + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 11227, + "kind": 32768, + "kindString": "Parameter", + "name": "request", + "type": { + "name": "Request", + "type": "reference", + "target": "409", + "ref": { + "id": 9244, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_request.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 145 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/Request" + } + }, + "parentId": 2391 + } + ], + "type": { + "name": "ProcessedRequest | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "ProcessedRequest", + "target": "1340", + "ref": { + "id": 10175, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 140 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/ProcessedRequest" + } + }, + { + "type": "literal", + "value": {} + } + ] + }, + "overwrites": { + "name": "RequestManager.mark_request_as_handled", + "target": 4148, + "type": "reference", + "ref": { + "id": 12983, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 53 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestManager#mark_request_as_handled" + } + }, + "parentId": 2390 + } + ], + "overwrites": { + "name": "RequestManager.mark_request_as_handled", + "target": 4148, + "type": "reference", + "ref": { + "id": 12983, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 53 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestManager#mark_request_as_handled" + } + }, + "permalink": "/python/api/class/RequestQueue#mark_request_as_handled", + "parentId": 2348 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Reclaim a failed request back to the queue for later processing.\n\nIf a request fails during processing, this method can be used to return it to the queue.\nThe request will be returned for processing again in a subsequent call\nto `RequestQueue.fetch_next_request`.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 11228, + "module": "storages._request_queue", + "name": "reclaim_request", + "parsedDocstring": { + "text": "Reclaim a failed request back to the queue for later processing.\n\nIf a request fails during processing, this method can be used to return it to the queue.\nThe request will be returned for processing again in a subsequent call\nto `RequestQueue.fetch_next_request`.\n", + "args": { + "request": "The request to return to the queue.", + "forefront": "If true, the request will be added to the beginning of the queue.\nOtherwise, it will be added to the end.\n" + }, + "returns": "Information about the queue operation." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_request_queue.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 294, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storages/_request_queue.py#L294" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "Information about the queue operation." + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Reclaim a failed request back to the queue for later processing.\n\nIf a request fails during processing, this method can be used to return it to the queue.\nThe request will be returned for processing again in a subsequent call\nto `RequestQueue.fetch_next_request`.\n" + } + ] + }, + "flags": {}, + "id": 11229, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "reclaim_request", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The request to return to the queue." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 11230, + "kind": 32768, + "kindString": "Parameter", + "name": "request", + "type": { + "name": "Request", + "type": "reference", + "target": "409", + "ref": { + "id": 9244, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_request.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 145 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/Request" + } + }, + "parentId": 2394 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "If true, the request will be added to the beginning of the queue.\nOtherwise, it will be added to the end.\n" + } + ] + }, + "defaultValue": "False", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 11231, + "kind": 32768, + "kindString": "Parameter", + "name": "forefront", + "type": { + "name": "bool", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2394 + } + ], + "type": { + "name": "ProcessedRequest | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "ProcessedRequest", + "target": "1340", + "ref": { + "id": 10175, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 140 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/ProcessedRequest" + } + }, + { + "type": "literal", + "value": {} + } + ] + }, + "overwrites": { + "name": "RequestManager.reclaim_request", + "target": 3018, + "type": "reference", + "ref": { + "id": 11853, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_manager.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 77 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestManager#reclaim_request" + } + }, + "parentId": 2393 + } + ], + "overwrites": { + "name": "RequestManager.reclaim_request", + "target": 3018, + "type": "reference", + "ref": { + "id": 11853, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_manager.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 77 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestManager#reclaim_request" + } + }, + "permalink": "/python/api/class/RequestQueue#reclaim_request", + "parentId": 2348 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Check if the request queue is empty.\n\nAn empty queue means that there are no requests currently in the queue, either pending or being processed.\nHowever, this does not necessarily mean that the crawling operation is finished, as there still might be\ntasks that could add additional requests to the queue.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 11232, + "module": "storages._request_queue", + "name": "is_empty", + "parsedDocstring": { + "text": "Check if the request queue is empty.\n\nAn empty queue means that there are no requests currently in the queue, either pending or being processed.\nHowever, this does not necessarily mean that the crawling operation is finished, as there still might be\ntasks that could add additional requests to the queue.\n", + "returns": "True if the request queue is empty, False otherwise." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_request_queue.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 316, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storages/_request_queue.py#L316" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "True if the request queue is empty, False otherwise." + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Check if the request queue is empty.\n\nAn empty queue means that there are no requests currently in the queue, either pending or being processed.\nHowever, this does not necessarily mean that the crawling operation is finished, as there still might be\ntasks that could add additional requests to the queue.\n" + } + ] + }, + "flags": {}, + "id": 11233, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "is_empty", + "parameters": [], + "type": { + "name": "bool", + "type": "reference", + "target": 0, + "ref": {} + }, + "overwrites": { + "name": "RequestManager.is_empty", + "target": 4145, + "type": "reference", + "ref": { + "id": 12980, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 37 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestManager#is_empty" + } + }, + "parentId": 2397 + } + ], + "overwrites": { + "name": "RequestManager.is_empty", + "target": 4145, + "type": "reference", + "ref": { + "id": 12980, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 37 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestManager#is_empty" + } + }, + "permalink": "/python/api/class/RequestQueue#is_empty", + "parentId": 2348 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Check if the request queue is finished.\n\nA finished queue means that all requests in the queue have been processed (the queue is empty) and there\nare no more tasks that could add additional requests to the queue. This is the definitive way to check\nif a crawling operation is complete.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 11234, + "module": "storages._request_queue", + "name": "is_finished", + "parsedDocstring": { + "text": "Check if the request queue is finished.\n\nA finished queue means that all requests in the queue have been processed (the queue is empty) and there\nare no more tasks that could add additional requests to the queue. This is the definitive way to check\nif a crawling operation is complete.\n", + "returns": "True if the request queue is finished (empty and no pending add operations), False otherwise." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_request_queue.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 328, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storages/_request_queue.py#L328" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "True if the request queue is finished (empty and no pending add operations), False otherwise." + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Check if the request queue is finished.\n\nA finished queue means that all requests in the queue have been processed (the queue is empty) and there\nare no more tasks that could add additional requests to the queue. This is the definitive way to check\nif a crawling operation is complete.\n" + } + ] + }, + "flags": {}, + "id": 11235, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "is_finished", + "parameters": [], + "type": { + "name": "bool", + "type": "reference", + "target": 0, + "ref": {} + }, + "overwrites": { + "name": "RequestManager.is_finished", + "target": 4146, + "type": "reference", + "ref": { + "id": 12981, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 41 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestManager#is_finished" + } + }, + "parentId": 2399 + } + ], + "overwrites": { + "name": "RequestManager.is_finished", + "target": 4146, + "type": "reference", + "ref": { + "id": 12981, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 41 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestManager#is_finished" + } + }, + "permalink": "/python/api/class/RequestQueue#is_finished", + "parentId": 2348 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Combine the loader with a request manager to support adding and reclaiming requests.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 12985, + "module": "request_loaders._request_loader", + "name": "to_tandem", + "parsedDocstring": { + "text": "Combine the loader with a request manager to support adding and reclaiming requests.\n", + "args": { + "request_manager": "Request manager to combine the loader with.\nIf None is given, the default request queue is used." + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 56, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/request_loaders/_request_loader.py#L56" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Combine the loader with a request manager to support adding and reclaiming requests.\n" + } + ] + }, + "flags": {}, + "id": 11784, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "to_tandem", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Request manager to combine the loader with.\nIf None is given, the default request queue is used." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": false + }, + "id": 11785, + "kind": 32768, + "kindString": "Parameter", + "name": "request_manager", + "type": { + "name": "RequestManager | None", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2949 + } + ], + "type": { + "name": "RequestManagerTandem", + "type": "reference", + "target": "2803", + "ref": { + "id": 11638, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_manager_tandem.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 24 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/RequestManagerTandem" + } + }, + "inheritedFrom": { + "name": "RequestLoader.to_tandem", + "target": 2948, + "type": "reference", + "ref": { + "id": 11783, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 56 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestLoader#to_tandem" + } + }, + "parentId": 4150 + } + ], + "inheritedFrom": { + "name": "RequestLoader.to_tandem", + "target": 2948, + "type": "reference", + "ref": { + "id": 11783, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 56 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestLoader#to_tandem" + } + }, + "permalink": "/python/api/class/RequestQueue#to_tandem", + "parentId": 2348 + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Request queue is a storage for managing HTTP requests.\n\nThe request queue class serves as a high-level interface for organizing and managing HTTP requests\nduring web crawling. It provides methods for adding, retrieving, and manipulating requests throughout\nthe crawling lifecycle, abstracting away the underlying storage implementation details.\n\nRequest queue maintains the state of each URL to be crawled, tracking whether it has been processed,\nis currently being handled, or is waiting in the queue. Each URL in the queue is uniquely identified\nby a `unique_key` property, which prevents duplicate processing unless explicitly configured otherwise.\n\nThe class supports both breadth-first and depth-first crawling strategies through its `forefront` parameter\nwhen adding requests. It also provides mechanisms for error handling and request reclamation when\nprocessing fails.\n\nYou can open a request queue using the `open` class method, specifying either a name or ID to identify\nthe queue. The underlying storage implementation is determined by the configured storage client.\n\n### Usage\n\n```python\nfrom crawlee.storages import RequestQueue\n\n# Open a request queue\nrq = await RequestQueue.open(name='my-queue')\n\n# Add a request\nawait rq.add_request('https://example.com')\n\n# Process requests\nrequest = await rq.fetch_next_request()\nif request:\n try:\n # Process the request\n # ...\n await rq.mark_request_as_handled(request)\n except Exception:\n await rq.reclaim_request(request)\n```" + } + ] + }, + "decorations": [ + { + "args": "('Storages')", + "name": "docs_group" + } + ], + "flags": {}, + "groups": [ + { + "children": [ + 11184, + 11208, + 11212, + 11204, + 11220, + 11193, + 11191, + 11222, + 11195, + 11232, + 11234, + 11225, + 11197, + 11206, + 11228, + 12985 + ], + "title": "Methods" + }, + { + "children": [ + 11189, + 11190 + ], + "title": "Properties" + } + ], + "id": 11183, + "module": "storages._request_queue", + "name": "RequestQueue", + "parsedDocstring": { + "text": "Request queue is a storage for managing HTTP requests.\n\nThe request queue class serves as a high-level interface for organizing and managing HTTP requests\nduring web crawling. It provides methods for adding, retrieving, and manipulating requests throughout\nthe crawling lifecycle, abstracting away the underlying storage implementation details.\n\nRequest queue maintains the state of each URL to be crawled, tracking whether it has been processed,\nis currently being handled, or is waiting in the queue. Each URL in the queue is uniquely identified\nby a `unique_key` property, which prevents duplicate processing unless explicitly configured otherwise.\n\nThe class supports both breadth-first and depth-first crawling strategies through its `forefront` parameter\nwhen adding requests. It also provides mechanisms for error handling and request reclamation when\nprocessing fails.\n\nYou can open a request queue using the `open` class method, specifying either a name or ID to identify\nthe queue. The underlying storage implementation is determined by the configured storage client.\n\n### Usage\n\n```python\nfrom crawlee.storages import RequestQueue\n\n# Open a request queue\nrq = await RequestQueue.open(name='my-queue')\n\n# Add a request\nawait rq.add_request('https://example.com')\n\n# Process requests\nrequest = await rq.fetch_next_request()\nif request:\n try:\n # Process the request\n # ...\n await rq.mark_request_as_handled(request)\n except Exception:\n await rq.reclaim_request(request)\n```" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_request_queue.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 33, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storages/_request_queue.py#L33" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "extendedTypes": [ + { + "name": "RequestManager", + "target": "3001", + "type": "reference", + "ref": { + "id": 11836, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_manager.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 18 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/RequestManager" + } + }, + { + "name": "Storage", + "target": "2330", + "type": "reference", + "ref": { + "id": 11165, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_base.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 15 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/Storage" + } + } + ], + "permalink": "/python/api/class/RequestQueue", + "categories": [] + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 14337, + "module": "storage_clients.models", + "name": "model_config", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 173, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/models.py#L173" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "permalink": "/python/api/class/AddRequestsResponse#model_config", + "parentId": 1351 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Successfully processed requests, including information about whether they were\nalready present in the queue and whether they had been handled previously." + } + ] + }, + "flags": {}, + "groups": [], + "id": 14338, + "module": "storage_clients.models", + "name": "processed_requests", + "parsedDocstring": { + "text": "Successfully processed requests, including information about whether they were\nalready present in the queue and whether they had been handled previously." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 175, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/models.py#L175" + } + ], + "type": { + "name": "list", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "ProcessedRequest", + "target": "1340", + "ref": { + "id": 14325, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 140 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/ProcessedRequest" + } + } + ], + "target": 0, + "ref": {} + }, + "permalink": "/python/api/class/AddRequestsResponse#processed_requests", + "parentId": 1351 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Requests that could not be processed, typically due to validation errors or other issues." + } + ] + }, + "flags": {}, + "groups": [], + "id": 14339, + "module": "storage_clients.models", + "name": "unprocessed_requests", + "parsedDocstring": { + "text": "Requests that could not be processed, typically due to validation errors or other issues." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 179, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/models.py#L179" + } + ], + "type": { + "name": "list", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "UnprocessedRequest", + "target": "1346", + "ref": { + "id": 14331, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 154 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/UnprocessedRequest" + } + } + ], + "target": 0, + "ref": {} + }, + "permalink": "/python/api/class/AddRequestsResponse#unprocessed_requests", + "parentId": 1351 + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Model for a response to add requests to a queue.\n\nContains detailed information about the processing results when adding multiple requests\nto a queue. This includes which requests were successfully processed and which ones\nencountered issues during processing." + } + ] + }, + "decorations": [ + { + "args": "('Storage data')", + "name": "docs_group" + } + ], + "flags": {}, + "groups": [ + { + "children": [ + 14337, + 14338, + 14339 + ], + "title": "Properties" + } + ], + "id": 14336, + "module": "storage_clients.models", + "name": "AddRequestsResponse", + "parsedDocstring": { + "text": "Model for a response to add requests to a queue.\n\nContains detailed information about the processing results when adding multiple requests\nto a queue. This includes which requests were successfully processed and which ones\nencountered issues during processing." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 165, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/models.py#L165" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "permalink": "/python/api/class/AddRequestsResponse", + "categories": [] + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 15673, + "module": "storage_clients.models", + "name": "model_config", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 113, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/models.py#L113" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "permalink": "/python/api/class/DatasetItemsListPage#model_config", + "parentId": 1333 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "The number of objects returned on this page." + } + ] + }, + "flags": {}, + "groups": [], + "id": 15674, + "module": "storage_clients.models", + "name": "count", + "parsedDocstring": { + "text": "The number of objects returned on this page." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 115, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/models.py#L115" + } + ], + "type": { + "name": "int", + "type": "reference", + "target": 0, + "ref": {} + }, + "permalink": "/python/api/class/DatasetItemsListPage#count", + "parentId": 1333 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "The starting position of the first object returned, as specified in the API call." + } + ] + }, + "flags": {}, + "groups": [], + "id": 15675, + "module": "storage_clients.models", + "name": "offset", + "parsedDocstring": { + "text": "The starting position of the first object returned, as specified in the API call." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 118, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/models.py#L118" + } + ], + "type": { + "name": "int", + "type": "reference", + "target": 0, + "ref": {} + }, + "permalink": "/python/api/class/DatasetItemsListPage#offset", + "parentId": 1333 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "The maximum number of objects to return, as specified in the API call." + } + ] + }, + "flags": {}, + "groups": [], + "id": 15676, + "module": "storage_clients.models", + "name": "limit", + "parsedDocstring": { + "text": "The maximum number of objects to return, as specified in the API call." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 121, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/models.py#L121" + } + ], + "type": { + "name": "int", + "type": "reference", + "target": 0, + "ref": {} + }, + "permalink": "/python/api/class/DatasetItemsListPage#limit", + "parentId": 1333 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "The total number of objects that match the criteria of the API call." + } + ] + }, + "flags": {}, + "groups": [], + "id": 15677, + "module": "storage_clients.models", + "name": "total", + "parsedDocstring": { + "text": "The total number of objects that match the criteria of the API call." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 124, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/models.py#L124" + } + ], + "type": { + "name": "int", + "type": "reference", + "target": 0, + "ref": {} + }, + "permalink": "/python/api/class/DatasetItemsListPage#total", + "parentId": 1333 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Indicates if the returned list is in descending order." + } + ] + }, + "flags": {}, + "groups": [], + "id": 15678, + "module": "storage_clients.models", + "name": "desc", + "parsedDocstring": { + "text": "Indicates if the returned list is in descending order." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 127, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/models.py#L127" + } + ], + "type": { + "name": "bool", + "type": "reference", + "target": 0, + "ref": {} + }, + "permalink": "/python/api/class/DatasetItemsListPage#desc", + "parentId": 1333 + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Model for a single page of dataset items returned from a collection list method." + } + ] + }, + "decorations": [ + { + "args": "('Storage data')", + "name": "docs_group" + } + ], + "flags": {}, + "groups": [ + { + "children": [ + 15674, + 15678, + 15676, + 15673, + 15675, + 15677 + ], + "title": "Properties" + } + ], + "id": 15672, + "module": "storage_clients.models", + "name": "DatasetItemsListPage", + "parsedDocstring": { + "text": "Model for a single page of dataset items returned from a collection list method." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 110, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/models.py#L110" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "permalink": "/python/api/class/DatasetItemsListPage", + "categories": [] + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 16993, + "module": "storage_clients.models", + "name": "model_config", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 45, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/models.py#L45" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "overwrites": { + "name": "StorageMetadata.model_config", + "target": 1308, + "type": "reference", + "ref": { + "id": 16986, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 23 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api/class/StorageMetadata#model_config" + } + }, + "permalink": "/python/api/class/DatasetMetadata#model_config", + "parentId": 1314 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "The number of items in the dataset." + } + ] + }, + "flags": {}, + "groups": [], + "id": 16994, + "module": "storage_clients.models", + "name": "item_count", + "parsedDocstring": { + "text": "The number of items in the dataset." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 47, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/models.py#L47" + } + ], + "type": { + "name": "int", + "type": "reference", + "target": 0, + "ref": {} + }, + "permalink": "/python/api/class/DatasetMetadata#item_count", + "parentId": 1314 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "The unique identifier of the storage." + } + ] + }, + "flags": {}, + "groups": [], + "id": 19887, + "module": "storage_clients.models", + "name": "id", + "parsedDocstring": { + "text": "The unique identifier of the storage." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 25, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/models.py#L25" + } + ], + "type": { + "name": "Annotated[str, Field(alias='id')]", + "type": "reference", + "target": 0, + "ref": {} + }, + "inheritedFrom": { + "name": "StorageMetadata.id", + "target": 1309, + "type": "reference", + "ref": { + "id": 16987, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 25 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api/class/StorageMetadata#id" + } + }, + "permalink": "/python/api/class/DatasetMetadata#id", + "parentId": 1314 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "The name of the storage." + } + ] + }, + "flags": {}, + "groups": [], + "id": 19888, + "module": "storage_clients.models", + "name": "name", + "parsedDocstring": { + "text": "The name of the storage." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 28, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/models.py#L28" + } + ], + "type": { + "name": "Annotated[str | None, Field(alias='name', default=None)]", + "type": "reference", + "target": 0, + "ref": {} + }, + "inheritedFrom": { + "name": "StorageMetadata.name", + "target": 1310, + "type": "reference", + "ref": { + "id": 16988, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 28 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api/class/StorageMetadata#name" + } + }, + "permalink": "/python/api/class/DatasetMetadata#name", + "parentId": 1314 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "The timestamp when the storage was last accessed." + } + ] + }, + "flags": {}, + "groups": [], + "id": 19889, + "module": "storage_clients.models", + "name": "accessed_at", + "parsedDocstring": { + "text": "The timestamp when the storage was last accessed." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 31, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/models.py#L31" + } + ], + "type": { + "name": "Annotated[datetime, Field(alias='accessedAt')]", + "type": "reference", + "target": 0, + "ref": {} + }, + "inheritedFrom": { + "name": "StorageMetadata.accessed_at", + "target": 1311, + "type": "reference", + "ref": { + "id": 16989, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 31 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api/class/StorageMetadata#accessed_at" + } + }, + "permalink": "/python/api/class/DatasetMetadata#accessed_at", + "parentId": 1314 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "The timestamp when the storage was created." + } + ] + }, + "flags": {}, + "groups": [], + "id": 19890, + "module": "storage_clients.models", + "name": "created_at", + "parsedDocstring": { + "text": "The timestamp when the storage was created." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 34, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/models.py#L34" + } + ], + "type": { + "name": "Annotated[datetime, Field(alias='createdAt')]", + "type": "reference", + "target": 0, + "ref": {} + }, + "inheritedFrom": { + "name": "StorageMetadata.created_at", + "target": 1312, + "type": "reference", + "ref": { + "id": 16990, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 34 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api/class/StorageMetadata#created_at" + } + }, + "permalink": "/python/api/class/DatasetMetadata#created_at", + "parentId": 1314 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "The timestamp when the storage was last modified." + } + ] + }, + "flags": {}, + "groups": [], + "id": 19891, + "module": "storage_clients.models", + "name": "modified_at", + "parsedDocstring": { + "text": "The timestamp when the storage was last modified." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 37, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/models.py#L37" + } + ], + "type": { + "name": "Annotated[datetime, Field(alias='modifiedAt')]", + "type": "reference", + "target": 0, + "ref": {} + }, + "inheritedFrom": { + "name": "StorageMetadata.modified_at", + "target": 1313, + "type": "reference", + "ref": { + "id": 16991, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 37 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api/class/StorageMetadata#modified_at" + } + }, + "permalink": "/python/api/class/DatasetMetadata#modified_at", + "parentId": 1314 + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Model for a dataset metadata." + } + ] + }, + "decorations": [ + { + "args": "('Storage data')", + "name": "docs_group" + } + ], + "flags": {}, + "groups": [ + { + "children": [ + 19889, + 19890, + 19887, + 16994, + 16993, + 19891, + 19888 + ], + "title": "Properties" + } + ], + "id": 16992, + "module": "storage_clients.models", + "name": "DatasetMetadata", + "parsedDocstring": { + "text": "Model for a dataset metadata." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 42, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/models.py#L42" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "extendedTypes": [ + { + "name": "StorageMetadata", + "target": "1307", + "type": "reference", + "ref": { + "id": 16985, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 17 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/StorageMetadata" + } + } + ], + "permalink": "/python/api/class/DatasetMetadata", + "categories": [] + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 21209, + "module": "storage_clients.models", + "name": "model_config", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 55, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/models.py#L55" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "overwrites": { + "name": "StorageMetadata.model_config", + "target": 1308, + "type": "reference", + "ref": { + "id": 21199, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 23 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api/class/StorageMetadata#model_config" + } + }, + "permalink": "/python/api/class/KeyValueStoreMetadata#model_config", + "parentId": 1317 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "The unique identifier of the storage." + } + ] + }, + "flags": {}, + "groups": [], + "id": 24105, + "module": "storage_clients.models", + "name": "id", + "parsedDocstring": { + "text": "The unique identifier of the storage." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 25, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/models.py#L25" + } + ], + "type": { + "name": "Annotated[str, Field(alias='id')]", + "type": "reference", + "target": 0, + "ref": {} + }, + "inheritedFrom": { + "name": "StorageMetadata.id", + "target": 1309, + "type": "reference", + "ref": { + "id": 21200, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 25 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api/class/StorageMetadata#id" + } + }, + "permalink": "/python/api/class/KeyValueStoreMetadata#id", + "parentId": 1317 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "The name of the storage." + } + ] + }, + "flags": {}, + "groups": [], + "id": 24106, + "module": "storage_clients.models", + "name": "name", + "parsedDocstring": { + "text": "The name of the storage." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 28, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/models.py#L28" + } + ], + "type": { + "name": "Annotated[str | None, Field(alias='name', default=None)]", + "type": "reference", + "target": 0, + "ref": {} + }, + "inheritedFrom": { + "name": "StorageMetadata.name", + "target": 1310, + "type": "reference", + "ref": { + "id": 21201, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 28 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api/class/StorageMetadata#name" + } + }, + "permalink": "/python/api/class/KeyValueStoreMetadata#name", + "parentId": 1317 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "The timestamp when the storage was last accessed." + } + ] + }, + "flags": {}, + "groups": [], + "id": 24107, + "module": "storage_clients.models", + "name": "accessed_at", + "parsedDocstring": { + "text": "The timestamp when the storage was last accessed." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 31, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/models.py#L31" + } + ], + "type": { + "name": "Annotated[datetime, Field(alias='accessedAt')]", + "type": "reference", + "target": 0, + "ref": {} + }, + "inheritedFrom": { + "name": "StorageMetadata.accessed_at", + "target": 1311, + "type": "reference", + "ref": { + "id": 21202, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 31 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api/class/StorageMetadata#accessed_at" + } + }, + "permalink": "/python/api/class/KeyValueStoreMetadata#accessed_at", + "parentId": 1317 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "The timestamp when the storage was created." + } + ] + }, + "flags": {}, + "groups": [], + "id": 24108, + "module": "storage_clients.models", + "name": "created_at", + "parsedDocstring": { + "text": "The timestamp when the storage was created." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 34, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/models.py#L34" + } + ], + "type": { + "name": "Annotated[datetime, Field(alias='createdAt')]", + "type": "reference", + "target": 0, + "ref": {} + }, + "inheritedFrom": { + "name": "StorageMetadata.created_at", + "target": 1312, + "type": "reference", + "ref": { + "id": 21203, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 34 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api/class/StorageMetadata#created_at" + } + }, + "permalink": "/python/api/class/KeyValueStoreMetadata#created_at", + "parentId": 1317 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "The timestamp when the storage was last modified." + } + ] + }, + "flags": {}, + "groups": [], + "id": 24109, + "module": "storage_clients.models", + "name": "modified_at", + "parsedDocstring": { + "text": "The timestamp when the storage was last modified." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 37, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/models.py#L37" + } + ], + "type": { + "name": "Annotated[datetime, Field(alias='modifiedAt')]", + "type": "reference", + "target": 0, + "ref": {} + }, + "inheritedFrom": { + "name": "StorageMetadata.modified_at", + "target": 1313, + "type": "reference", + "ref": { + "id": 21204, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 37 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api/class/StorageMetadata#modified_at" + } + }, + "permalink": "/python/api/class/KeyValueStoreMetadata#modified_at", + "parentId": 1317 + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Model for a key-value store metadata." + } + ] + }, + "decorations": [ + { + "args": "('Storage data')", + "name": "docs_group" + } + ], + "flags": {}, + "groups": [ + { + "children": [ + 24107, + 24108, + 24105, + 21209, + 24109, + 24106 + ], + "title": "Properties" + } + ], + "id": 21208, + "module": "storage_clients.models", + "name": "KeyValueStoreMetadata", + "parsedDocstring": { + "text": "Model for a key-value store metadata." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 52, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/models.py#L52" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "extendedTypes": [ + { + "name": "StorageMetadata", + "target": "1307", + "type": "reference", + "ref": { + "id": 21198, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 17 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/StorageMetadata" + } + } + ], + "permalink": "/python/api/class/KeyValueStoreMetadata", + "categories": [] + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 25440, + "module": "storage_clients.models", + "name": "model_config", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 103, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/models.py#L103" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "overwrites": { + "name": "KeyValueStoreRecordMetadata.model_config", + "target": 1326, + "type": "reference", + "ref": { + "id": 25435, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 81 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api/class/KeyValueStoreRecordMetadata#model_config" + } + }, + "permalink": "/python/api/class/KeyValueStoreRecord#model_config", + "parentId": 1330 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "The value of the record." + } + ] + }, + "flags": {}, + "groups": [], + "id": 25441, + "module": "storage_clients.models", + "name": "value", + "parsedDocstring": { + "text": "The value of the record." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 105, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/models.py#L105" + } + ], + "type": { + "name": "KvsValueType", + "type": "reference", + "target": "1306", + "ref": { + "id": 25415, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 13 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api#KvsValueType" + } + }, + "permalink": "/python/api/class/KeyValueStoreRecord#value", + "parentId": 1330 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "The key of the record.\n\nA unique identifier for the record in the key-value store." + } + ] + }, + "flags": {}, + "groups": [], + "id": 28315, + "module": "storage_clients.models", + "name": "key", + "parsedDocstring": { + "text": "The key of the record.\n\nA unique identifier for the record in the key-value store." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 83, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/models.py#L83" + } + ], + "type": { + "name": "Annotated[str, Field(alias='key')]", + "type": "reference", + "target": 0, + "ref": {} + }, + "inheritedFrom": { + "name": "KeyValueStoreRecordMetadata.key", + "target": 1327, + "type": "reference", + "ref": { + "id": 25436, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 83 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api/class/KeyValueStoreRecordMetadata#key" + } + }, + "permalink": "/python/api/class/KeyValueStoreRecord#key", + "parentId": 1330 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "The MIME type of the record.\n\nDescribe the format and type of data stored in the record, following the MIME specification." + } + ] + }, + "flags": {}, + "groups": [], + "id": 28316, + "module": "storage_clients.models", + "name": "content_type", + "parsedDocstring": { + "text": "The MIME type of the record.\n\nDescribe the format and type of data stored in the record, following the MIME specification." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 89, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/models.py#L89" + } + ], + "type": { + "name": "Annotated[str, Field(alias='contentType')]", + "type": "reference", + "target": 0, + "ref": {} + }, + "inheritedFrom": { + "name": "KeyValueStoreRecordMetadata.content_type", + "target": 1328, + "type": "reference", + "ref": { + "id": 25437, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 89 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api/class/KeyValueStoreRecordMetadata#content_type" + } + }, + "permalink": "/python/api/class/KeyValueStoreRecord#content_type", + "parentId": 1330 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "The size of the record in bytes." + } + ] + }, + "flags": {}, + "groups": [], + "id": 28317, + "module": "storage_clients.models", + "name": "size", + "parsedDocstring": { + "text": "The size of the record in bytes." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 95, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/models.py#L95" + } + ], + "type": { + "name": "Annotated[int | None, Field(alias='size', default=None)]", + "type": "reference", + "target": 0, + "ref": {} + }, + "inheritedFrom": { + "name": "KeyValueStoreRecordMetadata.size", + "target": 1329, + "type": "reference", + "ref": { + "id": 25438, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 95 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api/class/KeyValueStoreRecordMetadata#size" + } + }, + "permalink": "/python/api/class/KeyValueStoreRecord#size", + "parentId": 1330 + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Model for a key-value store record." + } + ] + }, + "decorations": [ + { + "args": "('Storage data')", + "name": "docs_group" + } + ], + "flags": {}, + "groups": [ + { + "children": [ + 28316, + 28315, + 25440, + 28317, + 25441 + ], + "title": "Properties" + } + ], + "id": 25439, + "module": "storage_clients.models", + "name": "KeyValueStoreRecord", + "parsedDocstring": { + "text": "Model for a key-value store record." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 100, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/models.py#L100" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "extendedTypes": [ + { + "name": "KeyValueStoreRecordMetadata", + "target": "1325", + "type": "reference", + "ref": { + "id": 25434, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 78 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/KeyValueStoreRecordMetadata" + } + } + ], + "permalink": "/python/api/class/KeyValueStoreRecord", + "categories": [] + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 29643, + "module": "storage_clients.models", + "name": "model_config", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 81, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/models.py#L81" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "permalink": "/python/api/class/KeyValueStoreRecordMetadata#model_config", + "parentId": 1325 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "The key of the record.\n\nA unique identifier for the record in the key-value store." + } + ] + }, + "flags": {}, + "groups": [], + "id": 29644, + "module": "storage_clients.models", + "name": "key", + "parsedDocstring": { + "text": "The key of the record.\n\nA unique identifier for the record in the key-value store." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 83, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/models.py#L83" + } + ], + "type": { + "name": "str", + "type": "reference", + "target": 0, + "ref": {} + }, + "permalink": "/python/api/class/KeyValueStoreRecordMetadata#key", + "parentId": 1325 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "The MIME type of the record.\n\nDescribe the format and type of data stored in the record, following the MIME specification." + } + ] + }, + "flags": {}, + "groups": [], + "id": 29645, + "module": "storage_clients.models", + "name": "content_type", + "parsedDocstring": { + "text": "The MIME type of the record.\n\nDescribe the format and type of data stored in the record, following the MIME specification." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 89, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/models.py#L89" + } + ], + "type": { + "name": "str", + "type": "reference", + "target": 0, + "ref": {} + }, + "permalink": "/python/api/class/KeyValueStoreRecordMetadata#content_type", + "parentId": 1325 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "The size of the record in bytes." + } + ] + }, + "flags": {}, + "groups": [], + "id": 29646, + "module": "storage_clients.models", + "name": "size", + "parsedDocstring": { + "text": "The size of the record in bytes." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 95, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/models.py#L95" + } + ], + "type": { + "name": "Annotated[int | None, Field(alias='size', default=None)]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "int", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "permalink": "/python/api/class/KeyValueStoreRecordMetadata#size", + "parentId": 1325 + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Model for a key-value store record metadata." + } + ] + }, + "decorations": [ + { + "args": "('Storage data')", + "name": "docs_group" + } + ], + "flags": {}, + "groups": [ + { + "children": [ + 29645, + 29644, + 29643, + 29646 + ], + "title": "Properties" + } + ], + "id": 29642, + "module": "storage_clients.models", + "name": "KeyValueStoreRecordMetadata", + "parsedDocstring": { + "text": "Model for a key-value store record metadata." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 78, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/models.py#L78" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "extendedBy": [ + { + "name": "KeyValueStoreRecord", + "target": "1330", + "type": "reference", + "ref": { + "id": 29647, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 100 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/KeyValueStoreRecord" + } + } + ], + "permalink": "/python/api/class/KeyValueStoreRecordMetadata", + "categories": [] + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 30988, + "module": "storage_clients.models", + "name": "model_config", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 143, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/models.py#L143" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "permalink": "/python/api/class/ProcessedRequest#model_config", + "parentId": 1340 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Internal representation of the request by the storage client. Only some clients use id." + } + ] + }, + "flags": {}, + "groups": [], + "id": 30989, + "module": "storage_clients.models", + "name": "id", + "parsedDocstring": { + "text": "Internal representation of the request by the storage client. Only some clients use id." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 145, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/models.py#L145" + } + ], + "type": { + "name": "Annotated[str | None, Field(alias='requestId', default=None)]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "permalink": "/python/api/class/ProcessedRequest#id", + "parentId": 1340 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 30990, + "module": "storage_clients.models", + "name": "unique_key", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 148, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/models.py#L148" + } + ], + "type": { + "name": "str", + "type": "reference", + "target": 0, + "ref": {} + }, + "permalink": "/python/api/class/ProcessedRequest#unique_key", + "parentId": 1340 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 30991, + "module": "storage_clients.models", + "name": "was_already_present", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 149, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/models.py#L149" + } + ], + "type": { + "name": "bool", + "type": "reference", + "target": 0, + "ref": {} + }, + "permalink": "/python/api/class/ProcessedRequest#was_already_present", + "parentId": 1340 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 30992, + "module": "storage_clients.models", + "name": "was_already_handled", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 150, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/models.py#L150" + } + ], + "type": { + "name": "bool", + "type": "reference", + "target": 0, + "ref": {} + }, + "permalink": "/python/api/class/ProcessedRequest#was_already_handled", + "parentId": 1340 + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Represents a processed request." + } + ] + }, + "decorations": [ + { + "args": "('Storage data')", + "name": "docs_group" + } + ], + "flags": {}, + "groups": [ + { + "children": [ + 30989, + 30988, + 30990, + 30992, + 30991 + ], + "title": "Properties" + } + ], + "id": 30987, + "module": "storage_clients.models", + "name": "ProcessedRequest", + "parsedDocstring": { + "text": "Represents a processed request." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 140, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/models.py#L140" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "permalink": "/python/api/class/ProcessedRequest", + "categories": [] + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 31402, + "module": "_request", + "name": "model_config", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_request.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 169, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/_request.py#L169" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "permalink": "/python/api/class/Request#model_config", + "parentId": 409 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "A unique key identifying the request. Two requests with the same `unique_key` are considered as pointing\nto the same URL.\n\nIf `unique_key` is not provided, then it is automatically generated by normalizing the URL.\nFor example, the URL of `HTTP://www.EXAMPLE.com/something/` will produce the `unique_key`\nof `http://www.example.com/something`.\n\nPass an arbitrary non-empty text value to the `unique_key` property to override the default behavior\nand specify which URLs shall be considered equal." + } + ] + }, + "flags": {}, + "groups": [], + "id": 31403, + "module": "_request", + "name": "unique_key", + "parsedDocstring": { + "text": "A unique key identifying the request. Two requests with the same `unique_key` are considered as pointing\nto the same URL.\n\nIf `unique_key` is not provided, then it is automatically generated by normalizing the URL.\nFor example, the URL of `HTTP://www.EXAMPLE.com/something/` will produce the `unique_key`\nof `http://www.example.com/something`.\n\nPass an arbitrary non-empty text value to the `unique_key` property to override the default behavior\nand specify which URLs shall be considered equal." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_request.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 171, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/_request.py#L171" + } + ], + "type": { + "name": "str", + "type": "reference", + "target": 0, + "ref": {} + }, + "permalink": "/python/api/class/Request#unique_key", + "parentId": 409 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "The URL of the web page to crawl. Must be a valid HTTP or HTTPS URL, and may include query parameters\nand fragments." + } + ] + }, + "flags": {}, + "groups": [], + "id": 31404, + "module": "_request", + "name": "url", + "parsedDocstring": { + "text": "The URL of the web page to crawl. Must be a valid HTTP or HTTPS URL, and may include query parameters\nand fragments." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_request.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 183, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/_request.py#L183" + } + ], + "type": { + "name": "str", + "type": "reference", + "target": 0, + "ref": {} + }, + "permalink": "/python/api/class/Request#url", + "parentId": 409 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "HTTP request method." + } + ] + }, + "flags": {}, + "groups": [], + "id": 31405, + "module": "_request", + "name": "method", + "parsedDocstring": { + "text": "HTTP request method." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_request.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 187, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/_request.py#L187" + } + ], + "type": { + "name": "HttpMethod", + "type": "reference", + "target": "84", + "ref": { + "id": 31076, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_types.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 38 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api#HttpMethod" + } + }, + "permalink": "/python/api/class/Request#method", + "parentId": 409 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "HTTP request payload." + } + ] + }, + "flags": {}, + "groups": [], + "id": 31406, + "module": "_request", + "name": "payload", + "parsedDocstring": { + "text": "HTTP request payload." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_request.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 190, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/_request.py#L190" + } + ], + "type": { + "name": "Annotated[ HttpPayload | None, BeforeValidator(lambda v: v.encode() if isinstance(v, str) else v), PlainSerializer(lambda v: v.decode() if isinstance(v, bytes) else v), Field(frozen=True), ]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "HttpPayload", + "target": "85", + "ref": { + "id": 31077, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_types.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 40 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api#HttpPayload" + } + }, + { + "type": "literal", + "value": {} + } + ] + }, + "permalink": "/python/api/class/Request#payload", + "parentId": 409 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Number of times the request has been retried." + } + ] + }, + "flags": {}, + "groups": [], + "id": 31407, + "module": "_request", + "name": "retry_count", + "parsedDocstring": { + "text": "Number of times the request has been retried." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_request.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 230, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/_request.py#L230" + } + ], + "type": { + "name": "int", + "type": "reference", + "target": 0, + "ref": {} + }, + "permalink": "/python/api/class/Request#retry_count", + "parentId": 409 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "If set to `True`, the request will not be retried in case of failure." + } + ] + }, + "flags": {}, + "groups": [], + "id": 31408, + "module": "_request", + "name": "no_retry", + "parsedDocstring": { + "text": "If set to `True`, the request will not be retried in case of failure." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_request.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 233, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/_request.py#L233" + } + ], + "type": { + "name": "bool", + "type": "reference", + "target": 0, + "ref": {} + }, + "permalink": "/python/api/class/Request#no_retry", + "parentId": 409 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "URL of the web page that was loaded. This can differ from the original URL in case of redirects." + } + ] + }, + "flags": {}, + "groups": [], + "id": 31409, + "module": "_request", + "name": "loaded_url", + "parsedDocstring": { + "text": "URL of the web page that was loaded. This can differ from the original URL in case of redirects." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_request.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 236, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/_request.py#L236" + } + ], + "type": { + "name": "Annotated[str | None, BeforeValidator(validate_http_url), Field(alias='loadedUrl')]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "permalink": "/python/api/class/Request#loaded_url", + "parentId": 409 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Timestamp when the request was handled." + } + ] + }, + "flags": {}, + "groups": [], + "id": 31410, + "module": "_request", + "name": "handled_at", + "parsedDocstring": { + "text": "Timestamp when the request was handled." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_request.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 239, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/_request.py#L239" + } + ], + "type": { + "name": "Annotated[datetime | None, Field(alias='handledAt')]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "datetime", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "permalink": "/python/api/class/Request#handled_at", + "parentId": 409 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Create a new `Request` instance from a URL.\n\nThis is recommended constructor for creating new `Request` instances. It generates a `Request` object from\na given URL with additional options to customize HTTP method, payload, unique key, and other request\nproperties. If no `unique_key` or `id` is provided, they are computed automatically based on the URL,\nmethod and payload. It depends on the `keep_url_fragment` and `use_extended_unique_key` flags.\n" + } + ] + }, + "decorations": [ + { + "name": "classmethod" + } + ], + "flags": {}, + "groups": [], + "id": 31411, + "module": "_request", + "name": "from_url", + "parsedDocstring": { + "text": "Create a new `Request` instance from a URL.\n\nThis is recommended constructor for creating new `Request` instances. It generates a `Request` object from\na given URL with additional options to customize HTTP method, payload, unique key, and other request\nproperties. If no `unique_key` or `id` is provided, they are computed automatically based on the URL,\nmethod and payload. It depends on the `keep_url_fragment` and `use_extended_unique_key` flags.\n", + "args": { + "url": "The URL of the request.", + "method": "The HTTP method of the request.", + "headers": "The HTTP headers of the request.", + "payload": "The data to be sent as the request body. Typically used with 'POST' or 'PUT' requests.", + "label": "A custom label to differentiate between request types. This is stored in `user_data`, and it is\nused for request routing (different requests go to different handlers).", + "session_id": "ID of a specific `Session` to which the request will be strictly bound.\nIf the session becomes unavailable when the request is processed, a `RequestCollisionError` will be\nraised.", + "unique_key": "A unique key identifying the request. If not provided, it is automatically computed based on\nthe URL and other parameters. Requests with the same `unique_key` are treated as identical.", + "keep_url_fragment": "Determines whether the URL fragment (e.g., ``section``) should be included in\nthe `unique_key` computation. This is only relevant when `unique_key` is not provided.", + "use_extended_unique_key": "Determines whether to include the HTTP method, ID Session and payload in the\n`unique_key` computation. This is only relevant when `unique_key` is not provided.", + "always_enqueue": "If set to `True`, the request will be enqueued even if it is already present in the queue.\nUsing this is not allowed when a custom `unique_key` is also provided and will result in a `ValueError`.", + "enqueue_strategy": "The strategy that will be used for enqueuing the request.", + "max_retries": "Maximum number of retries for this request. Allows to override the global `max_request_retries`\noption of `BasicCrawler`.", + "**kwargs": "Additional request properties." + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_request.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 243, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/_request.py#L243" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Create a new `Request` instance from a URL.\n\nThis is recommended constructor for creating new `Request` instances. It generates a `Request` object from\na given URL with additional options to customize HTTP method, payload, unique key, and other request\nproperties. If no `unique_key` or `id` is provided, they are computed automatically based on the URL,\nmethod and payload. It depends on the `keep_url_fragment` and `use_extended_unique_key` flags.\n" + } + ] + }, + "flags": {}, + "id": 31412, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "from_url", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The URL of the request." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 31413, + "kind": 32768, + "kindString": "Parameter", + "name": "url", + "type": { + "name": "str", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 420 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The HTTP method of the request." + } + ] + }, + "defaultValue": "'GET'", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 31414, + "kind": 32768, + "kindString": "Parameter", + "name": "method", + "type": { + "name": "HttpMethod", + "type": "reference", + "target": "84", + "ref": { + "id": 31076, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_types.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 38 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api#HttpMethod" + } + }, + "parentId": 420 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The HTTP headers of the request." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 31415, + "kind": 32768, + "kindString": "Parameter", + "name": "headers", + "type": { + "name": "HttpHeaders | dict[str, str] | None", + "type": "union", + "types": [ + { + "type": "union", + "types": [ + { + "type": "reference", + "name": "HttpHeaders", + "target": "91", + "ref": { + "id": 31083, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_types.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 63 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/HttpHeaders" + } + }, + { + "type": "reference", + "name": "dict", + "typeArguments": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + } + ] + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 420 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The data to be sent as the request body. Typically used with 'POST' or 'PUT' requests." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 31416, + "kind": 32768, + "kindString": "Parameter", + "name": "payload", + "type": { + "name": "HttpPayload | str | None", + "type": "union", + "types": [ + { + "type": "union", + "types": [ + { + "type": "reference", + "name": "HttpPayload", + "target": "85", + "ref": { + "id": 31077, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_types.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 40 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api#HttpPayload" + } + }, + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + } + ] + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 420 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "A custom label to differentiate between request types. This is stored in `user_data`, and it is\nused for request routing (different requests go to different handlers)." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 31417, + "kind": 32768, + "kindString": "Parameter", + "name": "label", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 420 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "ID of a specific `Session` to which the request will be strictly bound.\nIf the session becomes unavailable when the request is processed, a `RequestCollisionError` will be\nraised." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 31418, + "kind": 32768, + "kindString": "Parameter", + "name": "session_id", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 420 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "A unique key identifying the request. If not provided, it is automatically computed based on\nthe URL and other parameters. Requests with the same `unique_key` are treated as identical." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 31419, + "kind": 32768, + "kindString": "Parameter", + "name": "unique_key", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 420 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Determines whether the URL fragment (e.g., ``section``) should be included in\nthe `unique_key` computation. This is only relevant when `unique_key` is not provided." + } + ] + }, + "defaultValue": "False", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 31420, + "kind": 32768, + "kindString": "Parameter", + "name": "keep_url_fragment", + "type": { + "name": "bool", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 420 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Determines whether to include the HTTP method, ID Session and payload in the\n`unique_key` computation. This is only relevant when `unique_key` is not provided." + } + ] + }, + "defaultValue": "False", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 31421, + "kind": 32768, + "kindString": "Parameter", + "name": "use_extended_unique_key", + "type": { + "name": "bool", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 420 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "If set to `True`, the request will be enqueued even if it is already present in the queue.\nUsing this is not allowed when a custom `unique_key` is also provided and will result in a `ValueError`." + } + ] + }, + "defaultValue": "False", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 31422, + "kind": 32768, + "kindString": "Parameter", + "name": "always_enqueue", + "type": { + "name": "bool", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 420 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The strategy that will be used for enqueuing the request." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 31423, + "kind": 32768, + "kindString": "Parameter", + "name": "enqueue_strategy", + "type": { + "name": "EnqueueStrategy | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "EnqueueStrategy", + "target": "88", + "ref": { + "id": 31080, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_types.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 47 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api#EnqueueStrategy" + } + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 420 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Maximum number of retries for this request. Allows to override the global `max_request_retries`\noption of `BasicCrawler`." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 31424, + "kind": 32768, + "kindString": "Parameter", + "name": "max_retries", + "type": { + "name": "int | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "int", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 420 + }, + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 31425, + "kind": 32768, + "kindString": "Parameter", + "name": "kwargs", + "type": { + "name": "Any", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 420 + } + ], + "type": { + "name": "Self", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 419 + } + ], + "permalink": "/python/api/class/Request#from_url", + "parentId": 409 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Get the value of a specific query parameter from the URL." + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 31426, + "module": "_request", + "name": "get_query_param_from_url", + "parsedDocstring": { + "text": "Get the value of a specific query parameter from the URL." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_request.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 344, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/_request.py#L344" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Get the value of a specific query parameter from the URL." + } + ] + }, + "flags": {}, + "id": 31427, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "get_query_param_from_url", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 31428, + "kind": 32768, + "kindString": "Parameter", + "name": "param", + "type": { + "name": "str", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 435 + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 31429, + "kind": 32768, + "kindString": "Parameter", + "name": "default", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 435 + } + ], + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 434 + } + ], + "permalink": "/python/api/class/Request#get_query_param_from_url", + "parentId": 409 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "A string used to differentiate between arbitrary request types." + } + ] + }, + "decorations": [ + { + "name": "property" + } + ], + "flags": {}, + "groups": [], + "id": 31430, + "module": "_request", + "name": "label", + "parsedDocstring": { + "text": "A string used to differentiate between arbitrary request types." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_request.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 350, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/_request.py#L350" + } + ], + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "permalink": "/python/api/class/Request#label", + "parentId": 409 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "The ID of the bound session, if there is any." + } + ] + }, + "decorations": [ + { + "name": "property" + } + ], + "flags": {}, + "groups": [], + "id": 31431, + "module": "_request", + "name": "session_id", + "parsedDocstring": { + "text": "The ID of the bound session, if there is any." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_request.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 355, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/_request.py#L355" + } + ], + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "permalink": "/python/api/class/Request#session_id", + "parentId": 409 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Crawlee-specific configuration stored in the `user_data`." + } + ] + }, + "decorations": [ + { + "name": "property" + } + ], + "flags": {}, + "groups": [], + "id": 31432, + "module": "_request", + "name": "crawlee_data", + "parsedDocstring": { + "text": "Crawlee-specific configuration stored in the `user_data`." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_request.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 360, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/_request.py#L360" + } + ], + "type": { + "name": "CrawleeRequestData", + "type": "reference", + "target": "358", + "ref": { + "id": 31350, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_request.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 34 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/CrawleeRequestData" + } + }, + "permalink": "/python/api/class/Request#crawlee_data", + "parentId": 409 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "The depth of the request in the crawl tree." + } + ] + }, + "decorations": [ + { + "name": "property" + } + ], + "flags": {}, + "groups": [], + "id": 31433, + "module": "_request", + "name": "crawl_depth", + "parsedDocstring": { + "text": "The depth of the request in the crawl tree." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_request.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 369, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/_request.py#L369" + } + ], + "type": { + "name": "int", + "type": "reference", + "target": 0, + "ref": {} + }, + "permalink": "/python/api/class/Request#crawl_depth", + "parentId": 409 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "decorations": [ + { + "args": ".setter", + "name": "crawl_depth" + } + ], + "flags": {}, + "groups": [], + "id": 31434, + "module": "_request", + "name": "crawl_depth", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_request.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 374, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/_request.py#L374" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "flags": {}, + "id": 31435, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "crawl_depth", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 31436, + "kind": 32768, + "kindString": "Parameter", + "name": "new_value", + "type": { + "name": "int", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 443 + } + ], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "parentId": 442 + } + ], + "permalink": "/python/api/class/Request#crawl_depth", + "parentId": 409 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Crawlee-specific request handling state." + } + ] + }, + "decorations": [ + { + "name": "property" + } + ], + "flags": {}, + "groups": [], + "id": 31437, + "module": "_request", + "name": "state", + "parsedDocstring": { + "text": "Crawlee-specific request handling state." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_request.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 378, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/_request.py#L378" + } + ], + "type": { + "name": "RequestState", + "type": "reference", + "target": "349", + "ref": { + "id": 31341, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_request.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 21 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/RequestState" + } + }, + "permalink": "/python/api/class/Request#state", + "parentId": 409 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "decorations": [ + { + "args": ".setter", + "name": "state" + } + ], + "flags": {}, + "groups": [], + "id": 31438, + "module": "_request", + "name": "state", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_request.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 383, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/_request.py#L383" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "flags": {}, + "id": 31439, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "state", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 31440, + "kind": 32768, + "kindString": "Parameter", + "name": "new_state", + "type": { + "name": "RequestState", + "type": "reference", + "target": "349", + "ref": { + "id": 31341, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_request.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 21 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/RequestState" + } + }, + "parentId": 447 + } + ], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "parentId": 446 + } + ], + "permalink": "/python/api/class/Request#state", + "parentId": 409 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Crawlee-specific limit on the number of retries of the request." + } + ] + }, + "decorations": [ + { + "name": "property" + } + ], + "flags": {}, + "groups": [], + "id": 31441, + "module": "_request", + "name": "max_retries", + "parsedDocstring": { + "text": "Crawlee-specific limit on the number of retries of the request." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_request.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 387, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/_request.py#L387" + } + ], + "type": { + "name": "int | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "int", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "permalink": "/python/api/class/Request#max_retries", + "parentId": 409 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Crawlee-specific number of finished session rotations for the request." + } + ] + }, + "decorations": [ + { + "name": "property" + } + ], + "flags": {}, + "groups": [], + "id": 31442, + "module": "_request", + "name": "session_rotation_count", + "parsedDocstring": { + "text": "Crawlee-specific number of finished session rotations for the request." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_request.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 392, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/_request.py#L392" + } + ], + "type": { + "name": "int | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "int", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "permalink": "/python/api/class/Request#session_rotation_count", + "parentId": 409 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "decorations": [ + { + "args": ".setter", + "name": "session_rotation_count" + } + ], + "flags": {}, + "groups": [], + "id": 31443, + "module": "_request", + "name": "session_rotation_count", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_request.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 397, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/_request.py#L397" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "flags": {}, + "id": 31444, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "session_rotation_count", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 31445, + "kind": 32768, + "kindString": "Parameter", + "name": "new_session_rotation_count", + "type": { + "name": "int", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 452 + } + ], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "parentId": 451 + } + ], + "permalink": "/python/api/class/Request#session_rotation_count", + "parentId": 409 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "The strategy that was used for enqueuing the request." + } + ] + }, + "decorations": [ + { + "name": "property" + } + ], + "flags": {}, + "groups": [], + "id": 31446, + "module": "_request", + "name": "enqueue_strategy", + "parsedDocstring": { + "text": "The strategy that was used for enqueuing the request." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_request.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 401, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/_request.py#L401" + } + ], + "type": { + "name": "EnqueueStrategy", + "type": "reference", + "target": "88", + "ref": { + "id": 31080, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_types.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 47 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api#EnqueueStrategy" + } + }, + "permalink": "/python/api/class/Request#enqueue_strategy", + "parentId": 409 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "decorations": [ + { + "args": ".setter", + "name": "enqueue_strategy" + } + ], + "flags": {}, + "groups": [], + "id": 31447, + "module": "_request", + "name": "enqueue_strategy", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_request.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 406, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/_request.py#L406" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "flags": {}, + "id": 31448, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "enqueue_strategy", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 31449, + "kind": 32768, + "kindString": "Parameter", + "name": "new_enqueue_strategy", + "type": { + "name": "EnqueueStrategy", + "type": "reference", + "target": "88", + "ref": { + "id": 31080, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_types.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 47 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api#EnqueueStrategy" + } + }, + "parentId": 456 + } + ], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "parentId": 455 + } + ], + "permalink": "/python/api/class/Request#enqueue_strategy", + "parentId": 409 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "The last proxy tier used to process the request." + } + ] + }, + "decorations": [ + { + "name": "property" + } + ], + "flags": {}, + "groups": [], + "id": 31450, + "module": "_request", + "name": "last_proxy_tier", + "parsedDocstring": { + "text": "The last proxy tier used to process the request." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_request.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 410, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/_request.py#L410" + } + ], + "type": { + "name": "int | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "int", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "permalink": "/python/api/class/Request#last_proxy_tier", + "parentId": 409 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "decorations": [ + { + "args": ".setter", + "name": "last_proxy_tier" + } + ], + "flags": {}, + "groups": [], + "id": 31451, + "module": "_request", + "name": "last_proxy_tier", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_request.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 415, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/_request.py#L415" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "flags": {}, + "id": 31452, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "last_proxy_tier", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 31453, + "kind": 32768, + "kindString": "Parameter", + "name": "new_value", + "type": { + "name": "int", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 460 + } + ], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "parentId": 459 + } + ], + "permalink": "/python/api/class/Request#last_proxy_tier", + "parentId": 409 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Indicate whether the request should be enqueued at the front of the queue." + } + ] + }, + "decorations": [ + { + "name": "property" + } + ], + "flags": {}, + "groups": [], + "id": 31454, + "module": "_request", + "name": "forefront", + "parsedDocstring": { + "text": "Indicate whether the request should be enqueued at the front of the queue." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_request.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 419, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/_request.py#L419" + } + ], + "type": { + "name": "bool", + "type": "reference", + "target": 0, + "ref": {} + }, + "permalink": "/python/api/class/Request#forefront", + "parentId": 409 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "decorations": [ + { + "args": ".setter", + "name": "forefront" + } + ], + "flags": {}, + "groups": [], + "id": 31455, + "module": "_request", + "name": "forefront", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_request.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 424, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/_request.py#L424" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "flags": {}, + "id": 31456, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "forefront", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 31457, + "kind": 32768, + "kindString": "Parameter", + "name": "new_value", + "type": { + "name": "bool", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 464 + } + ], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "parentId": 463 + } + ], + "permalink": "/python/api/class/Request#forefront", + "parentId": 409 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Indicates whether the request was handled." + } + ] + }, + "decorations": [ + { + "name": "property" + } + ], + "flags": {}, + "groups": [], + "id": 31458, + "module": "_request", + "name": "was_already_handled", + "parsedDocstring": { + "text": "Indicates whether the request was handled." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_request.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 428, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/_request.py#L428" + } + ], + "type": { + "name": "bool", + "type": "reference", + "target": 0, + "ref": {} + }, + "permalink": "/python/api/class/Request#was_already_handled", + "parentId": 409 + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Represents a request in the Crawlee framework, containing the necessary information for crawling operations.\n\nThe `Request` class is one of the core components in Crawlee, utilized by various components such as request\nproviders, HTTP clients, crawlers, and more. It encapsulates the essential data for executing web requests,\nincluding the URL, HTTP method, headers, payload, and user data. The user data allows custom information\nto be stored and persisted throughout the request lifecycle, including its retries.\n\nKey functionalities include managing the request's identifier (`id`), unique key (`unique_key`) that is used\nfor request deduplication, controlling retries, handling state management, and enabling configuration for session\nrotation and proxy handling.\n\nThe recommended way to create a new instance is by using the `Request.from_url` constructor, which automatically\ngenerates a unique key and identifier based on the URL and request parameters.\n\n### Usage\n\n```python\nfrom crawlee import Request\n\nrequest = Request.from_url('https://crawlee.dev')\n```" + } + ] + }, + "decorations": [ + { + "args": "('Storage data')", + "name": "docs_group" + } + ], + "flags": {}, + "groups": [ + { + "children": [ + 31434, + 31447, + 31455, + 31411, + 31426, + 31451, + 31443, + 31438 + ], + "title": "Methods" + }, + { + "children": [ + 31433, + 31432, + 31446, + 31454, + 31410, + 31430, + 31450, + 31409, + 31441, + 31405, + 31402, + 31408, + 31406, + 31407, + 31431, + 31442, + 31437, + 31403, + 31404, + 31458 + ], + "title": "Properties" + } + ], + "id": 31401, + "module": "_request", + "name": "Request", + "parsedDocstring": { + "text": "Represents a request in the Crawlee framework, containing the necessary information for crawling operations.\n\nThe `Request` class is one of the core components in Crawlee, utilized by various components such as request\nproviders, HTTP clients, crawlers, and more. It encapsulates the essential data for executing web requests,\nincluding the URL, HTTP method, headers, payload, and user data. The user data allows custom information\nto be stored and persisted throughout the request lifecycle, including its retries.\n\nKey functionalities include managing the request's identifier (`id`), unique key (`unique_key`) that is used\nfor request deduplication, controlling retries, handling state management, and enabling configuration for session\nrotation and proxy handling.\n\nThe recommended way to create a new instance is by using the `Request.from_url` constructor, which automatically\ngenerates a unique key and identifier based on the URL and request parameters.\n\n### Usage\n\n```python\nfrom crawlee import Request\n\nrequest = Request.from_url('https://crawlee.dev')\n```" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_request.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 145, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/_request.py#L145" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "extendedBy": [ + { + "name": "RequestWithLock", + "target": "467", + "type": "reference", + "ref": { + "id": 31459, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_request.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 433 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/RequestWithLock" + } + } + ], + "permalink": "/python/api/class/Request", + "categories": [] + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 32779, + "module": "storage_clients.models", + "name": "model_config", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 62, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/models.py#L62" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "overwrites": { + "name": "StorageMetadata.model_config", + "target": 1308, + "type": "reference", + "ref": { + "id": 32767, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 23 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api/class/StorageMetadata#model_config" + } + }, + "permalink": "/python/api/class/RequestQueueMetadata#model_config", + "parentId": 1319 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Indicates whether the queue has been accessed by multiple clients (consumers)." + } + ] + }, + "flags": {}, + "groups": [], + "id": 32780, + "module": "storage_clients.models", + "name": "had_multiple_clients", + "parsedDocstring": { + "text": "Indicates whether the queue has been accessed by multiple clients (consumers)." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 64, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/models.py#L64" + } + ], + "type": { + "name": "bool", + "type": "reference", + "target": 0, + "ref": {} + }, + "permalink": "/python/api/class/RequestQueueMetadata#had_multiple_clients", + "parentId": 1319 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "The number of requests that have been handled from the queue." + } + ] + }, + "flags": {}, + "groups": [], + "id": 32781, + "module": "storage_clients.models", + "name": "handled_request_count", + "parsedDocstring": { + "text": "The number of requests that have been handled from the queue." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 67, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/models.py#L67" + } + ], + "type": { + "name": "int", + "type": "reference", + "target": 0, + "ref": {} + }, + "permalink": "/python/api/class/RequestQueueMetadata#handled_request_count", + "parentId": 1319 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "The number of requests that are still pending in the queue." + } + ] + }, + "flags": {}, + "groups": [], + "id": 32782, + "module": "storage_clients.models", + "name": "pending_request_count", + "parsedDocstring": { + "text": "The number of requests that are still pending in the queue." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 70, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/models.py#L70" + } + ], + "type": { + "name": "int", + "type": "reference", + "target": 0, + "ref": {} + }, + "permalink": "/python/api/class/RequestQueueMetadata#pending_request_count", + "parentId": 1319 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "The total number of requests that have been added to the queue." + } + ] + }, + "flags": {}, + "groups": [], + "id": 32783, + "module": "storage_clients.models", + "name": "total_request_count", + "parsedDocstring": { + "text": "The total number of requests that have been added to the queue." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 73, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/models.py#L73" + } + ], + "type": { + "name": "int", + "type": "reference", + "target": 0, + "ref": {} + }, + "permalink": "/python/api/class/RequestQueueMetadata#total_request_count", + "parentId": 1319 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "The unique identifier of the storage." + } + ] + }, + "flags": {}, + "groups": [], + "id": 35678, + "module": "storage_clients.models", + "name": "id", + "parsedDocstring": { + "text": "The unique identifier of the storage." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 25, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/models.py#L25" + } + ], + "type": { + "name": "Annotated[str, Field(alias='id')]", + "type": "reference", + "target": 0, + "ref": {} + }, + "inheritedFrom": { + "name": "StorageMetadata.id", + "target": 1309, + "type": "reference", + "ref": { + "id": 32768, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 25 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api/class/StorageMetadata#id" + } + }, + "permalink": "/python/api/class/RequestQueueMetadata#id", + "parentId": 1319 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "The name of the storage." + } + ] + }, + "flags": {}, + "groups": [], + "id": 35679, + "module": "storage_clients.models", + "name": "name", + "parsedDocstring": { + "text": "The name of the storage." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 28, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/models.py#L28" + } + ], + "type": { + "name": "Annotated[str | None, Field(alias='name', default=None)]", + "type": "reference", + "target": 0, + "ref": {} + }, + "inheritedFrom": { + "name": "StorageMetadata.name", + "target": 1310, + "type": "reference", + "ref": { + "id": 32769, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 28 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api/class/StorageMetadata#name" + } + }, + "permalink": "/python/api/class/RequestQueueMetadata#name", + "parentId": 1319 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "The timestamp when the storage was last accessed." + } + ] + }, + "flags": {}, + "groups": [], + "id": 35680, + "module": "storage_clients.models", + "name": "accessed_at", + "parsedDocstring": { + "text": "The timestamp when the storage was last accessed." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 31, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/models.py#L31" + } + ], + "type": { + "name": "Annotated[datetime, Field(alias='accessedAt')]", + "type": "reference", + "target": 0, + "ref": {} + }, + "inheritedFrom": { + "name": "StorageMetadata.accessed_at", + "target": 1311, + "type": "reference", + "ref": { + "id": 32770, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 31 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api/class/StorageMetadata#accessed_at" + } + }, + "permalink": "/python/api/class/RequestQueueMetadata#accessed_at", + "parentId": 1319 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "The timestamp when the storage was created." + } + ] + }, + "flags": {}, + "groups": [], + "id": 35681, + "module": "storage_clients.models", + "name": "created_at", + "parsedDocstring": { + "text": "The timestamp when the storage was created." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 34, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/models.py#L34" + } + ], + "type": { + "name": "Annotated[datetime, Field(alias='createdAt')]", + "type": "reference", + "target": 0, + "ref": {} + }, + "inheritedFrom": { + "name": "StorageMetadata.created_at", + "target": 1312, + "type": "reference", + "ref": { + "id": 32771, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 34 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api/class/StorageMetadata#created_at" + } + }, + "permalink": "/python/api/class/RequestQueueMetadata#created_at", + "parentId": 1319 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "The timestamp when the storage was last modified." + } + ] + }, + "flags": {}, + "groups": [], + "id": 35682, + "module": "storage_clients.models", + "name": "modified_at", + "parsedDocstring": { + "text": "The timestamp when the storage was last modified." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 37, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/models.py#L37" + } + ], + "type": { + "name": "Annotated[datetime, Field(alias='modifiedAt')]", + "type": "reference", + "target": 0, + "ref": {} + }, + "inheritedFrom": { + "name": "StorageMetadata.modified_at", + "target": 1313, + "type": "reference", + "ref": { + "id": 32772, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 37 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api/class/StorageMetadata#modified_at" + } + }, + "permalink": "/python/api/class/RequestQueueMetadata#modified_at", + "parentId": 1319 + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Model for a request queue metadata." + } + ] + }, + "decorations": [ + { + "args": "('Storage data')", + "name": "docs_group" + } + ], + "flags": {}, + "groups": [ + { + "children": [ + 35680, + 35681, + 32780, + 32781, + 35678, + 32779, + 35682, + 35679, + 32782, + 32783 + ], + "title": "Properties" + } + ], + "id": 32778, + "module": "storage_clients.models", + "name": "RequestQueueMetadata", + "parsedDocstring": { + "text": "Model for a request queue metadata." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 59, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/models.py#L59" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "extendedTypes": [ + { + "name": "StorageMetadata", + "target": "1307", + "type": "reference", + "ref": { + "id": 32766, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 17 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/StorageMetadata" + } + } + ], + "permalink": "/python/api/class/RequestQueueMetadata", + "categories": [] + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 36990, + "module": "storage_clients.models", + "name": "model_config", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 23, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/models.py#L23" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "permalink": "/python/api/class/StorageMetadata#model_config", + "parentId": 1307 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "The unique identifier of the storage." + } + ] + }, + "flags": {}, + "groups": [], + "id": 36991, + "module": "storage_clients.models", + "name": "id", + "parsedDocstring": { + "text": "The unique identifier of the storage." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 25, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/models.py#L25" + } + ], + "type": { + "name": "str", + "type": "reference", + "target": 0, + "ref": {} + }, + "permalink": "/python/api/class/StorageMetadata#id", + "parentId": 1307 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "The name of the storage." + } + ] + }, + "flags": {}, + "groups": [], + "id": 36992, + "module": "storage_clients.models", + "name": "name", + "parsedDocstring": { + "text": "The name of the storage." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 28, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/models.py#L28" + } + ], + "type": { + "name": "Annotated[str | None, Field(alias='name', default=None)]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "permalink": "/python/api/class/StorageMetadata#name", + "parentId": 1307 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "The timestamp when the storage was last accessed." + } + ] + }, + "flags": {}, + "groups": [], + "id": 36993, + "module": "storage_clients.models", + "name": "accessed_at", + "parsedDocstring": { + "text": "The timestamp when the storage was last accessed." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 31, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/models.py#L31" + } + ], + "type": { + "name": "datetime", + "type": "reference", + "target": 0, + "ref": {} + }, + "permalink": "/python/api/class/StorageMetadata#accessed_at", + "parentId": 1307 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "The timestamp when the storage was created." + } + ] + }, + "flags": {}, + "groups": [], + "id": 36994, + "module": "storage_clients.models", + "name": "created_at", + "parsedDocstring": { + "text": "The timestamp when the storage was created." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 34, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/models.py#L34" + } + ], + "type": { + "name": "datetime", + "type": "reference", + "target": 0, + "ref": {} + }, + "permalink": "/python/api/class/StorageMetadata#created_at", + "parentId": 1307 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "The timestamp when the storage was last modified." + } + ] + }, + "flags": {}, + "groups": [], + "id": 36995, + "module": "storage_clients.models", + "name": "modified_at", + "parsedDocstring": { + "text": "The timestamp when the storage was last modified." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 37, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/models.py#L37" + } + ], + "type": { + "name": "datetime", + "type": "reference", + "target": 0, + "ref": {} + }, + "permalink": "/python/api/class/StorageMetadata#modified_at", + "parentId": 1307 + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Represents the base model for storage metadata.\n\nIt contains common fields shared across all specific storage types." + } + ] + }, + "decorations": [ + { + "args": "('Storage data')", + "name": "docs_group" + } + ], + "flags": {}, + "groups": [ + { + "children": [ + 36993, + 36994, + 36991, + 36990, + 36995, + 36992 + ], + "title": "Properties" + } + ], + "id": 36989, + "module": "storage_clients.models", + "name": "StorageMetadata", + "parsedDocstring": { + "text": "Represents the base model for storage metadata.\n\nIt contains common fields shared across all specific storage types." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 17, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/models.py#L17" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "extendedBy": [ + { + "name": "DatasetMetadata", + "target": "1314", + "type": "reference", + "ref": { + "id": 36996, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 42 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/DatasetMetadata" + } + }, + { + "name": "KeyValueStoreMetadata", + "target": "1317", + "type": "reference", + "ref": { + "id": 36999, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 52 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/KeyValueStoreMetadata" + } + }, + { + "name": "RequestQueueMetadata", + "target": "1319", + "type": "reference", + "ref": { + "id": 37001, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 59 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/RequestQueueMetadata" + } + } + ], + "permalink": "/python/api/class/StorageMetadata", + "categories": [] + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Initialize a new instance.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 39157, + "module": "events._event_manager", + "name": "__init__", + "parsedDocstring": { + "text": "Initialize a new instance.\n", + "args": { + "persist_state_interval": "Interval between emitted `PersistState` events to maintain state persistence.", + "close_timeout": "Optional timeout for canceling pending event listeners if they exceed this duration." + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_event_manager.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 63, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/events/_event_manager.py#L63" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Initialize a new instance.\n" + } + ] + }, + "flags": {}, + "id": 39158, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "__init__", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Interval between emitted `PersistState` events to maintain state persistence." + } + ] + }, + "defaultValue": "timedelta(minutes=1)", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 39159, + "kind": 32768, + "kindString": "Parameter", + "name": "persist_state_interval", + "type": { + "name": "timedelta", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2157 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Optional timeout for canceling pending event listeners if they exceed this duration." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 39160, + "kind": 32768, + "kindString": "Parameter", + "name": "close_timeout", + "type": { + "name": "timedelta | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "timedelta", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2157 + } + ], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "parentId": 2156 + } + ], + "permalink": "/python/api/class/EventManager#__init__", + "parentId": 2155 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Indicate whether the context is active." + } + ] + }, + "decorations": [ + { + "name": "property" + } + ], + "flags": {}, + "groups": [], + "id": 39161, + "module": "events._event_manager", + "name": "active", + "parsedDocstring": { + "text": "Indicate whether the context is active." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_event_manager.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 100, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/events/_event_manager.py#L100" + } + ], + "type": { + "name": "bool", + "type": "reference", + "target": 0, + "ref": {} + }, + "permalink": "/python/api/class/EventManager#active", + "parentId": 2155 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Initialize the event manager upon entering the async context." + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 39162, + "module": "events._event_manager", + "name": "__aenter__", + "parsedDocstring": { + "text": "Initialize the event manager upon entering the async context." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_event_manager.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 104, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/events/_event_manager.py#L104" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Initialize the event manager upon entering the async context." + } + ] + }, + "flags": {}, + "id": 39163, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "__aenter__", + "parameters": [], + "type": { + "name": "EventManager", + "type": "reference", + "target": "2155", + "ref": { + "id": 39156, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_event_manager.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 55 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/EventManager" + } + }, + "parentId": 2161 + } + ], + "permalink": "/python/api/class/EventManager#__aenter__", + "parentId": 2155 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Close the local event manager upon exiting the async context.\n\nThis will stop listening for the events, and it will wait for all the event listeners to finish.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 39164, + "module": "events._event_manager", + "name": "__aexit__", + "parsedDocstring": { + "text": "Close the local event manager upon exiting the async context.\n\nThis will stop listening for the events, and it will wait for all the event listeners to finish.\n" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_event_manager.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 113, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/events/_event_manager.py#L113" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Close the local event manager upon exiting the async context.\n\nThis will stop listening for the events, and it will wait for all the event listeners to finish.\n" + } + ] + }, + "flags": {}, + "id": 39165, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "__aexit__", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 39166, + "kind": 32768, + "kindString": "Parameter", + "name": "exc_type", + "type": { + "name": "type[BaseException] | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "type", + "typeArguments": [ + { + "type": "reference", + "name": "BaseException", + "target": 0, + "ref": {} + } + ], + "target": "3422", + "ref": { + "id": 40423, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_utils/sitemap.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 62 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api/class/SitemapSource#type" + } + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2164 + }, + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 39167, + "kind": 32768, + "kindString": "Parameter", + "name": "exc_value", + "type": { + "name": "BaseException | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "BaseException", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2164 + }, + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 39168, + "kind": 32768, + "kindString": "Parameter", + "name": "exc_traceback", + "type": { + "name": "TracebackType | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "TracebackType", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2164 + } + ], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "parentId": 2163 + } + ], + "permalink": "/python/api/class/EventManager#__aexit__", + "parentId": 2155 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Register an event listener for a specific event.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 39169, + "module": "events._event_manager", + "name": "on", + "parsedDocstring": { + "text": "Register an event listener for a specific event.\n", + "args": { + "event": "The event for which to listen to.", + "listener": "The function (sync or async) which is to be called when the event is emitted." + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_event_manager.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 160, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/events/_event_manager.py#L160" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Register an event listener for a specific event.\n" + } + ] + }, + "flags": {}, + "id": 39170, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "on", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The event for which to listen to." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 39171, + "kind": 32768, + "kindString": "Parameter", + "name": "event", + "type": { + "name": "Event", + "type": "reference", + "target": "2116", + "ref": { + "id": 39117, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 15 + } + ], + "kind": 8, + "permalink": "https://crawlee.dev/python/api/enum/Event" + } + }, + "parentId": 2169 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The function (sync or async) which is to be called when the event is emitted." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 39172, + "kind": 32768, + "kindString": "Parameter", + "name": "listener", + "type": { + "name": "EventListener", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "Any", + "target": 0, + "ref": {} + } + ], + "target": "2150", + "ref": { + "id": 39151, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 112 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api#EventListener" + } + }, + "parentId": 2169 + } + ], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "parentId": 2168 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Register an event listener for a specific event.\n" + } + ] + }, + "flags": {}, + "id": 39184, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "on", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The event for which to listen to." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 39185, + "kind": 32768, + "kindString": "Parameter", + "name": "event", + "type": { + "name": "Literal", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "Event.PERSIST_STATE", + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + }, + "parentId": 2183 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The function (sync or async) which is to be called when the event is emitted." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 39186, + "kind": 32768, + "kindString": "Parameter", + "name": "listener", + "type": { + "name": "EventListener", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "EventPersistStateData", + "target": "2129", + "ref": { + "id": 39130, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 40 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/EventPersistStateData" + } + } + ], + "target": "2150", + "ref": { + "id": 39151, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 112 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api#EventListener" + } + }, + "parentId": 2183 + } + ], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "parentId": 2168 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Register an event listener for a specific event.\n" + } + ] + }, + "flags": {}, + "id": 39187, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "on", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The event for which to listen to." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 39188, + "kind": 32768, + "kindString": "Parameter", + "name": "event", + "type": { + "name": "Literal", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "Event.SYSTEM_INFO", + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + }, + "parentId": 2186 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The function (sync or async) which is to be called when the event is emitted." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 39189, + "kind": 32768, + "kindString": "Parameter", + "name": "listener", + "type": { + "name": "EventListener", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "EventSystemInfoData", + "target": "2132", + "ref": { + "id": 39133, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 49 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/EventSystemInfoData" + } + } + ], + "target": "2150", + "ref": { + "id": 39151, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 112 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api#EventListener" + } + }, + "parentId": 2186 + } + ], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "parentId": 2168 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Register an event listener for a specific event.\n" + } + ] + }, + "flags": {}, + "id": 39190, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "on", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The event for which to listen to." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 39191, + "kind": 32768, + "kindString": "Parameter", + "name": "event", + "type": { + "name": "Literal", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "Event.MIGRATING", + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + }, + "parentId": 2189 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The function (sync or async) which is to be called when the event is emitted." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 39192, + "kind": 32768, + "kindString": "Parameter", + "name": "listener", + "type": { + "name": "EventListener", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "EventMigratingData", + "target": "2136", + "ref": { + "id": 39137, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 62 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/EventMigratingData" + } + } + ], + "target": "2150", + "ref": { + "id": 39151, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 112 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api#EventListener" + } + }, + "parentId": 2189 + } + ], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "parentId": 2168 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Register an event listener for a specific event.\n" + } + ] + }, + "flags": {}, + "id": 39193, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "on", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The event for which to listen to." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 39194, + "kind": 32768, + "kindString": "Parameter", + "name": "event", + "type": { + "name": "Literal", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "Event.ABORTING", + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + }, + "parentId": 2192 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The function (sync or async) which is to be called when the event is emitted." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 39195, + "kind": 32768, + "kindString": "Parameter", + "name": "listener", + "type": { + "name": "EventListener", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "EventAbortingData", + "target": "2139", + "ref": { + "id": 39140, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 73 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/EventAbortingData" + } + } + ], + "target": "2150", + "ref": { + "id": 39151, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 112 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api#EventListener" + } + }, + "parentId": 2192 + } + ], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "parentId": 2168 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Register an event listener for a specific event.\n" + } + ] + }, + "flags": {}, + "id": 39196, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "on", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The event for which to listen to." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 39197, + "kind": 32768, + "kindString": "Parameter", + "name": "event", + "type": { + "name": "Literal", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "Event.EXIT", + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + }, + "parentId": 2195 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The function (sync or async) which is to be called when the event is emitted." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 39198, + "kind": 32768, + "kindString": "Parameter", + "name": "listener", + "type": { + "name": "EventListener", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "EventExitData", + "target": "2141", + "ref": { + "id": 39142, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 80 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/EventExitData" + } + } + ], + "target": "2150", + "ref": { + "id": 39151, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 112 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api#EventListener" + } + }, + "parentId": 2195 + } + ], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "parentId": 2168 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Register an event listener for a specific event.\n" + } + ] + }, + "flags": {}, + "id": 39199, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "on", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The event for which to listen to." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 39200, + "kind": 32768, + "kindString": "Parameter", + "name": "event", + "type": { + "name": "Literal", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "Event.CRAWLER_STATUS", + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + }, + "parentId": 2198 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The function (sync or async) which is to be called when the event is emitted." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 39201, + "kind": 32768, + "kindString": "Parameter", + "name": "listener", + "type": { + "name": "EventListener", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "EventCrawlerStatusData", + "target": "2143", + "ref": { + "id": 39144, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 87 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/EventCrawlerStatusData" + } + } + ], + "target": "2150", + "ref": { + "id": 39151, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 112 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api#EventListener" + } + }, + "parentId": 2198 + } + ], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "parentId": 2168 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Register an event listener for a specific event.\n" + } + ] + }, + "flags": {}, + "id": 39202, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "on", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The event for which to listen to." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 39203, + "kind": 32768, + "kindString": "Parameter", + "name": "event", + "type": { + "name": "Event", + "type": "reference", + "target": "2116", + "ref": { + "id": 39117, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 15 + } + ], + "kind": 8, + "permalink": "https://crawlee.dev/python/api/enum/Event" + } + }, + "parentId": 2201 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The function (sync or async) which is to be called when the event is emitted." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 39204, + "kind": 32768, + "kindString": "Parameter", + "name": "listener", + "type": { + "name": "EventListener", + "type": "reference", + "typeArguments": [ + { + "type": "literal", + "value": {} + } + ], + "target": "2150", + "ref": { + "id": 39151, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 112 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api#EventListener" + } + }, + "parentId": 2201 + } + ], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "parentId": 2168 + } + ], + "permalink": "/python/api/class/EventManager#on", + "parentId": 2155 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Remove a specific listener or all listeners for an event.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 39173, + "module": "events._event_manager", + "name": "off", + "parsedDocstring": { + "text": "Remove a specific listener or all listeners for an event.\n", + "args": { + "event": "The Actor event for which to remove listeners.", + "listener": "The listener which is supposed to be removed. If not passed, all listeners of this event\nare removed." + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_event_manager.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 210, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/events/_event_manager.py#L210" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Remove a specific listener or all listeners for an event.\n" + } + ] + }, + "flags": {}, + "id": 39174, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "off", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The Actor event for which to remove listeners." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 39175, + "kind": 32768, + "kindString": "Parameter", + "name": "event", + "type": { + "name": "Event", + "type": "reference", + "target": "2116", + "ref": { + "id": 39117, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 15 + } + ], + "kind": 8, + "permalink": "https://crawlee.dev/python/api/enum/Event" + } + }, + "parentId": 2173 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The listener which is supposed to be removed. If not passed, all listeners of this event\nare removed." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 39176, + "kind": 32768, + "kindString": "Parameter", + "name": "listener", + "type": { + "name": "EventListener[Any] | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "EventListener", + "typeArguments": [ + { + "type": "reference", + "name": "Any", + "target": 0, + "ref": {} + } + ], + "target": "2150", + "ref": { + "id": 39151, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 112 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api#EventListener" + } + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2173 + } + ], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "parentId": 2172 + } + ], + "permalink": "/python/api/class/EventManager#off", + "parentId": 2155 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Emit an event with the associated data to all registered listeners.\n" + } + ] + }, + "decorations": [ + { + "name": "ensure_context" + } + ], + "flags": {}, + "groups": [], + "id": 39177, + "module": "events._event_manager", + "name": "emit", + "parsedDocstring": { + "text": "Emit an event with the associated data to all registered listeners.\n", + "args": { + "event": "The event which will be emitted.", + "event_data": "The data which will be passed to the event listeners." + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_event_manager.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 242, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/events/_event_manager.py#L242" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Emit an event with the associated data to all registered listeners.\n" + } + ] + }, + "flags": {}, + "id": 39178, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "emit", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The event which will be emitted." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 39179, + "kind": 32768, + "kindString": "Parameter", + "name": "event", + "type": { + "name": "Event", + "type": "reference", + "target": "2116", + "ref": { + "id": 39117, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 15 + } + ], + "kind": 8, + "permalink": "https://crawlee.dev/python/api/enum/Event" + } + }, + "parentId": 2177 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The data which will be passed to the event listeners." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 39180, + "kind": 32768, + "kindString": "Parameter", + "name": "event_data", + "type": { + "name": "EventData", + "type": "reference", + "target": "2147", + "ref": { + "id": 39148, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 99 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api#EventData" + } + }, + "parentId": 2177 + } + ], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "parentId": 2176 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Emit an event with the associated data to all registered listeners.\n" + } + ] + }, + "flags": {}, + "id": 39205, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "emit", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The event which will be emitted." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 39206, + "kind": 32768, + "kindString": "Parameter", + "name": "event", + "type": { + "name": "Literal", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "Event.PERSIST_STATE", + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + }, + "parentId": 2204 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The data which will be passed to the event listeners." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 39207, + "kind": 32768, + "kindString": "Parameter", + "name": "event_data", + "type": { + "name": "EventPersistStateData", + "type": "reference", + "target": "2129", + "ref": { + "id": 39130, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 40 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/EventPersistStateData" + } + }, + "parentId": 2204 + } + ], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "parentId": 2176 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Emit an event with the associated data to all registered listeners.\n" + } + ] + }, + "flags": {}, + "id": 39208, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "emit", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The event which will be emitted." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 39209, + "kind": 32768, + "kindString": "Parameter", + "name": "event", + "type": { + "name": "Literal", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "Event.SYSTEM_INFO", + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + }, + "parentId": 2207 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The data which will be passed to the event listeners." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 39210, + "kind": 32768, + "kindString": "Parameter", + "name": "event_data", + "type": { + "name": "EventSystemInfoData", + "type": "reference", + "target": "2132", + "ref": { + "id": 39133, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 49 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/EventSystemInfoData" + } + }, + "parentId": 2207 + } + ], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "parentId": 2176 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Emit an event with the associated data to all registered listeners.\n" + } + ] + }, + "flags": {}, + "id": 39211, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "emit", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The event which will be emitted." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 39212, + "kind": 32768, + "kindString": "Parameter", + "name": "event", + "type": { + "name": "Literal", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "Event.MIGRATING", + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + }, + "parentId": 2210 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The data which will be passed to the event listeners." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 39213, + "kind": 32768, + "kindString": "Parameter", + "name": "event_data", + "type": { + "name": "EventMigratingData", + "type": "reference", + "target": "2136", + "ref": { + "id": 39137, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 62 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/EventMigratingData" + } + }, + "parentId": 2210 + } + ], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "parentId": 2176 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Emit an event with the associated data to all registered listeners.\n" + } + ] + }, + "flags": {}, + "id": 39214, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "emit", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The event which will be emitted." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 39215, + "kind": 32768, + "kindString": "Parameter", + "name": "event", + "type": { + "name": "Literal", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "Event.ABORTING", + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + }, + "parentId": 2213 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The data which will be passed to the event listeners." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 39216, + "kind": 32768, + "kindString": "Parameter", + "name": "event_data", + "type": { + "name": "EventAbortingData", + "type": "reference", + "target": "2139", + "ref": { + "id": 39140, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 73 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/EventAbortingData" + } + }, + "parentId": 2213 + } + ], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "parentId": 2176 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Emit an event with the associated data to all registered listeners.\n" + } + ] + }, + "flags": {}, + "id": 39217, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "emit", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The event which will be emitted." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 39218, + "kind": 32768, + "kindString": "Parameter", + "name": "event", + "type": { + "name": "Literal", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "Event.EXIT", + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + }, + "parentId": 2216 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The data which will be passed to the event listeners." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 39219, + "kind": 32768, + "kindString": "Parameter", + "name": "event_data", + "type": { + "name": "EventExitData", + "type": "reference", + "target": "2141", + "ref": { + "id": 39142, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 80 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/EventExitData" + } + }, + "parentId": 2216 + } + ], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "parentId": 2176 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Emit an event with the associated data to all registered listeners.\n" + } + ] + }, + "flags": {}, + "id": 39220, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "emit", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The event which will be emitted." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 39221, + "kind": 32768, + "kindString": "Parameter", + "name": "event", + "type": { + "name": "Literal", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "Event.CRAWLER_STATUS", + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + }, + "parentId": 2219 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The data which will be passed to the event listeners." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 39222, + "kind": 32768, + "kindString": "Parameter", + "name": "event_data", + "type": { + "name": "EventCrawlerStatusData", + "type": "reference", + "target": "2143", + "ref": { + "id": 39144, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 87 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/EventCrawlerStatusData" + } + }, + "parentId": 2219 + } + ], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "parentId": 2176 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Emit an event with the associated data to all registered listeners.\n" + } + ] + }, + "flags": {}, + "id": 39223, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "emit", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The event which will be emitted." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 39224, + "kind": 32768, + "kindString": "Parameter", + "name": "event", + "type": { + "name": "Event", + "type": "reference", + "target": "2116", + "ref": { + "id": 39117, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 15 + } + ], + "kind": 8, + "permalink": "https://crawlee.dev/python/api/enum/Event" + } + }, + "parentId": 2222 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The data which will be passed to the event listeners." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 39225, + "kind": 32768, + "kindString": "Parameter", + "name": "event_data", + "type": { + "name": "Any", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2222 + } + ], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "parentId": 2176 + } + ], + "permalink": "/python/api/class/EventManager#emit", + "parentId": 2155 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Wait for all currently executing event listeners to complete.\n" + } + ] + }, + "decorations": [ + { + "name": "ensure_context" + } + ], + "flags": {}, + "groups": [], + "id": 39181, + "module": "events._event_manager", + "name": "wait_for_all_listeners_to_complete", + "parsedDocstring": { + "text": "Wait for all currently executing event listeners to complete.\n", + "args": { + "timeout": "The maximum time to wait for the event listeners to finish. If they do not complete within\nthe specified timeout, they will be canceled." + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_event_manager.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 252, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/events/_event_manager.py#L252" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Wait for all currently executing event listeners to complete.\n" + } + ] + }, + "flags": {}, + "id": 39182, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "wait_for_all_listeners_to_complete", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The maximum time to wait for the event listeners to finish. If they do not complete within\nthe specified timeout, they will be canceled." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 39183, + "kind": 32768, + "kindString": "Parameter", + "name": "timeout", + "type": { + "name": "timedelta | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "timedelta", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2181 + } + ], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "parentId": 2180 + } + ], + "permalink": "/python/api/class/EventManager#wait_for_all_listeners_to_complete", + "parentId": 2155 + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Manage events and their listeners, enabling registration, emission, and execution control.\n\nIt allows for registering event listeners, emitting events, and ensuring all listeners complete their execution.\nBuilt on top of `pyee.asyncio.AsyncIOEventEmitter`. It implements additional features such as waiting for all\nlisteners to complete and emitting `PersistState` events at regular intervals." + } + ] + }, + "decorations": [ + { + "args": "('Event managers')", + "name": "docs_group" + } + ], + "flags": {}, + "groups": [ + { + "children": [ + 39162, + 39164, + 39157, + 39177, + 39173, + 39169, + 39181 + ], + "title": "Methods" + }, + { + "children": [ + 39161 + ], + "title": "Properties" + } + ], + "id": 39156, + "module": "events._event_manager", + "name": "EventManager", + "parsedDocstring": { + "text": "Manage events and their listeners, enabling registration, emission, and execution control.\n\nIt allows for registering event listeners, emitting events, and ensuring all listeners complete their execution.\nBuilt on top of `pyee.asyncio.AsyncIOEventEmitter`. It implements additional features such as waiting for all\nlisteners to complete and emitting `PersistState` events at regular intervals." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_event_manager.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 55, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/events/_event_manager.py#L55" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "extendedBy": [ + { + "name": "LocalEventManager", + "target": "2226", + "type": "reference", + "ref": { + "id": 39227, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_local_event_manager.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 26 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/LocalEventManager" + } + } + ], + "permalink": "/python/api/class/EventManager", + "categories": [] + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Initialize a new instance.\n\nIn most cases, you should use the `from_config` constructor to create a new instance based on\nthe provided configuration.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 42650, + "module": "events._local_event_manager", + "name": "__init__", + "parsedDocstring": { + "text": "Initialize a new instance.\n\nIn most cases, you should use the `from_config` constructor to create a new instance based on\nthe provided configuration.\n", + "args": { + "system_info_interval": "Interval at which `SystemInfo` events are emitted.", + "event_manager_options": "Additional options for the parent class." + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_local_event_manager.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 34, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/events/_local_event_manager.py#L34" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Initialize a new instance.\n\nIn most cases, you should use the `from_config` constructor to create a new instance based on\nthe provided configuration.\n" + } + ] + }, + "flags": {}, + "id": 42651, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "__init__", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Interval at which `SystemInfo` events are emitted." + } + ] + }, + "defaultValue": "timedelta(seconds=1)", + "flags": { + "isOptional": true, + "keyword-only": false + }, + "id": 42652, + "kind": 32768, + "kindString": "Parameter", + "name": "system_info_interval", + "type": { + "name": "timedelta", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2228 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Interval between emitted `PersistState` events to maintain state persistence." + } + ] + }, + "flags": { + "keyword-only": true, + "optional": true + }, + "groups": [], + "id": 42576, + "module": "events._event_manager", + "name": "persist_state_interval", + "parsedDocstring": { + "text": "Interval between emitted `PersistState` events to maintain state persistence." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_event_manager.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 47, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/events/_event_manager.py#L47" + } + ], + "type": { + "name": "NotRequired", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "timedelta", + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + }, + "parentId": 2228 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Optional timeout for canceling pending event listeners if they exceed this duration." + } + ] + }, + "flags": { + "keyword-only": true, + "optional": true + }, + "groups": [], + "id": 42577, + "module": "events._event_manager", + "name": "close_timeout", + "parsedDocstring": { + "text": "Optional timeout for canceling pending event listeners if they exceed this duration." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_event_manager.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 50, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/events/_event_manager.py#L50" + } + ], + "type": { + "name": "NotRequired", + "type": "reference", + "typeArguments": [ + { + "type": "union", + "types": [ + { + "type": "reference", + "name": "timedelta", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + } + ], + "target": 0, + "ref": {} + }, + "parentId": 2228 + } + ], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "overwrites": { + "name": "EventManager.__init__", + "target": 2156, + "type": "reference", + "ref": { + "id": 42579, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_event_manager.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 63 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/EventManager#__init__" + } + }, + "parentId": 2227 + } + ], + "overwrites": { + "name": "EventManager.__init__", + "target": 2156, + "type": "reference", + "ref": { + "id": 42579, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_event_manager.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 63 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/EventManager#__init__" + } + }, + "permalink": "/python/api/class/LocalEventManager#__init__", + "parentId": 2226 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Initialize a new instance based on the provided `Configuration`.\n" + } + ] + }, + "decorations": [ + { + "name": "classmethod" + } + ], + "flags": {}, + "groups": [], + "id": 42654, + "module": "events._local_event_manager", + "name": "from_config", + "parsedDocstring": { + "text": "Initialize a new instance based on the provided `Configuration`.\n", + "args": { + "config": "The `Configuration` instance. Uses the global (default) one if not provided." + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_local_event_manager.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 59, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/events/_local_event_manager.py#L59" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Initialize a new instance based on the provided `Configuration`.\n" + } + ] + }, + "flags": {}, + "id": 42655, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "from_config", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The `Configuration` instance. Uses the global (default) one if not provided." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": false + }, + "id": 42656, + "kind": 32768, + "kindString": "Parameter", + "name": "config", + "type": { + "name": "Configuration | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "Configuration", + "target": "2", + "ref": { + "id": 40425, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/configuration.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 20 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/Configuration" + } + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2232 + } + ], + "type": { + "name": "LocalEventManager", + "type": "reference", + "target": "2226", + "ref": { + "id": 42649, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_local_event_manager.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 26 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/LocalEventManager" + } + }, + "parentId": 2231 + } + ], + "permalink": "/python/api/class/LocalEventManager#from_config", + "parentId": 2226 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Initialize the local event manager upon entering the async context.\n\nIt starts emitting system info events at regular intervals." + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 42657, + "module": "events._local_event_manager", + "name": "__aenter__", + "parsedDocstring": { + "text": "Initialize the local event manager upon entering the async context.\n\nIt starts emitting system info events at regular intervals." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_local_event_manager.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 72, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/events/_local_event_manager.py#L72" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Initialize the local event manager upon entering the async context.\n\nIt starts emitting system info events at regular intervals." + } + ] + }, + "flags": {}, + "id": 42658, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "__aenter__", + "parameters": [], + "type": { + "name": "LocalEventManager", + "type": "reference", + "target": "2226", + "ref": { + "id": 42649, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_local_event_manager.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 26 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/LocalEventManager" + } + }, + "overwrites": { + "name": "EventManager.__aenter__", + "target": 2161, + "type": "reference", + "ref": { + "id": 42584, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_event_manager.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 104 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/EventManager#__aenter__" + } + }, + "parentId": 2234 + } + ], + "overwrites": { + "name": "EventManager.__aenter__", + "target": 2161, + "type": "reference", + "ref": { + "id": 42584, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_event_manager.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 104 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/EventManager#__aenter__" + } + }, + "permalink": "/python/api/class/LocalEventManager#__aenter__", + "parentId": 2226 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Close the local event manager upon exiting the async context.\n\nIt stops emitting system info events and closes the event manager." + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 42659, + "module": "events._local_event_manager", + "name": "__aexit__", + "parsedDocstring": { + "text": "Close the local event manager upon exiting the async context.\n\nIt stops emitting system info events and closes the event manager." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_local_event_manager.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 84, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/events/_local_event_manager.py#L84" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Close the local event manager upon exiting the async context.\n\nIt stops emitting system info events and closes the event manager." + } + ] + }, + "flags": {}, + "id": 42660, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "__aexit__", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 42661, + "kind": 32768, + "kindString": "Parameter", + "name": "exc_type", + "type": { + "name": "type[BaseException] | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "type", + "typeArguments": [ + { + "type": "reference", + "name": "BaseException", + "target": 0, + "ref": {} + } + ], + "target": "3422", + "ref": { + "id": 43845, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_utils/sitemap.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 62 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api/class/SitemapSource#type" + } + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2237 + }, + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 42662, + "kind": 32768, + "kindString": "Parameter", + "name": "exc_value", + "type": { + "name": "BaseException | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "BaseException", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2237 + }, + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 42663, + "kind": 32768, + "kindString": "Parameter", + "name": "exc_traceback", + "type": { + "name": "TracebackType | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "TracebackType", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2237 + } + ], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "overwrites": { + "name": "EventManager.__aexit__", + "target": 2163, + "type": "reference", + "ref": { + "id": 42586, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_event_manager.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 113 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/EventManager#__aexit__" + } + }, + "parentId": 2236 + } + ], + "overwrites": { + "name": "EventManager.__aexit__", + "target": 2163, + "type": "reference", + "ref": { + "id": 42586, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_event_manager.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 113 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/EventManager#__aexit__" + } + }, + "permalink": "/python/api/class/LocalEventManager#__aexit__", + "parentId": 2226 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Indicate whether the context is active." + } + ] + }, + "decorations": [ + { + "name": "property" + } + ], + "flags": {}, + "groups": [], + "id": 44577, + "module": "events._event_manager", + "name": "active", + "parsedDocstring": { + "text": "Indicate whether the context is active." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_event_manager.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 100, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/events/_event_manager.py#L100" + } + ], + "type": { + "name": "bool", + "type": "reference", + "target": 0, + "ref": {} + }, + "inheritedFrom": { + "name": "EventManager.active", + "target": 2160, + "type": "reference", + "ref": { + "id": 42583, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_event_manager.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 100 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api/class/EventManager#active" + } + }, + "permalink": "/python/api/class/LocalEventManager#active", + "parentId": 2226 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Register an event listener for a specific event.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 44578, + "module": "events._event_manager", + "name": "on", + "parsedDocstring": { + "text": "Register an event listener for a specific event.\n", + "args": { + "event": "The event for which to listen to.", + "listener": "The function (sync or async) which is to be called when the event is emitted." + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_event_manager.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 160, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/events/_event_manager.py#L160" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Register an event listener for a specific event.\n" + } + ] + }, + "flags": {}, + "id": 42592, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "on", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The event for which to listen to." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 42593, + "kind": 32768, + "kindString": "Parameter", + "name": "event", + "type": { + "name": "Event", + "type": "reference", + "target": "2116", + "ref": { + "id": 42539, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 15 + } + ], + "kind": 8, + "permalink": "https://crawlee.dev/python/api/enum/Event" + } + }, + "parentId": 2169 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The function (sync or async) which is to be called when the event is emitted." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 42594, + "kind": 32768, + "kindString": "Parameter", + "name": "listener", + "type": { + "name": "EventListener[Any]", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2169 + } + ], + "type": { + "name": "None", + "type": "reference", + "target": 0, + "ref": {} + }, + "inheritedFrom": { + "name": "EventManager.on", + "target": 2168, + "type": "reference", + "ref": { + "id": 42591, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_event_manager.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 160 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/EventManager#on" + } + }, + "parentId": 4155 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Register an event listener for a specific event.\n" + } + ] + }, + "flags": {}, + "id": 42606, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "on", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The event for which to listen to." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 42607, + "kind": 32768, + "kindString": "Parameter", + "name": "event", + "type": { + "name": "Literal[Event.PERSIST_STATE]", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2183 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The function (sync or async) which is to be called when the event is emitted." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 42608, + "kind": 32768, + "kindString": "Parameter", + "name": "listener", + "type": { + "name": "EventListener[EventPersistStateData]", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2183 + } + ], + "type": { + "name": "None", + "type": "reference", + "target": 0, + "ref": {} + }, + "inheritedFrom": { + "name": "EventManager.on", + "target": 2168, + "type": "reference", + "ref": { + "id": 42591, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_event_manager.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 160 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/EventManager#on" + } + }, + "parentId": 4155 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Register an event listener for a specific event.\n" + } + ] + }, + "flags": {}, + "id": 42609, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "on", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The event for which to listen to." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 42610, + "kind": 32768, + "kindString": "Parameter", + "name": "event", + "type": { + "name": "Literal[Event.SYSTEM_INFO]", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2186 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The function (sync or async) which is to be called when the event is emitted." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 42611, + "kind": 32768, + "kindString": "Parameter", + "name": "listener", + "type": { + "name": "EventListener[EventSystemInfoData]", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2186 + } + ], + "type": { + "name": "None", + "type": "reference", + "target": 0, + "ref": {} + }, + "inheritedFrom": { + "name": "EventManager.on", + "target": 2168, + "type": "reference", + "ref": { + "id": 42591, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_event_manager.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 160 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/EventManager#on" + } + }, + "parentId": 4155 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Register an event listener for a specific event.\n" + } + ] + }, + "flags": {}, + "id": 42612, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "on", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The event for which to listen to." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 42613, + "kind": 32768, + "kindString": "Parameter", + "name": "event", + "type": { + "name": "Literal[Event.MIGRATING]", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2189 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The function (sync or async) which is to be called when the event is emitted." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 42614, + "kind": 32768, + "kindString": "Parameter", + "name": "listener", + "type": { + "name": "EventListener[EventMigratingData]", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2189 + } + ], + "type": { + "name": "None", + "type": "reference", + "target": 0, + "ref": {} + }, + "inheritedFrom": { + "name": "EventManager.on", + "target": 2168, + "type": "reference", + "ref": { + "id": 42591, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_event_manager.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 160 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/EventManager#on" + } + }, + "parentId": 4155 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Register an event listener for a specific event.\n" + } + ] + }, + "flags": {}, + "id": 42615, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "on", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The event for which to listen to." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 42616, + "kind": 32768, + "kindString": "Parameter", + "name": "event", + "type": { + "name": "Literal[Event.ABORTING]", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2192 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The function (sync or async) which is to be called when the event is emitted." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 42617, + "kind": 32768, + "kindString": "Parameter", + "name": "listener", + "type": { + "name": "EventListener[EventAbortingData]", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2192 + } + ], + "type": { + "name": "None", + "type": "reference", + "target": 0, + "ref": {} + }, + "inheritedFrom": { + "name": "EventManager.on", + "target": 2168, + "type": "reference", + "ref": { + "id": 42591, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_event_manager.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 160 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/EventManager#on" + } + }, + "parentId": 4155 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Register an event listener for a specific event.\n" + } + ] + }, + "flags": {}, + "id": 42618, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "on", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The event for which to listen to." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 42619, + "kind": 32768, + "kindString": "Parameter", + "name": "event", + "type": { + "name": "Literal[Event.EXIT]", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2195 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The function (sync or async) which is to be called when the event is emitted." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 42620, + "kind": 32768, + "kindString": "Parameter", + "name": "listener", + "type": { + "name": "EventListener[EventExitData]", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2195 + } + ], + "type": { + "name": "None", + "type": "reference", + "target": 0, + "ref": {} + }, + "inheritedFrom": { + "name": "EventManager.on", + "target": 2168, + "type": "reference", + "ref": { + "id": 42591, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_event_manager.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 160 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/EventManager#on" + } + }, + "parentId": 4155 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Register an event listener for a specific event.\n" + } + ] + }, + "flags": {}, + "id": 42621, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "on", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The event for which to listen to." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 42622, + "kind": 32768, + "kindString": "Parameter", + "name": "event", + "type": { + "name": "Literal[Event.CRAWLER_STATUS]", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2198 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The function (sync or async) which is to be called when the event is emitted." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 42623, + "kind": 32768, + "kindString": "Parameter", + "name": "listener", + "type": { + "name": "EventListener[EventCrawlerStatusData]", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2198 + } + ], + "type": { + "name": "None", + "type": "reference", + "target": 0, + "ref": {} + }, + "inheritedFrom": { + "name": "EventManager.on", + "target": 2168, + "type": "reference", + "ref": { + "id": 42591, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_event_manager.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 160 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/EventManager#on" + } + }, + "parentId": 4155 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Register an event listener for a specific event.\n" + } + ] + }, + "flags": {}, + "id": 42624, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "on", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The event for which to listen to." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 42625, + "kind": 32768, + "kindString": "Parameter", + "name": "event", + "type": { + "name": "Event", + "type": "reference", + "target": "2116", + "ref": { + "id": 42539, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 15 + } + ], + "kind": 8, + "permalink": "https://crawlee.dev/python/api/enum/Event" + } + }, + "parentId": 2201 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The function (sync or async) which is to be called when the event is emitted." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 42626, + "kind": 32768, + "kindString": "Parameter", + "name": "listener", + "type": { + "name": "EventListener[None]", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2201 + } + ], + "type": { + "name": "None", + "type": "reference", + "target": 0, + "ref": {} + }, + "inheritedFrom": { + "name": "EventManager.on", + "target": 2168, + "type": "reference", + "ref": { + "id": 42591, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_event_manager.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 160 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/EventManager#on" + } + }, + "parentId": 4155 + } + ], + "inheritedFrom": { + "name": "EventManager.on", + "target": 2168, + "type": "reference", + "ref": { + "id": 42591, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_event_manager.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 160 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/EventManager#on" + } + }, + "permalink": "/python/api/class/LocalEventManager#on", + "parentId": 2226 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Remove a specific listener or all listeners for an event.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 44579, + "module": "events._event_manager", + "name": "off", + "parsedDocstring": { + "text": "Remove a specific listener or all listeners for an event.\n", + "args": { + "event": "The Actor event for which to remove listeners.", + "listener": "The listener which is supposed to be removed. If not passed, all listeners of this event\nare removed." + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_event_manager.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 210, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/events/_event_manager.py#L210" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Remove a specific listener or all listeners for an event.\n" + } + ] + }, + "flags": {}, + "id": 42596, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "off", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The Actor event for which to remove listeners." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 42597, + "kind": 32768, + "kindString": "Parameter", + "name": "event", + "type": { + "name": "Event", + "type": "reference", + "target": "2116", + "ref": { + "id": 42539, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 15 + } + ], + "kind": 8, + "permalink": "https://crawlee.dev/python/api/enum/Event" + } + }, + "parentId": 2173 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The listener which is supposed to be removed. If not passed, all listeners of this event\nare removed." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 42598, + "kind": 32768, + "kindString": "Parameter", + "name": "listener", + "type": { + "name": "EventListener[Any] | None", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2173 + } + ], + "type": { + "name": "None", + "type": "reference", + "target": 0, + "ref": {} + }, + "inheritedFrom": { + "name": "EventManager.off", + "target": 2172, + "type": "reference", + "ref": { + "id": 42595, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_event_manager.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 210 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/EventManager#off" + } + }, + "parentId": 4156 + } + ], + "inheritedFrom": { + "name": "EventManager.off", + "target": 2172, + "type": "reference", + "ref": { + "id": 42595, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_event_manager.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 210 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/EventManager#off" + } + }, + "permalink": "/python/api/class/LocalEventManager#off", + "parentId": 2226 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Emit an event with the associated data to all registered listeners.\n" + } + ] + }, + "decorations": [ + { + "name": "ensure_context" + } + ], + "flags": {}, + "groups": [], + "id": 44580, + "module": "events._event_manager", + "name": "emit", + "parsedDocstring": { + "text": "Emit an event with the associated data to all registered listeners.\n", + "args": { + "event": "The event which will be emitted.", + "event_data": "The data which will be passed to the event listeners." + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_event_manager.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 242, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/events/_event_manager.py#L242" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Emit an event with the associated data to all registered listeners.\n" + } + ] + }, + "flags": {}, + "id": 42600, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "emit", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The event which will be emitted." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 42601, + "kind": 32768, + "kindString": "Parameter", + "name": "event", + "type": { + "name": "Event", + "type": "reference", + "target": "2116", + "ref": { + "id": 42539, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 15 + } + ], + "kind": 8, + "permalink": "https://crawlee.dev/python/api/enum/Event" + } + }, + "parentId": 2177 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The data which will be passed to the event listeners." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 42602, + "kind": 32768, + "kindString": "Parameter", + "name": "event_data", + "type": { + "name": "EventData", + "type": "reference", + "target": "2147", + "ref": { + "id": 42570, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 99 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api#EventData" + } + }, + "parentId": 2177 + } + ], + "type": { + "name": "None", + "type": "reference", + "target": 0, + "ref": {} + }, + "inheritedFrom": { + "name": "EventManager.emit", + "target": 2176, + "type": "reference", + "ref": { + "id": 42599, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_event_manager.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 242 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/EventManager#emit" + } + }, + "parentId": 4157 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Emit an event with the associated data to all registered listeners.\n" + } + ] + }, + "flags": {}, + "id": 42627, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "emit", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The event which will be emitted." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 42628, + "kind": 32768, + "kindString": "Parameter", + "name": "event", + "type": { + "name": "Literal[Event.PERSIST_STATE]", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2204 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The data which will be passed to the event listeners." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 42629, + "kind": 32768, + "kindString": "Parameter", + "name": "event_data", + "type": { + "name": "EventPersistStateData", + "type": "reference", + "target": "2129", + "ref": { + "id": 42552, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 40 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/EventPersistStateData" + } + }, + "parentId": 2204 + } + ], + "type": { + "name": "None", + "type": "reference", + "target": 0, + "ref": {} + }, + "inheritedFrom": { + "name": "EventManager.emit", + "target": 2176, + "type": "reference", + "ref": { + "id": 42599, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_event_manager.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 242 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/EventManager#emit" + } + }, + "parentId": 4157 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Emit an event with the associated data to all registered listeners.\n" + } + ] + }, + "flags": {}, + "id": 42630, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "emit", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The event which will be emitted." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 42631, + "kind": 32768, + "kindString": "Parameter", + "name": "event", + "type": { + "name": "Literal[Event.SYSTEM_INFO]", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2207 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The data which will be passed to the event listeners." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 42632, + "kind": 32768, + "kindString": "Parameter", + "name": "event_data", + "type": { + "name": "EventSystemInfoData", + "type": "reference", + "target": "2132", + "ref": { + "id": 42555, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 49 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/EventSystemInfoData" + } + }, + "parentId": 2207 + } + ], + "type": { + "name": "None", + "type": "reference", + "target": 0, + "ref": {} + }, + "inheritedFrom": { + "name": "EventManager.emit", + "target": 2176, + "type": "reference", + "ref": { + "id": 42599, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_event_manager.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 242 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/EventManager#emit" + } + }, + "parentId": 4157 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Emit an event with the associated data to all registered listeners.\n" + } + ] + }, + "flags": {}, + "id": 42633, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "emit", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The event which will be emitted." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 42634, + "kind": 32768, + "kindString": "Parameter", + "name": "event", + "type": { + "name": "Literal[Event.MIGRATING]", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2210 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The data which will be passed to the event listeners." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 42635, + "kind": 32768, + "kindString": "Parameter", + "name": "event_data", + "type": { + "name": "EventMigratingData", + "type": "reference", + "target": "2136", + "ref": { + "id": 42559, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 62 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/EventMigratingData" + } + }, + "parentId": 2210 + } + ], + "type": { + "name": "None", + "type": "reference", + "target": 0, + "ref": {} + }, + "inheritedFrom": { + "name": "EventManager.emit", + "target": 2176, + "type": "reference", + "ref": { + "id": 42599, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_event_manager.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 242 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/EventManager#emit" + } + }, + "parentId": 4157 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Emit an event with the associated data to all registered listeners.\n" + } + ] + }, + "flags": {}, + "id": 42636, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "emit", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The event which will be emitted." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 42637, + "kind": 32768, + "kindString": "Parameter", + "name": "event", + "type": { + "name": "Literal[Event.ABORTING]", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2213 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The data which will be passed to the event listeners." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 42638, + "kind": 32768, + "kindString": "Parameter", + "name": "event_data", + "type": { + "name": "EventAbortingData", + "type": "reference", + "target": "2139", + "ref": { + "id": 42562, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 73 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/EventAbortingData" + } + }, + "parentId": 2213 + } + ], + "type": { + "name": "None", + "type": "reference", + "target": 0, + "ref": {} + }, + "inheritedFrom": { + "name": "EventManager.emit", + "target": 2176, + "type": "reference", + "ref": { + "id": 42599, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_event_manager.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 242 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/EventManager#emit" + } + }, + "parentId": 4157 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Emit an event with the associated data to all registered listeners.\n" + } + ] + }, + "flags": {}, + "id": 42639, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "emit", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The event which will be emitted." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 42640, + "kind": 32768, + "kindString": "Parameter", + "name": "event", + "type": { + "name": "Literal[Event.EXIT]", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2216 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The data which will be passed to the event listeners." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 42641, + "kind": 32768, + "kindString": "Parameter", + "name": "event_data", + "type": { + "name": "EventExitData", + "type": "reference", + "target": "2141", + "ref": { + "id": 42564, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 80 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/EventExitData" + } + }, + "parentId": 2216 + } + ], + "type": { + "name": "None", + "type": "reference", + "target": 0, + "ref": {} + }, + "inheritedFrom": { + "name": "EventManager.emit", + "target": 2176, + "type": "reference", + "ref": { + "id": 42599, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_event_manager.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 242 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/EventManager#emit" + } + }, + "parentId": 4157 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Emit an event with the associated data to all registered listeners.\n" + } + ] + }, + "flags": {}, + "id": 42642, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "emit", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The event which will be emitted." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 42643, + "kind": 32768, + "kindString": "Parameter", + "name": "event", + "type": { + "name": "Literal[Event.CRAWLER_STATUS]", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2219 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The data which will be passed to the event listeners." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 42644, + "kind": 32768, + "kindString": "Parameter", + "name": "event_data", + "type": { + "name": "EventCrawlerStatusData", + "type": "reference", + "target": "2143", + "ref": { + "id": 42566, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 87 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/EventCrawlerStatusData" + } + }, + "parentId": 2219 + } + ], + "type": { + "name": "None", + "type": "reference", + "target": 0, + "ref": {} + }, + "inheritedFrom": { + "name": "EventManager.emit", + "target": 2176, + "type": "reference", + "ref": { + "id": 42599, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_event_manager.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 242 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/EventManager#emit" + } + }, + "parentId": 4157 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Emit an event with the associated data to all registered listeners.\n" + } + ] + }, + "flags": {}, + "id": 42645, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "emit", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The event which will be emitted." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 42646, + "kind": 32768, + "kindString": "Parameter", + "name": "event", + "type": { + "name": "Event", + "type": "reference", + "target": "2116", + "ref": { + "id": 42539, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 15 + } + ], + "kind": 8, + "permalink": "https://crawlee.dev/python/api/enum/Event" + } + }, + "parentId": 2222 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The data which will be passed to the event listeners." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 42647, + "kind": 32768, + "kindString": "Parameter", + "name": "event_data", + "type": { + "name": "Any", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2222 + } + ], + "type": { + "name": "None", + "type": "reference", + "target": 0, + "ref": {} + }, + "inheritedFrom": { + "name": "EventManager.emit", + "target": 2176, + "type": "reference", + "ref": { + "id": 42599, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_event_manager.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 242 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/EventManager#emit" + } + }, + "parentId": 4157 + } + ], + "inheritedFrom": { + "name": "EventManager.emit", + "target": 2176, + "type": "reference", + "ref": { + "id": 42599, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_event_manager.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 242 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/EventManager#emit" + } + }, + "permalink": "/python/api/class/LocalEventManager#emit", + "parentId": 2226 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Wait for all currently executing event listeners to complete.\n" + } + ] + }, + "decorations": [ + { + "name": "ensure_context" + } + ], + "flags": {}, + "groups": [], + "id": 44581, + "module": "events._event_manager", + "name": "wait_for_all_listeners_to_complete", + "parsedDocstring": { + "text": "Wait for all currently executing event listeners to complete.\n", + "args": { + "timeout": "The maximum time to wait for the event listeners to finish. If they do not complete within\nthe specified timeout, they will be canceled." + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_event_manager.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 252, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/events/_event_manager.py#L252" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Wait for all currently executing event listeners to complete.\n" + } + ] + }, + "flags": {}, + "id": 42604, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "wait_for_all_listeners_to_complete", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The maximum time to wait for the event listeners to finish. If they do not complete within\nthe specified timeout, they will be canceled." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 42605, + "kind": 32768, + "kindString": "Parameter", + "name": "timeout", + "type": { + "name": "timedelta | None", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2181 + } + ], + "type": { + "name": "None", + "type": "reference", + "target": 0, + "ref": {} + }, + "inheritedFrom": { + "name": "EventManager.wait_for_all_listeners_to_complete", + "target": 2180, + "type": "reference", + "ref": { + "id": 42603, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_event_manager.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 252 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/EventManager#wait_for_all_listeners_to_complete" + } + }, + "parentId": 4158 + } + ], + "inheritedFrom": { + "name": "EventManager.wait_for_all_listeners_to_complete", + "target": 2180, + "type": "reference", + "ref": { + "id": 42603, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_event_manager.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 252 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/EventManager#wait_for_all_listeners_to_complete" + } + }, + "permalink": "/python/api/class/LocalEventManager#wait_for_all_listeners_to_complete", + "parentId": 2226 + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Event manager for local environments.\n\nIt extends the `EventManager` to emit `SystemInfo` events at regular intervals. The `LocalEventManager`\nis intended to be used in local environments, where the system metrics are required managing the `Snapshotter`\nand `AutoscaledPool`." + } + ] + }, + "decorations": [ + { + "args": "('Event managers')", + "name": "docs_group" + } + ], + "flags": {}, + "groups": [ + { + "children": [ + 42657, + 42659, + 42650, + 44580, + 42654, + 44579, + 44578, + 44581 + ], + "title": "Methods" + }, + { + "children": [ + 44577 + ], + "title": "Properties" + } + ], + "id": 42649, + "module": "events._local_event_manager", + "name": "LocalEventManager", + "parsedDocstring": { + "text": "Event manager for local environments.\n\nIt extends the `EventManager` to emit `SystemInfo` events at regular intervals. The `LocalEventManager`\nis intended to be used in local environments, where the system metrics are required managing the `Snapshotter`\nand `AutoscaledPool`." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_local_event_manager.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 26, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/events/_local_event_manager.py#L26" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "extendedTypes": [ + { + "name": "EventManager", + "target": "2155", + "type": "reference", + "ref": { + "id": 42578, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_event_manager.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 55 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/EventManager" + } + } + ], + "permalink": "/python/api/class/LocalEventManager", + "categories": [] + }, + { + "kind": 8, + "kindString": "Enumeration", + "children": [ + { + "kind": 16, + "kindString": "Enumeration Member", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 46698, + "module": "events._types", + "name": "PERSIST_STATE", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 19, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/events/_types.py#L19" + } + ], + "type": { + "type": "literal", + "value": "'persistState'" + }, + "permalink": "/python/api/enum/Event#PERSIST_STATE", + "parentId": 2116 + }, + { + "kind": 16, + "kindString": "Enumeration Member", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 46699, + "module": "events._types", + "name": "SYSTEM_INFO", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 20, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/events/_types.py#L20" + } + ], + "type": { + "type": "literal", + "value": "'systemInfo'" + }, + "permalink": "/python/api/enum/Event#SYSTEM_INFO", + "parentId": 2116 + }, + { + "kind": 16, + "kindString": "Enumeration Member", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 46700, + "module": "events._types", + "name": "MIGRATING", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 21, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/events/_types.py#L21" + } + ], + "type": { + "type": "literal", + "value": "'migrating'" + }, + "permalink": "/python/api/enum/Event#MIGRATING", + "parentId": 2116 + }, + { + "kind": 16, + "kindString": "Enumeration Member", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 46701, + "module": "events._types", + "name": "ABORTING", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 22, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/events/_types.py#L22" + } + ], + "type": { + "type": "literal", + "value": "'aborting'" + }, + "permalink": "/python/api/enum/Event#ABORTING", + "parentId": 2116 + }, + { + "kind": 16, + "kindString": "Enumeration Member", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 46702, + "module": "events._types", + "name": "EXIT", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 23, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/events/_types.py#L23" + } + ], + "type": { + "type": "literal", + "value": "'exit'" + }, + "permalink": "/python/api/enum/Event#EXIT", + "parentId": 2116 + }, + { + "kind": 16, + "kindString": "Enumeration Member", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 46703, + "module": "events._types", + "name": "SESSION_RETIRED", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 26, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/events/_types.py#L26" + } + ], + "type": { + "type": "literal", + "value": "'sessionRetired'" + }, + "permalink": "/python/api/enum/Event#SESSION_RETIRED", + "parentId": 2116 + }, + { + "kind": 16, + "kindString": "Enumeration Member", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 46704, + "module": "events._types", + "name": "BROWSER_LAUNCHED", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 29, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/events/_types.py#L29" + } + ], + "type": { + "type": "literal", + "value": "'browserLaunched'" + }, + "permalink": "/python/api/enum/Event#BROWSER_LAUNCHED", + "parentId": 2116 + }, + { + "kind": 16, + "kindString": "Enumeration Member", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 46705, + "module": "events._types", + "name": "BROWSER_RETIRED", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 30, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/events/_types.py#L30" + } + ], + "type": { + "type": "literal", + "value": "'browserRetired'" + }, + "permalink": "/python/api/enum/Event#BROWSER_RETIRED", + "parentId": 2116 + }, + { + "kind": 16, + "kindString": "Enumeration Member", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 46706, + "module": "events._types", + "name": "BROWSER_CLOSED", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 31, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/events/_types.py#L31" + } + ], + "type": { + "type": "literal", + "value": "'browserClosed'" + }, + "permalink": "/python/api/enum/Event#BROWSER_CLOSED", + "parentId": 2116 + }, + { + "kind": 16, + "kindString": "Enumeration Member", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 46707, + "module": "events._types", + "name": "PAGE_CREATED", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 32, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/events/_types.py#L32" + } + ], + "type": { + "type": "literal", + "value": "'pageCreated'" + }, + "permalink": "/python/api/enum/Event#PAGE_CREATED", + "parentId": 2116 + }, + { + "kind": 16, + "kindString": "Enumeration Member", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 46708, + "module": "events._types", + "name": "PAGE_CLOSED", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 33, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/events/_types.py#L33" + } + ], + "type": { + "type": "literal", + "value": "'pageClosed'" + }, + "permalink": "/python/api/enum/Event#PAGE_CLOSED", + "parentId": 2116 + }, + { + "kind": 16, + "kindString": "Enumeration Member", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 46709, + "module": "events._types", + "name": "CRAWLER_STATUS", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 36, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/events/_types.py#L36" + } + ], + "type": { + "type": "literal", + "value": "'crawlerStatus'" + }, + "permalink": "/python/api/enum/Event#CRAWLER_STATUS", + "parentId": 2116 + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Names of all possible events that can be emitted using an `EventManager`." + } + ] + }, + "decorations": [ + { + "args": "('Event data')", + "name": "docs_group" + } + ], + "flags": {}, + "groups": [ + { + "children": [ + 46701, + 46706, + 46704, + 46705, + 46709, + 46702, + 46700, + 46708, + 46707, + 46698, + 46703, + 46699 + ], + "title": "Enumeration members" + } + ], + "id": 46697, + "module": "events._types", + "name": "Event", + "parsedDocstring": { + "text": "Names of all possible events that can be emitted using an `EventManager`." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 15, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/events/_types.py#L15" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "permalink": "/python/api/enum/Event", + "categories": [] + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 48849, + "module": "events._types", + "name": "model_config", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 76, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/events/_types.py#L76" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "permalink": "/python/api/class/EventAbortingData#model_config", + "parentId": 2139 + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Data for the aborting event." + } + ] + }, + "decorations": [ + { + "args": "('Event data')", + "name": "docs_group" + } + ], + "flags": {}, + "groups": [ + { + "children": [ + 48849 + ], + "title": "Properties" + } + ], + "id": 48848, + "module": "events._types", + "name": "EventAbortingData", + "parsedDocstring": { + "text": "Data for the aborting event." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 73, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/events/_types.py#L73" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "permalink": "/python/api/class/EventAbortingData", + "categories": [] + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 50991, + "module": "events._types", + "name": "model_config", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 83, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/events/_types.py#L83" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "permalink": "/python/api/class/EventExitData#model_config", + "parentId": 2141 + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Data for the exit event." + } + ] + }, + "decorations": [ + { + "args": "('Event data')", + "name": "docs_group" + } + ], + "flags": {}, + "groups": [ + { + "children": [ + 50991 + ], + "title": "Properties" + } + ], + "id": 50990, + "module": "events._types", + "name": "EventExitData", + "parsedDocstring": { + "text": "Data for the exit event." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 80, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/events/_types.py#L80" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "permalink": "/python/api/class/EventExitData", + "categories": [] + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 53128, + "module": "events._types", + "name": "model_config", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 65, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/events/_types.py#L65" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "permalink": "/python/api/class/EventMigratingData#model_config", + "parentId": 2136 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 53129, + "module": "events._types", + "name": "time_remaining", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 69, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/events/_types.py#L69" + } + ], + "type": { + "name": "Annotated[timedelta_secs | None, Field(alias='timeRemainingSecs')]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "timedelta_secs", + "target": "3321", + "ref": { + "id": 54312, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_utils/models.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 68 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api#timedelta_secs" + } + }, + { + "type": "literal", + "value": {} + } + ] + }, + "permalink": "/python/api/class/EventMigratingData#time_remaining", + "parentId": 2136 + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Data for the migrating event." + } + ] + }, + "decorations": [ + { + "args": "('Event data')", + "name": "docs_group" + } + ], + "flags": {}, + "groups": [ + { + "children": [ + 53128, + 53129 + ], + "title": "Properties" + } + ], + "id": 53127, + "module": "events._types", + "name": "EventMigratingData", + "parsedDocstring": { + "text": "Data for the migrating event." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 62, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/events/_types.py#L62" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "permalink": "/python/api/class/EventMigratingData", + "categories": [] + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 56442, + "module": "events._types", + "name": "model_config", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 43, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/events/_types.py#L43" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "permalink": "/python/api/class/EventPersistStateData#model_config", + "parentId": 2129 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 56443, + "module": "events._types", + "name": "is_migrating", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 45, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/events/_types.py#L45" + } + ], + "type": { + "name": "bool", + "type": "reference", + "target": 0, + "ref": {} + }, + "permalink": "/python/api/class/EventPersistStateData#is_migrating", + "parentId": 2129 + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Data for the persist state event." + } + ] + }, + "decorations": [ + { + "args": "('Event data')", + "name": "docs_group" + } + ], + "flags": {}, + "groups": [ + { + "children": [ + 56443, + 56442 + ], + "title": "Properties" + } + ], + "id": 56441, + "module": "events._types", + "name": "EventPersistStateData", + "parsedDocstring": { + "text": "Data for the persist state event." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 40, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/events/_types.py#L40" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "permalink": "/python/api/class/EventPersistStateData", + "categories": [] + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 58576, + "module": "events._types", + "name": "model_config", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 52, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/events/_types.py#L52" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "permalink": "/python/api/class/EventSystemInfoData#model_config", + "parentId": 2132 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 58577, + "module": "events._types", + "name": "cpu_info", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 54, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/events/_types.py#L54" + } + ], + "type": { + "name": "CpuInfo", + "type": "reference", + "target": "3342", + "ref": { + "id": 59785, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_utils/system.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 36 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/CpuInfo" + } + }, + "permalink": "/python/api/class/EventSystemInfoData#cpu_info", + "parentId": 2132 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 58578, + "module": "events._types", + "name": "memory_info", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 55, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/events/_types.py#L55" + } + ], + "type": { + "name": "MemoryUsageInfo", + "type": "reference", + "target": "3345", + "ref": { + "id": 59788, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_utils/system.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 59 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/MemoryUsageInfo" + } + }, + "permalink": "/python/api/class/EventSystemInfoData#memory_info", + "parentId": 2132 + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Data for the system info event." + } + ] + }, + "decorations": [ + { + "args": "('Event data')", + "name": "docs_group" + } + ], + "flags": {}, + "groups": [ + { + "children": [ + 58577, + 58578, + 58576 + ], + "title": "Properties" + } + ], + "id": 58575, + "module": "events._types", + "name": "EventSystemInfoData", + "parsedDocstring": { + "text": "Data for the system info event." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 49, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/events/_types.py#L49" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "permalink": "/python/api/class/EventSystemInfoData", + "categories": [] + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Return a cache key that can differentiate between different storages of this and other clients.\n\nCan be based on configuration or on the client itself. By default, returns a module and name of the client\nclass." + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 61071, + "module": "storage_clients._base._storage_client", + "name": "get_storage_client_cache_key", + "parsedDocstring": { + "text": "Return a cache key that can differentiate between different storages of this and other clients.\n\nCan be based on configuration or on the client itself. By default, returns a module and name of the client\nclass." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 33, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/_base/_storage_client.py#L33" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Return a cache key that can differentiate between different storages of this and other clients.\n\nCan be based on configuration or on the client itself. By default, returns a module and name of the client\nclass." + } + ] + }, + "flags": {}, + "id": 61072, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "get_storage_client_cache_key", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 61073, + "kind": 32768, + "kindString": "Parameter", + "name": "configuration", + "type": { + "name": "Configuration", + "type": "reference", + "target": "2", + "ref": { + "id": 59790, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/configuration.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 20 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/Configuration" + } + }, + "parentId": 1284 + } + ], + "type": { + "name": "Hashable", + "type": "reference", + "target": 0, + "ref": {} + }, + "overwrites": { + "name": "StorageClient.get_storage_client_cache_key", + "target": 1283, + "type": "reference", + "ref": { + "id": 61071, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 33 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/StorageClient#get_storage_client_cache_key" + } + }, + "parentId": 1283 + } + ], + "permalink": "/python/api/class/StorageClient#get_storage_client_cache_key", + "parentId": 1282 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Create a dataset client." + } + ] + }, + "decorations": [ + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 61074, + "module": "storage_clients._base._storage_client", + "name": "create_dataset_client", + "parsedDocstring": { + "text": "Create a dataset client." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 42, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/_base/_storage_client.py#L42" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Create a dataset client." + } + ] + }, + "flags": {}, + "id": 61075, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "create_dataset_client", + "parameters": [ + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 61076, + "kind": 32768, + "kindString": "Parameter", + "name": "id", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 1287 + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 61077, + "kind": 32768, + "kindString": "Parameter", + "name": "name", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 1287 + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 61078, + "kind": 32768, + "kindString": "Parameter", + "name": "alias", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 1287 + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 61079, + "kind": 32768, + "kindString": "Parameter", + "name": "configuration", + "type": { + "name": "Configuration | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "Configuration", + "target": "2", + "ref": { + "id": 59790, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/configuration.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 20 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/Configuration" + } + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 1287 + } + ], + "type": { + "name": "DatasetClient", + "type": "reference", + "target": "1220", + "ref": { + "id": 61008, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_dataset_client.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 13 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/DatasetClient" + } + }, + "overwrites": { + "name": "StorageClient.create_dataset_client", + "target": 1286, + "type": "reference", + "ref": { + "id": 61074, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 42 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/StorageClient#create_dataset_client" + } + }, + "parentId": 1286 + } + ], + "permalink": "/python/api/class/StorageClient#create_dataset_client", + "parentId": 1282 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Create a key-value store client." + } + ] + }, + "decorations": [ + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 61080, + "module": "storage_clients._base._storage_client", + "name": "create_kvs_client", + "parsedDocstring": { + "text": "Create a key-value store client." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 53, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/_base/_storage_client.py#L53" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Create a key-value store client." + } + ] + }, + "flags": {}, + "id": 61081, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "create_kvs_client", + "parameters": [ + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 61082, + "kind": 32768, + "kindString": "Parameter", + "name": "id", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 1293 + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 61083, + "kind": 32768, + "kindString": "Parameter", + "name": "name", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 1293 + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 61084, + "kind": 32768, + "kindString": "Parameter", + "name": "alias", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 1293 + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 61085, + "kind": 32768, + "kindString": "Parameter", + "name": "configuration", + "type": { + "name": "Configuration | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "Configuration", + "target": "2", + "ref": { + "id": 59790, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/configuration.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 20 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/Configuration" + } + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 1293 + } + ], + "type": { + "name": "KeyValueStoreClient", + "type": "reference", + "target": "1254", + "ref": { + "id": 61042, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_key_value_store_client.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 14 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/KeyValueStoreClient" + } + }, + "overwrites": { + "name": "StorageClient.create_kvs_client", + "target": 1292, + "type": "reference", + "ref": { + "id": 61080, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 53 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/StorageClient#create_kvs_client" + } + }, + "parentId": 1292 + } + ], + "permalink": "/python/api/class/StorageClient#create_kvs_client", + "parentId": 1282 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Create a request queue client." + } + ] + }, + "decorations": [ + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 61086, + "module": "storage_clients._base._storage_client", + "name": "create_rq_client", + "parsedDocstring": { + "text": "Create a request queue client." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 64, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/_base/_storage_client.py#L64" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Create a request queue client." + } + ] + }, + "flags": {}, + "id": 61087, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "create_rq_client", + "parameters": [ + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 61088, + "kind": 32768, + "kindString": "Parameter", + "name": "id", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 1299 + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 61089, + "kind": 32768, + "kindString": "Parameter", + "name": "name", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 1299 + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 61090, + "kind": 32768, + "kindString": "Parameter", + "name": "alias", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 1299 + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 61091, + "kind": 32768, + "kindString": "Parameter", + "name": "configuration", + "type": { + "name": "Configuration | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "Configuration", + "target": "2", + "ref": { + "id": 59790, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/configuration.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 20 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/Configuration" + } + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 1299 + } + ], + "type": { + "name": "RequestQueueClient", + "type": "reference", + "target": "1195", + "ref": { + "id": 60983, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_request_queue_client.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 13 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/RequestQueueClient" + } + }, + "overwrites": { + "name": "StorageClient.create_rq_client", + "target": 1298, + "type": "reference", + "ref": { + "id": 61086, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 64 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/StorageClient#create_rq_client" + } + }, + "parentId": 1298 + } + ], + "permalink": "/python/api/class/StorageClient#create_rq_client", + "parentId": 1282 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Return statistics about rate limit errors encountered by the HTTP client in storage client." + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 61092, + "module": "storage_clients._base._storage_client", + "name": "get_rate_limit_errors", + "parsedDocstring": { + "text": "Return statistics about rate limit errors encountered by the HTTP client in storage client." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 74, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/_base/_storage_client.py#L74" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Return statistics about rate limit errors encountered by the HTTP client in storage client." + } + ] + }, + "flags": {}, + "id": 61093, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "get_rate_limit_errors", + "parameters": [], + "type": { + "name": "dict", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "int", + "target": 0, + "ref": {} + }, + { + "type": "reference", + "name": "int", + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + }, + "parentId": 1304 + } + ], + "permalink": "/python/api/class/StorageClient#get_rate_limit_errors", + "parentId": 1282 + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Base class for storage clients.\n\nThe `StorageClient` serves as an abstract base class that defines the interface for accessing Crawlee's\nstorage types: datasets, key-value stores, and request queues. It provides methods to open clients for\neach of these storage types and handles common functionality.\n\nStorage clients implementations can be provided for various backends (file system, memory, databases,\nvarious cloud providers, etc.) to support different use cases from development to production environments.\n\nEach storage client implementation is responsible for ensuring proper initialization, data persistence\n(where applicable), and consistent access patterns across all storage types it supports." + } + ] + }, + "decorations": [ + { + "args": "('Storage clients')", + "name": "docs_group" + } + ], + "flags": {}, + "groups": [ + { + "children": [ + 61074, + 61080, + 61086, + 61092, + 61071 + ], + "title": "Methods" + } + ], + "id": 61070, + "module": "storage_clients._base._storage_client", + "name": "StorageClient", + "parsedDocstring": { + "text": "Base class for storage clients.\n\nThe `StorageClient` serves as an abstract base class that defines the interface for accessing Crawlee's\nstorage types: datasets, key-value stores, and request queues. It provides methods to open clients for\neach of these storage types and handles common functionality.\n\nStorage clients implementations can be provided for various backends (file system, memory, databases,\nvarious cloud providers, etc.) to support different use cases from development to production environments.\n\nEach storage client implementation is responsible for ensuring proper initialization, data persistence\n(where applicable), and consistent access patterns across all storage types it supports." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 19, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/_base/_storage_client.py#L19" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "extendedBy": [ + { + "name": "MemoryStorageClient", + "target": "1468", + "type": "reference", + "ref": { + "id": 61256, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_memory/_storage_client.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 15 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/MemoryStorageClient" + } + }, + { + "name": "FileSystemStorageClient", + "target": "1624", + "type": "reference", + "ref": { + "id": 61412, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_file_system/_storage_client.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 20 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/FileSystemStorageClient" + } + }, + { + "name": "SqlStorageClient", + "target": "1893", + "type": "reference", + "ref": { + "id": 61681, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_sql/_storage_client.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 35 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/SqlStorageClient" + } + }, + { + "name": "RedisStorageClient", + "target": "2091", + "type": "reference", + "ref": { + "id": 61879, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_redis/_storage_client.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 19 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/RedisStorageClient" + } + } + ], + "permalink": "/python/api/class/StorageClient", + "categories": [] + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Create a dataset client." + } + ] + }, + "decorations": [ + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 63348, + "module": "storage_clients._base._storage_client", + "name": "create_dataset_client", + "parsedDocstring": { + "text": "Create a dataset client." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_memory/_storage_client.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 31, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/_memory/_storage_client.py#L31" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Create a dataset client." + } + ] + }, + "flags": {}, + "id": 63166, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "create_dataset_client", + "parameters": [ + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 63167, + "kind": 32768, + "kindString": "Parameter", + "name": "id", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 1287 + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 63168, + "kind": 32768, + "kindString": "Parameter", + "name": "name", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 1287 + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 63169, + "kind": 32768, + "kindString": "Parameter", + "name": "alias", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 1287 + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 63170, + "kind": 32768, + "kindString": "Parameter", + "name": "configuration", + "type": { + "name": "Configuration | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "Configuration", + "target": "2", + "ref": { + "id": 61881, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/configuration.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 20 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/Configuration" + } + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 1287 + } + ], + "type": { + "name": "DatasetClient", + "type": "reference", + "target": "1220", + "ref": { + "id": 63099, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_dataset_client.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 13 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/DatasetClient" + } + }, + "overwrites": { + "name": "StorageClient.create_dataset_client", + "target": 1286, + "type": "reference", + "ref": { + "id": 63165, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 42 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/StorageClient#create_dataset_client" + } + }, + "parentId": 1469 + } + ], + "overwrites": { + "name": "StorageClient.create_dataset_client", + "target": 1286, + "type": "reference", + "ref": { + "id": 63165, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 42 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/StorageClient#create_dataset_client" + } + }, + "permalink": "/python/api/class/MemoryStorageClient#create_dataset_client", + "parentId": 1468 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Create a key-value store client." + } + ] + }, + "decorations": [ + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 63354, + "module": "storage_clients._base._storage_client", + "name": "create_kvs_client", + "parsedDocstring": { + "text": "Create a key-value store client." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_memory/_storage_client.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 45, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/_memory/_storage_client.py#L45" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Create a key-value store client." + } + ] + }, + "flags": {}, + "id": 63172, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "create_kvs_client", + "parameters": [ + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 63173, + "kind": 32768, + "kindString": "Parameter", + "name": "id", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 1293 + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 63174, + "kind": 32768, + "kindString": "Parameter", + "name": "name", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 1293 + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 63175, + "kind": 32768, + "kindString": "Parameter", + "name": "alias", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 1293 + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 63176, + "kind": 32768, + "kindString": "Parameter", + "name": "configuration", + "type": { + "name": "Configuration | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "Configuration", + "target": "2", + "ref": { + "id": 61881, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/configuration.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 20 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/Configuration" + } + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 1293 + } + ], + "type": { + "name": "KeyValueStoreClient", + "type": "reference", + "target": "1254", + "ref": { + "id": 63133, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_key_value_store_client.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 14 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/KeyValueStoreClient" + } + }, + "overwrites": { + "name": "StorageClient.create_kvs_client", + "target": 1292, + "type": "reference", + "ref": { + "id": 63171, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 53 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/StorageClient#create_kvs_client" + } + }, + "parentId": 1475 + } + ], + "overwrites": { + "name": "StorageClient.create_kvs_client", + "target": 1292, + "type": "reference", + "ref": { + "id": 63171, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 53 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/StorageClient#create_kvs_client" + } + }, + "permalink": "/python/api/class/MemoryStorageClient#create_kvs_client", + "parentId": 1468 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Create a request queue client." + } + ] + }, + "decorations": [ + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 63360, + "module": "storage_clients._base._storage_client", + "name": "create_rq_client", + "parsedDocstring": { + "text": "Create a request queue client." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_memory/_storage_client.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 59, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/_memory/_storage_client.py#L59" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Create a request queue client." + } + ] + }, + "flags": {}, + "id": 63178, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "create_rq_client", + "parameters": [ + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 63179, + "kind": 32768, + "kindString": "Parameter", + "name": "id", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 1299 + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 63180, + "kind": 32768, + "kindString": "Parameter", + "name": "name", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 1299 + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 63181, + "kind": 32768, + "kindString": "Parameter", + "name": "alias", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 1299 + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 63182, + "kind": 32768, + "kindString": "Parameter", + "name": "configuration", + "type": { + "name": "Configuration | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "Configuration", + "target": "2", + "ref": { + "id": 61881, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/configuration.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 20 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/Configuration" + } + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 1299 + } + ], + "type": { + "name": "RequestQueueClient", + "type": "reference", + "target": "1195", + "ref": { + "id": 63074, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_request_queue_client.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 13 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/RequestQueueClient" + } + }, + "overwrites": { + "name": "StorageClient.create_rq_client", + "target": 1298, + "type": "reference", + "ref": { + "id": 63177, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 64 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/StorageClient#create_rq_client" + } + }, + "parentId": 1481 + } + ], + "overwrites": { + "name": "StorageClient.create_rq_client", + "target": 1298, + "type": "reference", + "ref": { + "id": 63177, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 64 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/StorageClient#create_rq_client" + } + }, + "permalink": "/python/api/class/MemoryStorageClient#create_rq_client", + "parentId": 1468 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Return a cache key that can differentiate between different storages of this and other clients.\n\nCan be based on configuration or on the client itself. By default, returns a module and name of the client\nclass." + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 66078, + "module": "storage_clients._base._storage_client", + "name": "get_storage_client_cache_key", + "parsedDocstring": { + "text": "Return a cache key that can differentiate between different storages of this and other clients.\n\nCan be based on configuration or on the client itself. By default, returns a module and name of the client\nclass." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 33, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/_base/_storage_client.py#L33" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Return a cache key that can differentiate between different storages of this and other clients.\n\nCan be based on configuration or on the client itself. By default, returns a module and name of the client\nclass." + } + ] + }, + "flags": {}, + "id": 63163, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "get_storage_client_cache_key", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 63164, + "kind": 32768, + "kindString": "Parameter", + "name": "configuration", + "type": { + "name": "Configuration", + "type": "reference", + "target": "2", + "ref": { + "id": 61881, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/configuration.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 20 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/Configuration" + } + }, + "parentId": 1284 + } + ], + "type": { + "name": "Hashable", + "type": "reference", + "target": 0, + "ref": {} + }, + "inheritedFrom": { + "name": "StorageClient.get_storage_client_cache_key", + "target": 1283, + "type": "reference", + "ref": { + "id": 63162, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 33 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/StorageClient#get_storage_client_cache_key" + } + }, + "parentId": 4199 + } + ], + "inheritedFrom": { + "name": "StorageClient.get_storage_client_cache_key", + "target": 1283, + "type": "reference", + "ref": { + "id": 63162, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 33 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/StorageClient#get_storage_client_cache_key" + } + }, + "permalink": "/python/api/class/MemoryStorageClient#get_storage_client_cache_key", + "parentId": 1468 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Return statistics about rate limit errors encountered by the HTTP client in storage client." + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 66079, + "module": "storage_clients._base._storage_client", + "name": "get_rate_limit_errors", + "parsedDocstring": { + "text": "Return statistics about rate limit errors encountered by the HTTP client in storage client." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 74, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/_base/_storage_client.py#L74" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Return statistics about rate limit errors encountered by the HTTP client in storage client." + } + ] + }, + "flags": {}, + "id": 63184, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "get_rate_limit_errors", + "parameters": [], + "type": { + "name": "dict[int, int]", + "type": "reference", + "target": 0, + "ref": {} + }, + "inheritedFrom": { + "name": "StorageClient.get_rate_limit_errors", + "target": 1304, + "type": "reference", + "ref": { + "id": 63183, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 74 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/StorageClient#get_rate_limit_errors" + } + }, + "parentId": 4200 + } + ], + "inheritedFrom": { + "name": "StorageClient.get_rate_limit_errors", + "target": 1304, + "type": "reference", + "ref": { + "id": 63183, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 74 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/StorageClient#get_rate_limit_errors" + } + }, + "permalink": "/python/api/class/MemoryStorageClient#get_rate_limit_errors", + "parentId": 1468 + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Memory implementation of the storage client.\n\nThis storage client provides access to datasets, key-value stores, and request queues that store all data\nin memory using Python data structures (lists and dictionaries). No data is persisted between process runs,\nmeaning all stored data is lost when the program terminates.\n\nThe memory implementation provides fast access to data but is limited by available memory and does not\nsupport data sharing across different processes. All storage operations happen entirely in memory with\nno disk operations.\n\nThe memory storage client is useful for testing and development environments, or short-lived crawler\noperations where persistence is not required." + } + ] + }, + "decorations": [ + { + "args": "('Storage clients')", + "name": "docs_group" + } + ], + "flags": {}, + "groups": [ + { + "children": [ + 63348, + 63354, + 63360, + 66079, + 66078 + ], + "title": "Methods" + } + ], + "id": 63347, + "module": "storage_clients._memory._storage_client", + "name": "MemoryStorageClient", + "parsedDocstring": { + "text": "Memory implementation of the storage client.\n\nThis storage client provides access to datasets, key-value stores, and request queues that store all data\nin memory using Python data structures (lists and dictionaries). No data is persisted between process runs,\nmeaning all stored data is lost when the program terminates.\n\nThe memory implementation provides fast access to data but is limited by available memory and does not\nsupport data sharing across different processes. All storage operations happen entirely in memory with\nno disk operations.\n\nThe memory storage client is useful for testing and development environments, or short-lived crawler\noperations where persistence is not required." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_memory/_storage_client.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 15, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/_memory/_storage_client.py#L15" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "extendedTypes": [ + { + "name": "StorageClient", + "target": "1282", + "type": "reference", + "ref": { + "id": 63161, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 19 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/StorageClient" + } + } + ], + "permalink": "/python/api/class/MemoryStorageClient", + "categories": [] + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Return a cache key that can differentiate between different storages of this and other clients.\n\nCan be based on configuration or on the client itself. By default, returns a module and name of the client\nclass." + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 67704, + "module": "storage_clients._base._storage_client", + "name": "get_storage_client_cache_key", + "parsedDocstring": { + "text": "Return a cache key that can differentiate between different storages of this and other clients.\n\nCan be based on configuration or on the client itself. By default, returns a module and name of the client\nclass." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_file_system/_storage_client.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 38, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/_file_system/_storage_client.py#L38" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Return a cache key that can differentiate between different storages of this and other clients.\n\nCan be based on configuration or on the client itself. By default, returns a module and name of the client\nclass." + } + ] + }, + "flags": {}, + "id": 67363, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "get_storage_client_cache_key", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 67364, + "kind": 32768, + "kindString": "Parameter", + "name": "configuration", + "type": { + "name": "Configuration", + "type": "reference", + "target": "2", + "ref": { + "id": 66081, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/configuration.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 20 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/Configuration" + } + }, + "parentId": 1284 + } + ], + "type": { + "name": "Hashable", + "type": "reference", + "target": 0, + "ref": {} + }, + "overwrites": { + "name": "StorageClient.get_storage_client_cache_key", + "target": 1283, + "type": "reference", + "ref": { + "id": 67362, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 33 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/StorageClient#get_storage_client_cache_key" + } + }, + "parentId": 1625 + } + ], + "overwrites": { + "name": "StorageClient.get_storage_client_cache_key", + "target": 1283, + "type": "reference", + "ref": { + "id": 67362, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 33 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/StorageClient#get_storage_client_cache_key" + } + }, + "permalink": "/python/api/class/FileSystemStorageClient#get_storage_client_cache_key", + "parentId": 1624 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Create a dataset client." + } + ] + }, + "decorations": [ + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 67707, + "module": "storage_clients._base._storage_client", + "name": "create_dataset_client", + "parsedDocstring": { + "text": "Create a dataset client." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_file_system/_storage_client.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 43, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/_file_system/_storage_client.py#L43" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Create a dataset client." + } + ] + }, + "flags": {}, + "id": 67366, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "create_dataset_client", + "parameters": [ + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 67367, + "kind": 32768, + "kindString": "Parameter", + "name": "id", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 1287 + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 67368, + "kind": 32768, + "kindString": "Parameter", + "name": "name", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 1287 + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 67369, + "kind": 32768, + "kindString": "Parameter", + "name": "alias", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 1287 + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 67370, + "kind": 32768, + "kindString": "Parameter", + "name": "configuration", + "type": { + "name": "Configuration | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "Configuration", + "target": "2", + "ref": { + "id": 66081, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/configuration.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 20 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/Configuration" + } + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 1287 + } + ], + "type": { + "name": "DatasetClient", + "type": "reference", + "target": "1220", + "ref": { + "id": 67299, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_dataset_client.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 13 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/DatasetClient" + } + }, + "overwrites": { + "name": "StorageClient.create_dataset_client", + "target": 1286, + "type": "reference", + "ref": { + "id": 67365, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 42 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/StorageClient#create_dataset_client" + } + }, + "parentId": 1628 + } + ], + "overwrites": { + "name": "StorageClient.create_dataset_client", + "target": 1286, + "type": "reference", + "ref": { + "id": 67365, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 42 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/StorageClient#create_dataset_client" + } + }, + "permalink": "/python/api/class/FileSystemStorageClient#create_dataset_client", + "parentId": 1624 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Create a key-value store client." + } + ] + }, + "decorations": [ + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 67713, + "module": "storage_clients._base._storage_client", + "name": "create_kvs_client", + "parsedDocstring": { + "text": "Create a key-value store client." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_file_system/_storage_client.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 57, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/_file_system/_storage_client.py#L57" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Create a key-value store client." + } + ] + }, + "flags": {}, + "id": 67372, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "create_kvs_client", + "parameters": [ + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 67373, + "kind": 32768, + "kindString": "Parameter", + "name": "id", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 1293 + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 67374, + "kind": 32768, + "kindString": "Parameter", + "name": "name", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 1293 + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 67375, + "kind": 32768, + "kindString": "Parameter", + "name": "alias", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 1293 + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 67376, + "kind": 32768, + "kindString": "Parameter", + "name": "configuration", + "type": { + "name": "Configuration | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "Configuration", + "target": "2", + "ref": { + "id": 66081, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/configuration.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 20 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/Configuration" + } + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 1293 + } + ], + "type": { + "name": "KeyValueStoreClient", + "type": "reference", + "target": "1254", + "ref": { + "id": 67333, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_key_value_store_client.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 14 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/KeyValueStoreClient" + } + }, + "overwrites": { + "name": "StorageClient.create_kvs_client", + "target": 1292, + "type": "reference", + "ref": { + "id": 67371, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 53 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/StorageClient#create_kvs_client" + } + }, + "parentId": 1634 + } + ], + "overwrites": { + "name": "StorageClient.create_kvs_client", + "target": 1292, + "type": "reference", + "ref": { + "id": 67371, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 53 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/StorageClient#create_kvs_client" + } + }, + "permalink": "/python/api/class/FileSystemStorageClient#create_kvs_client", + "parentId": 1624 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Create a request queue client." + } + ] + }, + "decorations": [ + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 67719, + "module": "storage_clients._base._storage_client", + "name": "create_rq_client", + "parsedDocstring": { + "text": "Create a request queue client." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_file_system/_storage_client.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 71, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/_file_system/_storage_client.py#L71" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Create a request queue client." + } + ] + }, + "flags": {}, + "id": 67378, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "create_rq_client", + "parameters": [ + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 67379, + "kind": 32768, + "kindString": "Parameter", + "name": "id", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 1299 + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 67380, + "kind": 32768, + "kindString": "Parameter", + "name": "name", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 1299 + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 67381, + "kind": 32768, + "kindString": "Parameter", + "name": "alias", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 1299 + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 67382, + "kind": 32768, + "kindString": "Parameter", + "name": "configuration", + "type": { + "name": "Configuration | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "Configuration", + "target": "2", + "ref": { + "id": 66081, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/configuration.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 20 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/Configuration" + } + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 1299 + } + ], + "type": { + "name": "RequestQueueClient", + "type": "reference", + "target": "1195", + "ref": { + "id": 67274, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_request_queue_client.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 13 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/RequestQueueClient" + } + }, + "overwrites": { + "name": "StorageClient.create_rq_client", + "target": 1298, + "type": "reference", + "ref": { + "id": 67377, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 64 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/StorageClient#create_rq_client" + } + }, + "parentId": 1640 + } + ], + "overwrites": { + "name": "StorageClient.create_rq_client", + "target": 1298, + "type": "reference", + "ref": { + "id": 67377, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 64 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/StorageClient#create_rq_client" + } + }, + "permalink": "/python/api/class/FileSystemStorageClient#create_rq_client", + "parentId": 1624 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Return statistics about rate limit errors encountered by the HTTP client in storage client." + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 70280, + "module": "storage_clients._base._storage_client", + "name": "get_rate_limit_errors", + "parsedDocstring": { + "text": "Return statistics about rate limit errors encountered by the HTTP client in storage client." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 74, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/_base/_storage_client.py#L74" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Return statistics about rate limit errors encountered by the HTTP client in storage client." + } + ] + }, + "flags": {}, + "id": 67384, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "get_rate_limit_errors", + "parameters": [], + "type": { + "name": "dict[int, int]", + "type": "reference", + "target": 0, + "ref": {} + }, + "inheritedFrom": { + "name": "StorageClient.get_rate_limit_errors", + "target": 1304, + "type": "reference", + "ref": { + "id": 67383, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 74 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/StorageClient#get_rate_limit_errors" + } + }, + "parentId": 4201 + } + ], + "inheritedFrom": { + "name": "StorageClient.get_rate_limit_errors", + "target": 1304, + "type": "reference", + "ref": { + "id": 67383, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 74 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/StorageClient#get_rate_limit_errors" + } + }, + "permalink": "/python/api/class/FileSystemStorageClient#get_rate_limit_errors", + "parentId": 1624 + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "File system implementation of the storage client.\n\nThis storage client provides access to datasets, key-value stores, and request queues that persist data\nto the local file system. Each storage type is implemented with its own specific file system client\nthat stores data in a structured directory hierarchy.\n\nData is stored in JSON format in predictable file paths, making it easy to inspect and manipulate\nthe stored data outside of the Crawlee application if needed.\n\nAll data persists between program runs but is limited to access from the local machine\nwhere the files are stored.\n\nWarning: This storage client is not safe for concurrent access from multiple crawler processes.\nUse it only when running a single crawler process at a time." + } + ] + }, + "decorations": [ + { + "args": "('Storage clients')", + "name": "docs_group" + } + ], + "flags": {}, + "groups": [ + { + "children": [ + 67707, + 67713, + 67719, + 70280, + 67704 + ], + "title": "Methods" + } + ], + "id": 67703, + "module": "storage_clients._file_system._storage_client", + "name": "FileSystemStorageClient", + "parsedDocstring": { + "text": "File system implementation of the storage client.\n\nThis storage client provides access to datasets, key-value stores, and request queues that persist data\nto the local file system. Each storage type is implemented with its own specific file system client\nthat stores data in a structured directory hierarchy.\n\nData is stored in JSON format in predictable file paths, making it easy to inspect and manipulate\nthe stored data outside of the Crawlee application if needed.\n\nAll data persists between program runs but is limited to access from the local machine\nwhere the files are stored.\n\nWarning: This storage client is not safe for concurrent access from multiple crawler processes.\nUse it only when running a single crawler process at a time." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_file_system/_storage_client.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 20, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/_file_system/_storage_client.py#L20" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "extendedTypes": [ + { + "name": "StorageClient", + "target": "1282", + "type": "reference", + "ref": { + "id": 67361, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 19 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/StorageClient" + } + } + ], + "permalink": "/python/api/class/FileSystemStorageClient", + "categories": [] + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Initialize the SQL storage client.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 72174, + "module": "storage_clients._sql._storage_client", + "name": "__init__", + "parsedDocstring": { + "text": "Initialize the SQL storage client.\n", + "args": { + "connection_string": "Database connection string (e.g., \"sqlite+aiosqlite:///crawlee.db\").\nIf not provided, defaults to SQLite database in the storage directory.", + "engine": "Pre-configured AsyncEngine instance. If provided, connection_string is ignored." + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_sql/_storage_client.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 57, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/_sql/_storage_client.py#L57" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Initialize the SQL storage client.\n" + } + ] + }, + "flags": {}, + "id": 72175, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "__init__", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Database connection string (e.g., \"sqlite+aiosqlite:///crawlee.db\").\nIf not provided, defaults to SQLite database in the storage directory." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 72176, + "kind": 32768, + "kindString": "Parameter", + "name": "connection_string", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 1895 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Pre-configured AsyncEngine instance. If provided, connection_string is ignored." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 72177, + "kind": 32768, + "kindString": "Parameter", + "name": "engine", + "type": { + "name": "AsyncEngine | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "AsyncEngine", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 1895 + } + ], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "parentId": 1894 + } + ], + "permalink": "/python/api/class/SqlStorageClient#__init__", + "parentId": 1893 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Async context manager entry." + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 72178, + "module": "storage_clients._sql._storage_client", + "name": "__aenter__", + "parsedDocstring": { + "text": "Async context manager entry." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_sql/_storage_client.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 88, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/_sql/_storage_client.py#L88" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Async context manager entry." + } + ] + }, + "flags": {}, + "id": 72179, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "__aenter__", + "parameters": [], + "type": { + "name": "SqlStorageClient", + "type": "reference", + "target": "1893", + "ref": { + "id": 72173, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_sql/_storage_client.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 35 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/SqlStorageClient" + } + }, + "parentId": 1898 + } + ], + "permalink": "/python/api/class/SqlStorageClient#__aenter__", + "parentId": 1893 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Async context manager exit." + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 72180, + "module": "storage_clients._sql._storage_client", + "name": "__aexit__", + "parsedDocstring": { + "text": "Async context manager exit." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_sql/_storage_client.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 92, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/_sql/_storage_client.py#L92" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Async context manager exit." + } + ] + }, + "flags": {}, + "id": 72181, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "__aexit__", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 72182, + "kind": 32768, + "kindString": "Parameter", + "name": "exc_type", + "type": { + "name": "type[BaseException] | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "type", + "typeArguments": [ + { + "type": "reference", + "name": "BaseException", + "target": 0, + "ref": {} + } + ], + "target": "3422", + "ref": { + "id": 73702, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_utils/sitemap.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 62 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api/class/SitemapSource#type" + } + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 1901 + }, + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 72183, + "kind": 32768, + "kindString": "Parameter", + "name": "exc_value", + "type": { + "name": "BaseException | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "BaseException", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 1901 + }, + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 72184, + "kind": 32768, + "kindString": "Parameter", + "name": "exc_traceback", + "type": { + "name": "TracebackType | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "TracebackType", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 1901 + } + ], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "parentId": 1900 + } + ], + "permalink": "/python/api/class/SqlStorageClient#__aexit__", + "parentId": 1893 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Get the SQLAlchemy AsyncEngine instance." + } + ] + }, + "decorations": [ + { + "name": "property" + } + ], + "flags": {}, + "groups": [], + "id": 72185, + "module": "storage_clients._sql._storage_client", + "name": "engine", + "parsedDocstring": { + "text": "Get the SQLAlchemy AsyncEngine instance." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_sql/_storage_client.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 102, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/_sql/_storage_client.py#L102" + } + ], + "type": { + "name": "AsyncEngine", + "type": "reference", + "target": 0, + "ref": {} + }, + "permalink": "/python/api/class/SqlStorageClient#engine", + "parentId": 1893 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Get the database dialect name." + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 72186, + "module": "storage_clients._sql._storage_client", + "name": "get_dialect_name", + "parsedDocstring": { + "text": "Get the database dialect name." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_sql/_storage_client.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 108, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/_sql/_storage_client.py#L108" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Get the database dialect name." + } + ] + }, + "flags": {}, + "id": 72187, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "get_dialect_name", + "parameters": [], + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 1906 + } + ], + "permalink": "/python/api/class/SqlStorageClient#get_dialect_name", + "parentId": 1893 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Initialize the database schema.\n\nThis method creates all necessary tables if they don't exist.\nShould be called before using the storage client." + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 72188, + "module": "storage_clients._sql._storage_client", + "name": "initialize", + "parsedDocstring": { + "text": "Initialize the database schema.\n\nThis method creates all necessary tables if they don't exist.\nShould be called before using the storage client." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_sql/_storage_client.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 112, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/_sql/_storage_client.py#L112" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Initialize the database schema.\n\nThis method creates all necessary tables if they don't exist.\nShould be called before using the storage client." + } + ] + }, + "flags": {}, + "id": 72189, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "initialize", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 72190, + "kind": 32768, + "kindString": "Parameter", + "name": "configuration", + "type": { + "name": "Configuration", + "type": "reference", + "target": "2", + "ref": { + "id": 70282, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/configuration.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 20 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/Configuration" + } + }, + "parentId": 1909 + } + ], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "parentId": 1908 + } + ], + "permalink": "/python/api/class/SqlStorageClient#initialize", + "parentId": 1893 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Close the database connection pool." + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 72191, + "module": "storage_clients._sql._storage_client", + "name": "close", + "parsedDocstring": { + "text": "Close the database connection pool." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_sql/_storage_client.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 157, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/_sql/_storage_client.py#L157" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Close the database connection pool." + } + ] + }, + "flags": {}, + "id": 72192, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "close", + "parameters": [], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "parentId": 1911 + } + ], + "permalink": "/python/api/class/SqlStorageClient#close", + "parentId": 1893 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Create a new database session.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 72193, + "module": "storage_clients._sql._storage_client", + "name": "create_session", + "parsedDocstring": { + "text": "Create a new database session.\n", + "returns": "A new AsyncSession instance." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_sql/_storage_client.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 167, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/_sql/_storage_client.py#L167" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "A new AsyncSession instance." + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Create a new database session.\n" + } + ] + }, + "flags": {}, + "id": 72194, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "create_session", + "parameters": [], + "type": { + "name": "AsyncSession", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 1913 + } + ], + "permalink": "/python/api/class/SqlStorageClient#create_session", + "parentId": 1893 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Create a dataset client." + } + ] + }, + "decorations": [ + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 72195, + "module": "storage_clients._base._storage_client", + "name": "create_dataset_client", + "parsedDocstring": { + "text": "Create a dataset client." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_sql/_storage_client.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 178, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/_sql/_storage_client.py#L178" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Create a dataset client." + } + ] + }, + "flags": {}, + "id": 71567, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "create_dataset_client", + "parameters": [ + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 71568, + "kind": 32768, + "kindString": "Parameter", + "name": "id", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 1287 + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 71569, + "kind": 32768, + "kindString": "Parameter", + "name": "name", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 1287 + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 71570, + "kind": 32768, + "kindString": "Parameter", + "name": "alias", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 1287 + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 71571, + "kind": 32768, + "kindString": "Parameter", + "name": "configuration", + "type": { + "name": "Configuration | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "Configuration", + "target": "2", + "ref": { + "id": 70282, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/configuration.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 20 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/Configuration" + } + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 1287 + } + ], + "type": { + "name": "DatasetClient", + "type": "reference", + "target": "1220", + "ref": { + "id": 71500, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_dataset_client.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 13 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/DatasetClient" + } + }, + "overwrites": { + "name": "StorageClient.create_dataset_client", + "target": 1286, + "type": "reference", + "ref": { + "id": 71566, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 42 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/StorageClient#create_dataset_client" + } + }, + "parentId": 1915 + } + ], + "overwrites": { + "name": "StorageClient.create_dataset_client", + "target": 1286, + "type": "reference", + "ref": { + "id": 71566, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 42 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/StorageClient#create_dataset_client" + } + }, + "permalink": "/python/api/class/SqlStorageClient#create_dataset_client", + "parentId": 1893 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Create a key-value store client." + } + ] + }, + "decorations": [ + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 72201, + "module": "storage_clients._base._storage_client", + "name": "create_kvs_client", + "parsedDocstring": { + "text": "Create a key-value store client." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_sql/_storage_client.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 200, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/_sql/_storage_client.py#L200" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Create a key-value store client." + } + ] + }, + "flags": {}, + "id": 71573, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "create_kvs_client", + "parameters": [ + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 71574, + "kind": 32768, + "kindString": "Parameter", + "name": "id", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 1293 + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 71575, + "kind": 32768, + "kindString": "Parameter", + "name": "name", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 1293 + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 71576, + "kind": 32768, + "kindString": "Parameter", + "name": "alias", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 1293 + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 71577, + "kind": 32768, + "kindString": "Parameter", + "name": "configuration", + "type": { + "name": "Configuration | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "Configuration", + "target": "2", + "ref": { + "id": 70282, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/configuration.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 20 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/Configuration" + } + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 1293 + } + ], + "type": { + "name": "KeyValueStoreClient", + "type": "reference", + "target": "1254", + "ref": { + "id": 71534, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_key_value_store_client.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 14 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/KeyValueStoreClient" + } + }, + "overwrites": { + "name": "StorageClient.create_kvs_client", + "target": 1292, + "type": "reference", + "ref": { + "id": 71572, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 53 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/StorageClient#create_kvs_client" + } + }, + "parentId": 1921 + } + ], + "overwrites": { + "name": "StorageClient.create_kvs_client", + "target": 1292, + "type": "reference", + "ref": { + "id": 71572, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 53 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/StorageClient#create_kvs_client" + } + }, + "permalink": "/python/api/class/SqlStorageClient#create_kvs_client", + "parentId": 1893 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Create a request queue client." + } + ] + }, + "decorations": [ + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 72207, + "module": "storage_clients._base._storage_client", + "name": "create_rq_client", + "parsedDocstring": { + "text": "Create a request queue client." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_sql/_storage_client.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 222, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/_sql/_storage_client.py#L222" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Create a request queue client." + } + ] + }, + "flags": {}, + "id": 71579, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "create_rq_client", + "parameters": [ + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 71580, + "kind": 32768, + "kindString": "Parameter", + "name": "id", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 1299 + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 71581, + "kind": 32768, + "kindString": "Parameter", + "name": "name", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 1299 + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 71582, + "kind": 32768, + "kindString": "Parameter", + "name": "alias", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 1299 + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 71583, + "kind": 32768, + "kindString": "Parameter", + "name": "configuration", + "type": { + "name": "Configuration | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "Configuration", + "target": "2", + "ref": { + "id": 70282, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/configuration.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 20 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/Configuration" + } + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 1299 + } + ], + "type": { + "name": "RequestQueueClient", + "type": "reference", + "target": "1195", + "ref": { + "id": 71475, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_request_queue_client.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 13 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/RequestQueueClient" + } + }, + "overwrites": { + "name": "StorageClient.create_rq_client", + "target": 1298, + "type": "reference", + "ref": { + "id": 71578, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 64 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/StorageClient#create_rq_client" + } + }, + "parentId": 1927 + } + ], + "overwrites": { + "name": "StorageClient.create_rq_client", + "target": 1298, + "type": "reference", + "ref": { + "id": 71578, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 64 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/StorageClient#create_rq_client" + } + }, + "permalink": "/python/api/class/SqlStorageClient#create_rq_client", + "parentId": 1893 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Return a cache key that can differentiate between different storages of this and other clients.\n\nCan be based on configuration or on the client itself. By default, returns a module and name of the client\nclass." + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 74482, + "module": "storage_clients._base._storage_client", + "name": "get_storage_client_cache_key", + "parsedDocstring": { + "text": "Return a cache key that can differentiate between different storages of this and other clients.\n\nCan be based on configuration or on the client itself. By default, returns a module and name of the client\nclass." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 33, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/_base/_storage_client.py#L33" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Return a cache key that can differentiate between different storages of this and other clients.\n\nCan be based on configuration or on the client itself. By default, returns a module and name of the client\nclass." + } + ] + }, + "flags": {}, + "id": 71564, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "get_storage_client_cache_key", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 71565, + "kind": 32768, + "kindString": "Parameter", + "name": "configuration", + "type": { + "name": "Configuration", + "type": "reference", + "target": "2", + "ref": { + "id": 70282, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/configuration.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 20 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/Configuration" + } + }, + "parentId": 1284 + } + ], + "type": { + "name": "Hashable", + "type": "reference", + "target": 0, + "ref": {} + }, + "inheritedFrom": { + "name": "StorageClient.get_storage_client_cache_key", + "target": 1283, + "type": "reference", + "ref": { + "id": 71563, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 33 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/StorageClient#get_storage_client_cache_key" + } + }, + "parentId": 4202 + } + ], + "inheritedFrom": { + "name": "StorageClient.get_storage_client_cache_key", + "target": 1283, + "type": "reference", + "ref": { + "id": 71563, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 33 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/StorageClient#get_storage_client_cache_key" + } + }, + "permalink": "/python/api/class/SqlStorageClient#get_storage_client_cache_key", + "parentId": 1893 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Return statistics about rate limit errors encountered by the HTTP client in storage client." + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 74483, + "module": "storage_clients._base._storage_client", + "name": "get_rate_limit_errors", + "parsedDocstring": { + "text": "Return statistics about rate limit errors encountered by the HTTP client in storage client." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 74, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/_base/_storage_client.py#L74" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Return statistics about rate limit errors encountered by the HTTP client in storage client." + } + ] + }, + "flags": {}, + "id": 71585, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "get_rate_limit_errors", + "parameters": [], + "type": { + "name": "dict[int, int]", + "type": "reference", + "target": 0, + "ref": {} + }, + "inheritedFrom": { + "name": "StorageClient.get_rate_limit_errors", + "target": 1304, + "type": "reference", + "ref": { + "id": 71584, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 74 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/StorageClient#get_rate_limit_errors" + } + }, + "parentId": 4203 + } + ], + "inheritedFrom": { + "name": "StorageClient.get_rate_limit_errors", + "target": 1304, + "type": "reference", + "ref": { + "id": 71584, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 74 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/StorageClient#get_rate_limit_errors" + } + }, + "permalink": "/python/api/class/SqlStorageClient#get_rate_limit_errors", + "parentId": 1893 + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "SQL implementation of the storage client.\n\nThis storage client provides access to datasets, key-value stores, and request queues that persist data\nto a SQL database using SQLAlchemy 2+. Each storage type uses two tables: one for metadata and one for\nrecords.\n\nThe client accepts either a database connection string or a pre-configured AsyncEngine. If neither is\nprovided, it creates a default SQLite database 'crawlee.db' in the storage directory.\n\nDatabase schema is automatically created during initialization. SQLite databases receive performance\noptimizations including WAL mode and increased cache size.\n\n\n:::warning Warning\nThis is an experimental feature. The behavior and interface may change in future versions.\n:::" + } + ] + }, + "decorations": [ + { + "args": "('Storage clients')", + "name": "docs_group" + } + ], + "flags": {}, + "groups": [ + { + "children": [ + 72178, + 72180, + 72174, + 72191, + 72195, + 72201, + 72207, + 72193, + 72186, + 74483, + 74482, + 72188 + ], + "title": "Methods" + }, + { + "children": [ + 72185 + ], + "title": "Properties" + } + ], + "id": 72173, + "module": "storage_clients._sql._storage_client", + "name": "SqlStorageClient", + "parsedDocstring": { + "text": "SQL implementation of the storage client.\n\nThis storage client provides access to datasets, key-value stores, and request queues that persist data\nto a SQL database using SQLAlchemy 2+. Each storage type uses two tables: one for metadata and one for\nrecords.\n\nThe client accepts either a database connection string or a pre-configured AsyncEngine. If neither is\nprovided, it creates a default SQLite database 'crawlee.db' in the storage directory.\n\nDatabase schema is automatically created during initialization. SQLite databases receive performance\noptimizations including WAL mode and increased cache size.\n\n\n:::warning Warning\nThis is an experimental feature. The behavior and interface may change in future versions.\n:::" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_sql/_storage_client.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 35, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/_sql/_storage_client.py#L35" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "extendedTypes": [ + { + "name": "StorageClient", + "target": "1282", + "type": "reference", + "ref": { + "id": 71562, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 19 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/StorageClient" + } + } + ], + "permalink": "/python/api/class/SqlStorageClient", + "categories": [] + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Get the number of requests in the loader that have been handled." + } + ] + }, + "decorations": [ + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 77418, + "module": "request_loaders._request_loader", + "name": "get_handled_count", + "parsedDocstring": { + "text": "Get the number of requests in the loader that have been handled." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 29, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/request_loaders/_request_loader.py#L29" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Get the number of requests in the loader that have been handled." + } + ] + }, + "flags": {}, + "id": 77419, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "get_handled_count", + "parameters": [], + "type": { + "name": "int", + "type": "reference", + "target": 0, + "ref": {} + }, + "overwrites": { + "name": "RequestLoader.get_handled_count", + "target": 2935, + "type": "reference", + "ref": { + "id": 77418, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 29 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestLoader#get_handled_count" + } + }, + "parentId": 2935 + } + ], + "permalink": "/python/api/class/RequestLoader#get_handled_count", + "parentId": 2934 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Get an offline approximation of the total number of requests in the loader (i.e. pending + handled)." + } + ] + }, + "decorations": [ + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 77420, + "module": "request_loaders._request_loader", + "name": "get_total_count", + "parsedDocstring": { + "text": "Get an offline approximation of the total number of requests in the loader (i.e. pending + handled)." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 33, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/request_loaders/_request_loader.py#L33" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Get an offline approximation of the total number of requests in the loader (i.e. pending + handled)." + } + ] + }, + "flags": {}, + "id": 77421, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "get_total_count", + "parameters": [], + "type": { + "name": "int", + "type": "reference", + "target": 0, + "ref": {} + }, + "overwrites": { + "name": "RequestLoader.get_total_count", + "target": 2937, + "type": "reference", + "ref": { + "id": 77420, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 33 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestLoader#get_total_count" + } + }, + "parentId": 2937 + } + ], + "permalink": "/python/api/class/RequestLoader#get_total_count", + "parentId": 2934 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Return True if there are no more requests in the loader (there might still be unfinished requests)." + } + ] + }, + "decorations": [ + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 77422, + "module": "request_loaders._request_loader", + "name": "is_empty", + "parsedDocstring": { + "text": "Return True if there are no more requests in the loader (there might still be unfinished requests)." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 37, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/request_loaders/_request_loader.py#L37" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Return True if there are no more requests in the loader (there might still be unfinished requests)." + } + ] + }, + "flags": {}, + "id": 77423, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "is_empty", + "parameters": [], + "type": { + "name": "bool", + "type": "reference", + "target": 0, + "ref": {} + }, + "overwrites": { + "name": "RequestLoader.is_empty", + "target": 2939, + "type": "reference", + "ref": { + "id": 77422, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 37 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestLoader#is_empty" + } + }, + "parentId": 2939 + } + ], + "permalink": "/python/api/class/RequestLoader#is_empty", + "parentId": 2934 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Return True if all requests have been handled." + } + ] + }, + "decorations": [ + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 77424, + "module": "request_loaders._request_loader", + "name": "is_finished", + "parsedDocstring": { + "text": "Return True if all requests have been handled." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 41, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/request_loaders/_request_loader.py#L41" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Return True if all requests have been handled." + } + ] + }, + "flags": {}, + "id": 77425, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "is_finished", + "parameters": [], + "type": { + "name": "bool", + "type": "reference", + "target": 0, + "ref": {} + }, + "overwrites": { + "name": "RequestLoader.is_finished", + "target": 2941, + "type": "reference", + "ref": { + "id": 77424, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 41 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestLoader#is_finished" + } + }, + "parentId": 2941 + } + ], + "permalink": "/python/api/class/RequestLoader#is_finished", + "parentId": 2934 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Return the next request to be processed, or `None` if there are no more pending requests.\n\nThe method should return `None` if and only if `is_finished` would return `True`. In other cases, the method\nshould wait until a request appears." + } + ] + }, + "decorations": [ + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 77426, + "module": "request_loaders._request_loader", + "name": "fetch_next_request", + "parsedDocstring": { + "text": "Return the next request to be processed, or `None` if there are no more pending requests.\n\nThe method should return `None` if and only if `is_finished` would return `True`. In other cases, the method\nshould wait until a request appears." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 45, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/request_loaders/_request_loader.py#L45" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Return the next request to be processed, or `None` if there are no more pending requests.\n\nThe method should return `None` if and only if `is_finished` would return `True`. In other cases, the method\nshould wait until a request appears." + } + ] + }, + "flags": {}, + "id": 77427, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "fetch_next_request", + "parameters": [], + "type": { + "name": "Request | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "Request", + "target": "409", + "ref": { + "id": 74892, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_request.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 145 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/Request" + } + }, + { + "type": "literal", + "value": {} + } + ] + }, + "overwrites": { + "name": "RequestLoader.fetch_next_request", + "target": 2943, + "type": "reference", + "ref": { + "id": 77426, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 45 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestLoader#fetch_next_request" + } + }, + "parentId": 2943 + } + ], + "permalink": "/python/api/class/RequestLoader#fetch_next_request", + "parentId": 2934 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Mark a request as handled after a successful processing (or after giving up retrying)." + } + ] + }, + "decorations": [ + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 77428, + "module": "request_loaders._request_loader", + "name": "mark_request_as_handled", + "parsedDocstring": { + "text": "Mark a request as handled after a successful processing (or after giving up retrying)." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 53, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/request_loaders/_request_loader.py#L53" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Mark a request as handled after a successful processing (or after giving up retrying)." + } + ] + }, + "flags": {}, + "id": 77429, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "mark_request_as_handled", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 77430, + "kind": 32768, + "kindString": "Parameter", + "name": "request", + "type": { + "name": "Request", + "type": "reference", + "target": "409", + "ref": { + "id": 74892, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_request.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 145 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/Request" + } + }, + "parentId": 2946 + } + ], + "type": { + "name": "ProcessedRequest | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "ProcessedRequest", + "target": "1340", + "ref": { + "id": 75823, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 140 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/ProcessedRequest" + } + }, + { + "type": "literal", + "value": {} + } + ] + }, + "overwrites": { + "name": "RequestLoader.mark_request_as_handled", + "target": 2945, + "type": "reference", + "ref": { + "id": 77428, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 53 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestLoader#mark_request_as_handled" + } + }, + "parentId": 2945 + } + ], + "permalink": "/python/api/class/RequestLoader#mark_request_as_handled", + "parentId": 2934 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Combine the loader with a request manager to support adding and reclaiming requests.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 77431, + "module": "request_loaders._request_loader", + "name": "to_tandem", + "parsedDocstring": { + "text": "Combine the loader with a request manager to support adding and reclaiming requests.\n", + "args": { + "request_manager": "Request manager to combine the loader with.\nIf None is given, the default request queue is used." + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 56, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/request_loaders/_request_loader.py#L56" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Combine the loader with a request manager to support adding and reclaiming requests.\n" + } + ] + }, + "flags": {}, + "id": 77432, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "to_tandem", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Request manager to combine the loader with.\nIf None is given, the default request queue is used." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": false + }, + "id": 77433, + "kind": 32768, + "kindString": "Parameter", + "name": "request_manager", + "type": { + "name": "RequestManager | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "RequestManager", + "target": "3001", + "ref": { + "id": 77484, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_manager.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 18 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/RequestManager" + } + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2949 + } + ], + "type": { + "name": "RequestManagerTandem", + "type": "reference", + "target": "2803", + "ref": { + "id": 77286, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_manager_tandem.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 24 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/RequestManagerTandem" + } + }, + "parentId": 2948 + } + ], + "permalink": "/python/api/class/RequestLoader#to_tandem", + "parentId": 2934 + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "An abstract class defining the interface for classes that provide access to a read-only stream of requests.\n\nRequest loaders are used to manage and provide access to a storage of crawling requests.\n\nKey responsibilities:\n- Fetching the next request to be processed.\n- Marking requests as successfully handled after processing.\n- Managing state information such as the total and handled request counts." + } + ] + }, + "decorations": [ + { + "args": "('Request loaders')", + "name": "docs_group" + } + ], + "flags": {}, + "groups": [ + { + "children": [ + 77426, + 77418, + 77420, + 77422, + 77424, + 77428, + 77431 + ], + "title": "Methods" + } + ], + "id": 77417, + "module": "request_loaders._request_loader", + "name": "RequestLoader", + "parsedDocstring": { + "text": "An abstract class defining the interface for classes that provide access to a read-only stream of requests.\n\nRequest loaders are used to manage and provide access to a storage of crawling requests.\n\nKey responsibilities:\n- Fetching the next request to be processed.\n- Marking requests as successfully handled after processing.\n- Managing state information such as the total and handled request counts." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 17, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/request_loaders/_request_loader.py#L17" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "extendedBy": [ + { + "name": "RequestList", + "target": "2913", + "type": "reference", + "ref": { + "id": 77396, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_list.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 32 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/RequestList" + } + }, + { + "name": "SitemapRequestLoader", + "target": "2963", + "type": "reference", + "ref": { + "id": 77446, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_sitemap_request_loader.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 94 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/SitemapRequestLoader" + } + }, + { + "name": "RequestManager", + "target": "3001", + "type": "reference", + "ref": { + "id": 77484, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_manager.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 18 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/RequestManager" + } + } + ], + "permalink": "/python/api/class/RequestLoader", + "categories": [] + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Remove persistent state either from the Apify Cloud storage or from the local database." + } + ] + }, + "decorations": [ + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 80486, + "module": "request_loaders._request_manager", + "name": "drop", + "parsedDocstring": { + "text": "Remove persistent state either from the Apify Cloud storage or from the local database." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_manager.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 22, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/request_loaders/_request_manager.py#L22" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Remove persistent state either from the Apify Cloud storage or from the local database." + } + ] + }, + "flags": {}, + "id": 80487, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "drop", + "parameters": [], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "overwrites": { + "name": "Storage.drop", + "target": 2342, + "type": "reference", + "ref": { + "id": 79826, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_base.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 57 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/Storage#drop" + } + }, + "parentId": 3002 + } + ], + "permalink": "/python/api/class/RequestManager#drop", + "parentId": 3001 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Remove all queued requests from this manager while keeping it usable.\n\nDefault implementation is a no-op. Subclasses with persistent state should override to clear it." + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 80488, + "module": "request_loaders._request_manager", + "name": "purge", + "parsedDocstring": { + "text": "Remove all queued requests from this manager while keeping it usable.\n\nDefault implementation is a no-op. Subclasses with persistent state should override to clear it." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_manager.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 25, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/request_loaders/_request_manager.py#L25" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Remove all queued requests from this manager while keeping it usable.\n\nDefault implementation is a no-op. Subclasses with persistent state should override to clear it." + } + ] + }, + "flags": {}, + "id": 80489, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "purge", + "parameters": [], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "overwrites": { + "name": "Storage.purge", + "target": 2344, + "type": "reference", + "ref": { + "id": 79828, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_base.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 61 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/Storage#purge" + } + }, + "parentId": 3004 + } + ], + "permalink": "/python/api/class/RequestManager#purge", + "parentId": 3001 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Add a single request to the manager and store it in underlying resource client.\n" + } + ] + }, + "decorations": [ + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 80490, + "module": "request_loaders._request_manager", + "name": "add_request", + "parsedDocstring": { + "text": "Add a single request to the manager and store it in underlying resource client.\n", + "args": { + "request": "The request object (or its string representation) to be added to the manager.", + "forefront": "Determines whether the request should be added to the beginning (if True) or the end (if False)\nof the manager.\n" + }, + "returns": "Information about the request addition to the manager or None if the request was not added." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_manager.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 32, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/request_loaders/_request_manager.py#L32" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "Information about the request addition to the manager or None if the request was not added." + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Add a single request to the manager and store it in underlying resource client.\n" + } + ] + }, + "flags": {}, + "id": 80491, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "add_request", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The request object (or its string representation) to be added to the manager." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 80492, + "kind": 32768, + "kindString": "Parameter", + "name": "request", + "type": { + "name": "str | Request", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "reference", + "name": "Request", + "target": "409", + "ref": { + "id": 77893, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_request.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 145 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/Request" + } + } + ] + }, + "parentId": 3007 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Determines whether the request should be added to the beginning (if True) or the end (if False)\nof the manager.\n" + } + ] + }, + "defaultValue": "False", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 80493, + "kind": 32768, + "kindString": "Parameter", + "name": "forefront", + "type": { + "name": "bool", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 3007 + } + ], + "type": { + "name": "ProcessedRequest | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "ProcessedRequest", + "target": "1340", + "ref": { + "id": 78824, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 140 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/ProcessedRequest" + } + }, + { + "type": "literal", + "value": {} + } + ] + }, + "overwrites": { + "name": "RequestManager.add_request", + "target": 3006, + "type": "reference", + "ref": { + "id": 80490, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_manager.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 32 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestManager#add_request" + } + }, + "parentId": 3006 + } + ], + "permalink": "/python/api/class/RequestManager#add_request", + "parentId": 3001 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Add requests to the manager in batches.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 80494, + "module": "request_loaders._request_manager", + "name": "add_requests", + "parsedDocstring": { + "text": "Add requests to the manager in batches.\n", + "args": { + "requests": "Requests to enqueue.", + "forefront": "If True, add requests to the beginning of the queue.", + "batch_size": "The number of requests to add in one batch.", + "wait_time_between_batches": "Time to wait between adding batches.", + "wait_for_all_requests_to_be_added": "If True, wait for all requests to be added before returning.", + "wait_for_all_requests_to_be_added_timeout": "Timeout for waiting for all requests to be added." + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_manager.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 49, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/request_loaders/_request_manager.py#L49" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Add requests to the manager in batches.\n" + } + ] + }, + "flags": {}, + "id": 80495, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "add_requests", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Requests to enqueue." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 80496, + "kind": 32768, + "kindString": "Parameter", + "name": "requests", + "type": { + "name": "Sequence", + "type": "reference", + "typeArguments": [ + { + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "reference", + "name": "Request", + "target": "409", + "ref": { + "id": 77893, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_request.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 145 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/Request" + } + } + ] + } + ], + "target": 0, + "ref": {} + }, + "parentId": 3011 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "If True, add requests to the beginning of the queue." + } + ] + }, + "defaultValue": "False", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 80497, + "kind": 32768, + "kindString": "Parameter", + "name": "forefront", + "type": { + "name": "bool", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 3011 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The number of requests to add in one batch." + } + ] + }, + "defaultValue": "1000", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 80498, + "kind": 32768, + "kindString": "Parameter", + "name": "batch_size", + "type": { + "name": "int", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 3011 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Time to wait between adding batches." + } + ] + }, + "defaultValue": "timedelta(seconds=1)", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 80499, + "kind": 32768, + "kindString": "Parameter", + "name": "wait_time_between_batches", + "type": { + "name": "timedelta", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 3011 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "If True, wait for all requests to be added before returning." + } + ] + }, + "defaultValue": "False", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 80500, + "kind": 32768, + "kindString": "Parameter", + "name": "wait_for_all_requests_to_be_added", + "type": { + "name": "bool", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 3011 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Timeout for waiting for all requests to be added." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 80501, + "kind": 32768, + "kindString": "Parameter", + "name": "wait_for_all_requests_to_be_added_timeout", + "type": { + "name": "timedelta | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "timedelta", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 3011 + } + ], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "overwrites": { + "name": "RequestManager.add_requests", + "target": 3010, + "type": "reference", + "ref": { + "id": 80494, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_manager.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 49 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestManager#add_requests" + } + }, + "parentId": 3010 + } + ], + "permalink": "/python/api/class/RequestManager#add_requests", + "parentId": 3001 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Reclaims a failed request back to the source, so that it can be returned for processing later again.\n\nIt is possible to modify the request data by supplying an updated request as a parameter." + } + ] + }, + "decorations": [ + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 80502, + "module": "request_loaders._request_manager", + "name": "reclaim_request", + "parsedDocstring": { + "text": "Reclaims a failed request back to the source, so that it can be returned for processing later again.\n\nIt is possible to modify the request data by supplying an updated request as a parameter." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_manager.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 77, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/request_loaders/_request_manager.py#L77" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Reclaims a failed request back to the source, so that it can be returned for processing later again.\n\nIt is possible to modify the request data by supplying an updated request as a parameter." + } + ] + }, + "flags": {}, + "id": 80503, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "reclaim_request", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 80504, + "kind": 32768, + "kindString": "Parameter", + "name": "request", + "type": { + "name": "Request", + "type": "reference", + "target": "409", + "ref": { + "id": 77893, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_request.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 145 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/Request" + } + }, + "parentId": 3019 + }, + { + "defaultValue": "False", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 80505, + "kind": 32768, + "kindString": "Parameter", + "name": "forefront", + "type": { + "name": "bool", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 3019 + } + ], + "type": { + "name": "ProcessedRequest | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "ProcessedRequest", + "target": "1340", + "ref": { + "id": 78824, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 140 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/ProcessedRequest" + } + }, + { + "type": "literal", + "value": {} + } + ] + }, + "overwrites": { + "name": "RequestManager.reclaim_request", + "target": 3018, + "type": "reference", + "ref": { + "id": 80502, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_manager.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 77 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestManager#reclaim_request" + } + }, + "parentId": 3018 + } + ], + "permalink": "/python/api/class/RequestManager#reclaim_request", + "parentId": 3001 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Get the number of requests in the loader that have been handled." + } + ] + }, + "decorations": [ + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 81627, + "module": "request_loaders._request_loader", + "name": "get_handled_count", + "parsedDocstring": { + "text": "Get the number of requests in the loader that have been handled." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 29, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/request_loaders/_request_loader.py#L29" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Get the number of requests in the loader that have been handled." + } + ] + }, + "flags": {}, + "id": 80420, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "get_handled_count", + "parameters": [], + "type": { + "name": "int", + "type": "reference", + "target": 0, + "ref": {} + }, + "overwrites": { + "name": "RequestManager.get_handled_count", + "target": 4143, + "type": "reference", + "ref": { + "id": 81627, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 29 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestManager#get_handled_count" + } + }, + "parentId": 4143 + } + ], + "inheritedFrom": { + "name": "RequestLoader.get_handled_count", + "target": 2935, + "type": "reference", + "ref": { + "id": 80419, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 29 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestLoader#get_handled_count" + } + }, + "permalink": "/python/api/class/RequestManager#get_handled_count", + "parentId": 3001 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Get an offline approximation of the total number of requests in the loader (i.e. pending + handled)." + } + ] + }, + "decorations": [ + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 81628, + "module": "request_loaders._request_loader", + "name": "get_total_count", + "parsedDocstring": { + "text": "Get an offline approximation of the total number of requests in the loader (i.e. pending + handled)." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 33, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/request_loaders/_request_loader.py#L33" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Get an offline approximation of the total number of requests in the loader (i.e. pending + handled)." + } + ] + }, + "flags": {}, + "id": 80422, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "get_total_count", + "parameters": [], + "type": { + "name": "int", + "type": "reference", + "target": 0, + "ref": {} + }, + "overwrites": { + "name": "RequestManager.get_total_count", + "target": 4144, + "type": "reference", + "ref": { + "id": 81628, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 33 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestManager#get_total_count" + } + }, + "parentId": 4144 + } + ], + "inheritedFrom": { + "name": "RequestLoader.get_total_count", + "target": 2937, + "type": "reference", + "ref": { + "id": 80421, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 33 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestLoader#get_total_count" + } + }, + "permalink": "/python/api/class/RequestManager#get_total_count", + "parentId": 3001 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Return True if there are no more requests in the loader (there might still be unfinished requests)." + } + ] + }, + "decorations": [ + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 81629, + "module": "request_loaders._request_loader", + "name": "is_empty", + "parsedDocstring": { + "text": "Return True if there are no more requests in the loader (there might still be unfinished requests)." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 37, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/request_loaders/_request_loader.py#L37" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Return True if there are no more requests in the loader (there might still be unfinished requests)." + } + ] + }, + "flags": {}, + "id": 80424, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "is_empty", + "parameters": [], + "type": { + "name": "bool", + "type": "reference", + "target": 0, + "ref": {} + }, + "overwrites": { + "name": "RequestManager.is_empty", + "target": 4145, + "type": "reference", + "ref": { + "id": 81629, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 37 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestManager#is_empty" + } + }, + "parentId": 4145 + } + ], + "inheritedFrom": { + "name": "RequestLoader.is_empty", + "target": 2939, + "type": "reference", + "ref": { + "id": 80423, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 37 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestLoader#is_empty" + } + }, + "permalink": "/python/api/class/RequestManager#is_empty", + "parentId": 3001 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Return True if all requests have been handled." + } + ] + }, + "decorations": [ + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 81630, + "module": "request_loaders._request_loader", + "name": "is_finished", + "parsedDocstring": { + "text": "Return True if all requests have been handled." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 41, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/request_loaders/_request_loader.py#L41" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Return True if all requests have been handled." + } + ] + }, + "flags": {}, + "id": 80426, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "is_finished", + "parameters": [], + "type": { + "name": "bool", + "type": "reference", + "target": 0, + "ref": {} + }, + "overwrites": { + "name": "RequestManager.is_finished", + "target": 4146, + "type": "reference", + "ref": { + "id": 81630, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 41 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestManager#is_finished" + } + }, + "parentId": 4146 + } + ], + "inheritedFrom": { + "name": "RequestLoader.is_finished", + "target": 2941, + "type": "reference", + "ref": { + "id": 80425, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 41 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestLoader#is_finished" + } + }, + "permalink": "/python/api/class/RequestManager#is_finished", + "parentId": 3001 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Return the next request to be processed, or `None` if there are no more pending requests.\n\nThe method should return `None` if and only if `is_finished` would return `True`. In other cases, the method\nshould wait until a request appears." + } + ] + }, + "decorations": [ + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 81631, + "module": "request_loaders._request_loader", + "name": "fetch_next_request", + "parsedDocstring": { + "text": "Return the next request to be processed, or `None` if there are no more pending requests.\n\nThe method should return `None` if and only if `is_finished` would return `True`. In other cases, the method\nshould wait until a request appears." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 45, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/request_loaders/_request_loader.py#L45" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Return the next request to be processed, or `None` if there are no more pending requests.\n\nThe method should return `None` if and only if `is_finished` would return `True`. In other cases, the method\nshould wait until a request appears." + } + ] + }, + "flags": {}, + "id": 80428, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "fetch_next_request", + "parameters": [], + "type": { + "name": "Request | None", + "type": "reference", + "target": 0, + "ref": {} + }, + "overwrites": { + "name": "RequestManager.fetch_next_request", + "target": 4147, + "type": "reference", + "ref": { + "id": 81631, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 45 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestManager#fetch_next_request" + } + }, + "parentId": 4147 + } + ], + "inheritedFrom": { + "name": "RequestLoader.fetch_next_request", + "target": 2943, + "type": "reference", + "ref": { + "id": 80427, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 45 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestLoader#fetch_next_request" + } + }, + "permalink": "/python/api/class/RequestManager#fetch_next_request", + "parentId": 3001 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Mark a request as handled after a successful processing (or after giving up retrying)." + } + ] + }, + "decorations": [ + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 81632, + "module": "request_loaders._request_loader", + "name": "mark_request_as_handled", + "parsedDocstring": { + "text": "Mark a request as handled after a successful processing (or after giving up retrying)." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 53, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/request_loaders/_request_loader.py#L53" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Mark a request as handled after a successful processing (or after giving up retrying)." + } + ] + }, + "flags": {}, + "id": 80430, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "mark_request_as_handled", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 80431, + "kind": 32768, + "kindString": "Parameter", + "name": "request", + "type": { + "name": "Request", + "type": "reference", + "target": "409", + "ref": { + "id": 77893, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_request.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 145 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/Request" + } + }, + "parentId": 2946 + } + ], + "type": { + "name": "ProcessedRequest | None", + "type": "reference", + "target": 0, + "ref": {} + }, + "overwrites": { + "name": "RequestManager.mark_request_as_handled", + "target": 4148, + "type": "reference", + "ref": { + "id": 81632, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 53 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestManager#mark_request_as_handled" + } + }, + "parentId": 4148 + } + ], + "inheritedFrom": { + "name": "RequestLoader.mark_request_as_handled", + "target": 2945, + "type": "reference", + "ref": { + "id": 80429, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 53 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestLoader#mark_request_as_handled" + } + }, + "permalink": "/python/api/class/RequestManager#mark_request_as_handled", + "parentId": 3001 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Combine the loader with a request manager to support adding and reclaiming requests.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 81633, + "module": "request_loaders._request_loader", + "name": "to_tandem", + "parsedDocstring": { + "text": "Combine the loader with a request manager to support adding and reclaiming requests.\n", + "args": { + "request_manager": "Request manager to combine the loader with.\nIf None is given, the default request queue is used." + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 56, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/request_loaders/_request_loader.py#L56" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Combine the loader with a request manager to support adding and reclaiming requests.\n" + } + ] + }, + "flags": {}, + "id": 80433, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "to_tandem", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Request manager to combine the loader with.\nIf None is given, the default request queue is used." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": false + }, + "id": 80434, + "kind": 32768, + "kindString": "Parameter", + "name": "request_manager", + "type": { + "name": "RequestManager | None", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2949 + } + ], + "type": { + "name": "RequestManagerTandem", + "type": "reference", + "target": "2803", + "ref": { + "id": 80287, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_manager_tandem.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 24 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/RequestManagerTandem" + } + }, + "inheritedFrom": { + "name": "RequestLoader.to_tandem", + "target": 2948, + "type": "reference", + "ref": { + "id": 80432, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 56 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestLoader#to_tandem" + } + }, + "parentId": 4149 + } + ], + "inheritedFrom": { + "name": "RequestLoader.to_tandem", + "target": 2948, + "type": "reference", + "ref": { + "id": 80432, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 56 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestLoader#to_tandem" + } + }, + "permalink": "/python/api/class/RequestManager#to_tandem", + "parentId": 3001 + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Base class that extends `RequestLoader` with the capability to enqueue new requests and reclaim failed ones." + } + ] + }, + "decorations": [ + { + "args": "('Request loaders')", + "name": "docs_group" + } + ], + "flags": {}, + "groups": [ + { + "children": [ + 80490, + 80494, + 80486, + 81631, + 81627, + 81628, + 81629, + 81630, + 81632, + 80488, + 80502, + 81633 + ], + "title": "Methods" + } + ], + "id": 80485, + "module": "request_loaders._request_manager", + "name": "RequestManager", + "parsedDocstring": { + "text": "Base class that extends `RequestLoader` with the capability to enqueue new requests and reclaim failed ones." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_manager.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 18, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/request_loaders/_request_manager.py#L18" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "extendedTypes": [ + { + "name": "RequestLoader", + "target": "2934", + "type": "reference", + "ref": { + "id": 80418, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 17 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/RequestLoader" + } + } + ], + "extendedBy": [ + { + "name": "RequestQueue", + "target": "2348", + "type": "reference", + "ref": { + "id": 79832, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_request_queue.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 33 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/RequestQueue" + } + }, + { + "name": "RequestManagerTandem", + "target": "2803", + "type": "reference", + "ref": { + "id": 80287, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_manager_tandem.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 24 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/RequestManagerTandem" + } + }, + { + "name": "ThrottlingRequestManager", + "target": "2841", + "type": "reference", + "ref": { + "id": 80325, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_throttling_request_manager.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 35 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/ThrottlingRequestManager" + } + } + ], + "permalink": "/python/api/class/RequestManager", + "categories": [] + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 84437, + "module": "request_loaders._request_manager_tandem", + "name": "__init__", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_manager_tandem.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 31, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/request_loaders/_request_manager_tandem.py#L31" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "flags": {}, + "id": 84438, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "__init__", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 84439, + "kind": 32768, + "kindString": "Parameter", + "name": "request_loader", + "type": { + "name": "RequestLoader", + "type": "reference", + "target": "2934", + "ref": { + "id": 84567, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 17 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/RequestLoader" + } + }, + "parentId": 2805 + }, + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 84440, + "kind": 32768, + "kindString": "Parameter", + "name": "request_manager", + "type": { + "name": "RequestManager", + "type": "reference", + "target": "3001", + "ref": { + "id": 84634, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_manager.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 18 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/RequestManager" + } + }, + "parentId": 2805 + } + ], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "parentId": 2804 + } + ], + "permalink": "/python/api/class/RequestManagerTandem#__init__", + "parentId": 2803 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Get the number of requests in the loader that have been handled." + } + ] + }, + "decorations": [ + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 84441, + "module": "request_loaders._request_loader", + "name": "get_handled_count", + "parsedDocstring": { + "text": "Get the number of requests in the loader that have been handled." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_manager_tandem.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 36, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/request_loaders/_request_manager_tandem.py#L36" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Get the number of requests in the loader that have been handled." + } + ] + }, + "flags": {}, + "id": 84569, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "get_handled_count", + "parameters": [], + "type": { + "name": "int", + "type": "reference", + "target": 0, + "ref": {} + }, + "overwrites": { + "name": "RequestManager.get_handled_count", + "target": 4143, + "type": "reference", + "ref": { + "id": 85776, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 29 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestManager#get_handled_count" + } + }, + "parentId": 2808 + } + ], + "overwrites": { + "name": "RequestManager.get_handled_count", + "target": 4143, + "type": "reference", + "ref": { + "id": 85776, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 29 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestManager#get_handled_count" + } + }, + "permalink": "/python/api/class/RequestManagerTandem#get_handled_count", + "parentId": 2803 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Get an offline approximation of the total number of requests in the loader (i.e. pending + handled)." + } + ] + }, + "decorations": [ + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 84443, + "module": "request_loaders._request_loader", + "name": "get_total_count", + "parsedDocstring": { + "text": "Get an offline approximation of the total number of requests in the loader (i.e. pending + handled)." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_manager_tandem.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 40, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/request_loaders/_request_manager_tandem.py#L40" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Get an offline approximation of the total number of requests in the loader (i.e. pending + handled)." + } + ] + }, + "flags": {}, + "id": 84571, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "get_total_count", + "parameters": [], + "type": { + "name": "int", + "type": "reference", + "target": 0, + "ref": {} + }, + "overwrites": { + "name": "RequestManager.get_total_count", + "target": 4144, + "type": "reference", + "ref": { + "id": 85777, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 33 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestManager#get_total_count" + } + }, + "parentId": 2810 + } + ], + "overwrites": { + "name": "RequestManager.get_total_count", + "target": 4144, + "type": "reference", + "ref": { + "id": 85777, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 33 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestManager#get_total_count" + } + }, + "permalink": "/python/api/class/RequestManagerTandem#get_total_count", + "parentId": 2803 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Return True if there are no more requests in the loader (there might still be unfinished requests)." + } + ] + }, + "decorations": [ + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 84445, + "module": "request_loaders._request_loader", + "name": "is_empty", + "parsedDocstring": { + "text": "Return True if there are no more requests in the loader (there might still be unfinished requests)." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_manager_tandem.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 44, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/request_loaders/_request_manager_tandem.py#L44" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Return True if there are no more requests in the loader (there might still be unfinished requests)." + } + ] + }, + "flags": {}, + "id": 84573, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "is_empty", + "parameters": [], + "type": { + "name": "bool", + "type": "reference", + "target": 0, + "ref": {} + }, + "overwrites": { + "name": "RequestManager.is_empty", + "target": 4145, + "type": "reference", + "ref": { + "id": 85778, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 37 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestManager#is_empty" + } + }, + "parentId": 2812 + } + ], + "overwrites": { + "name": "RequestManager.is_empty", + "target": 4145, + "type": "reference", + "ref": { + "id": 85778, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 37 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestManager#is_empty" + } + }, + "permalink": "/python/api/class/RequestManagerTandem#is_empty", + "parentId": 2803 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Return True if all requests have been handled." + } + ] + }, + "decorations": [ + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 84447, + "module": "request_loaders._request_loader", + "name": "is_finished", + "parsedDocstring": { + "text": "Return True if all requests have been handled." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_manager_tandem.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 48, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/request_loaders/_request_manager_tandem.py#L48" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Return True if all requests have been handled." + } + ] + }, + "flags": {}, + "id": 84575, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "is_finished", + "parameters": [], + "type": { + "name": "bool", + "type": "reference", + "target": 0, + "ref": {} + }, + "overwrites": { + "name": "RequestManager.is_finished", + "target": 4146, + "type": "reference", + "ref": { + "id": 85779, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 41 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestManager#is_finished" + } + }, + "parentId": 2814 + } + ], + "overwrites": { + "name": "RequestManager.is_finished", + "target": 4146, + "type": "reference", + "ref": { + "id": 85779, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 41 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestManager#is_finished" + } + }, + "permalink": "/python/api/class/RequestManagerTandem#is_finished", + "parentId": 2803 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Add a single request to the manager and store it in underlying resource client.\n" + } + ] + }, + "decorations": [ + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 84449, + "module": "request_loaders._request_manager", + "name": "add_request", + "parsedDocstring": { + "text": "Add a single request to the manager and store it in underlying resource client.\n", + "args": { + "request": "The request object (or its string representation) to be added to the manager.", + "forefront": "Determines whether the request should be added to the beginning (if True) or the end (if False)\nof the manager.\n" + }, + "returns": "Information about the request addition to the manager or None if the request was not added." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_manager_tandem.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 52, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/request_loaders/_request_manager_tandem.py#L52" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "Information about the request addition to the manager or None if the request was not added." + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Add a single request to the manager and store it in underlying resource client.\n" + } + ] + }, + "flags": {}, + "id": 84640, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "add_request", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The request object (or its string representation) to be added to the manager." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 84641, + "kind": 32768, + "kindString": "Parameter", + "name": "request", + "type": { + "name": "str | Request", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "reference", + "name": "Request", + "target": "409", + "ref": { + "id": 82042, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_request.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 145 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/Request" + } + } + ] + }, + "parentId": 3007 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Determines whether the request should be added to the beginning (if True) or the end (if False)\nof the manager.\n" + } + ] + }, + "defaultValue": "False", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 84642, + "kind": 32768, + "kindString": "Parameter", + "name": "forefront", + "type": { + "name": "bool", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 3007 + } + ], + "type": { + "name": "ProcessedRequest | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "ProcessedRequest", + "target": "1340", + "ref": { + "id": 82973, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 140 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/ProcessedRequest" + } + }, + { + "type": "literal", + "value": {} + } + ] + }, + "overwrites": { + "name": "RequestManager.add_request", + "target": 3006, + "type": "reference", + "ref": { + "id": 84639, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_manager.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 32 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestManager#add_request" + } + }, + "parentId": 2816 + } + ], + "overwrites": { + "name": "RequestManager.add_request", + "target": 3006, + "type": "reference", + "ref": { + "id": 84639, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_manager.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 32 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestManager#add_request" + } + }, + "permalink": "/python/api/class/RequestManagerTandem#add_request", + "parentId": 2803 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Add requests to the manager in batches.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 84453, + "module": "request_loaders._request_manager", + "name": "add_requests", + "parsedDocstring": { + "text": "Add requests to the manager in batches.\n", + "args": { + "requests": "Requests to enqueue.", + "forefront": "If True, add requests to the beginning of the queue.", + "batch_size": "The number of requests to add in one batch.", + "wait_time_between_batches": "Time to wait between adding batches.", + "wait_for_all_requests_to_be_added": "If True, wait for all requests to be added before returning.", + "wait_for_all_requests_to_be_added_timeout": "Timeout for waiting for all requests to be added." + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_manager_tandem.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 56, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/request_loaders/_request_manager_tandem.py#L56" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Add requests to the manager in batches.\n" + } + ] + }, + "flags": {}, + "id": 84644, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "add_requests", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Requests to enqueue." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 84645, + "kind": 32768, + "kindString": "Parameter", + "name": "requests", + "type": { + "name": "Sequence", + "type": "reference", + "typeArguments": [ + { + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "reference", + "name": "Request", + "target": "409", + "ref": { + "id": 82042, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_request.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 145 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/Request" + } + } + ] + } + ], + "target": 0, + "ref": {} + }, + "parentId": 3011 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "If True, add requests to the beginning of the queue." + } + ] + }, + "defaultValue": "False", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 84646, + "kind": 32768, + "kindString": "Parameter", + "name": "forefront", + "type": { + "name": "bool", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 3011 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The number of requests to add in one batch." + } + ] + }, + "defaultValue": "1000", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 84647, + "kind": 32768, + "kindString": "Parameter", + "name": "batch_size", + "type": { + "name": "int", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 3011 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Time to wait between adding batches." + } + ] + }, + "defaultValue": "timedelta(seconds=1)", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 84648, + "kind": 32768, + "kindString": "Parameter", + "name": "wait_time_between_batches", + "type": { + "name": "timedelta", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 3011 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "If True, wait for all requests to be added before returning." + } + ] + }, + "defaultValue": "False", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 84649, + "kind": 32768, + "kindString": "Parameter", + "name": "wait_for_all_requests_to_be_added", + "type": { + "name": "bool", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 3011 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Timeout for waiting for all requests to be added." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 84650, + "kind": 32768, + "kindString": "Parameter", + "name": "wait_for_all_requests_to_be_added_timeout", + "type": { + "name": "timedelta | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "timedelta", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 3011 + } + ], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "overwrites": { + "name": "RequestManager.add_requests", + "target": 3010, + "type": "reference", + "ref": { + "id": 84643, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_manager.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 49 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestManager#add_requests" + } + }, + "parentId": 2820 + } + ], + "overwrites": { + "name": "RequestManager.add_requests", + "target": 3010, + "type": "reference", + "ref": { + "id": 84643, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_manager.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 49 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestManager#add_requests" + } + }, + "permalink": "/python/api/class/RequestManagerTandem#add_requests", + "parentId": 2803 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Return the next request to be processed, or `None` if there are no more pending requests.\n\nThe method should return `None` if and only if `is_finished` would return `True`. In other cases, the method\nshould wait until a request appears." + } + ] + }, + "decorations": [ + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 84461, + "module": "request_loaders._request_loader", + "name": "fetch_next_request", + "parsedDocstring": { + "text": "Return the next request to be processed, or `None` if there are no more pending requests.\n\nThe method should return `None` if and only if `is_finished` would return `True`. In other cases, the method\nshould wait until a request appears." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_manager_tandem.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 76, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/request_loaders/_request_manager_tandem.py#L76" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Return the next request to be processed, or `None` if there are no more pending requests.\n\nThe method should return `None` if and only if `is_finished` would return `True`. In other cases, the method\nshould wait until a request appears." + } + ] + }, + "flags": {}, + "id": 84577, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "fetch_next_request", + "parameters": [], + "type": { + "name": "Request | None", + "type": "reference", + "target": 0, + "ref": {} + }, + "overwrites": { + "name": "RequestManager.fetch_next_request", + "target": 4147, + "type": "reference", + "ref": { + "id": 85780, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 45 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestManager#fetch_next_request" + } + }, + "parentId": 2828 + } + ], + "overwrites": { + "name": "RequestManager.fetch_next_request", + "target": 4147, + "type": "reference", + "ref": { + "id": 85780, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 45 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestManager#fetch_next_request" + } + }, + "permalink": "/python/api/class/RequestManagerTandem#fetch_next_request", + "parentId": 2803 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Reclaims a failed request back to the source, so that it can be returned for processing later again.\n\nIt is possible to modify the request data by supplying an updated request as a parameter." + } + ] + }, + "decorations": [ + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 84463, + "module": "request_loaders._request_manager", + "name": "reclaim_request", + "parsedDocstring": { + "text": "Reclaims a failed request back to the source, so that it can be returned for processing later again.\n\nIt is possible to modify the request data by supplying an updated request as a parameter." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_manager_tandem.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 102, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/request_loaders/_request_manager_tandem.py#L102" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Reclaims a failed request back to the source, so that it can be returned for processing later again.\n\nIt is possible to modify the request data by supplying an updated request as a parameter." + } + ] + }, + "flags": {}, + "id": 84652, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "reclaim_request", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 84653, + "kind": 32768, + "kindString": "Parameter", + "name": "request", + "type": { + "name": "Request", + "type": "reference", + "target": "409", + "ref": { + "id": 82042, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_request.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 145 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/Request" + } + }, + "parentId": 3019 + }, + { + "defaultValue": "False", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 84654, + "kind": 32768, + "kindString": "Parameter", + "name": "forefront", + "type": { + "name": "bool", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 3019 + } + ], + "type": { + "name": "ProcessedRequest | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "ProcessedRequest", + "target": "1340", + "ref": { + "id": 82973, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 140 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/ProcessedRequest" + } + }, + { + "type": "literal", + "value": {} + } + ] + }, + "overwrites": { + "name": "RequestManager.reclaim_request", + "target": 3018, + "type": "reference", + "ref": { + "id": 84651, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_manager.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 77 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestManager#reclaim_request" + } + }, + "parentId": 2830 + } + ], + "overwrites": { + "name": "RequestManager.reclaim_request", + "target": 3018, + "type": "reference", + "ref": { + "id": 84651, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_manager.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 77 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestManager#reclaim_request" + } + }, + "permalink": "/python/api/class/RequestManagerTandem#reclaim_request", + "parentId": 2803 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Mark a request as handled after a successful processing (or after giving up retrying)." + } + ] + }, + "decorations": [ + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 84467, + "module": "request_loaders._request_loader", + "name": "mark_request_as_handled", + "parsedDocstring": { + "text": "Mark a request as handled after a successful processing (or after giving up retrying)." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_manager_tandem.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 106, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/request_loaders/_request_manager_tandem.py#L106" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Mark a request as handled after a successful processing (or after giving up retrying)." + } + ] + }, + "flags": {}, + "id": 84579, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "mark_request_as_handled", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 84580, + "kind": 32768, + "kindString": "Parameter", + "name": "request", + "type": { + "name": "Request", + "type": "reference", + "target": "409", + "ref": { + "id": 82042, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_request.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 145 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/Request" + } + }, + "parentId": 2946 + } + ], + "type": { + "name": "ProcessedRequest | None", + "type": "reference", + "target": 0, + "ref": {} + }, + "overwrites": { + "name": "RequestManager.mark_request_as_handled", + "target": 4148, + "type": "reference", + "ref": { + "id": 85781, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 53 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestManager#mark_request_as_handled" + } + }, + "parentId": 2834 + } + ], + "overwrites": { + "name": "RequestManager.mark_request_as_handled", + "target": 4148, + "type": "reference", + "ref": { + "id": 85781, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 53 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestManager#mark_request_as_handled" + } + }, + "permalink": "/python/api/class/RequestManagerTandem#mark_request_as_handled", + "parentId": 2803 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Remove persistent state either from the Apify Cloud storage or from the local database." + } + ] + }, + "decorations": [ + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 84470, + "module": "request_loaders._request_manager", + "name": "drop", + "parsedDocstring": { + "text": "Remove persistent state either from the Apify Cloud storage or from the local database." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_manager_tandem.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 110, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/request_loaders/_request_manager_tandem.py#L110" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Remove persistent state either from the Apify Cloud storage or from the local database." + } + ] + }, + "flags": {}, + "id": 84636, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "drop", + "parameters": [], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "overwrites": { + "name": "Storage.drop", + "target": 2342, + "type": "reference", + "ref": { + "id": 83975, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_base.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 57 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/Storage#drop" + } + }, + "parentId": 2837 + } + ], + "overwrites": { + "name": "RequestManager.drop", + "target": 3002, + "type": "reference", + "ref": { + "id": 84635, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_manager.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 22 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestManager#drop" + } + }, + "permalink": "/python/api/class/RequestManagerTandem#drop", + "parentId": 2803 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Remove all queued requests from this manager while keeping it usable.\n\nDefault implementation is a no-op. Subclasses with persistent state should override to clear it." + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 85784, + "module": "request_loaders._request_manager", + "name": "purge", + "parsedDocstring": { + "text": "Remove all queued requests from this manager while keeping it usable.\n\nDefault implementation is a no-op. Subclasses with persistent state should override to clear it." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_manager.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 25, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/request_loaders/_request_manager.py#L25" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Remove all queued requests from this manager while keeping it usable.\n\nDefault implementation is a no-op. Subclasses with persistent state should override to clear it." + } + ] + }, + "flags": {}, + "id": 84638, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "purge", + "parameters": [], + "type": { + "name": "None", + "type": "reference", + "target": 0, + "ref": {} + }, + "inheritedFrom": { + "name": "RequestManager.purge", + "target": 3004, + "type": "reference", + "ref": { + "id": 84637, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_manager.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 25 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestManager#purge" + } + }, + "parentId": 4151 + } + ], + "inheritedFrom": { + "name": "RequestManager.purge", + "target": 3004, + "type": "reference", + "ref": { + "id": 84637, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_manager.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 25 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestManager#purge" + } + }, + "permalink": "/python/api/class/RequestManagerTandem#purge", + "parentId": 2803 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Combine the loader with a request manager to support adding and reclaiming requests.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 85785, + "module": "request_loaders._request_loader", + "name": "to_tandem", + "parsedDocstring": { + "text": "Combine the loader with a request manager to support adding and reclaiming requests.\n", + "args": { + "request_manager": "Request manager to combine the loader with.\nIf None is given, the default request queue is used." + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 56, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/request_loaders/_request_loader.py#L56" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Combine the loader with a request manager to support adding and reclaiming requests.\n" + } + ] + }, + "flags": {}, + "id": 84582, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "to_tandem", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Request manager to combine the loader with.\nIf None is given, the default request queue is used." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": false + }, + "id": 84583, + "kind": 32768, + "kindString": "Parameter", + "name": "request_manager", + "type": { + "name": "RequestManager | None", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2949 + } + ], + "type": { + "name": "RequestManagerTandem", + "type": "reference", + "target": "2803", + "ref": { + "id": 84436, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_manager_tandem.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 24 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/RequestManagerTandem" + } + }, + "inheritedFrom": { + "name": "RequestLoader.to_tandem", + "target": 2948, + "type": "reference", + "ref": { + "id": 84581, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 56 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestLoader#to_tandem" + } + }, + "parentId": 4152 + } + ], + "inheritedFrom": { + "name": "RequestLoader.to_tandem", + "target": 2948, + "type": "reference", + "ref": { + "id": 84581, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 56 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestLoader#to_tandem" + } + }, + "permalink": "/python/api/class/RequestManagerTandem#to_tandem", + "parentId": 2803 + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Implements a tandem behaviour for a pair of `RequestLoader` and `RequestManager`.\n\nIn this scenario, the contents of the \"loader\" get transferred into the \"manager\", allowing processing the requests\nfrom both sources and also enqueueing new requests (not possible with plain `RequestManager`)." + } + ] + }, + "decorations": [ + { + "args": "('Request loaders')", + "name": "docs_group" + } + ], + "flags": {}, + "groups": [ + { + "children": [ + 84437, + 84449, + 84453, + 84470, + 84461, + 84441, + 84443, + 84445, + 84447, + 84467, + 85784, + 84463, + 85785 + ], + "title": "Methods" + } + ], + "id": 84436, + "module": "request_loaders._request_manager_tandem", + "name": "RequestManagerTandem", + "parsedDocstring": { + "text": "Implements a tandem behaviour for a pair of `RequestLoader` and `RequestManager`.\n\nIn this scenario, the contents of the \"loader\" get transferred into the \"manager\", allowing processing the requests\nfrom both sources and also enqueueing new requests (not possible with plain `RequestManager`)." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_manager_tandem.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 24, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/request_loaders/_request_manager_tandem.py#L24" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "extendedTypes": [ + { + "name": "RequestManager", + "target": "3001", + "type": "reference", + "ref": { + "id": 84634, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_manager.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 18 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/RequestManager" + } + } + ], + "permalink": "/python/api/class/RequestManagerTandem", + "categories": [] + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Initialize the sitemap request loader.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 88749, + "module": "request_loaders._sitemap_request_loader", + "name": "__init__", + "parsedDocstring": { + "text": "Initialize the sitemap request loader.\n", + "args": { + "sitemap_urls": "Configuration options for the loader.", + "proxy_info": "Optional proxy to use for fetching sitemaps.", + "include": "List of glob or regex patterns to include URLs.", + "exclude": "List of glob or regex patterns to exclude URLs.", + "enqueue_strategy": "Strategy used to decide which sitemap-derived URLs (both nested-sitemap entries and\nURL entries) are kept relative to the parent sitemap URL. Defaults to `'same-hostname'`, matching\nthe sitemap protocol's same-host expectation and the `enqueue_links` default; pass `'all'` to\ndisable filtering. Note: regardless of `enqueue_strategy`, entries with non-`http(s)` schemes are\nalways filtered out.", + "max_buffer_size": "Maximum number of URLs to buffer in memory.", + "http_client": "the instance of `HttpClient` to use for fetching sitemaps.", + "persist_state_key": "A key for persisting the loader's state in the KeyValueStore.\nWhen provided, allows resuming from where it left off after interruption.\nIf None, no state persistence occurs.", + "transform_request_function": "An optional function to transform requests\ngenerated by the loader. It receives `RequestOptions` with `url` and should return either\nmodified `RequestOptions` or a `RequestTransformAction`." + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_sitemap_request_loader.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 109, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/request_loaders/_sitemap_request_loader.py#L109" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Initialize the sitemap request loader.\n" + } + ] + }, + "flags": {}, + "id": 88750, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "__init__", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Configuration options for the loader." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 88751, + "kind": 32768, + "kindString": "Parameter", + "name": "sitemap_urls", + "type": { + "name": "list", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + }, + "parentId": 2965 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "the instance of `HttpClient` to use for fetching sitemaps." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 88752, + "kind": 32768, + "kindString": "Parameter", + "name": "http_client", + "type": { + "name": "HttpClient", + "type": "reference", + "target": "3137", + "ref": { + "id": 88922, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/http_clients/_base.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 75 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/HttpClient" + } + }, + "parentId": 2965 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Optional proxy to use for fetching sitemaps." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 88753, + "kind": 32768, + "kindString": "Parameter", + "name": "proxy_info", + "type": { + "name": "ProxyInfo | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "ProxyInfo", + "target": "21", + "ref": { + "id": 85806, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/proxy_configuration.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 26 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/ProxyInfo" + } + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2965 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "List of glob or regex patterns to include URLs." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 88754, + "kind": 32768, + "kindString": "Parameter", + "name": "include", + "type": { + "name": "list[re.Pattern[Any] | Glob] | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "list", + "typeArguments": [ + { + "type": "union", + "types": [ + { + "type": "reference", + "name": "re.Pattern", + "typeArguments": [ + { + "type": "reference", + "name": "Any", + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + }, + { + "type": "reference", + "name": "Glob", + "target": "3253", + "ref": { + "id": 89038, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_utils/globs.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 11 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/Glob" + } + } + ] + } + ], + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2965 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "List of glob or regex patterns to exclude URLs." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 88755, + "kind": 32768, + "kindString": "Parameter", + "name": "exclude", + "type": { + "name": "list[re.Pattern[Any] | Glob] | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "list", + "typeArguments": [ + { + "type": "union", + "types": [ + { + "type": "reference", + "name": "re.Pattern", + "typeArguments": [ + { + "type": "reference", + "name": "Any", + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + }, + { + "type": "reference", + "name": "Glob", + "target": "3253", + "ref": { + "id": 89038, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_utils/globs.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 11 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/Glob" + } + } + ] + } + ], + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2965 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Strategy used to decide which sitemap-derived URLs (both nested-sitemap entries and\nURL entries) are kept relative to the parent sitemap URL. Defaults to `'same-hostname'`, matching\nthe sitemap protocol's same-host expectation and the `enqueue_links` default; pass `'all'` to\ndisable filtering. Note: regardless of `enqueue_strategy`, entries with non-`http(s)` schemes are\nalways filtered out." + } + ] + }, + "defaultValue": "'same-hostname'", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 88756, + "kind": 32768, + "kindString": "Parameter", + "name": "enqueue_strategy", + "type": { + "name": "EnqueueStrategy", + "type": "reference", + "target": "88", + "ref": { + "id": 85873, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_types.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 47 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api#EnqueueStrategy" + } + }, + "parentId": 2965 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Maximum number of URLs to buffer in memory." + } + ] + }, + "defaultValue": "200", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 88757, + "kind": 32768, + "kindString": "Parameter", + "name": "max_buffer_size", + "type": { + "name": "int", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2965 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "A key for persisting the loader's state in the KeyValueStore.\nWhen provided, allows resuming from where it left off after interruption.\nIf None, no state persistence occurs." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 88758, + "kind": 32768, + "kindString": "Parameter", + "name": "persist_state_key", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2965 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "An optional function to transform requests\ngenerated by the loader. It receives `RequestOptions` with `url` and should return either\nmodified `RequestOptions` or a `RequestTransformAction`." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 88759, + "kind": 32768, + "kindString": "Parameter", + "name": "transform_request_function", + "type": { + "name": "Callable[[RequestOptions], RequestOptions | RequestTransformAction] | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "Callable", + "typeArguments": [ + { + "type": "reference", + "name": "[RequestOptions]", + "target": 0, + "ref": {} + }, + { + "type": "union", + "types": [ + { + "type": "reference", + "name": "RequestOptions", + "target": "393", + "ref": { + "id": 86178, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_request.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 121 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/RequestOptions" + } + }, + { + "type": "reference", + "name": "RequestTransformAction", + "target": "87", + "ref": { + "id": 85872, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_types.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 45 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api#RequestTransformAction" + } + } + ] + } + ], + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2965 + } + ], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "parentId": 2964 + } + ], + "permalink": "/python/api/class/SitemapRequestLoader#__init__", + "parentId": 2963 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Enter the context manager." + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 88760, + "module": "request_loaders._sitemap_request_loader", + "name": "__aenter__", + "parsedDocstring": { + "text": "Enter the context manager." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_sitemap_request_loader.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 171, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/request_loaders/_sitemap_request_loader.py#L171" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Enter the context manager." + } + ] + }, + "flags": {}, + "id": 88761, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "__aenter__", + "parameters": [], + "type": { + "name": "SitemapRequestLoader", + "type": "reference", + "target": "2963", + "ref": { + "id": 88748, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_sitemap_request_loader.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 94 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/SitemapRequestLoader" + } + }, + "parentId": 2975 + } + ], + "permalink": "/python/api/class/SitemapRequestLoader#__aenter__", + "parentId": 2963 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Exit the context manager." + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 88762, + "module": "request_loaders._sitemap_request_loader", + "name": "__aexit__", + "parsedDocstring": { + "text": "Exit the context manager." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_sitemap_request_loader.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 176, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/request_loaders/_sitemap_request_loader.py#L176" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Exit the context manager." + } + ] + }, + "flags": {}, + "id": 88763, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "__aexit__", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 88764, + "kind": 32768, + "kindString": "Parameter", + "name": "exc_type", + "type": { + "name": "type[BaseException] | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "type", + "typeArguments": [ + { + "type": "reference", + "name": "BaseException", + "target": 0, + "ref": {} + } + ], + "target": "3422", + "ref": { + "id": 89207, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_utils/sitemap.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 62 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api/class/SitemapSource#type" + } + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2978 + }, + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 88765, + "kind": 32768, + "kindString": "Parameter", + "name": "exc_value", + "type": { + "name": "BaseException | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "BaseException", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2978 + }, + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 88766, + "kind": 32768, + "kindString": "Parameter", + "name": "exc_traceback", + "type": { + "name": "TracebackType | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "TracebackType", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2978 + } + ], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "parentId": 2977 + } + ], + "permalink": "/python/api/class/SitemapRequestLoader#__aexit__", + "parentId": 2963 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Return the total number of URLs found so far." + } + ] + }, + "decorations": [ + { + "name": "override" + } + ], + "flags": {}, + "groups": [], + "id": 88767, + "module": "request_loaders._sitemap_request_loader", + "name": "get_total_count", + "parsedDocstring": { + "text": "Return the total number of URLs found so far." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_sitemap_request_loader.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 186, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/request_loaders/_sitemap_request_loader.py#L186" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Return the total number of URLs found so far." + } + ] + }, + "flags": {}, + "id": 88768, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "get_total_count", + "parameters": [], + "type": { + "name": "int", + "type": "reference", + "target": 0, + "ref": {} + }, + "overwrites": { + "name": "RequestLoader.get_total_count", + "target": 2937, + "type": "reference", + "ref": { + "id": 88722, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 33 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestLoader#get_total_count" + } + }, + "parentId": 2982 + } + ], + "overwrites": { + "name": "RequestLoader.get_total_count", + "target": 2937, + "type": "reference", + "ref": { + "id": 88722, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 33 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestLoader#get_total_count" + } + }, + "permalink": "/python/api/class/SitemapRequestLoader#get_total_count", + "parentId": 2963 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Return the number of URLs that have been handled." + } + ] + }, + "decorations": [ + { + "name": "override" + } + ], + "flags": {}, + "groups": [], + "id": 88769, + "module": "request_loaders._sitemap_request_loader", + "name": "get_handled_count", + "parsedDocstring": { + "text": "Return the number of URLs that have been handled." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_sitemap_request_loader.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 192, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/request_loaders/_sitemap_request_loader.py#L192" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Return the number of URLs that have been handled." + } + ] + }, + "flags": {}, + "id": 88770, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "get_handled_count", + "parameters": [], + "type": { + "name": "int", + "type": "reference", + "target": 0, + "ref": {} + }, + "overwrites": { + "name": "RequestLoader.get_handled_count", + "target": 2935, + "type": "reference", + "ref": { + "id": 88720, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 29 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestLoader#get_handled_count" + } + }, + "parentId": 2984 + } + ], + "overwrites": { + "name": "RequestLoader.get_handled_count", + "target": 2935, + "type": "reference", + "ref": { + "id": 88720, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 29 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestLoader#get_handled_count" + } + }, + "permalink": "/python/api/class/SitemapRequestLoader#get_handled_count", + "parentId": 2963 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Check if there are no more URLs to process." + } + ] + }, + "decorations": [ + { + "name": "override" + } + ], + "flags": {}, + "groups": [], + "id": 88771, + "module": "request_loaders._sitemap_request_loader", + "name": "is_empty", + "parsedDocstring": { + "text": "Check if there are no more URLs to process." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_sitemap_request_loader.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 198, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/request_loaders/_sitemap_request_loader.py#L198" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Check if there are no more URLs to process." + } + ] + }, + "flags": {}, + "id": 88772, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "is_empty", + "parameters": [], + "type": { + "name": "bool", + "type": "reference", + "target": 0, + "ref": {} + }, + "overwrites": { + "name": "RequestLoader.is_empty", + "target": 2939, + "type": "reference", + "ref": { + "id": 88724, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 37 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestLoader#is_empty" + } + }, + "parentId": 2986 + } + ], + "overwrites": { + "name": "RequestLoader.is_empty", + "target": 2939, + "type": "reference", + "ref": { + "id": 88724, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 37 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestLoader#is_empty" + } + }, + "permalink": "/python/api/class/SitemapRequestLoader#is_empty", + "parentId": 2963 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Check if all URLs have been processed." + } + ] + }, + "decorations": [ + { + "name": "override" + } + ], + "flags": {}, + "groups": [], + "id": 88773, + "module": "request_loaders._sitemap_request_loader", + "name": "is_finished", + "parsedDocstring": { + "text": "Check if all URLs have been processed." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_sitemap_request_loader.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 204, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/request_loaders/_sitemap_request_loader.py#L204" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Check if all URLs have been processed." + } + ] + }, + "flags": {}, + "id": 88774, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "is_finished", + "parameters": [], + "type": { + "name": "bool", + "type": "reference", + "target": 0, + "ref": {} + }, + "overwrites": { + "name": "RequestLoader.is_finished", + "target": 2941, + "type": "reference", + "ref": { + "id": 88726, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 41 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestLoader#is_finished" + } + }, + "parentId": 2988 + } + ], + "overwrites": { + "name": "RequestLoader.is_finished", + "target": 2941, + "type": "reference", + "ref": { + "id": 88726, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 41 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestLoader#is_finished" + } + }, + "permalink": "/python/api/class/SitemapRequestLoader#is_finished", + "parentId": 2963 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Fetch the next request to process." + } + ] + }, + "decorations": [ + { + "name": "override" + } + ], + "flags": {}, + "groups": [], + "id": 88775, + "module": "request_loaders._sitemap_request_loader", + "name": "fetch_next_request", + "parsedDocstring": { + "text": "Fetch the next request to process." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_sitemap_request_loader.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 210, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/request_loaders/_sitemap_request_loader.py#L210" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Fetch the next request to process." + } + ] + }, + "flags": {}, + "id": 88776, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "fetch_next_request", + "parameters": [], + "type": { + "name": "Request | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "Request", + "target": "409", + "ref": { + "id": 86194, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_request.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 145 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/Request" + } + }, + { + "type": "literal", + "value": {} + } + ] + }, + "overwrites": { + "name": "RequestLoader.fetch_next_request", + "target": 2943, + "type": "reference", + "ref": { + "id": 88728, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 45 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestLoader#fetch_next_request" + } + }, + "parentId": 2990 + } + ], + "overwrites": { + "name": "RequestLoader.fetch_next_request", + "target": 2943, + "type": "reference", + "ref": { + "id": 88728, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 45 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestLoader#fetch_next_request" + } + }, + "permalink": "/python/api/class/SitemapRequestLoader#fetch_next_request", + "parentId": 2963 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Mark a request as successfully handled." + } + ] + }, + "decorations": [ + { + "name": "override" + } + ], + "flags": {}, + "groups": [], + "id": 88777, + "module": "request_loaders._sitemap_request_loader", + "name": "mark_request_as_handled", + "parsedDocstring": { + "text": "Mark a request as successfully handled." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_sitemap_request_loader.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 245, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/request_loaders/_sitemap_request_loader.py#L245" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Mark a request as successfully handled." + } + ] + }, + "flags": {}, + "id": 88778, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "mark_request_as_handled", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 88779, + "kind": 32768, + "kindString": "Parameter", + "name": "request", + "type": { + "name": "Request", + "type": "reference", + "target": "409", + "ref": { + "id": 86194, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_request.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 145 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/Request" + } + }, + "parentId": 2993 + } + ], + "type": { + "name": "ProcessedRequest | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "ProcessedRequest", + "target": "1340", + "ref": { + "id": 87125, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 140 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/ProcessedRequest" + } + }, + { + "type": "literal", + "value": {} + } + ] + }, + "overwrites": { + "name": "RequestLoader.mark_request_as_handled", + "target": 2945, + "type": "reference", + "ref": { + "id": 88730, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 53 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestLoader#mark_request_as_handled" + } + }, + "parentId": 2992 + } + ], + "overwrites": { + "name": "RequestLoader.mark_request_as_handled", + "target": 2945, + "type": "reference", + "ref": { + "id": 88730, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 53 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestLoader#mark_request_as_handled" + } + }, + "permalink": "/python/api/class/SitemapRequestLoader#mark_request_as_handled", + "parentId": 2963 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Start the sitemap loading process." + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 88780, + "module": "request_loaders._sitemap_request_loader", + "name": "start", + "parsedDocstring": { + "text": "Start the sitemap loading process." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_sitemap_request_loader.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 253, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/request_loaders/_sitemap_request_loader.py#L253" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Start the sitemap loading process." + } + ] + }, + "flags": {}, + "id": 88781, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "start", + "parameters": [], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "parentId": 2995 + } + ], + "permalink": "/python/api/class/SitemapRequestLoader#start", + "parentId": 2963 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Abort the sitemap loading process." + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 88782, + "module": "request_loaders._sitemap_request_loader", + "name": "abort_loading", + "parsedDocstring": { + "text": "Abort the sitemap loading process." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_sitemap_request_loader.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 259, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/request_loaders/_sitemap_request_loader.py#L259" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Abort the sitemap loading process." + } + ] + }, + "flags": {}, + "id": 88783, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "abort_loading", + "parameters": [], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "parentId": 2997 + } + ], + "permalink": "/python/api/class/SitemapRequestLoader#abort_loading", + "parentId": 2963 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Close the request loader." + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 88784, + "module": "request_loaders._sitemap_request_loader", + "name": "close", + "parsedDocstring": { + "text": "Close the request loader." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_sitemap_request_loader.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 266, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/request_loaders/_sitemap_request_loader.py#L266" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Close the request loader." + } + ] + }, + "flags": {}, + "id": 88785, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "close", + "parameters": [], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "parentId": 2999 + } + ], + "permalink": "/python/api/class/SitemapRequestLoader#close", + "parentId": 2963 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Combine the loader with a request manager to support adding and reclaiming requests.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 89927, + "module": "request_loaders._request_loader", + "name": "to_tandem", + "parsedDocstring": { + "text": "Combine the loader with a request manager to support adding and reclaiming requests.\n", + "args": { + "request_manager": "Request manager to combine the loader with.\nIf None is given, the default request queue is used." + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 56, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/request_loaders/_request_loader.py#L56" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Combine the loader with a request manager to support adding and reclaiming requests.\n" + } + ] + }, + "flags": {}, + "id": 88734, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "to_tandem", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Request manager to combine the loader with.\nIf None is given, the default request queue is used." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": false + }, + "id": 88735, + "kind": 32768, + "kindString": "Parameter", + "name": "request_manager", + "type": { + "name": "RequestManager | None", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2949 + } + ], + "type": { + "name": "RequestManagerTandem", + "type": "reference", + "target": "2803", + "ref": { + "id": 88588, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_manager_tandem.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 24 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/RequestManagerTandem" + } + }, + "inheritedFrom": { + "name": "RequestLoader.to_tandem", + "target": 2948, + "type": "reference", + "ref": { + "id": 88733, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 56 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestLoader#to_tandem" + } + }, + "parentId": 4142 + } + ], + "inheritedFrom": { + "name": "RequestLoader.to_tandem", + "target": 2948, + "type": "reference", + "ref": { + "id": 88733, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 56 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestLoader#to_tandem" + } + }, + "permalink": "/python/api/class/SitemapRequestLoader#to_tandem", + "parentId": 2963 + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "A request loader that reads URLs from sitemap(s).\n\nThe loader is designed to handle sitemaps that follow the format described in the Sitemaps protocol\n(https://www.sitemaps.org/protocol.html). It supports both XML and plain text sitemap formats.\nNote that HTML pages containing links are not supported - those should be handled by regular crawlers\nand the `enqueue_links` functionality.\n\nThe loader fetches and parses sitemaps in the background, allowing crawling to start\nbefore all URLs are loaded. It supports filtering URLs using glob and regex patterns.\n\nThe loader supports state persistence, allowing it to resume from where it left off\nafter interruption when a `persist_state_key` is provided during initialization." + } + ] + }, + "decorations": [ + { + "args": "('Request loaders')", + "name": "docs_group" + } + ], + "flags": {}, + "groups": [ + { + "children": [ + 88760, + 88762, + 88749, + 88782, + 88784, + 88775, + 88769, + 88767, + 88771, + 88773, + 88777, + 88780, + 89927 + ], + "title": "Methods" + } + ], + "id": 88748, + "module": "request_loaders._sitemap_request_loader", + "name": "SitemapRequestLoader", + "parsedDocstring": { + "text": "A request loader that reads URLs from sitemap(s).\n\nThe loader is designed to handle sitemaps that follow the format described in the Sitemaps protocol\n(https://www.sitemaps.org/protocol.html). It supports both XML and plain text sitemap formats.\nNote that HTML pages containing links are not supported - those should be handled by regular crawlers\nand the `enqueue_links` functionality.\n\nThe loader fetches and parses sitemaps in the background, allowing crawling to start\nbefore all URLs are loaded. It supports filtering URLs using glob and regex patterns.\n\nThe loader supports state persistence, allowing it to resume from where it left off\nafter interruption when a `persist_state_key` is provided during initialization." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_sitemap_request_loader.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 94, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/request_loaders/_sitemap_request_loader.py#L94" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "extendedTypes": [ + { + "name": "RequestLoader", + "target": "2934", + "type": "reference", + "ref": { + "id": 88719, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09", + "line": 17 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/RequestLoader" + } + } + ], + "permalink": "/python/api/class/SitemapRequestLoader", + "categories": [] + } + ], + "flags": {}, + "groups": [ + { + "children": [ + 26, + 34, + 271, + 1, + 511, + 572, + 286, + 277, + 636 + ], + "title": "Classes" + }, + { + "children": [ + 318, + 302, + 311, + 171, + 299, + 308, + 332, + 361, + 374, + 423, + 457, + 487, + 604, + 617, + 321, + 323, + 314, + 473, + 176, + 339, + 337, + 385, + 434, + 468, + 454, + 484, + 187, + 380, + 429, + 463, + 493, + 411, + 355, + 367, + 497, + 504, + 440, + 443, + 477, + 609, + 174, + 327, + 400, + 344, + 342, + 353, + 365, + 495, + 502, + 470, + 387, + 436, + 183, + 179, + 623, + 419, + 349, + 347, + 612, + 606, + 449, + 415, + 446, + 382, + 431, + 465, + 480, + 325, + 191 + ], + "title": "Methods" + }, + { + "children": [ + 273, + 632, + 274, + 265, + 615, + 616, + 251, + 275, + 252, + 261, + 422, + 298, + 297, + 510, + 266, + 267, + 170, + 169, + 168, + 264, + 250, + 306, + 249, + 570, + 571, + 194, + 372, + 331, + 373, + 270, + 269, + 25, + 629, + 268, + 633, + 307, + 330, + 329, + 631, + 630, + 634, + 262, + 263, + 360, + 509, + 276, + 635, + 253, + 296, + 569 + ], + "title": "Properties" + }, + { + "children": [ + 257, + 195, + 254, + 200 + ], + "title": "Constants" + }, + { + "title": "Storages", + "children": [ + 2996, + 5470, + 8759, + 11183 + ] + }, + { + "title": "Storage data", + "children": [ + 14336, + 15672, + 16992, + 21208, + 25439, + 29642, + 30987, + 31401, + 32778, + 36989 + ] + }, + { + "title": "Event managers", + "children": [ + 39156, + 42649 + ] + }, + { + "title": "Events", + "children": [ + 46697 + ] + }, + { + "title": "Event data", + "children": [ + 48848, + 50990, + 53127, + 56441, + 58575 + ] + }, + { + "title": "Storage clients", + "children": [ + 61070, + 63347, + 67703, + 72173 + ] + }, + { + "title": "Request loaders", + "children": [ + 77417, + 80485, + 84436, + 88748 + ] + } + ], + "id": 0, + "kind": 1, + "kindString": "Project", + "name": "apify-client", + "sources": [ + { + "character": 0, + "fileName": "src/index.ts", + "line": 1, + "gitRevision": "2ea5d2eb8c45babbe8dfde7c26e57d07c0bb9b09" + } + ], + "symbolIdMap": { + "1": { + "qualifiedName": "Configuration", + "sourceFileName": "/src/apify/config.py" + }, + "2": { + "qualifiedName": "__init__", + "sourceFileName": "/src/apify/config.py" + }, + "23": { + "qualifiedName": "get_global_configuration", + "sourceFileName": "/src/apify/config.py" + }, + "25": { + "qualifiedName": "MainReturnType", + "sourceFileName": "/src/apify/actor.py" + }, + "26": { + "qualifiedName": "_ActorContextManager", + "sourceFileName": "/src/apify/actor.py" + }, + "27": { + "qualifiedName": "__aenter__", + "sourceFileName": "/src/apify/actor.py" + }, + "29": { + "qualifiedName": "__aexit__", + "sourceFileName": "/src/apify/actor.py" + }, + "34": { + "qualifiedName": "Actor", + "sourceFileName": "/src/apify/actor.py" + }, + "35": { + "qualifiedName": "__init__", + "sourceFileName": "/src/apify/actor.py" + }, + "38": { + "qualifiedName": "apify_client", + "sourceFileName": "/src/apify/actor.py" + }, + "39": { + "qualifiedName": "config", + "sourceFileName": "/src/apify/actor.py" + }, + "40": { + "qualifiedName": "event_manager", + "sourceFileName": "/src/apify/actor.py" + }, + "41": { + "qualifiedName": "log", + "sourceFileName": "/src/apify/actor.py" + }, + "42": { + "qualifiedName": "init", + "sourceFileName": "/src/apify/actor.py" + }, + "44": { + "qualifiedName": "exit", + "sourceFileName": "/src/apify/actor.py" + }, + "48": { + "qualifiedName": "fail", + "sourceFileName": "/src/apify/actor.py" + }, + "52": { + "qualifiedName": "main", + "sourceFileName": "/src/apify/actor.py" + }, + "55": { + "qualifiedName": "new_client", + "sourceFileName": "/src/apify/actor.py" + }, + "62": { + "qualifiedName": "open_dataset", + "sourceFileName": "/src/apify/actor.py" + }, + "67": { + "qualifiedName": "open_key_value_store", + "sourceFileName": "/src/apify/actor.py" + }, + "72": { + "qualifiedName": "open_request_queue", + "sourceFileName": "/src/apify/actor.py" + }, + "77": { + "qualifiedName": "push_data", + "sourceFileName": "/src/apify/actor.py" + }, + "80": { + "qualifiedName": "get_input", + "sourceFileName": "/src/apify/actor.py" + }, + "82": { + "qualifiedName": "get_value", + "sourceFileName": "/src/apify/actor.py" + }, + "85": { + "qualifiedName": "set_value", + "sourceFileName": "/src/apify/actor.py" + }, + "90": { + "qualifiedName": "on", + "sourceFileName": "/src/apify/actor.py" + }, + "94": { + "qualifiedName": "off", + "sourceFileName": "/src/apify/actor.py" + }, + "98": { + "qualifiedName": "is_at_home", + "sourceFileName": "/src/apify/actor.py" + }, + "100": { + "qualifiedName": "get_env", + "sourceFileName": "/src/apify/actor.py" + }, + "102": { + "qualifiedName": "start", + "sourceFileName": "/src/apify/actor.py" + }, + "113": { + "qualifiedName": "abort", + "sourceFileName": "/src/apify/actor.py" + }, + "118": { + "qualifiedName": "call", + "sourceFileName": "/src/apify/actor.py" + }, + "129": { + "qualifiedName": "call_task", + "sourceFileName": "/src/apify/actor.py" + }, + "139": { + "qualifiedName": "metamorph", + "sourceFileName": "/src/apify/actor.py" + }, + "146": { + "qualifiedName": "reboot", + "sourceFileName": "/src/apify/actor.py" + }, + "149": { + "qualifiedName": "add_webhook", + "sourceFileName": "/src/apify/actor.py" + }, + "157": { + "qualifiedName": "set_status_message", + "sourceFileName": "/src/apify/actor.py" + }, + "160": { + "qualifiedName": "create_proxy_configuration", + "sourceFileName": "/src/apify/actor.py" + }, + "168": { + "qualifiedName": "ENCRYPTION_KEY_LENGTH", + "sourceFileName": "/src/apify/_crypto.py" + }, + "169": { + "qualifiedName": "ENCRYPTION_IV_LENGTH", + "sourceFileName": "/src/apify/_crypto.py" + }, + "170": { + "qualifiedName": "ENCRYPTION_AUTH_TAG_LENGTH", + "sourceFileName": "/src/apify/_crypto.py" + }, + "171": { + "qualifiedName": "__init__", + "sourceFileName": "/src/apify/event_manager.py" + }, + "174": { + "qualifiedName": "init", + "sourceFileName": "/src/apify/event_manager.py" + }, + "176": { + "qualifiedName": "close", + "sourceFileName": "/src/apify/event_manager.py" + }, + "179": { + "qualifiedName": "on", + "sourceFileName": "/src/apify/event_manager.py" + }, + "183": { + "qualifiedName": "off", + "sourceFileName": "/src/apify/event_manager.py" + }, + "187": { + "qualifiedName": "emit", + "sourceFileName": "/src/apify/event_manager.py" + }, + "191": { + "qualifiedName": "wait_for_all_listeners_to_complete", + "sourceFileName": "/src/apify/event_manager.py" + }, + "194": { + "qualifiedName": "JSONSerializable", + "sourceFileName": "/src/apify/_types.py" + }, + "195": { + "qualifiedName": "ActorEventTypes", + "sourceFileName": "/src/apify/consts.py" + }, + "196": { + "qualifiedName": "SYSTEM_INFO", + "sourceFileName": "/src/apify/consts.py" + }, + "197": { + "qualifiedName": "MIGRATING", + "sourceFileName": "/src/apify/consts.py" + }, + "198": { + "qualifiedName": "PERSIST_STATE", + "sourceFileName": "/src/apify/consts.py" + }, + "199": { + "qualifiedName": "ABORTING", + "sourceFileName": "/src/apify/consts.py" + }, + "200": { + "qualifiedName": "ApifyEnvVars", + "sourceFileName": "/src/apify/consts.py" + }, + "201": { + "qualifiedName": "ACT_ID", + "sourceFileName": "/src/apify/consts.py" + }, + "202": { + "qualifiedName": "ACT_RUN_ID", + "sourceFileName": "/src/apify/consts.py" + }, + "203": { + "qualifiedName": "ACTOR_BUILD_ID", + "sourceFileName": "/src/apify/consts.py" + }, + "204": { + "qualifiedName": "ACTOR_BUILD_NUMBER", + "sourceFileName": "/src/apify/consts.py" + }, + "205": { + "qualifiedName": "ACTOR_EVENTS_WS_URL", + "sourceFileName": "/src/apify/consts.py" + }, + "206": { + "qualifiedName": "ACTOR_ID", + "sourceFileName": "/src/apify/consts.py" + }, + "207": { + "qualifiedName": "ACTOR_RUN_ID", + "sourceFileName": "/src/apify/consts.py" + }, + "208": { + "qualifiedName": "ACTOR_TASK_ID", + "sourceFileName": "/src/apify/consts.py" + }, + "209": { + "qualifiedName": "API_BASE_URL", + "sourceFileName": "/src/apify/consts.py" + }, + "210": { + "qualifiedName": "API_PUBLIC_BASE_URL", + "sourceFileName": "/src/apify/consts.py" + }, + "211": { + "qualifiedName": "CHROME_EXECUTABLE_PATH", + "sourceFileName": "/src/apify/consts.py" + }, + "212": { + "qualifiedName": "CONTAINER_PORT", + "sourceFileName": "/src/apify/consts.py" + }, + "213": { + "qualifiedName": "CONTAINER_URL", + "sourceFileName": "/src/apify/consts.py" + }, + "214": { + "qualifiedName": "DEDICATED_CPUS", + "sourceFileName": "/src/apify/consts.py" + }, + "215": { + "qualifiedName": "DEFAULT_BROWSER_PATH", + "sourceFileName": "/src/apify/consts.py" + }, + "216": { + "qualifiedName": "DEFAULT_DATASET_ID", + "sourceFileName": "/src/apify/consts.py" + }, + "217": { + "qualifiedName": "DEFAULT_KEY_VALUE_STORE_ID", + "sourceFileName": "/src/apify/consts.py" + }, + "218": { + "qualifiedName": "DEFAULT_REQUEST_QUEUE_ID", + "sourceFileName": "/src/apify/consts.py" + }, + "219": { + "qualifiedName": "DISABLE_BROWSER_SANDBOX", + "sourceFileName": "/src/apify/consts.py" + }, + "220": { + "qualifiedName": "DISABLE_OUTDATED_WARNING", + "sourceFileName": "/src/apify/consts.py" + }, + "221": { + "qualifiedName": "FACT", + "sourceFileName": "/src/apify/consts.py" + }, + "222": { + "qualifiedName": "HEADLESS", + "sourceFileName": "/src/apify/consts.py" + }, + "223": { + "qualifiedName": "INPUT_KEY", + "sourceFileName": "/src/apify/consts.py" + }, + "224": { + "qualifiedName": "INPUT_SECRETS_PRIVATE_KEY_FILE", + "sourceFileName": "/src/apify/consts.py" + }, + "225": { + "qualifiedName": "INPUT_SECRETS_PRIVATE_KEY_PASSPHRASE", + "sourceFileName": "/src/apify/consts.py" + }, + "226": { + "qualifiedName": "IS_AT_HOME", + "sourceFileName": "/src/apify/consts.py" + }, + "227": { + "qualifiedName": "LOCAL_STORAGE_DIR", + "sourceFileName": "/src/apify/consts.py" + }, + "228": { + "qualifiedName": "LOG_FORMAT", + "sourceFileName": "/src/apify/consts.py" + }, + "229": { + "qualifiedName": "LOG_LEVEL", + "sourceFileName": "/src/apify/consts.py" + }, + "230": { + "qualifiedName": "MAX_USED_CPU_RATIO", + "sourceFileName": "/src/apify/consts.py" + }, + "231": { + "qualifiedName": "MEMORY_MBYTES", + "sourceFileName": "/src/apify/consts.py" + }, + "232": { + "qualifiedName": "META_ORIGIN", + "sourceFileName": "/src/apify/consts.py" + }, + "233": { + "qualifiedName": "PERSIST_STORAGE", + "sourceFileName": "/src/apify/consts.py" + }, + "234": { + "qualifiedName": "PROXY_HOSTNAME", + "sourceFileName": "/src/apify/consts.py" + }, + "235": { + "qualifiedName": "PROXY_PASSWORD", + "sourceFileName": "/src/apify/consts.py" + }, + "236": { + "qualifiedName": "PROXY_PORT", + "sourceFileName": "/src/apify/consts.py" + }, + "237": { + "qualifiedName": "PROXY_STATUS_URL", + "sourceFileName": "/src/apify/consts.py" + }, + "238": { + "qualifiedName": "SDK_LATEST_VERSION", + "sourceFileName": "/src/apify/consts.py" + }, + "239": { + "qualifiedName": "STARTED_AT", + "sourceFileName": "/src/apify/consts.py" + }, + "240": { + "qualifiedName": "TIMEOUT_AT", + "sourceFileName": "/src/apify/consts.py" + }, + "241": { + "qualifiedName": "TOKEN", + "sourceFileName": "/src/apify/consts.py" + }, + "242": { + "qualifiedName": "USER_ID", + "sourceFileName": "/src/apify/consts.py" + }, + "243": { + "qualifiedName": "WORKFLOW_KEY", + "sourceFileName": "/src/apify/consts.py" + }, + "244": { + "qualifiedName": "XVFB", + "sourceFileName": "/src/apify/consts.py" + }, + "245": { + "qualifiedName": "METAMORPH_AFTER_SLEEP_MILLIS", + "sourceFileName": "/src/apify/consts.py" + }, + "246": { + "qualifiedName": "PERSIST_STATE_INTERVAL_MILLIS", + "sourceFileName": "/src/apify/consts.py" + }, + "247": { + "qualifiedName": "PURGE_ON_START", + "sourceFileName": "/src/apify/consts.py" + }, + "248": { + "qualifiedName": "SYSTEM_INFO_INTERVAL_MILLIS", + "sourceFileName": "/src/apify/consts.py" + }, + "249": { + "qualifiedName": "INTEGER_ENV_VARS", + "sourceFileName": "/src/apify/consts.py" + }, + "250": { + "qualifiedName": "FLOAT_ENV_VARS", + "sourceFileName": "/src/apify/consts.py" + }, + "251": { + "qualifiedName": "BOOL_ENV_VARS", + "sourceFileName": "/src/apify/consts.py" + }, + "252": { + "qualifiedName": "DATETIME_ENV_VARS", + "sourceFileName": "/src/apify/consts.py" + }, + "253": { + "qualifiedName": "STRING_ENV_VARS", + "sourceFileName": "/src/apify/consts.py" + }, + "254": { + "qualifiedName": "ActorExitCodes", + "sourceFileName": "/src/apify/consts.py" + }, + "255": { + "qualifiedName": "SUCCESS", + "sourceFileName": "/src/apify/consts.py" + }, + "256": { + "qualifiedName": "ERROR_USER_FUNCTION_THREW", + "sourceFileName": "/src/apify/consts.py" + }, + "257": { + "qualifiedName": "_StorageTypes", + "sourceFileName": "/src/apify/consts.py" + }, + "258": { + "qualifiedName": "DATASET", + "sourceFileName": "/src/apify/consts.py" + }, + "259": { + "qualifiedName": "KEY_VALUE_STORE", + "sourceFileName": "/src/apify/consts.py" + }, + "260": { + "qualifiedName": "REQUEST_QUEUE", + "sourceFileName": "/src/apify/consts.py" + }, + "261": { + "qualifiedName": "DEFAULT_API_PARAM_LIMIT", + "sourceFileName": "/src/apify/consts.py" + }, + "262": { + "qualifiedName": "REQUEST_ID_LENGTH", + "sourceFileName": "/src/apify/consts.py" + }, + "263": { + "qualifiedName": "REQUEST_QUEUE_HEAD_MAX_LIMIT", + "sourceFileName": "/src/apify/consts.py" + }, + "264": { + "qualifiedName": "EVENT_LISTENERS_TIMEOUT_SECS", + "sourceFileName": "/src/apify/consts.py" + }, + "265": { + "qualifiedName": "BASE64_REGEXP", + "sourceFileName": "/src/apify/consts.py" + }, + "266": { + "qualifiedName": "ENCRYPTED_INPUT_VALUE_PREFIX", + "sourceFileName": "/src/apify/consts.py" + }, + "267": { + "qualifiedName": "ENCRYPTED_INPUT_VALUE_REGEXP", + "sourceFileName": "/src/apify/consts.py" + }, + "268": { + "qualifiedName": "MAX_PAYLOAD_SIZE_BYTES", + "sourceFileName": "/src/apify/consts.py" + }, + "269": { + "qualifiedName": "logger_name", + "sourceFileName": "/src/apify/log.py" + }, + "270": { + "qualifiedName": "logger", + "sourceFileName": "/src/apify/log.py" + }, + "271": { + "qualifiedName": "ActorLogFormatter", + "sourceFileName": "/src/apify/log.py" + }, + "272": { + "qualifiedName": "empty_record", + "sourceFileName": "/src/apify/log.py" + }, + "273": { + "qualifiedName": "__version__", + "sourceFileName": "/src/apify/_version.py" + }, + "274": { + "qualifiedName": "APIFY_PROXY_VALUE_REGEX", + "sourceFileName": "/src/apify/proxy_configuration.py" + }, + "275": { + "qualifiedName": "COUNTRY_CODE_REGEX", + "sourceFileName": "/src/apify/proxy_configuration.py" + }, + "276": { + "qualifiedName": "SESSION_ID_MAX_LENGTH", + "sourceFileName": "/src/apify/proxy_configuration.py" + }, + "277": { + "qualifiedName": "ProxyInfo", + "sourceFileName": "/src/apify/proxy_configuration.py" + }, + "278": { + "qualifiedName": "url", + "sourceFileName": "/src/apify/proxy_configuration.py" + }, + "279": { + "qualifiedName": "hostname", + "sourceFileName": "/src/apify/proxy_configuration.py" + }, + "280": { + "qualifiedName": "port", + "sourceFileName": "/src/apify/proxy_configuration.py" + }, + "281": { + "qualifiedName": "username", + "sourceFileName": "/src/apify/proxy_configuration.py" + }, + "282": { + "qualifiedName": "password", + "sourceFileName": "/src/apify/proxy_configuration.py" + }, + "283": { + "qualifiedName": "groups", + "sourceFileName": "/src/apify/proxy_configuration.py" + }, + "284": { + "qualifiedName": "country_code", + "sourceFileName": "/src/apify/proxy_configuration.py" + }, + "285": { + "qualifiedName": "session_id", + "sourceFileName": "/src/apify/proxy_configuration.py" + }, + "286": { + "qualifiedName": "ProxyConfiguration", + "sourceFileName": "/src/apify/proxy_configuration.py" + }, + "287": { + "qualifiedName": "is_man_in_the_middle", + "sourceFileName": "/src/apify/proxy_configuration.py" + }, + "288": { + "qualifiedName": "initialize", + "sourceFileName": "/src/apify/proxy_configuration.py" + }, + "290": { + "qualifiedName": "new_url", + "sourceFileName": "/src/apify/proxy_configuration.py" + }, + "293": { + "qualifiedName": "new_proxy_info", + "sourceFileName": "/src/apify/proxy_configuration.py" + }, + "296": { + "qualifiedName": "T", + "sourceFileName": "/src/apify/_utils.py" + }, + "297": { + "qualifiedName": "DualPropertyType", + "sourceFileName": "/src/apify/_utils.py" + }, + "298": { + "qualifiedName": "DualPropertyOwner", + "sourceFileName": "/src/apify/_utils.py" + }, + "299": { + "qualifiedName": "__init__", + "sourceFileName": "/src/apify/_utils.py" + }, + "302": { + "qualifiedName": "__get__", + "sourceFileName": "/src/apify/_utils.py" + }, + "306": { + "qualifiedName": "ImplementationType", + "sourceFileName": "/src/apify/_utils.py" + }, + "307": { + "qualifiedName": "MetadataType", + "sourceFileName": "/src/apify/_utils.py" + }, + "308": { + "qualifiedName": "__init__", + "sourceFileName": "/src/apify/_utils.py" + }, + "311": { + "qualifiedName": "__getitem__", + "sourceFileName": "/src/apify/_utils.py" + }, + "314": { + "qualifiedName": "__setitem__", + "sourceFileName": "/src/apify/_utils.py" + }, + "318": { + "qualifiedName": "__delitem__", + "sourceFileName": "/src/apify/_utils.py" + }, + "321": { + "qualifiedName": "__iter__", + "sourceFileName": "/src/apify/_utils.py" + }, + "323": { + "qualifiedName": "__len__", + "sourceFileName": "/src/apify/_utils.py" + }, + "325": { + "qualifiedName": "values", + "sourceFileName": "/src/apify/_utils.py" + }, + "327": { + "qualifiedName": "items", + "sourceFileName": "/src/apify/_utils.py" + }, + "329": { + "qualifiedName": "PARSE_DATE_FIELDS_MAX_DEPTH", + "sourceFileName": "/src/apify/_utils.py" + }, + "330": { + "qualifiedName": "PARSE_DATE_FIELDS_KEY_SUFFIX", + "sourceFileName": "/src/apify/_utils.py" + }, + "331": { + "qualifiedName": "ListOrDictOrAny", + "sourceFileName": "/src/apify/_utils.py" + }, + "332": { + "qualifiedName": "__init__", + "sourceFileName": "/src/apify/_memory_storage/memory_storage_client.py" + }, + "337": { + "qualifiedName": "datasets", + "sourceFileName": "/src/apify/_memory_storage/memory_storage_client.py" + }, + "339": { + "qualifiedName": "dataset", + "sourceFileName": "/src/apify/_memory_storage/memory_storage_client.py" + }, + "342": { + "qualifiedName": "key_value_stores", + "sourceFileName": "/src/apify/_memory_storage/memory_storage_client.py" + }, + "344": { + "qualifiedName": "key_value_store", + "sourceFileName": "/src/apify/_memory_storage/memory_storage_client.py" + }, + "347": { + "qualifiedName": "request_queues", + "sourceFileName": "/src/apify/_memory_storage/memory_storage_client.py" + }, + "349": { + "qualifiedName": "request_queue", + "sourceFileName": "/src/apify/_memory_storage/memory_storage_client.py" + }, + "353": { + "qualifiedName": "list", + "sourceFileName": "/src/apify/_memory_storage/resource_clients/key_value_store_collection.py" + }, + "355": { + "qualifiedName": "get_or_create", + "sourceFileName": "/src/apify/_memory_storage/resource_clients/key_value_store_collection.py" + }, + "360": { + "qualifiedName": "ResourceClientType", + "sourceFileName": "/src/apify/_memory_storage/resource_clients/base_resource_collection_client.py" + }, + "361": { + "qualifiedName": "__init__", + "sourceFileName": "/src/apify/_memory_storage/resource_clients/base_resource_collection_client.py" + }, + "365": { + "qualifiedName": "list", + "sourceFileName": "/src/apify/_memory_storage/resource_clients/base_resource_collection_client.py" + }, + "367": { + "qualifiedName": "get_or_create", + "sourceFileName": "/src/apify/_memory_storage/resource_clients/base_resource_collection_client.py" + }, + "372": { + "qualifiedName": "LIST_ITEMS_LIMIT", + "sourceFileName": "/src/apify/_memory_storage/resource_clients/dataset.py" + }, + "373": { + "qualifiedName": "LOCAL_ENTRY_NAME_DIGITS", + "sourceFileName": "/src/apify/_memory_storage/resource_clients/dataset.py" + }, + "374": { + "qualifiedName": "__init__", + "sourceFileName": "/src/apify/_memory_storage/resource_clients/dataset.py" + }, + "380": { + "qualifiedName": "get", + "sourceFileName": "/src/apify/_memory_storage/resource_clients/dataset.py" + }, + "382": { + "qualifiedName": "update", + "sourceFileName": "/src/apify/_memory_storage/resource_clients/dataset.py" + }, + "385": { + "qualifiedName": "delete", + "sourceFileName": "/src/apify/_memory_storage/resource_clients/dataset.py" + }, + "387": { + "qualifiedName": "list_items", + "sourceFileName": "/src/apify/_memory_storage/resource_clients/dataset.py" + }, + "400": { + "qualifiedName": "iterate_items", + "sourceFileName": "/src/apify/_memory_storage/resource_clients/dataset.py" + }, + "411": { + "qualifiedName": "get_items_as_bytes", + "sourceFileName": "/src/apify/_memory_storage/resource_clients/dataset.py" + }, + "415": { + "qualifiedName": "stream_items", + "sourceFileName": "/src/apify/_memory_storage/resource_clients/dataset.py" + }, + "419": { + "qualifiedName": "push_items", + "sourceFileName": "/src/apify/_memory_storage/resource_clients/dataset.py" + }, + "422": { + "qualifiedName": "DEFAULT_LOCAL_FILE_EXTENSION", + "sourceFileName": "/src/apify/_memory_storage/resource_clients/key_value_store.py" + }, + "423": { + "qualifiedName": "__init__", + "sourceFileName": "/src/apify/_memory_storage/resource_clients/key_value_store.py" + }, + "429": { + "qualifiedName": "get", + "sourceFileName": "/src/apify/_memory_storage/resource_clients/key_value_store.py" + }, + "431": { + "qualifiedName": "update", + "sourceFileName": "/src/apify/_memory_storage/resource_clients/key_value_store.py" + }, + "434": { + "qualifiedName": "delete", + "sourceFileName": "/src/apify/_memory_storage/resource_clients/key_value_store.py" + }, + "436": { + "qualifiedName": "list_keys", + "sourceFileName": "/src/apify/_memory_storage/resource_clients/key_value_store.py" + }, + "440": { + "qualifiedName": "get_record", + "sourceFileName": "/src/apify/_memory_storage/resource_clients/key_value_store.py" + }, + "443": { + "qualifiedName": "get_record_as_bytes", + "sourceFileName": "/src/apify/_memory_storage/resource_clients/key_value_store.py" + }, + "446": { + "qualifiedName": "stream_record", + "sourceFileName": "/src/apify/_memory_storage/resource_clients/key_value_store.py" + }, + "449": { + "qualifiedName": "set_record", + "sourceFileName": "/src/apify/_memory_storage/resource_clients/key_value_store.py" + }, + "454": { + "qualifiedName": "delete_record", + "sourceFileName": "/src/apify/_memory_storage/resource_clients/key_value_store.py" + }, + "457": { + "qualifiedName": "__init__", + "sourceFileName": "/src/apify/_memory_storage/resource_clients/request_queue.py" + }, + "463": { + "qualifiedName": "get", + "sourceFileName": "/src/apify/_memory_storage/resource_clients/request_queue.py" + }, + "465": { + "qualifiedName": "update", + "sourceFileName": "/src/apify/_memory_storage/resource_clients/request_queue.py" + }, + "468": { + "qualifiedName": "delete", + "sourceFileName": "/src/apify/_memory_storage/resource_clients/request_queue.py" + }, + "470": { + "qualifiedName": "list_head", + "sourceFileName": "/src/apify/_memory_storage/resource_clients/request_queue.py" + }, + "473": { + "qualifiedName": "add_request", + "sourceFileName": "/src/apify/_memory_storage/resource_clients/request_queue.py" + }, + "477": { + "qualifiedName": "get_request", + "sourceFileName": "/src/apify/_memory_storage/resource_clients/request_queue.py" + }, + "480": { + "qualifiedName": "update_request", + "sourceFileName": "/src/apify/_memory_storage/resource_clients/request_queue.py" + }, + "484": { + "qualifiedName": "delete_request", + "sourceFileName": "/src/apify/_memory_storage/resource_clients/request_queue.py" + }, + "487": { + "qualifiedName": "__init__", + "sourceFileName": "/src/apify/_memory_storage/resource_clients/base_resource_client.py" + }, + "493": { + "qualifiedName": "get", + "sourceFileName": "/src/apify/_memory_storage/resource_clients/base_resource_client.py" + }, + "495": { + "qualifiedName": "list", + "sourceFileName": "/src/apify/_memory_storage/resource_clients/request_queue_collection.py" + }, + "497": { + "qualifiedName": "get_or_create", + "sourceFileName": "/src/apify/_memory_storage/resource_clients/request_queue_collection.py" + }, + "502": { + "qualifiedName": "list", + "sourceFileName": "/src/apify/_memory_storage/resource_clients/dataset_collection.py" + }, + "504": { + "qualifiedName": "get_or_create", + "sourceFileName": "/src/apify/_memory_storage/resource_clients/dataset_collection.py" + }, + "509": { + "qualifiedName": "SAFETY_BUFFER_PERCENT", + "sourceFileName": "/src/apify/storages/dataset.py" + }, + "510": { + "qualifiedName": "EFFECTIVE_LIMIT_BYTES", + "sourceFileName": "/src/apify/storages/dataset.py" + }, + "511": { + "qualifiedName": "Dataset", + "sourceFileName": "/src/apify/storages/dataset.py" + }, + "512": { + "qualifiedName": "push_data", + "sourceFileName": "/src/apify/storages/dataset.py" + }, + "515": { + "qualifiedName": "get_data", + "sourceFileName": "/src/apify/storages/dataset.py" + }, + "528": { + "qualifiedName": "export_to", + "sourceFileName": "/src/apify/storages/dataset.py" + }, + "534": { + "qualifiedName": "export_to_json", + "sourceFileName": "/src/apify/storages/dataset.py" + }, + "541": { + "qualifiedName": "export_to_csv", + "sourceFileName": "/src/apify/storages/dataset.py" + }, + "548": { + "qualifiedName": "get_info", + "sourceFileName": "/src/apify/storages/dataset.py" + }, + "550": { + "qualifiedName": "iterate_items", + "sourceFileName": "/src/apify/storages/dataset.py" + }, + "561": { + "qualifiedName": "drop", + "sourceFileName": "/src/apify/storages/dataset.py" + }, + "563": { + "qualifiedName": "open", + "sourceFileName": "/src/apify/storages/dataset.py" + }, + "569": { + "qualifiedName": "T", + "sourceFileName": "/src/apify/storages/key_value_store.py" + }, + "570": { + "qualifiedName": "IterateKeysInfo", + "sourceFileName": "/src/apify/storages/key_value_store.py" + }, + "571": { + "qualifiedName": "IterateKeysTuple", + "sourceFileName": "/src/apify/storages/key_value_store.py" + }, + "572": { + "qualifiedName": "KeyValueStore", + "sourceFileName": "/src/apify/storages/key_value_store.py" + }, + "573": { + "qualifiedName": "get_value", + "sourceFileName": "/src/apify/storages/key_value_store.py" + }, + "577": { + "qualifiedName": "iterate_keys", + "sourceFileName": "/src/apify/storages/key_value_store.py" + }, + "580": { + "qualifiedName": "set_value", + "sourceFileName": "/src/apify/storages/key_value_store.py" + }, + "585": { + "qualifiedName": "get_public_url", + "sourceFileName": "/src/apify/storages/key_value_store.py" + }, + "588": { + "qualifiedName": "drop", + "sourceFileName": "/src/apify/storages/key_value_store.py" + }, + "590": { + "qualifiedName": "open", + "sourceFileName": "/src/apify/storages/key_value_store.py" + }, + "604": { + "qualifiedName": "__init__", + "sourceFileName": "/src/apify/storages/storage_client_manager.py" + }, + "606": { + "qualifiedName": "set_config", + "sourceFileName": "/src/apify/storages/storage_client_manager.py" + }, + "609": { + "qualifiedName": "get_storage_client", + "sourceFileName": "/src/apify/storages/storage_client_manager.py" + }, + "612": { + "qualifiedName": "set_cloud_client", + "sourceFileName": "/src/apify/storages/storage_client_manager.py" + }, + "615": { + "qualifiedName": "BaseResourceClientType", + "sourceFileName": "/src/apify/storages/base_storage.py" + }, + "616": { + "qualifiedName": "BaseResourceCollectionClientType", + "sourceFileName": "/src/apify/storages/base_storage.py" + }, + "617": { + "qualifiedName": "__init__", + "sourceFileName": "/src/apify/storages/base_storage.py" + }, + "623": { + "qualifiedName": "open", + "sourceFileName": "/src/apify/storages/base_storage.py" + }, + "629": { + "qualifiedName": "MAX_CACHED_REQUESTS", + "sourceFileName": "/src/apify/storages/request_queue.py" + }, + "630": { + "qualifiedName": "QUERY_HEAD_MIN_LENGTH", + "sourceFileName": "/src/apify/storages/request_queue.py" + }, + "631": { + "qualifiedName": "QUERY_HEAD_BUFFER", + "sourceFileName": "/src/apify/storages/request_queue.py" + }, + "632": { + "qualifiedName": "API_PROCESSED_REQUESTS_DELAY_MILLIS", + "sourceFileName": "/src/apify/storages/request_queue.py" + }, + "633": { + "qualifiedName": "MAX_QUERIES_FOR_CONSISTENCY", + "sourceFileName": "/src/apify/storages/request_queue.py" + }, + "634": { + "qualifiedName": "RECENTLY_HANDLED_CACHE_SIZE", + "sourceFileName": "/src/apify/storages/request_queue.py" + }, + "635": { + "qualifiedName": "STORAGE_CONSISTENCY_DELAY_MILLIS", + "sourceFileName": "/src/apify/storages/request_queue.py" + }, + "636": { + "qualifiedName": "RequestQueue", + "sourceFileName": "/src/apify/storages/request_queue.py" + }, + "637": { + "qualifiedName": "add_request", + "sourceFileName": "/src/apify/storages/request_queue.py" + }, + "641": { + "qualifiedName": "get_request", + "sourceFileName": "/src/apify/storages/request_queue.py" + }, + "644": { + "qualifiedName": "fetch_next_request", + "sourceFileName": "/src/apify/storages/request_queue.py" + }, + "646": { + "qualifiedName": "mark_request_as_handled", + "sourceFileName": "/src/apify/storages/request_queue.py" + }, + "649": { + "qualifiedName": "reclaim_request", + "sourceFileName": "/src/apify/storages/request_queue.py" + }, + "653": { + "qualifiedName": "is_empty", + "sourceFileName": "/src/apify/storages/request_queue.py" + }, + "655": { + "qualifiedName": "is_finished", + "sourceFileName": "/src/apify/storages/request_queue.py" + }, + "657": { + "qualifiedName": "drop", + "sourceFileName": "/src/apify/storages/request_queue.py" + }, + "659": { + "qualifiedName": "get_info", + "sourceFileName": "/src/apify/storages/request_queue.py" + }, + "661": { + "qualifiedName": "open", + "sourceFileName": "/src/apify/storages/request_queue.py" } - ] + } } \ No newline at end of file diff --git a/website/versioned_docs/version-1.7/api-packages.json b/website/versioned_docs/version-1.7/api-packages.json index e2e41f0d..765ea7e7 100644 --- a/website/versioned_docs/version-1.7/api-packages.json +++ b/website/versioned_docs/version-1.7/api-packages.json @@ -1 +1 @@ -[{"entryPoints":{"index":{"label":"Index","path":"src/index.ts"}},"packageRoot":".","packagePath":".","packageSlug":".","packageName":"apify-sdk-python"}] +[{"entryPoints":{"index":{"label":"Index","path":"src/index.ts"}},"packageRoot":".","packagePath":".","packageSlug":".","packageName":"apify-sdk-python-website"}] \ No newline at end of file diff --git a/website/versioned_docs/version-1.7/api-typedoc.json b/website/versioned_docs/version-1.7/api-typedoc.json index f63cd0da..8724eb13 100644 --- a/website/versioned_docs/version-1.7/api-typedoc.json +++ b/website/versioned_docs/version-1.7/api-typedoc.json @@ -1,8083 +1,60549 @@ { - "id": 0, - "name": "apify", - "kind": 1, - "kindString": "Project", - "flags": {}, - "originalName": "", "children": [ { - "id": 1, - "name": "crypto_random_object_id", - "module": "_crypto", - "kind": 2048, - "kindString": "Method", - "flags": {}, - "comment": { - "summary": [ - { - "kind": "text", - "text": "{\"content\": [\"Python reimplementation of cryptoRandomObjectId from `@apify/utilities`.\"]}" - } - ] - }, - "children": [], - "groups": [], - "sources": [ - { - "filename": "/tmp-old-source-v17/src/apify/_crypto.py", - "line": 128, - "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/_crypto.py#L128" - } - ], - "signatures": [ + "kind": 128, + "kindString": "Class", + "children": [ { - "id": 2, - "name": "crypto_random_object_id", - "modifiers": [], - "kind": 4096, - "kindString": "Call signature", - "flags": {}, + "kind": 2048, + "kindString": "Method", + "children": [], "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Python reimplementation of cryptoRandomObjectId from `@apify/utilities`.\"]}" + "text": "Create a `Configuration` instance.\n\nAll the parameters are loaded by default from environment variables when running on the Apify platform.\nYou can override them here in the Configuration constructor, which might be useful for local testing of your actors.\n" } ] }, - "type": { - "type": "intrinsic", - "name": "str" - }, - "parameters": [ - { - "id": 3, - "name": "length", - "kind": 32768, - "kindString": "Parameter", - "flags": {}, - "type": { - "type": "intrinsic", - "name": "int" - }, - "defaultValue": "17" - } - ] - } - ] - }, - { - "id": 4, - "name": "decrypt_input_secrets", - "module": "_crypto", - "kind": 2048, - "kindString": "Method", - "flags": {}, - "comment": { - "summary": [ - { - "kind": "text", - "text": "{\"content\": [\"Decrypt input secrets.\"]}" - } - ] - }, - "children": [], - "groups": [], - "sources": [ - { - "filename": "/tmp-old-source-v17/src/apify/_crypto.py", - "line": 134, - "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/_crypto.py#L134" - } - ], - "signatures": [ - { - "id": 5, - "name": "decrypt_input_secrets", - "modifiers": [], - "kind": 4096, - "kindString": "Call signature", + "decorations": [], "flags": {}, - "comment": { - "summary": [ - { - "kind": "text", - "text": "{\"content\": [\"Decrypt input secrets.\"]}" - } - ] - }, - "type": { - "type": "reference", - "name": "Any" - }, - "parameters": [ - { - "id": 6, - "name": "private_key", - "kind": 32768, - "kindString": "Parameter", - "flags": {}, - "type": { - "type": "reference", - "name": "rsa.RSAPrivateKey" - } - }, - { - "id": 7, - "name": "input", - "kind": 32768, - "kindString": "Parameter", - "flags": {}, - "type": { - "type": "reference", - "name": "Any" - } + "groups": [], + "id": 2, + "module": "config", + "name": "__init__", + "parsedDocstring": { + "text": "Create a `Configuration` instance.\n\nAll the parameters are loaded by default from environment variables when running on the Apify platform.\nYou can override them here in the Configuration constructor, which might be useful for local testing of your actors.\n", + "args": { + "api_base_url": "The URL of the Apify API.\nThis is the URL actually used for connecting to the API, so it can contain an IP address when running in a container on the platform.", + "api_public_base_url": "The public URL of the Apify API.\nThis will always contain the public URL of the API, even when running in a container on the platform.\nUseful for generating shareable URLs to key-value store records or datasets.", + "container_port": "The port on which the container can listen for HTTP requests.", + "container_url": "The URL on which the container can listen for HTTP requests.", + "default_dataset_id": "The ID of the default dataset for the actor.", + "default_key_value_store_id": "The ID of the default key-value store for the actor.", + "default_request_queue_id": "The ID of the default request queue for the actor.", + "input_key": "The key of the input record in the actor's default key-value store", + "max_used_cpu_ratio": "The CPU usage above which the SYSTEM_INFO event will report the CPU is overloaded.", + "metamorph_after_sleep_millis": "How long should the actor sleep after calling metamorph.", + "persist_state_interval_millis": "How often should the actor emit the PERSIST_STATE event.", + "persist_storage": "Whether the actor should persist its used storages to the filesystem when running locally.", + "proxy_hostname": "The hostname of Apify Proxy.", + "proxy_password": "The password for Apify Proxy.", + "proxy_port": "The port of Apify Proxy.", + "proxy_status_url": "The URL on which the Apify Proxy status page is available.", + "purge_on_start": "Whether the actor should purge its default storages on startup, when running locally.", + "token": "The API token for the Apify API this actor should use.", + "system_info_interval_millis": "How often should the actor emit the SYSTEM_INFO event when running locally.", + "standby_port": "The port on which the container can listen for Actor Standby HTTP requests." } - ] - } - ] - }, - { - "id": 8, - "name": "force_remove", - "module": "_utils", - "kind": 2048, - "kindString": "Method", - "flags": {}, - "comment": { - "summary": [ - { - "kind": "text", - "text": "{\"content\": [\"JS-like rm(filename, { force: true }).\"]}" - } - ] - }, - "children": [], - "groups": [], - "sources": [ - { - "filename": "/tmp-old-source-v17/src/apify/_utils.py", - "line": 254, - "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/_utils.py#L254" - } - ], - "signatures": [ - { - "id": 9, - "name": "force_remove", - "modifiers": [ - "async" - ], - "kind": 4096, - "kindString": "Call signature", - "flags": {}, - "comment": { - "summary": [ - { - "kind": "text", - "text": "{\"content\": [\"JS-like rm(filename, { force: true }).\"]}" - } - ] - }, - "type": { - "type": "reference", - "name": "None" }, - "parameters": [ + "sources": [ { - "id": 10, - "name": "filename", - "kind": 32768, - "kindString": "Parameter", - "flags": {}, - "type": { - "type": "intrinsic", - "name": "str" - } + "character": 1, + "fileName": "/src/apify/config.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 17 } - ] - } - ] - }, - { - "id": 11, - "name": "guess_file_extension", - "module": "_utils", - "kind": 2048, - "kindString": "Method", - "flags": {}, - "comment": { - "summary": [ - { - "kind": "text", - "text": "{\"content\": [\"Guess the file extension based on content type.\"]}" - } - ] - }, - "children": [], - "groups": [], - "sources": [ - { - "filename": "/tmp-old-source-v17/src/apify/_utils.py", - "line": 270, - "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/_utils.py#L270" - } - ], - "signatures": [ - { - "id": 12, - "name": "guess_file_extension", - "modifiers": [], - "kind": 4096, - "kindString": "Call signature", - "flags": {}, - "comment": { - "summary": [ - { - "kind": "text", - "text": "{\"content\": [\"Guess the file extension based on content type.\"]}" - } - ] - }, + ], "type": { - "type": "reference", - "name": "str | None" + "name": "Undefined", + "type": "reference" }, - "parameters": [ + "signatures": [ { - "id": 13, - "name": "content_type", - "kind": 32768, - "kindString": "Parameter", + "comment": { + "summary": [ + { + "kind": "text", + "text": "Create a `Configuration` instance.\n\nAll the parameters are loaded by default from environment variables when running on the Apify platform.\nYou can override them here in the Configuration constructor, which might be useful for local testing of your actors.\n" + } + ] + }, "flags": {}, - "type": { - "type": "intrinsic", - "name": "str" - } - } - ] - } - ] - }, - { - "id": 14, - "name": "unique_key_to_request_id", - "module": "_utils", - "kind": 2048, - "kindString": "Method", - "flags": {}, - "comment": { - "summary": [ - { - "kind": "text", - "text": "{\"content\": [\"Generate request ID based on unique key in a deterministic way.\"]}" - } - ] - }, - "children": [], - "groups": [], - "sources": [ - { - "filename": "/tmp-old-source-v17/src/apify/_utils.py", - "line": 296, - "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/_utils.py#L296" - } - ], - "signatures": [ - { - "id": 15, - "name": "unique_key_to_request_id", - "modifiers": [], - "kind": 4096, - "kindString": "Call signature", - "flags": {}, - "comment": { - "summary": [ - { - "kind": "text", - "text": "{\"content\": [\"Generate request ID based on unique key in a deterministic way.\"]}" - } + "id": 3, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "__init__", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The URL of the Apify API.\nThis is the URL actually used for connecting to the API, so it can contain an IP address when running in a container on the platform." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 4, + "kind": 32768, + "kindString": "Parameter", + "name": "api_base_url", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The public URL of the Apify API.\nThis will always contain the public URL of the API, even when running in a container on the platform.\nUseful for generating shareable URLs to key-value store records or datasets." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 5, + "kind": 32768, + "kindString": "Parameter", + "name": "api_public_base_url", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The port on which the container can listen for HTTP requests." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 6, + "kind": 32768, + "kindString": "Parameter", + "name": "container_port", + "type": { + "name": "int | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "int" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The URL on which the container can listen for HTTP requests." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 7, + "kind": 32768, + "kindString": "Parameter", + "name": "container_url", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The ID of the default dataset for the actor." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 8, + "kind": 32768, + "kindString": "Parameter", + "name": "default_dataset_id", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The ID of the default key-value store for the actor." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 9, + "kind": 32768, + "kindString": "Parameter", + "name": "default_key_value_store_id", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The ID of the default request queue for the actor." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 10, + "kind": 32768, + "kindString": "Parameter", + "name": "default_request_queue_id", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The key of the input record in the actor's default key-value store" + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 11, + "kind": 32768, + "kindString": "Parameter", + "name": "input_key", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The CPU usage above which the SYSTEM_INFO event will report the CPU is overloaded." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 12, + "kind": 32768, + "kindString": "Parameter", + "name": "max_used_cpu_ratio", + "type": { + "name": "float | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "float" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "How long should the actor sleep after calling metamorph." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 13, + "kind": 32768, + "kindString": "Parameter", + "name": "metamorph_after_sleep_millis", + "type": { + "name": "int | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "int" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "How often should the actor emit the PERSIST_STATE event." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 14, + "kind": 32768, + "kindString": "Parameter", + "name": "persist_state_interval_millis", + "type": { + "name": "int | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "int" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Whether the actor should persist its used storages to the filesystem when running locally." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 15, + "kind": 32768, + "kindString": "Parameter", + "name": "persist_storage", + "type": { + "name": "bool | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "bool" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The hostname of Apify Proxy." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 16, + "kind": 32768, + "kindString": "Parameter", + "name": "proxy_hostname", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The password for Apify Proxy." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 17, + "kind": 32768, + "kindString": "Parameter", + "name": "proxy_password", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The port of Apify Proxy." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 18, + "kind": 32768, + "kindString": "Parameter", + "name": "proxy_port", + "type": { + "name": "int | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "int" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The URL on which the Apify Proxy status page is available." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 19, + "kind": 32768, + "kindString": "Parameter", + "name": "proxy_status_url", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Whether the actor should purge its default storages on startup, when running locally." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 20, + "kind": 32768, + "kindString": "Parameter", + "name": "purge_on_start", + "type": { + "name": "bool | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "bool" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The API token for the Apify API this actor should use." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 21, + "kind": 32768, + "kindString": "Parameter", + "name": "token", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The port on which the container can listen for Actor Standby HTTP requests." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 22, + "kind": 32768, + "kindString": "Parameter", + "name": "standby_port", + "type": { + "name": "int | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "int" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "How often should the actor emit the SYSTEM_INFO event when running locally." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 23, + "kind": 32768, + "kindString": "Parameter", + "name": "system_info_interval_millis", + "type": { + "name": "int | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "int" + }, + { + "type": "literal", + "value": null + } + ] + } + } + ], + "type": { + "name": "None", + "type": "literal", + "value": null + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Retrive the global configuration.\n\nThe global configuration applies when you call actor methods via their static versions, e.g. `Actor.init()`.\nAlso accessible via `Actor.config`." + } + ] + }, + "decorations": [ + { + "name": "classmethod" + } + ], + "flags": {}, + "groups": [], + "id": 24, + "module": "config", + "name": "get_global_configuration", + "parsedDocstring": { + "text": "Retrive the global configuration.\n\nThe global configuration applies when you call actor methods via their static versions, e.g. `Actor.init()`.\nAlso accessible via `Actor.config`." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/config.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 124 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Retrive the global configuration.\n\nThe global configuration applies when you call actor methods via their static versions, e.g. `Actor.init()`.\nAlso accessible via `Actor.config`." + } + ] + }, + "flags": {}, + "id": 25, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "get_global_configuration", + "parameters": [], + "type": { + "name": "Configuration", + "type": "reference", + "target": "1" + } + } + ] + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "A class for specifying the configuration of an actor.\n\nCan be used either globally via `Configuration.get_global_configuration()`,\nor it can be specific to each `Actor` instance on the `actor.config` property." + } + ] + }, + "flags": {}, + "groups": [ + { + "children": [ + 2, + 24 + ], + "title": "Methods" + } + ], + "id": 1, + "module": "config", + "name": "Configuration", + "parsedDocstring": { + "text": "A class for specifying the configuration of an actor.\n\nCan be used either globally via `Configuration.get_global_configuration()`,\nor it can be specific to each `Actor` instance on the `actor.config` property." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/config.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 8 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 26, + "module": "actor", + "name": "T", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/actor.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 39 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 27, + "module": "actor", + "name": "MainReturnType", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/actor.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 40 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "decorations": [ + { + "name": "staticmethod" + } + ], + "flags": {}, + "groups": [], + "id": 29, + "module": "actor", + "name": "__aenter__", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/actor.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 48 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "flags": {}, + "id": 30, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "__aenter__", + "parameters": [], + "type": { + "name": "type", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "Actor", + "target": "36" + } + ] + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "decorations": [ + { + "name": "staticmethod" + } + ], + "flags": {}, + "groups": [], + "id": 31, + "module": "actor", + "name": "__aexit__", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/actor.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 53 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "flags": {}, + "id": 32, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "__aexit__", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 33, + "kind": 32768, + "kindString": "Parameter", + "name": "_exc_type", + "type": { + "name": "type[BaseException] | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "type", + "typeArguments": [ + { + "type": "reference", + "name": "BaseException" + } + ] + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 34, + "kind": 32768, + "kindString": "Parameter", + "name": "exc_value", + "type": { + "name": "BaseException | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "BaseException" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 35, + "kind": 32768, + "kindString": "Parameter", + "name": "_exc_traceback", + "type": { + "name": "TracebackType | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "TracebackType" + }, + { + "type": "literal", + "value": null + } + ] + } + } + ], + "type": { + "name": "None", + "type": "literal", + "value": null + } + } + ] + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [ + { + "children": [ + 29, + 31 + ], + "title": "Methods" + } + ], + "id": 28, + "module": "actor", + "name": "_ActorContextManager", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/actor.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 46 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Create an Actor instance.\n\nNote that you don't have to do this, all the methods on this class function as classmethods too,\nand that is their preferred usage.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 37, + "module": "actor", + "name": "__init__", + "parsedDocstring": { + "text": "Create an Actor instance.\n\nNote that you don't have to do this, all the methods on this class function as classmethods too,\nand that is their preferred usage.\n", + "args": { + "config": "The actor configuration to be used. If not passed, a new Configuration instance will be created." + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/actor.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 81 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Create an Actor instance.\n\nNote that you don't have to do this, all the methods on this class function as classmethods too,\nand that is their preferred usage.\n" + } + ] + }, + "flags": {}, + "id": 38, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "__init__", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The actor configuration to be used. If not passed, a new Configuration instance will be created." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": false + }, + "id": 39, + "kind": 32768, + "kindString": "Parameter", + "name": "config", + "type": { + "name": "Configuration | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "Configuration", + "target": "1" + }, + { + "type": "literal", + "value": null + } + ] + } + } + ], + "type": { + "name": "None", + "type": "literal", + "value": null + } + } + ] + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "The ApifyClientAsync instance the Actor instance uses." + } + ] + }, + "decorations": [ + { + "name": "dualproperty" + } + ], + "flags": {}, + "groups": [], + "id": 40, + "module": "actor", + "name": "apify_client", + "parsedDocstring": { + "text": "The ApifyClientAsync instance the Actor instance uses." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/actor.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 174 + } + ], + "type": { + "name": "ApifyClientAsync", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "The Configuration instance the Actor instance uses." + } + ] + }, + "decorations": [ + { + "name": "dualproperty" + } + ], + "flags": {}, + "groups": [], + "id": 41, + "module": "actor", + "name": "config", + "parsedDocstring": { + "text": "The Configuration instance the Actor instance uses." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/actor.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 181 + } + ], + "type": { + "name": "Configuration", + "type": "reference", + "target": "1" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "The EventManager instance the Actor instance uses." + } + ] + }, + "decorations": [ + { + "name": "dualproperty" + } + ], + "flags": {}, + "groups": [], + "id": 42, + "module": "actor", + "name": "event_manager", + "parsedDocstring": { + "text": "The EventManager instance the Actor instance uses." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/actor.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 188 + } + ], + "type": { + "name": "EventManager", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "The logging.Logger instance the Actor uses." + } + ] + }, + "decorations": [ + { + "name": "dualproperty" + } + ], + "flags": {}, + "groups": [], + "id": 43, + "module": "actor", + "name": "log", + "parsedDocstring": { + "text": "The logging.Logger instance the Actor uses." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/actor.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 196 + } + ], + "type": { + "name": "logging.Logger", + "type": "reference" + } + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Initialize the actor instance.\n\nThis initializes the Actor instance.\nIt configures the right storage client based on whether the actor is running locally or on the Apify platform,\nit initializes the event manager for processing actor events,\nand starts an interval for regularly sending `PERSIST_STATE` events,\nso that the actor can regularly persist its state in response to these events.\n\nThis method should be called immediately before performing any additional actor actions,\nand it should be called only once." + } + ] + }, + "decorations": [ + { + "name": "classmethod" + } + ], + "flags": {}, + "groups": [], + "id": 44, + "module": "actor", + "name": "init", + "parsedDocstring": { + "text": "Initialize the actor instance.\n\nThis initializes the Actor instance.\nIt configures the right storage client based on whether the actor is running locally or on the Apify platform,\nit initializes the event manager for processing actor events,\nand starts an interval for regularly sending `PERSIST_STATE` events,\nso that the actor can regularly persist its state in response to these events.\n\nThis method should be called immediately before performing any additional actor actions,\nand it should be called only once." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/actor.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 205 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Initialize the actor instance.\n\nThis initializes the Actor instance.\nIt configures the right storage client based on whether the actor is running locally or on the Apify platform,\nit initializes the event manager for processing actor events,\nand starts an interval for regularly sending `PERSIST_STATE` events,\nso that the actor can regularly persist its state in response to these events.\n\nThis method should be called immediately before performing any additional actor actions,\nand it should be called only once." + } + ] + }, + "flags": {}, + "id": 45, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "init", + "parameters": [], + "type": { + "name": "None", + "type": "literal", + "value": null + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Get the current system info." + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 46, + "module": "actor", + "name": "get_system_info", + "parsedDocstring": { + "text": "Get the current system info." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/actor.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 261 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Get the current system info." + } + ] + }, + "flags": {}, + "id": 47, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "get_system_info", + "parameters": [], + "type": { + "name": "dict", + "type": "reference" + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Exit the actor instance.\n\nThis stops the Actor instance.\nIt cancels all the intervals for regularly sending `PERSIST_STATE` events,\nsends a final `PERSIST_STATE` event,\nwaits for all the event listeners to finish,\nand stops the event manager.\n" + } + ] + }, + "decorations": [ + { + "name": "classmethod" + } + ], + "flags": {}, + "groups": [], + "id": 48, + "module": "actor", + "name": "exit", + "parsedDocstring": { + "text": "Exit the actor instance.\n\nThis stops the Actor instance.\nIt cancels all the intervals for regularly sending `PERSIST_STATE` events,\nsends a final `PERSIST_STATE` event,\nwaits for all the event listeners to finish,\nand stops the event manager.\n", + "args": { + "exit_code": "The exit code with which the actor should fail (defaults to `0`).", + "event_listeners_timeout_secs": "How long should the actor wait for actor event listeners to finish before exiting.", + "status_message": "The final status message that the actor should display.", + "cleanup_timeout": "How long we should wait for event listeners." + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/actor.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 298 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Exit the actor instance.\n\nThis stops the Actor instance.\nIt cancels all the intervals for regularly sending `PERSIST_STATE` events,\nsends a final `PERSIST_STATE` event,\nwaits for all the event listeners to finish,\nand stops the event manager.\n" + } + ] + }, + "flags": {}, + "id": 49, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "exit", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The exit code with which the actor should fail (defaults to `0`)." + } + ] + }, + "defaultValue": "0", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 50, + "kind": 32768, + "kindString": "Parameter", + "name": "exit_code", + "type": { + "name": "int", + "type": "reference" + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "How long should the actor wait for actor event listeners to finish before exiting." + } + ] + }, + "defaultValue": "EVENT_LISTENERS_TIMEOUT_SECS", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 51, + "kind": 32768, + "kindString": "Parameter", + "name": "event_listeners_timeout_secs", + "type": { + "name": "float | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "float" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The final status message that the actor should display." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 52, + "kind": 32768, + "kindString": "Parameter", + "name": "status_message", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "How long we should wait for event listeners." + } + ] + }, + "defaultValue": "timedelta(seconds=30)", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 53, + "kind": 32768, + "kindString": "Parameter", + "name": "cleanup_timeout", + "type": { + "name": "timedelta", + "type": "reference" + } + } + ], + "type": { + "name": "None", + "type": "literal", + "value": null + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Fail the actor instance.\n\nThis performs all the same steps as Actor.exit(),\nbut it additionally sets the exit code to `1` (by default).\n" + } + ] + }, + "decorations": [ + { + "name": "classmethod" + } + ], + "flags": {}, + "groups": [], + "id": 54, + "module": "actor", + "name": "fail", + "parsedDocstring": { + "text": "Fail the actor instance.\n\nThis performs all the same steps as Actor.exit(),\nbut it additionally sets the exit code to `1` (by default).\n", + "args": { + "exit_code": "The exit code with which the actor should fail (defaults to `1`).", + "exception": "The exception with which the actor failed.", + "status_message": "The final status message that the actor should display." + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/actor.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 372 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Fail the actor instance.\n\nThis performs all the same steps as Actor.exit(),\nbut it additionally sets the exit code to `1` (by default).\n" + } + ] + }, + "flags": {}, + "id": 55, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "fail", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The exit code with which the actor should fail (defaults to `1`)." + } + ] + }, + "defaultValue": "1", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 56, + "kind": 32768, + "kindString": "Parameter", + "name": "exit_code", + "type": { + "name": "int", + "type": "reference" + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The exception with which the actor failed." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 57, + "kind": 32768, + "kindString": "Parameter", + "name": "exception", + "type": { + "name": "BaseException | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "BaseException" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The final status message that the actor should display." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 58, + "kind": 32768, + "kindString": "Parameter", + "name": "status_message", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + } + ], + "type": { + "name": "None", + "type": "literal", + "value": null + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Initialize the actor, run the passed function and finish the actor cleanly.\n\n**The `Actor.main()` function is optional** and is provided merely for your convenience.\nIt is mainly useful when you're running your code as an actor on the [Apify platform](https://apify.com/actors).\n\nThe `Actor.main()` function performs the following actions:\n\n- When running on the Apify platform (i.e. `APIFY_IS_AT_HOME` environment variable is set),\nit sets up a connection to listen for platform events.\nFor example, to get a notification about an imminent migration to another server.\n- It invokes the user function passed as the `main_actor_function` parameter.\n- If the user function was an async function, it awaits it.\n- If the user function throws an exception or some other error is encountered,\nit prints error details to console so that they are stored to the log,\nand finishes the actor cleanly.\n- Finally, it exits the Python process, with zero exit code on success and non-zero on errors.\n" + } + ] + }, + "decorations": [ + { + "name": "classmethod" + } + ], + "flags": {}, + "groups": [], + "id": 59, + "module": "actor", + "name": "main", + "parsedDocstring": { + "text": "Initialize the actor, run the passed function and finish the actor cleanly.\n\n**The `Actor.main()` function is optional** and is provided merely for your convenience.\nIt is mainly useful when you're running your code as an actor on the [Apify platform](https://apify.com/actors).\n\nThe `Actor.main()` function performs the following actions:\n\n- When running on the Apify platform (i.e. `APIFY_IS_AT_HOME` environment variable is set),\nit sets up a connection to listen for platform events.\nFor example, to get a notification about an imminent migration to another server.\n- It invokes the user function passed as the `main_actor_function` parameter.\n- If the user function was an async function, it awaits it.\n- If the user function throws an exception or some other error is encountered,\nit prints error details to console so that they are stored to the log,\nand finishes the actor cleanly.\n- Finally, it exits the Python process, with zero exit code on success and non-zero on errors.\n", + "args": { + "main_actor_function": "The user function which should be run in the actor" + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/actor.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 412 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Initialize the actor, run the passed function and finish the actor cleanly.\n\n**The `Actor.main()` function is optional** and is provided merely for your convenience.\nIt is mainly useful when you're running your code as an actor on the [Apify platform](https://apify.com/actors).\n\nThe `Actor.main()` function performs the following actions:\n\n- When running on the Apify platform (i.e. `APIFY_IS_AT_HOME` environment variable is set),\nit sets up a connection to listen for platform events.\nFor example, to get a notification about an imminent migration to another server.\n- It invokes the user function passed as the `main_actor_function` parameter.\n- If the user function was an async function, it awaits it.\n- If the user function throws an exception or some other error is encountered,\nit prints error details to console so that they are stored to the log,\nand finishes the actor cleanly.\n- Finally, it exits the Python process, with zero exit code on success and non-zero on errors.\n" + } + ] + }, + "flags": {}, + "id": 60, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "main", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The user function which should be run in the actor" + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 61, + "kind": 32768, + "kindString": "Parameter", + "name": "main_actor_function", + "type": { + "name": "Callable", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "[]" + }, + { + "type": "reference", + "name": "MainReturnType", + "target": "27" + } + ] + } + } + ], + "type": { + "name": "MainReturnType | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "MainReturnType", + "target": "27" + }, + { + "type": "literal", + "value": null + } + ] + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Return a new instance of the Apify API client.\n\nThe `ApifyClientAsync` class is provided by the [apify-client](https://github.com/apify/apify-client-python) package,\nand it is automatically configured using the `APIFY_API_BASE_URL` and `APIFY_TOKEN` environment variables.\n\nYou can override the token via the available options.\nThat's useful if you want to use the client as a different Apify user than the SDK internals are using.\n" + } + ] + }, + "decorations": [ + { + "name": "classmethod" + } + ], + "flags": {}, + "groups": [], + "id": 62, + "module": "actor", + "name": "new_client", + "parsedDocstring": { + "text": "Return a new instance of the Apify API client.\n\nThe `ApifyClientAsync` class is provided by the [apify-client](https://github.com/apify/apify-client-python) package,\nand it is automatically configured using the `APIFY_API_BASE_URL` and `APIFY_TOKEN` environment variables.\n\nYou can override the token via the available options.\nThat's useful if you want to use the client as a different Apify user than the SDK internals are using.\n", + "args": { + "token": "The Apify API token", + "api_url": "The URL of the Apify API server to which to connect to. Defaults to https://api.apify.com", + "max_retries": "How many times to retry a failed request at most", + "min_delay_between_retries_millis": "How long will the client wait between retrying requests\n(increases exponentially from this value)", + "timeout_secs": "The socket timeout of the HTTP requests sent to the Apify API" + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/actor.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 457 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Return a new instance of the Apify API client.\n\nThe `ApifyClientAsync` class is provided by the [apify-client](https://github.com/apify/apify-client-python) package,\nand it is automatically configured using the `APIFY_API_BASE_URL` and `APIFY_TOKEN` environment variables.\n\nYou can override the token via the available options.\nThat's useful if you want to use the client as a different Apify user than the SDK internals are using.\n" + } + ] + }, + "flags": {}, + "id": 63, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "new_client", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The Apify API token" + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 64, + "kind": 32768, + "kindString": "Parameter", + "name": "token", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The URL of the Apify API server to which to connect to. Defaults to https://api.apify.com" + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 65, + "kind": 32768, + "kindString": "Parameter", + "name": "api_url", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "How many times to retry a failed request at most" + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 66, + "kind": 32768, + "kindString": "Parameter", + "name": "max_retries", + "type": { + "name": "int | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "int" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "How long will the client wait between retrying requests\n(increases exponentially from this value)" + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 67, + "kind": 32768, + "kindString": "Parameter", + "name": "min_delay_between_retries_millis", + "type": { + "name": "int | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "int" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The socket timeout of the HTTP requests sent to the Apify API" + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 68, + "kind": 32768, + "kindString": "Parameter", + "name": "timeout_secs", + "type": { + "name": "int | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "int" + }, + { + "type": "literal", + "value": null + } + ] + } + } + ], + "type": { + "name": "ApifyClientAsync", + "type": "reference" + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Open a dataset.\n\nDatasets are used to store structured data where each object stored has the same attributes,\nsuch as online store products or real estate offers.\nThe actual data is stored either on the local filesystem or in the Apify cloud.\n" + } + ] + }, + "decorations": [ + { + "name": "classmethod" + } + ], + "flags": {}, + "groups": [], + "id": 69, + "module": "actor", + "name": "open_dataset", + "parsedDocstring": { + "text": "Open a dataset.\n\nDatasets are used to store structured data where each object stored has the same attributes,\nsuch as online store products or real estate offers.\nThe actual data is stored either on the local filesystem or in the Apify cloud.\n", + "args": { + "id": "ID of the dataset to be opened.\nIf neither `id` nor `name` are provided, the method returns the default dataset associated with the actor run.", + "name": "Name of the dataset to be opened.\nIf neither `id` nor `name` are provided, the method returns the default dataset associated with the actor run.", + "force_cloud": "If set to `True` then the Apify cloud storage is always used.\nThis way it is possible to combine local and cloud storage.\n" + }, + "returns": "Dataset: An instance of the `Dataset` class for the given ID or name." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/actor.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 513 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "Dataset: An instance of the `Dataset` class for the given ID or name." + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Open a dataset.\n\nDatasets are used to store structured data where each object stored has the same attributes,\nsuch as online store products or real estate offers.\nThe actual data is stored either on the local filesystem or in the Apify cloud.\n" + } + ] + }, + "flags": {}, + "id": 70, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "open_dataset", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "ID of the dataset to be opened.\nIf neither `id` nor `name` are provided, the method returns the default dataset associated with the actor run." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 71, + "kind": 32768, + "kindString": "Parameter", + "name": "id", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Name of the dataset to be opened.\nIf neither `id` nor `name` are provided, the method returns the default dataset associated with the actor run." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 72, + "kind": 32768, + "kindString": "Parameter", + "name": "name", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "If set to `True` then the Apify cloud storage is always used.\nThis way it is possible to combine local and cloud storage.\n" + } + ] + }, + "defaultValue": "False", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 73, + "kind": 32768, + "kindString": "Parameter", + "name": "force_cloud", + "type": { + "name": "bool", + "type": "reference" + } + } + ], + "type": { + "name": "Dataset", + "type": "reference", + "target": "636" + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Open a key-value store.\n\nKey-value stores are used to store records or files, along with their MIME content type.\nThe records are stored and retrieved using a unique key.\nThe actual data is stored either on a local filesystem or in the Apify cloud.\n" + } + ] + }, + "decorations": [ + { + "name": "classmethod" + } + ], + "flags": {}, + "groups": [], + "id": 74, + "module": "actor", + "name": "open_key_value_store", + "parsedDocstring": { + "text": "Open a key-value store.\n\nKey-value stores are used to store records or files, along with their MIME content type.\nThe records are stored and retrieved using a unique key.\nThe actual data is stored either on a local filesystem or in the Apify cloud.\n", + "args": { + "id": "ID of the key-value store to be opened.\nIf neither `id` nor `name` are provided, the method returns the default key-value store associated with the actor run.", + "name": "Name of the key-value store to be opened.\nIf neither `id` nor `name` are provided, the method returns the default key-value store associated with the actor run.", + "force_cloud": "If set to `True` then the Apify cloud storage is always used.\nThis way it is possible to combine local and cloud storage.\n" + }, + "returns": "KeyValueStore: An instance of the `KeyValueStore` class for the given ID or name." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/actor.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 552 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "KeyValueStore: An instance of the `KeyValueStore` class for the given ID or name." + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Open a key-value store.\n\nKey-value stores are used to store records or files, along with their MIME content type.\nThe records are stored and retrieved using a unique key.\nThe actual data is stored either on a local filesystem or in the Apify cloud.\n" + } + ] + }, + "flags": {}, + "id": 75, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "open_key_value_store", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "ID of the key-value store to be opened.\nIf neither `id` nor `name` are provided, the method returns the default key-value store associated with the actor run." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 76, + "kind": 32768, + "kindString": "Parameter", + "name": "id", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Name of the key-value store to be opened.\nIf neither `id` nor `name` are provided, the method returns the default key-value store associated with the actor run." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 77, + "kind": 32768, + "kindString": "Parameter", + "name": "name", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "If set to `True` then the Apify cloud storage is always used.\nThis way it is possible to combine local and cloud storage.\n" + } + ] + }, + "defaultValue": "False", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 78, + "kind": 32768, + "kindString": "Parameter", + "name": "force_cloud", + "type": { + "name": "bool", + "type": "reference" + } + } + ], + "type": { + "name": "KeyValueStore", + "type": "reference", + "target": "700" + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Open a request queue.\n\nRequest queue represents a queue of URLs to crawl, which is stored either on local filesystem or in the Apify cloud.\nThe queue is used for deep crawling of websites, where you start with several URLs and then\nrecursively follow links to other pages. The data structure supports both breadth-first\nand depth-first crawling orders.\n" + } + ] + }, + "decorations": [ + { + "name": "classmethod" + } + ], + "flags": {}, + "groups": [], + "id": 79, + "module": "actor", + "name": "open_request_queue", + "parsedDocstring": { + "text": "Open a request queue.\n\nRequest queue represents a queue of URLs to crawl, which is stored either on local filesystem or in the Apify cloud.\nThe queue is used for deep crawling of websites, where you start with several URLs and then\nrecursively follow links to other pages. The data structure supports both breadth-first\nand depth-first crawling orders.\n", + "args": { + "id": "ID of the request queue to be opened.\nIf neither `id` nor `name` are provided, the method returns the default request queue associated with the actor run.", + "name": "Name of the request queue to be opened.\nIf neither `id` nor `name` are provided, the method returns the default request queue associated with the actor run.", + "force_cloud": "If set to `True` then the Apify cloud storage is always used.\nThis way it is possible to combine local and cloud storage.\n" + }, + "returns": "RequestQueue: An instance of the `RequestQueue` class for the given ID or name." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/actor.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 590 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "RequestQueue: An instance of the `RequestQueue` class for the given ID or name." + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Open a request queue.\n\nRequest queue represents a queue of URLs to crawl, which is stored either on local filesystem or in the Apify cloud.\nThe queue is used for deep crawling of websites, where you start with several URLs and then\nrecursively follow links to other pages. The data structure supports both breadth-first\nand depth-first crawling orders.\n" + } + ] + }, + "flags": {}, + "id": 80, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "open_request_queue", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "ID of the request queue to be opened.\nIf neither `id` nor `name` are provided, the method returns the default request queue associated with the actor run." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 81, + "kind": 32768, + "kindString": "Parameter", + "name": "id", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Name of the request queue to be opened.\nIf neither `id` nor `name` are provided, the method returns the default request queue associated with the actor run." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 82, + "kind": 32768, + "kindString": "Parameter", + "name": "name", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "If set to `True` then the Apify cloud storage is always used.\nThis way it is possible to combine local and cloud storage.\n" + } + ] + }, + "defaultValue": "False", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 83, + "kind": 32768, + "kindString": "Parameter", + "name": "force_cloud", + "type": { + "name": "bool", + "type": "reference" + } + } + ], + "type": { + "name": "RequestQueue", + "type": "reference", + "target": "764" + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Store an object or a list of objects to the default dataset of the current actor run.\n" + } + ] + }, + "decorations": [ + { + "name": "classmethod" + } + ], + "flags": {}, + "groups": [], + "id": 84, + "module": "actor", + "name": "push_data", + "parsedDocstring": { + "text": "Store an object or a list of objects to the default dataset of the current actor run.\n", + "args": { + "data": "The data to push to the default dataset." + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/actor.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 629 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Store an object or a list of objects to the default dataset of the current actor run.\n" + } + ] + }, + "flags": {}, + "id": 85, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "push_data", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The data to push to the default dataset." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 86, + "kind": 32768, + "kindString": "Parameter", + "name": "data", + "type": { + "name": "Any", + "type": "reference" + } + } + ], + "type": { + "name": "None", + "type": "literal", + "value": null + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Get the actor input value from the default key-value store associated with the current actor run." + } + ] + }, + "decorations": [ + { + "name": "classmethod" + } + ], + "flags": {}, + "groups": [], + "id": 87, + "module": "actor", + "name": "get_input", + "parsedDocstring": { + "text": "Get the actor input value from the default key-value store associated with the current actor run." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/actor.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 647 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Get the actor input value from the default key-value store associated with the current actor run." + } + ] + }, + "flags": {}, + "id": 88, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "get_input", + "parameters": [], + "type": { + "name": "Any", + "type": "reference" + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Get a value from the default key-value store associated with the current actor run.\n" + } + ] + }, + "decorations": [ + { + "name": "classmethod" + } + ], + "flags": {}, + "groups": [], + "id": 89, + "module": "actor", + "name": "get_value", + "parsedDocstring": { + "text": "Get a value from the default key-value store associated with the current actor run.\n", + "args": { + "key": "The key of the record which to retrieve.", + "default_value": "Default value returned in case the record does not exist." + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/actor.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 667 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Get a value from the default key-value store associated with the current actor run.\n" + } + ] + }, + "flags": {}, + "id": 90, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "get_value", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The key of the record which to retrieve." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 91, + "kind": 32768, + "kindString": "Parameter", + "name": "key", + "type": { + "name": "str", + "type": "reference" + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Default value returned in case the record does not exist." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": false + }, + "id": 92, + "kind": 32768, + "kindString": "Parameter", + "name": "default_value", + "type": { + "name": "Any", + "type": "reference" + } + } + ], + "type": { + "name": "Any", + "type": "reference" + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Set or delete a value in the default key-value store associated with the current actor run.\n" + } + ] + }, + "decorations": [ + { + "name": "classmethod" + } + ], + "flags": {}, + "groups": [], + "id": 93, + "module": "actor", + "name": "set_value", + "parsedDocstring": { + "text": "Set or delete a value in the default key-value store associated with the current actor run.\n", + "args": { + "key": "The key of the record which to set.", + "value": "The value of the record which to set, or None, if the record should be deleted.", + "content_type": "The content type which should be set to the value." + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/actor.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 683 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Set or delete a value in the default key-value store associated with the current actor run.\n" + } + ] + }, + "flags": {}, + "id": 94, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "set_value", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The key of the record which to set." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 95, + "kind": 32768, + "kindString": "Parameter", + "name": "key", + "type": { + "name": "str", + "type": "reference" + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The value of the record which to set, or None, if the record should be deleted." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 96, + "kind": 32768, + "kindString": "Parameter", + "name": "value", + "type": { + "name": "Any", + "type": "reference" + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The content type which should be set to the value." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 97, + "kind": 32768, + "kindString": "Parameter", + "name": "content_type", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + } + ], + "type": { + "name": "None", + "type": "literal", + "value": null + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Add an event listener to the actor's event manager.\n\nThe following events can be emitted:\n- `ActorEventTypes.SYSTEM_INFO`:\nEmitted every minute, the event data contains info about the resource usage of the actor.\n- `ActorEventTypes.MIGRATING`:\nEmitted when the actor running on the Apify platform is going to be migrated to another worker server soon.\nYou can use it to persist the state of the actor and gracefully stop your in-progress tasks,\nso that they are not interrupted by the migration..\n- `ActorEventTypes.PERSIST_STATE`:\nEmitted in regular intervals (by default 60 seconds) to notify the actor that it should persist its state,\nin order to avoid repeating all work when the actor restarts.\nThis event is automatically emitted together with the migrating event,\nin which case the `isMigrating` flag in the event data is set to True, otherwise the flag is False.\nNote that this event is provided merely for your convenience,\nyou can achieve the same effect using an interval and listening for the migrating event.\n- `ActorEventTypes.ABORTING`:\nWhen a user aborts an actor run on the Apify platform,\nthey can choose to abort it gracefully, to allow the actor some time before getting terminated.\nThis graceful abort emits the aborting event, which you can use to clean up the actor state.\n" + } + ] + }, + "decorations": [ + { + "name": "classmethod" + } + ], + "flags": {}, + "groups": [], + "id": 98, + "module": "actor", + "name": "on", + "parsedDocstring": { + "text": "Add an event listener to the actor's event manager.\n\nThe following events can be emitted:\n- `ActorEventTypes.SYSTEM_INFO`:\nEmitted every minute, the event data contains info about the resource usage of the actor.\n- `ActorEventTypes.MIGRATING`:\nEmitted when the actor running on the Apify platform is going to be migrated to another worker server soon.\nYou can use it to persist the state of the actor and gracefully stop your in-progress tasks,\nso that they are not interrupted by the migration..\n- `ActorEventTypes.PERSIST_STATE`:\nEmitted in regular intervals (by default 60 seconds) to notify the actor that it should persist its state,\nin order to avoid repeating all work when the actor restarts.\nThis event is automatically emitted together with the migrating event,\nin which case the `isMigrating` flag in the event data is set to True, otherwise the flag is False.\nNote that this event is provided merely for your convenience,\nyou can achieve the same effect using an interval and listening for the migrating event.\n- `ActorEventTypes.ABORTING`:\nWhen a user aborts an actor run on the Apify platform,\nthey can choose to abort it gracefully, to allow the actor some time before getting terminated.\nThis graceful abort emits the aborting event, which you can use to clean up the actor state.\n", + "args": { + "event_name": "The actor event for which to listen to.", + "listener": "The function which is to be called when the event is emitted (can be async)." + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/actor.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 716 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Add an event listener to the actor's event manager.\n\nThe following events can be emitted:\n- `ActorEventTypes.SYSTEM_INFO`:\nEmitted every minute, the event data contains info about the resource usage of the actor.\n- `ActorEventTypes.MIGRATING`:\nEmitted when the actor running on the Apify platform is going to be migrated to another worker server soon.\nYou can use it to persist the state of the actor and gracefully stop your in-progress tasks,\nso that they are not interrupted by the migration..\n- `ActorEventTypes.PERSIST_STATE`:\nEmitted in regular intervals (by default 60 seconds) to notify the actor that it should persist its state,\nin order to avoid repeating all work when the actor restarts.\nThis event is automatically emitted together with the migrating event,\nin which case the `isMigrating` flag in the event data is set to True, otherwise the flag is False.\nNote that this event is provided merely for your convenience,\nyou can achieve the same effect using an interval and listening for the migrating event.\n- `ActorEventTypes.ABORTING`:\nWhen a user aborts an actor run on the Apify platform,\nthey can choose to abort it gracefully, to allow the actor some time before getting terminated.\nThis graceful abort emits the aborting event, which you can use to clean up the actor state.\n" + } + ] + }, + "flags": {}, + "id": 99, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "on", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The actor event for which to listen to." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 100, + "kind": 32768, + "kindString": "Parameter", + "name": "event_name", + "type": { + "name": "ActorEventTypes", + "type": "reference" + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The function which is to be called when the event is emitted (can be async)." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 101, + "kind": 32768, + "kindString": "Parameter", + "name": "listener", + "type": { + "name": "Callable", + "type": "reference" + } + } + ], + "type": { + "name": "Callable", + "type": "reference" + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Remove a listener, or all listeners, from an actor event.\n" + } + ] + }, + "decorations": [ + { + "name": "classmethod" + } + ], + "flags": {}, + "groups": [], + "id": 102, + "module": "actor", + "name": "off", + "parsedDocstring": { + "text": "Remove a listener, or all listeners, from an actor event.\n", + "args": { + "event_name": "The actor event for which to remove listeners.", + "listener": "The listener which is supposed to be removed. If not passed, all listeners of this event are removed." + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/actor.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 750 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Remove a listener, or all listeners, from an actor event.\n" + } + ] + }, + "flags": {}, + "id": 103, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "off", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The actor event for which to remove listeners." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 104, + "kind": 32768, + "kindString": "Parameter", + "name": "event_name", + "type": { + "name": "ActorEventTypes", + "type": "reference" + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The listener which is supposed to be removed. If not passed, all listeners of this event are removed." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": false + }, + "id": 105, + "kind": 32768, + "kindString": "Parameter", + "name": "listener", + "type": { + "name": "Callable | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "Callable" + }, + { + "type": "literal", + "value": null + } + ] + } + } + ], + "type": { + "name": "None", + "type": "literal", + "value": null + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Return `True` when the actor is running on the Apify platform, and `False` otherwise (for example when running locally)." + } + ] + }, + "decorations": [ + { + "name": "classmethod" + } + ], + "flags": {}, + "groups": [], + "id": 106, + "module": "actor", + "name": "is_at_home", + "parsedDocstring": { + "text": "Return `True` when the actor is running on the Apify platform, and `False` otherwise (for example when running locally)." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/actor.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 765 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Return `True` when the actor is running on the Apify platform, and `False` otherwise (for example when running locally)." + } + ] + }, + "flags": {}, + "id": 107, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "is_at_home", + "parameters": [], + "type": { + "name": "bool", + "type": "reference" + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Return a dictionary with information parsed from all the `APIFY_XXX` environment variables.\n\nFor a list of all the environment variables,\nsee the [Actor documentation](https://docs.apify.com/actors/development/environment-variables).\nIf some variables are not defined or are invalid, the corresponding value in the resulting dictionary will be None." + } + ] + }, + "decorations": [ + { + "name": "classmethod" + } + ], + "flags": {}, + "groups": [], + "id": 108, + "module": "actor", + "name": "get_env", + "parsedDocstring": { + "text": "Return a dictionary with information parsed from all the `APIFY_XXX` environment variables.\n\nFor a list of all the environment variables,\nsee the [Actor documentation](https://docs.apify.com/actors/development/environment-variables).\nIf some variables are not defined or are invalid, the corresponding value in the resulting dictionary will be None." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/actor.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 773 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Return a dictionary with information parsed from all the `APIFY_XXX` environment variables.\n\nFor a list of all the environment variables,\nsee the [Actor documentation](https://docs.apify.com/actors/development/environment-variables).\nIf some variables are not defined or are invalid, the corresponding value in the resulting dictionary will be None." + } + ] + }, + "flags": {}, + "id": 109, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "get_env", + "parameters": [], + "type": { + "name": "dict", + "type": "reference" + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Run an actor on the Apify platform.\n\nUnlike `Actor.call`, this method just starts the run without waiting for finish.\n" + } + ] + }, + "decorations": [ + { + "name": "classmethod" + } + ], + "flags": {}, + "groups": [], + "id": 110, + "module": "actor", + "name": "start", + "parsedDocstring": { + "text": "Run an actor on the Apify platform.\n\nUnlike `Actor.call`, this method just starts the run without waiting for finish.\n", + "args": { + "actor_id": "The ID of the actor to be run.", + "run_input": "The input to pass to the actor run.", + "token": "The Apify API token to use for this request (defaults to the `APIFY_TOKEN` environment variable).", + "content_type": "The content type of the input.", + "build": "Specifies the actor build to run. It can be either a build tag or build number.\nBy default, the run uses the build specified in the default run configuration for the actor (typically latest).", + "memory_mbytes": "Memory limit for the run, in megabytes.\nBy default, the run uses a memory limit specified in the default run configuration for the actor.", + "timeout_secs": "Optional timeout for the run, in seconds.\nBy default, the run uses timeout specified in the default run configuration for the actor.", + "wait_for_finish": "The maximum number of seconds the server waits for the run to finish.\nBy default, it is 0, the maximum value is 300.", + "webhooks": "Optional ad-hoc webhooks (https://docs.apify.com/webhooks/ad-hoc-webhooks)\nassociated with the actor run which can be used to receive a notification,\ne.g. when the actor finished or failed.\nIf you already have a webhook set up for the actor or task, you do not have to add it again here.\nEach webhook is represented by a dictionary containing these items:\n* ``event_types``: list of ``WebhookEventType`` values which trigger the webhook\n* ``request_url``: URL to which to send the webhook HTTP request\n* ``payload_template`` (optional): Optional template for the request payload\n" + }, + "returns": "dict: Info about the started actor run" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/actor.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 788 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "dict: Info about the started actor run" + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Run an actor on the Apify platform.\n\nUnlike `Actor.call`, this method just starts the run without waiting for finish.\n" + } + ] + }, + "flags": {}, + "id": 111, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "start", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The ID of the actor to be run." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 112, + "kind": 32768, + "kindString": "Parameter", + "name": "actor_id", + "type": { + "name": "str", + "type": "reference" + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The input to pass to the actor run." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": false + }, + "id": 113, + "kind": 32768, + "kindString": "Parameter", + "name": "run_input", + "type": { + "name": "Any", + "type": "reference" + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The Apify API token to use for this request (defaults to the `APIFY_TOKEN` environment variable)." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 114, + "kind": 32768, + "kindString": "Parameter", + "name": "token", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The content type of the input." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 115, + "kind": 32768, + "kindString": "Parameter", + "name": "content_type", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Specifies the actor build to run. It can be either a build tag or build number.\nBy default, the run uses the build specified in the default run configuration for the actor (typically latest)." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 116, + "kind": 32768, + "kindString": "Parameter", + "name": "build", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Memory limit for the run, in megabytes.\nBy default, the run uses a memory limit specified in the default run configuration for the actor." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 117, + "kind": 32768, + "kindString": "Parameter", + "name": "memory_mbytes", + "type": { + "name": "int | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "int" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Optional timeout for the run, in seconds.\nBy default, the run uses timeout specified in the default run configuration for the actor." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 118, + "kind": 32768, + "kindString": "Parameter", + "name": "timeout_secs", + "type": { + "name": "int | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "int" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The maximum number of seconds the server waits for the run to finish.\nBy default, it is 0, the maximum value is 300." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 119, + "kind": 32768, + "kindString": "Parameter", + "name": "wait_for_finish", + "type": { + "name": "int | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "int" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Optional ad-hoc webhooks (https://docs.apify.com/webhooks/ad-hoc-webhooks)\nassociated with the actor run which can be used to receive a notification,\ne.g. when the actor finished or failed.\nIf you already have a webhook set up for the actor or task, you do not have to add it again here.\nEach webhook is represented by a dictionary containing these items:\n* ``event_types``: list of ``WebhookEventType`` values which trigger the webhook\n* ``request_url``: URL to which to send the webhook HTTP request\n* ``payload_template`` (optional): Optional template for the request payload\n" + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 120, + "kind": 32768, + "kindString": "Parameter", + "name": "webhooks", + "type": { + "name": "list[dict] | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "list", + "typeArguments": [ + { + "type": "reference", + "name": "dict" + } + ], + "target": "474" + }, + { + "type": "literal", + "value": null + } + ] + } + } + ], + "type": { + "name": "dict", + "type": "reference" + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Abort given actor run on the Apify platform using the current user account (determined by the `APIFY_TOKEN` environment variable).\n" + } + ] + }, + "decorations": [ + { + "name": "classmethod" + } + ], + "flags": {}, + "groups": [], + "id": 121, + "module": "actor", + "name": "abort", + "parsedDocstring": { + "text": "Abort given actor run on the Apify platform using the current user account (determined by the `APIFY_TOKEN` environment variable).\n", + "args": { + "run_id": "The ID of the actor run to be aborted.", + "token": "The Apify API token to use for this request (defaults to the `APIFY_TOKEN` environment variable).", + "gracefully": "If True, the actor run will abort gracefully.\nIt will send ``aborting`` and ``persistStates`` events into the run and force-stop the run after 30 seconds.\nIt is helpful in cases where you plan to resurrect the run later.\n" + }, + "returns": "dict: Info about the aborted actor run" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/actor.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 870 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "dict: Info about the aborted actor run" + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Abort given actor run on the Apify platform using the current user account (determined by the `APIFY_TOKEN` environment variable).\n" + } + ] + }, + "flags": {}, + "id": 122, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "abort", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The ID of the actor run to be aborted." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 123, + "kind": 32768, + "kindString": "Parameter", + "name": "run_id", + "type": { + "name": "str", + "type": "reference" + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The Apify API token to use for this request (defaults to the `APIFY_TOKEN` environment variable)." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 124, + "kind": 32768, + "kindString": "Parameter", + "name": "token", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "If True, the actor run will abort gracefully.\nIt will send ``aborting`` and ``persistStates`` events into the run and force-stop the run after 30 seconds.\nIt is helpful in cases where you plan to resurrect the run later.\n" + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 125, + "kind": 32768, + "kindString": "Parameter", + "name": "gracefully", + "type": { + "name": "bool | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "bool" + }, + { + "type": "literal", + "value": null + } + ] + } + } + ], + "type": { + "name": "dict", + "type": "reference" + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Start an actor on the Apify Platform and wait for it to finish before returning.\n\nIt waits indefinitely, unless the wait_secs argument is provided.\n" + } + ] + }, + "decorations": [ + { + "name": "classmethod" + } + ], + "flags": {}, + "groups": [], + "id": 126, + "module": "actor", + "name": "call", + "parsedDocstring": { + "text": "Start an actor on the Apify Platform and wait for it to finish before returning.\n\nIt waits indefinitely, unless the wait_secs argument is provided.\n", + "args": { + "actor_id": "The ID of the actor to be run.", + "run_input": "The input to pass to the actor run.", + "token": "The Apify API token to use for this request (defaults to the `APIFY_TOKEN` environment variable).", + "content_type": "The content type of the input.", + "build": "Specifies the actor build to run. It can be either a build tag or build number.\nBy default, the run uses the build specified in the default run configuration for the actor (typically latest).", + "memory_mbytes": "Memory limit for the run, in megabytes.\nBy default, the run uses a memory limit specified in the default run configuration for the actor.", + "timeout_secs": "Optional timeout for the run, in seconds.\nBy default, the run uses timeout specified in the default run configuration for the actor.", + "webhooks": "Optional webhooks (https://docs.apify.com/webhooks) associated with the actor run,\nwhich can be used to receive a notification, e.g. when the actor finished or failed.\nIf you already have a webhook set up for the actor, you do not have to add it again here.", + "wait_secs": "The maximum number of seconds the server waits for the run to finish. If not provided, waits indefinitely.\n" + }, + "returns": "dict: Info about the started actor run" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/actor.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 913 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "dict: Info about the started actor run" + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Start an actor on the Apify Platform and wait for it to finish before returning.\n\nIt waits indefinitely, unless the wait_secs argument is provided.\n" + } + ] + }, + "flags": {}, + "id": 127, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "call", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The ID of the actor to be run." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 128, + "kind": 32768, + "kindString": "Parameter", + "name": "actor_id", + "type": { + "name": "str", + "type": "reference" + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The input to pass to the actor run." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": false + }, + "id": 129, + "kind": 32768, + "kindString": "Parameter", + "name": "run_input", + "type": { + "name": "Any", + "type": "reference" + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The Apify API token to use for this request (defaults to the `APIFY_TOKEN` environment variable)." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 130, + "kind": 32768, + "kindString": "Parameter", + "name": "token", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The content type of the input." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 131, + "kind": 32768, + "kindString": "Parameter", + "name": "content_type", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Specifies the actor build to run. It can be either a build tag or build number.\nBy default, the run uses the build specified in the default run configuration for the actor (typically latest)." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 132, + "kind": 32768, + "kindString": "Parameter", + "name": "build", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Memory limit for the run, in megabytes.\nBy default, the run uses a memory limit specified in the default run configuration for the actor." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 133, + "kind": 32768, + "kindString": "Parameter", + "name": "memory_mbytes", + "type": { + "name": "int | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "int" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Optional timeout for the run, in seconds.\nBy default, the run uses timeout specified in the default run configuration for the actor." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 134, + "kind": 32768, + "kindString": "Parameter", + "name": "timeout_secs", + "type": { + "name": "int | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "int" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Optional webhooks (https://docs.apify.com/webhooks) associated with the actor run,\nwhich can be used to receive a notification, e.g. when the actor finished or failed.\nIf you already have a webhook set up for the actor, you do not have to add it again here." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 135, + "kind": 32768, + "kindString": "Parameter", + "name": "webhooks", + "type": { + "name": "list[dict] | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "list", + "typeArguments": [ + { + "type": "reference", + "name": "dict" + } + ], + "target": "474" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The maximum number of seconds the server waits for the run to finish. If not provided, waits indefinitely.\n" + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 136, + "kind": 32768, + "kindString": "Parameter", + "name": "wait_secs", + "type": { + "name": "int | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "int" + }, + { + "type": "literal", + "value": null + } + ] + } + } + ], + "type": { + "name": "dict | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "dict" + }, + { + "type": "literal", + "value": null + } + ] + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Start an actor task on the Apify Platform and wait for it to finish before returning.\n\nIt waits indefinitely, unless the wait_secs argument is provided.\n\nNote that an actor task is a saved input configuration and options for an actor.\nIf you want to run an actor directly rather than an actor task, please use the `Actor.call`\n" + } + ] + }, + "decorations": [ + { + "name": "classmethod" + } + ], + "flags": {}, + "groups": [], + "id": 137, + "module": "actor", + "name": "call_task", + "parsedDocstring": { + "text": "Start an actor task on the Apify Platform and wait for it to finish before returning.\n\nIt waits indefinitely, unless the wait_secs argument is provided.\n\nNote that an actor task is a saved input configuration and options for an actor.\nIf you want to run an actor directly rather than an actor task, please use the `Actor.call`\n", + "args": { + "task_id": "The ID of the actor to be run.", + "task_input": "Overrides the input to pass to the actor run.", + "token": "The Apify API token to use for this request (defaults to the `APIFY_TOKEN` environment variable).", + "content_type": "The content type of the input.", + "build": "Specifies the actor build to run. It can be either a build tag or build number.\nBy default, the run uses the build specified in the default run configuration for the actor (typically latest).", + "memory_mbytes": "Memory limit for the run, in megabytes.\nBy default, the run uses a memory limit specified in the default run configuration for the actor.", + "timeout_secs": "Optional timeout for the run, in seconds.\nBy default, the run uses timeout specified in the default run configuration for the actor.", + "webhooks": "Optional webhooks (https://docs.apify.com/webhooks) associated with the actor run,\nwhich can be used to receive a notification, e.g. when the actor finished or failed.\nIf you already have a webhook set up for the actor, you do not have to add it again here.", + "wait_secs": "The maximum number of seconds the server waits for the run to finish. If not provided, waits indefinitely.\n" + }, + "returns": "dict: Info about the started actor run" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/actor.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 989 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "dict: Info about the started actor run" + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Start an actor task on the Apify Platform and wait for it to finish before returning.\n\nIt waits indefinitely, unless the wait_secs argument is provided.\n\nNote that an actor task is a saved input configuration and options for an actor.\nIf you want to run an actor directly rather than an actor task, please use the `Actor.call`\n" + } + ] + }, + "flags": {}, + "id": 138, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "call_task", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The ID of the actor to be run." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 139, + "kind": 32768, + "kindString": "Parameter", + "name": "task_id", + "type": { + "name": "str", + "type": "reference" + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Overrides the input to pass to the actor run." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": false + }, + "id": 140, + "kind": 32768, + "kindString": "Parameter", + "name": "task_input", + "type": { + "name": "dict | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "dict" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Specifies the actor build to run. It can be either a build tag or build number.\nBy default, the run uses the build specified in the default run configuration for the actor (typically latest)." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 141, + "kind": 32768, + "kindString": "Parameter", + "name": "build", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Memory limit for the run, in megabytes.\nBy default, the run uses a memory limit specified in the default run configuration for the actor." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 142, + "kind": 32768, + "kindString": "Parameter", + "name": "memory_mbytes", + "type": { + "name": "int | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "int" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Optional timeout for the run, in seconds.\nBy default, the run uses timeout specified in the default run configuration for the actor." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 143, + "kind": 32768, + "kindString": "Parameter", + "name": "timeout_secs", + "type": { + "name": "int | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "int" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Optional webhooks (https://docs.apify.com/webhooks) associated with the actor run,\nwhich can be used to receive a notification, e.g. when the actor finished or failed.\nIf you already have a webhook set up for the actor, you do not have to add it again here." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 144, + "kind": 32768, + "kindString": "Parameter", + "name": "webhooks", + "type": { + "name": "list[dict] | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "list", + "typeArguments": [ + { + "type": "reference", + "name": "dict" + } + ], + "target": "474" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The maximum number of seconds the server waits for the run to finish. If not provided, waits indefinitely.\n" + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 145, + "kind": 32768, + "kindString": "Parameter", + "name": "wait_secs", + "type": { + "name": "int | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "int" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The Apify API token to use for this request (defaults to the `APIFY_TOKEN` environment variable)." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 146, + "kind": 32768, + "kindString": "Parameter", + "name": "token", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + } + ], + "type": { + "name": "dict | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "dict" + }, + { + "type": "literal", + "value": null + } + ] + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Transform this actor run to an actor run of a different actor.\n\nThe platform stops the current actor container and starts a new container with the new actor instead.\nAll the default storages are preserved,\nand the new input is stored under the `INPUT-METAMORPH-1` key in the same default key-value store.\n" + } + ] + }, + "decorations": [ + { + "name": "classmethod" + } + ], + "flags": {}, + "groups": [], + "id": 147, + "module": "actor", + "name": "metamorph", + "parsedDocstring": { + "text": "Transform this actor run to an actor run of a different actor.\n\nThe platform stops the current actor container and starts a new container with the new actor instead.\nAll the default storages are preserved,\nand the new input is stored under the `INPUT-METAMORPH-1` key in the same default key-value store.\n", + "args": { + "target_actor_id": "ID of the target actor that the run should be transformed into", + "run_input": "The input to pass to the new run.", + "target_actor_build": "The build of the target actor. It can be either a build tag or build number.\nBy default, the run uses the build specified in the default run configuration for the target actor (typically the latest build).", + "content_type": "The content type of the input.", + "custom_after_sleep_millis": "How long to sleep for after the metamorph, to wait for the container to be stopped.\n" + }, + "returns": "dict: The actor run data." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/actor.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 1064 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "dict: The actor run data." + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Transform this actor run to an actor run of a different actor.\n\nThe platform stops the current actor container and starts a new container with the new actor instead.\nAll the default storages are preserved,\nand the new input is stored under the `INPUT-METAMORPH-1` key in the same default key-value store.\n" + } + ] + }, + "flags": {}, + "id": 148, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "metamorph", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "ID of the target actor that the run should be transformed into" + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 149, + "kind": 32768, + "kindString": "Parameter", + "name": "target_actor_id", + "type": { + "name": "str", + "type": "reference" + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The input to pass to the new run." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": false + }, + "id": 150, + "kind": 32768, + "kindString": "Parameter", + "name": "run_input", + "type": { + "name": "Any", + "type": "reference" + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The build of the target actor. It can be either a build tag or build number.\nBy default, the run uses the build specified in the default run configuration for the target actor (typically the latest build)." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 151, + "kind": 32768, + "kindString": "Parameter", + "name": "target_actor_build", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The content type of the input." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 152, + "kind": 32768, + "kindString": "Parameter", + "name": "content_type", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "How long to sleep for after the metamorph, to wait for the container to be stopped.\n" + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 153, + "kind": 32768, + "kindString": "Parameter", + "name": "custom_after_sleep_millis", + "type": { + "name": "int | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "int" + }, + { + "type": "literal", + "value": null + } + ] + } + } + ], + "type": { + "name": "None", + "type": "literal", + "value": null + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Internally reboot this actor.\n\nThe system stops the current container and starts a new one, with the same run ID and default storages.\n" + } + ] + }, + "decorations": [ + { + "name": "classmethod" + } + ], + "flags": {}, + "groups": [], + "id": 154, + "module": "actor", + "name": "reboot", + "parsedDocstring": { + "text": "Internally reboot this actor.\n\nThe system stops the current container and starts a new one, with the same run ID and default storages.\n", + "args": { + "event_listeners_timeout_secs": "How long should the actor wait for actor event listeners to finish before exiting", + "custom_after_sleep_millis": "How long to sleep for after the reboot, to wait for the container to be stopped." + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/actor.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 1130 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Internally reboot this actor.\n\nThe system stops the current container and starts a new one, with the same run ID and default storages.\n" + } + ] + }, + "flags": {}, + "id": 155, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "reboot", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "How long should the actor wait for actor event listeners to finish before exiting" + } + ] + }, + "defaultValue": "EVENT_LISTENERS_TIMEOUT_SECS", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 156, + "kind": 32768, + "kindString": "Parameter", + "name": "event_listeners_timeout_secs", + "type": { + "name": "int | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "int" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "How long to sleep for after the reboot, to wait for the container to be stopped." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 157, + "kind": 32768, + "kindString": "Parameter", + "name": "custom_after_sleep_millis", + "type": { + "name": "int | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "int" + }, + { + "type": "literal", + "value": null + } + ] + } + } + ], + "type": { + "name": "None", + "type": "literal", + "value": null + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Create an ad-hoc webhook for the current actor run.\n\nThis webhook lets you receive a notification when the actor run finished or failed.\n\nNote that webhooks are only supported for actors running on the Apify platform.\nWhen running the actor locally, the function will print a warning and have no effect.\n\nFor more information about Apify actor webhooks, please see the [documentation](https://docs.apify.com/webhooks).\n" + } + ] + }, + "decorations": [ + { + "name": "classmethod" + } + ], + "flags": {}, + "groups": [], + "id": 158, + "module": "actor", + "name": "add_webhook", + "parsedDocstring": { + "text": "Create an ad-hoc webhook for the current actor run.\n\nThis webhook lets you receive a notification when the actor run finished or failed.\n\nNote that webhooks are only supported for actors running on the Apify platform.\nWhen running the actor locally, the function will print a warning and have no effect.\n\nFor more information about Apify actor webhooks, please see the [documentation](https://docs.apify.com/webhooks).\n", + "args": { + "event_types": "List of event types that should trigger the webhook. At least one is required.", + "request_url": "URL that will be invoked once the webhook is triggered.", + "payload_template": "Specification of the payload that will be sent to request_url", + "ignore_ssl_errors": "Whether the webhook should ignore SSL errors returned by request_url", + "do_not_retry": "Whether the webhook should retry sending the payload to request_url upon\nfailure.", + "idempotency_key": "A unique identifier of a webhook. You can use it to ensure that you won't\ncreate the same webhook multiple times.\n" + }, + "returns": "dict: The created webhook" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/actor.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 1178 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "dict: The created webhook" + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Create an ad-hoc webhook for the current actor run.\n\nThis webhook lets you receive a notification when the actor run finished or failed.\n\nNote that webhooks are only supported for actors running on the Apify platform.\nWhen running the actor locally, the function will print a warning and have no effect.\n\nFor more information about Apify actor webhooks, please see the [documentation](https://docs.apify.com/webhooks).\n" + } + ] + }, + "flags": {}, + "id": 159, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "add_webhook", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "List of event types that should trigger the webhook. At least one is required." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 160, + "kind": 32768, + "kindString": "Parameter", + "name": "event_types", + "type": { + "name": "list", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "WebhookEventType" + } + ], + "target": "474" + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "URL that will be invoked once the webhook is triggered." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 161, + "kind": 32768, + "kindString": "Parameter", + "name": "request_url", + "type": { + "name": "str", + "type": "reference" + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Specification of the payload that will be sent to request_url" + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 162, + "kind": 32768, + "kindString": "Parameter", + "name": "payload_template", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Whether the webhook should ignore SSL errors returned by request_url" + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 163, + "kind": 32768, + "kindString": "Parameter", + "name": "ignore_ssl_errors", + "type": { + "name": "bool | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "bool" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Whether the webhook should retry sending the payload to request_url upon\nfailure." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 164, + "kind": 32768, + "kindString": "Parameter", + "name": "do_not_retry", + "type": { + "name": "bool | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "bool" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "A unique identifier of a webhook. You can use it to ensure that you won't\ncreate the same webhook multiple times.\n" + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 165, + "kind": 32768, + "kindString": "Parameter", + "name": "idempotency_key", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + } + ], + "type": { + "name": "dict", + "type": "reference" + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Set the status message for the current actor run.\n" + } + ] + }, + "decorations": [ + { + "name": "classmethod" + } + ], + "flags": {}, + "groups": [], + "id": 166, + "module": "actor", + "name": "set_status_message", + "parsedDocstring": { + "text": "Set the status message for the current actor run.\n", + "args": { + "status_message": "The status message to set to the run.", + "is_terminal": "Set this flag to True if this is the final status message of the Actor run.\n" + }, + "returns": "dict: The updated actor run object" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/actor.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 1249 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "dict: The updated actor run object" + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Set the status message for the current actor run.\n" + } + ] + }, + "flags": {}, + "id": 167, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "set_status_message", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The status message to set to the run." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 168, + "kind": 32768, + "kindString": "Parameter", + "name": "status_message", + "type": { + "name": "str", + "type": "reference" + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Set this flag to True if this is the final status message of the Actor run.\n" + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 169, + "kind": 32768, + "kindString": "Parameter", + "name": "is_terminal", + "type": { + "name": "bool | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "bool" + }, + { + "type": "literal", + "value": null + } + ] + } + } + ], + "type": { + "name": "dict | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "dict" + }, + { + "type": "literal", + "value": null + } + ] + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Create a ProxyConfiguration object with the passed proxy configuration.\n\nConfigures connection to a proxy server with the provided options.\nProxy servers are used to prevent target websites from blocking your crawlers based on IP address rate limits or blacklists.\n\nFor more details and code examples, see the `ProxyConfiguration` class.\n" + } + ] + }, + "decorations": [ + { + "name": "classmethod" + } + ], + "flags": {}, + "groups": [], + "id": 170, + "module": "actor", + "name": "create_proxy_configuration", + "parsedDocstring": { + "text": "Create a ProxyConfiguration object with the passed proxy configuration.\n\nConfigures connection to a proxy server with the provided options.\nProxy servers are used to prevent target websites from blocking your crawlers based on IP address rate limits or blacklists.\n\nFor more details and code examples, see the `ProxyConfiguration` class.\n", + "args": { + "actor_proxy_input": "Proxy configuration field from the actor input, if actor has such input field.\nIf you pass this argument, all the other arguments will be inferred from it.", + "password": "Password for the Apify Proxy. If not provided, will use os.environ['APIFY_PROXY_PASSWORD'], if available.", + "groups": "Proxy groups which the Apify Proxy should use, if provided.", + "country_code": "Country which the Apify Proxy should use, if provided.", + "proxy_urls": "Custom proxy server URLs which should be rotated through.", + "new_url_function": "Function which returns a custom proxy URL to be used.\n" + }, + "returns": "ProxyConfiguration, optional: ProxyConfiguration object with the passed configuration,\nor None, if no proxy should be used based on the configuration." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/actor.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 1285 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "ProxyConfiguration, optional: ProxyConfiguration object with the passed configuration,\nor None, if no proxy should be used based on the configuration." + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Create a ProxyConfiguration object with the passed proxy configuration.\n\nConfigures connection to a proxy server with the provided options.\nProxy servers are used to prevent target websites from blocking your crawlers based on IP address rate limits or blacklists.\n\nFor more details and code examples, see the `ProxyConfiguration` class.\n" + } + ] + }, + "flags": {}, + "id": 171, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "create_proxy_configuration", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Proxy configuration field from the actor input, if actor has such input field.\nIf you pass this argument, all the other arguments will be inferred from it." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 172, + "kind": 32768, + "kindString": "Parameter", + "name": "actor_proxy_input", + "type": { + "name": "dict | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "dict" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Password for the Apify Proxy. If not provided, will use os.environ['APIFY_PROXY_PASSWORD'], if available." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 173, + "kind": 32768, + "kindString": "Parameter", + "name": "password", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Proxy groups which the Apify Proxy should use, if provided." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 174, + "kind": 32768, + "kindString": "Parameter", + "name": "groups", + "type": { + "name": "list[str] | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "list", + "typeArguments": [ + { + "type": "reference", + "name": "str" + } + ], + "target": "474" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Country which the Apify Proxy should use, if provided." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 175, + "kind": 32768, + "kindString": "Parameter", + "name": "country_code", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Custom proxy server URLs which should be rotated through." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 176, + "kind": 32768, + "kindString": "Parameter", + "name": "proxy_urls", + "type": { + "name": "list[str] | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "list", + "typeArguments": [ + { + "type": "reference", + "name": "str" + } + ], + "target": "474" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Function which returns a custom proxy URL to be used.\n" + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 177, + "kind": 32768, + "kindString": "Parameter", + "name": "new_url_function", + "type": { + "name": "Callable[[str | None], str] | Callable[[str | None], Awaitable[str]] | None", + "type": "union", + "types": [ + { + "type": "union", + "types": [ + { + "type": "reference", + "name": "Callable", + "typeArguments": [ + { + "type": "reference", + "name": "[str | None]" + }, + { + "type": "reference", + "name": "str" + } + ] + }, + { + "type": "reference", + "name": "Callable", + "typeArguments": [ + { + "type": "reference", + "name": "[str | None]" + }, + { + "type": "reference", + "name": "Awaitable", + "typeArguments": [ + { + "type": "reference", + "name": "str" + } + ] + } + ] + } + ] + }, + { + "type": "literal", + "value": null + } + ] + } + } + ], + "type": { + "name": "ProxyConfiguration | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "ProxyConfiguration", + "target": "257" + }, + { + "type": "literal", + "value": null + } + ] + } + } + ] + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "The main class of the SDK, through which all the actor operations should be done." + } + ] + }, + "flags": {}, + "groups": [ + { + "children": [ + 37, + 121, + 158, + 126, + 137, + 170, + 48, + 54, + 108, + 87, + 46, + 89, + 44, + 106, + 59, + 147, + 62, + 102, + 98, + 69, + 74, + 79, + 84, + 154, + 166, + 93, + 110 + ], + "title": "Methods" + }, + { + "children": [ + 40, + 41, + 42, + 43 + ], + "title": "Properties" + } + ], + "id": 36, + "module": "actor", + "name": "Actor", + "parsedDocstring": { + "text": "The main class of the SDK, through which all the actor operations should be done." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/actor.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 68 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 178, + "module": "_crypto", + "name": "ENCRYPTION_KEY_LENGTH", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_crypto.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 15 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 179, + "module": "_crypto", + "name": "ENCRYPTION_IV_LENGTH", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_crypto.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 16 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 180, + "module": "_crypto", + "name": "ENCRYPTION_AUTH_TAG_LENGTH", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_crypto.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 17 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 181, + "module": "_crypto", + "name": "load_private_key", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_crypto.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 109 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "flags": {}, + "id": 182, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "load_private_key", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 183, + "kind": 32768, + "kindString": "Parameter", + "name": "private_key_file_base64", + "type": { + "name": "str", + "type": "reference" + } + }, + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 184, + "kind": 32768, + "kindString": "Parameter", + "name": "private_key_password", + "type": { + "name": "str", + "type": "reference" + } + } + ], + "type": { + "name": "rsa.RSAPrivateKey", + "type": "reference" + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Python reimplementation of cryptoRandomObjectId from `@apify/utilities`." + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 185, + "module": "_crypto", + "name": "crypto_random_object_id", + "parsedDocstring": { + "text": "Python reimplementation of cryptoRandomObjectId from `@apify/utilities`." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_crypto.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 128 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Python reimplementation of cryptoRandomObjectId from `@apify/utilities`." + } + ] + }, + "flags": {}, + "id": 186, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "crypto_random_object_id", + "parameters": [ + { + "defaultValue": "17", + "flags": { + "isOptional": true, + "keyword-only": false + }, + "id": 187, + "kind": 32768, + "kindString": "Parameter", + "name": "length", + "type": { + "name": "int", + "type": "reference" + } + } + ], + "type": { + "name": "str", + "type": "reference" + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Decrypt input secrets." + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 188, + "module": "_crypto", + "name": "decrypt_input_secrets", + "parsedDocstring": { + "text": "Decrypt input secrets." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_crypto.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 134 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Decrypt input secrets." + } + ] + }, + "flags": {}, + "id": 189, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "decrypt_input_secrets", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 190, + "kind": 32768, + "kindString": "Parameter", + "name": "private_key", + "type": { + "name": "rsa.RSAPrivateKey", + "type": "reference" + } + }, + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 191, + "kind": 32768, + "kindString": "Parameter", + "name": "input", + "type": { + "name": "Any", + "type": "reference" + } + } + ], + "type": { + "name": "Any", + "type": "reference" + } + } + ] + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 192, + "module": "event_manager", + "name": "ListenerType", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/event_manager.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 21 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Create an instance of the EventManager.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 193, + "module": "event_manager", + "name": "__init__", + "parsedDocstring": { + "text": "Create an instance of the EventManager.\n", + "args": { + "config": "The actor configuration to be used in this event manager." + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/event_manager.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 40 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Create an instance of the EventManager.\n" + } + ] + }, + "flags": {}, + "id": 194, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "__init__", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The actor configuration to be used in this event manager." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 195, + "kind": 32768, + "kindString": "Parameter", + "name": "config", + "type": { + "name": "Configuration", + "type": "reference", + "target": "1" + } + } + ], + "type": { + "name": "None", + "type": "literal", + "value": null + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Initialize the event manager.\n\nWhen running this on the Apify Platform, this will start processing events\nsend by the platform to the events websocket and emitting them as events\nthat can be listened to by the `Actor.on()` method." + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 196, + "module": "event_manager", + "name": "init", + "parsedDocstring": { + "text": "Initialize the event manager.\n\nWhen running this on the Apify Platform, this will start processing events\nsend by the platform to the events websocket and emitting them as events\nthat can be listened to by the `Actor.on()` method." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/event_manager.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 52 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Initialize the event manager.\n\nWhen running this on the Apify Platform, this will start processing events\nsend by the platform to the events websocket and emitting them as events\nthat can be listened to by the `Actor.on()` method." + } + ] + }, + "flags": {}, + "id": 197, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "init", + "parameters": [], + "type": { + "name": "None", + "type": "literal", + "value": null + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Initialize the event manager.\n\nThis will stop listening for the platform events,\nand it will wait for all the event listeners to finish.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 198, + "module": "event_manager", + "name": "close", + "parsedDocstring": { + "text": "Initialize the event manager.\n\nThis will stop listening for the platform events,\nand it will wait for all the event listeners to finish.\n", + "args": { + "event_listeners_timeout_secs": "Optional timeout after which the pending event listeners are canceled." + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/event_manager.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 74 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Initialize the event manager.\n\nThis will stop listening for the platform events,\nand it will wait for all the event listeners to finish.\n" + } + ] + }, + "flags": {}, + "id": 199, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "close", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Optional timeout after which the pending event listeners are canceled." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": false + }, + "id": 200, + "kind": 32768, + "kindString": "Parameter", + "name": "event_listeners_timeout_secs", + "type": { + "name": "float | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "float" + }, + { + "type": "literal", + "value": null + } + ] + } + } + ], + "type": { + "name": "None", + "type": "literal", + "value": null + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Add an event listener to the event manager.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 201, + "module": "event_manager", + "name": "on", + "parsedDocstring": { + "text": "Add an event listener to the event manager.\n", + "args": { + "event_name": "The actor event for which to listen to.", + "listener": "The function which is to be called when the event is emitted (can be async).\nMust accept either zero or one arguments (the first argument will be the event data)." + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/event_manager.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 98 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Add an event listener to the event manager.\n" + } + ] + }, + "flags": {}, + "id": 202, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "on", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The actor event for which to listen to." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 203, + "kind": 32768, + "kindString": "Parameter", + "name": "event_name", + "type": { + "name": "ActorEventTypes", + "type": "reference" + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The function which is to be called when the event is emitted (can be async).\nMust accept either zero or one arguments (the first argument will be the event data)." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 204, + "kind": 32768, + "kindString": "Parameter", + "name": "listener", + "type": { + "name": "ListenerType", + "type": "reference", + "target": "192" + } + } + ], + "type": { + "name": "Callable", + "type": "reference" + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Remove a listener, or all listeners, from an actor event.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 205, + "module": "event_manager", + "name": "off", + "parsedDocstring": { + "text": "Remove a listener, or all listeners, from an actor event.\n", + "args": { + "event_name": "The actor event for which to remove listeners.", + "listener": "The listener which is supposed to be removed. If not passed, all listeners of this event are removed." + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/event_manager.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 158 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Remove a listener, or all listeners, from an actor event.\n" + } + ] + }, + "flags": {}, + "id": 206, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "off", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The actor event for which to remove listeners." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 207, + "kind": 32768, + "kindString": "Parameter", + "name": "event_name", + "type": { + "name": "ActorEventTypes", + "type": "reference" + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The listener which is supposed to be removed. If not passed, all listeners of this event are removed." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": false + }, + "id": 208, + "kind": 32768, + "kindString": "Parameter", + "name": "listener", + "type": { + "name": "Callable | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "Callable" + }, + { + "type": "literal", + "value": null + } + ] + } + } + ], + "type": { + "name": "None", + "type": "literal", + "value": null + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Emit an actor event manually.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 209, + "module": "event_manager", + "name": "emit", + "parsedDocstring": { + "text": "Emit an actor event manually.\n", + "args": { + "event_name": "The actor event which should be emitted.", + "data": "The data that should be emitted with the event." + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/event_manager.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 178 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Emit an actor event manually.\n" + } + ] + }, + "flags": {}, + "id": 210, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "emit", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The actor event which should be emitted." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 211, + "kind": 32768, + "kindString": "Parameter", + "name": "event_name", + "type": { + "name": "ActorEventTypes", + "type": "reference" + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The data that should be emitted with the event." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 212, + "kind": 32768, + "kindString": "Parameter", + "name": "data", + "type": { + "name": "Any", + "type": "reference" + } + } + ], + "type": { + "name": "None", + "type": "literal", + "value": null + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Wait for all event listeners which are currently being executed to complete.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 213, + "module": "event_manager", + "name": "wait_for_all_listeners_to_complete", + "parsedDocstring": { + "text": "Wait for all event listeners which are currently being executed to complete.\n", + "args": { + "timeout_secs": "Timeout for the wait. If the event listeners don't finish until the timeout, they will be canceled." + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/event_manager.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 189 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Wait for all event listeners which are currently being executed to complete.\n" + } + ] + }, + "flags": {}, + "id": 214, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "wait_for_all_listeners_to_complete", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Timeout for the wait. If the event listeners don't finish until the timeout, they will be canceled." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 215, + "kind": 32768, + "kindString": "Parameter", + "name": "timeout_secs", + "type": { + "name": "float | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "float" + }, + { + "type": "literal", + "value": null + } + ] + } + } + ], + "type": { + "name": "None", + "type": "literal", + "value": null + } + } + ] + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 216, + "module": "consts", + "name": "DEPRECATED_NAMES", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/consts.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 17 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 217, + "module": "consts", + "name": "__getattr__", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/consts.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 32 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "flags": {}, + "id": 218, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "__getattr__", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 219, + "kind": 32768, + "kindString": "Parameter", + "name": "name", + "type": { + "name": "str", + "type": "reference" + } + } + ], + "type": { + "name": "Any", + "type": "reference" + } + } + ] + }, + { + "kind": 8, + "kindString": "Enumeration", + "children": [ + { + "kind": 16, + "kindString": "Enumeration Member", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 221, + "module": "consts", + "name": "DATASET", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/consts.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 49 + } + ], + "type": { + "type": "literal", + "value": "'Dataset'" + } + }, + { + "kind": 16, + "kindString": "Enumeration Member", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 222, + "module": "consts", + "name": "KEY_VALUE_STORE", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/consts.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 50 + } + ], + "type": { + "type": "literal", + "value": "'Key-value store'" + } + }, + { + "kind": 16, + "kindString": "Enumeration Member", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 223, + "module": "consts", + "name": "REQUEST_QUEUE", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/consts.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 51 + } + ], + "type": { + "type": "literal", + "value": "'Request queue'" + } + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Possible Apify storage types." + } + ] + }, + "flags": {}, + "groups": [ + { + "children": [ + 221, + 222, + 223 + ], + "title": "Enumeration members" + } + ], + "id": 220, + "module": "consts", + "name": "StorageTypes", + "parsedDocstring": { + "text": "Possible Apify storage types." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/consts.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 46 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 224, + "module": "consts", + "name": "DEFAULT_API_PARAM_LIMIT", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/consts.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 54 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 225, + "module": "consts", + "name": "REQUEST_ID_LENGTH", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/consts.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 56 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 226, + "module": "consts", + "name": "REQUEST_QUEUE_HEAD_MAX_LIMIT", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/consts.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 58 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 227, + "module": "consts", + "name": "EVENT_LISTENERS_TIMEOUT_SECS", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/consts.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 60 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 228, + "module": "consts", + "name": "BASE64_REGEXP", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/consts.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 62 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 229, + "module": "consts", + "name": "ENCRYPTED_INPUT_VALUE_PREFIX", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/consts.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 63 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 230, + "module": "consts", + "name": "ENCRYPTED_INPUT_VALUE_REGEXP", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/consts.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 64 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 231, + "module": "consts", + "name": "MAX_PAYLOAD_SIZE_BYTES", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/consts.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 67 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 232, + "module": "log", + "name": "logger_name", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/log.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 16 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 233, + "module": "log", + "name": "logger", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/log.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 19 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 235, + "module": "log", + "name": "empty_record", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/log.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 57 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Create an instance of the ActorLogFormatter.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 236, + "module": "log", + "name": "__init__", + "parsedDocstring": { + "text": "Create an instance of the ActorLogFormatter.\n", + "args": { + "include_logger_name": "Include logger name at the beginning of the log line. Defaults to False.", + "args": "Arguments passed to the parent class.", + "kwargs": "Keyword arguments passed to the parent class." + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/log.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 59 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Create an instance of the ActorLogFormatter.\n" + } + ] + }, + "flags": {}, + "id": 237, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "__init__", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Include logger name at the beginning of the log line. Defaults to False." + } + ] + }, + "defaultValue": "False", + "flags": { + "isOptional": true, + "keyword-only": false + }, + "id": 238, + "kind": 32768, + "kindString": "Parameter", + "name": "include_logger_name", + "type": { + "name": "bool", + "type": "reference" + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Arguments passed to the parent class." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 239, + "kind": 32768, + "kindString": "Parameter", + "name": "args", + "type": { + "name": "Any", + "type": "reference" + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Keyword arguments passed to the parent class." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 240, + "kind": 32768, + "kindString": "Parameter", + "name": "kwargs", + "type": { + "name": "Any", + "type": "reference" + } + } + ], + "type": { + "name": "None", + "type": "literal", + "value": null + } + } + ] + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Log formatter that prints out the log message nicely formatted, with colored level and stringified extra fields.\n\nIt formats the log records so that they:\n- start with the level (colorized, and padded to 5 chars so that it is nicely aligned)\n- then have the actual log message, if it's multiline then it's nicely indented\n- then have the stringified extra log fields\n- then, if an exception is a part of the log record, prints the formatted exception." + } + ] + }, + "flags": {}, + "groups": [ + { + "children": [ + 236 + ], + "title": "Methods" + }, + { + "children": [ + 235 + ], + "title": "Properties" + } + ], + "id": 234, + "module": "log", + "name": "ActorLogFormatter", + "parsedDocstring": { + "text": "Log formatter that prints out the log message nicely formatted, with colored level and stringified extra fields.\n\nIt formats the log records so that they:\n- start with the level (colorized, and padded to 5 chars so that it is nicely aligned)\n- then have the actual log message, if it's multiline then it's nicely indented\n- then have the stringified extra log fields\n- then, if an exception is a part of the log record, prints the formatted exception." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/log.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 42 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 241, + "module": "__init__", + "name": "__version__", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/__init__.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 7 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 242, + "module": "proxy_configuration", + "name": "APIFY_PROXY_VALUE_REGEX", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/proxy_configuration.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 20 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 243, + "module": "proxy_configuration", + "name": "COUNTRY_CODE_REGEX", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/proxy_configuration.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 21 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 244, + "module": "proxy_configuration", + "name": "SESSION_ID_MAX_LENGTH", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/proxy_configuration.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 22 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Check if the given string is a valid URL." + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 245, + "module": "proxy_configuration", + "name": "is_url", + "parsedDocstring": { + "text": "Check if the given string is a valid URL." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/proxy_configuration.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 25 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Check if the given string is a valid URL." + } + ] + }, + "flags": {}, + "id": 246, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "is_url", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 247, + "kind": 32768, + "kindString": "Parameter", + "name": "url", + "type": { + "name": "str", + "type": "reference" + } + } + ], + "type": { + "name": "bool", + "type": "reference" + } + } + ] + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "The URL of the proxy." + } + ] + }, + "flags": {}, + "groups": [], + "id": 249, + "module": "proxy_configuration", + "name": "url", + "parsedDocstring": { + "text": "The URL of the proxy." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/proxy_configuration.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 68 + } + ], + "type": { + "name": "str", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "The hostname of the proxy." + } + ] + }, + "flags": {}, + "groups": [], + "id": 250, + "module": "proxy_configuration", + "name": "hostname", + "parsedDocstring": { + "text": "The hostname of the proxy." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/proxy_configuration.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 71 + } + ], + "type": { + "name": "str", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "The proxy port." + } + ] + }, + "flags": {}, + "groups": [], + "id": 251, + "module": "proxy_configuration", + "name": "port", + "parsedDocstring": { + "text": "The proxy port." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/proxy_configuration.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 74 + } + ], + "type": { + "name": "int", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "The username for the proxy." + } + ] + }, + "flags": {}, + "groups": [], + "id": 252, + "module": "proxy_configuration", + "name": "username", + "parsedDocstring": { + "text": "The username for the proxy." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/proxy_configuration.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 77 + } + ], + "type": { + "name": "NotRequired", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "str" + } + ] + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "The password for the proxy." + } + ] + }, + "flags": {}, + "groups": [], + "id": 253, + "module": "proxy_configuration", + "name": "password", + "parsedDocstring": { + "text": "The password for the proxy." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/proxy_configuration.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 80 + } + ], + "type": { + "name": "str", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "An array of proxy groups to be used by the [Apify Proxy](https://docs.apify.com/proxy).\nIf not provided, the proxy will select the groups automatically." + } + ] + }, + "flags": {}, + "groups": [], + "id": 254, + "module": "proxy_configuration", + "name": "groups", + "parsedDocstring": { + "text": "An array of proxy groups to be used by the [Apify Proxy](https://docs.apify.com/proxy).\nIf not provided, the proxy will select the groups automatically." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/proxy_configuration.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 83 + } + ], + "type": { + "name": "NotRequired", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "list", + "typeArguments": [ + { + "type": "reference", + "name": "str" + } + ], + "target": "474" + } + ] + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "If set and relevant proxies are available in your Apify account, all proxied requests will\nuse IP addresses that are geolocated to the specified country. For example `GB` for IPs\nfrom Great Britain. Note that online services often have their own rules for handling\ngeolocation and thus the country selection is a best attempt at geolocation, rather than\na guaranteed hit. This parameter is optional, by default, each proxied request is assigned\nan IP address from a random country. The country code needs to be a two letter ISO country code.\nSee the [full list of available country codes](https://en.wikipedia.org/wiki/ISO_3166-1_alpha-2#Officially_assigned_code_elements).\nThis parameter is optional, by default, the proxy uses all available proxy servers from all countries." + } + ] + }, + "flags": {}, + "groups": [], + "id": 255, + "module": "proxy_configuration", + "name": "country_code", + "parsedDocstring": { + "text": "If set and relevant proxies are available in your Apify account, all proxied requests will\nuse IP addresses that are geolocated to the specified country. For example `GB` for IPs\nfrom Great Britain. Note that online services often have their own rules for handling\ngeolocation and thus the country selection is a best attempt at geolocation, rather than\na guaranteed hit. This parameter is optional, by default, each proxied request is assigned\nan IP address from a random country. The country code needs to be a two letter ISO country code.\nSee the [full list of available country codes](https://en.wikipedia.org/wiki/ISO_3166-1_alpha-2#Officially_assigned_code_elements).\nThis parameter is optional, by default, the proxy uses all available proxy servers from all countries." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/proxy_configuration.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 88 + } + ], + "type": { + "name": "NotRequired", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "str" + } + ] + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "The identifier of the used proxy session, if used. Using the same session ID guarantees getting the same proxy URL." + } + ] + }, + "flags": {}, + "groups": [], + "id": 256, + "module": "proxy_configuration", + "name": "session_id", + "parsedDocstring": { + "text": "The identifier of the used proxy session, if used. Using the same session ID guarantees getting the same proxy URL." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/proxy_configuration.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 99 + } + ], + "type": { + "name": "NotRequired", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "str" + } + ] + } + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Provides information about a proxy connection that is used for requests." + } + ] + }, + "flags": {}, + "groups": [ + { + "children": [ + 255, + 254, + 250, + 253, + 251, + 256, + 249, + 252 + ], + "title": "Properties" + } + ], + "id": 248, + "module": "proxy_configuration", + "name": "ProxyInfo", + "parsedDocstring": { + "text": "Provides information about a proxy connection that is used for requests." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/proxy_configuration.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 65 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 258, + "module": "proxy_configuration", + "name": "is_man_in_the_middle", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/proxy_configuration.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 115 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Load the Apify Proxy password if the API token is provided and check access to Apify Proxy and provided proxy groups.\n\nOnly called if Apify Proxy configuration is used.\nAlso checks if country has access to Apify Proxy groups if the country code is provided.\n\nYou should use the Actor.create_proxy_configuration function\nto create a pre-initialized `ProxyConfiguration` instance instead of calling this manually." + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 259, + "module": "proxy_configuration", + "name": "initialize", + "parsedDocstring": { + "text": "Load the Apify Proxy password if the API token is provided and check access to Apify Proxy and provided proxy groups.\n\nOnly called if Apify Proxy configuration is used.\nAlso checks if country has access to Apify Proxy groups if the country code is provided.\n\nYou should use the Actor.create_proxy_configuration function\nto create a pre-initialized `ProxyConfiguration` instance instead of calling this manually." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/proxy_configuration.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 195 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Load the Apify Proxy password if the API token is provided and check access to Apify Proxy and provided proxy groups.\n\nOnly called if Apify Proxy configuration is used.\nAlso checks if country has access to Apify Proxy groups if the country code is provided.\n\nYou should use the Actor.create_proxy_configuration function\nto create a pre-initialized `ProxyConfiguration` instance instead of calling this manually." + } + ] + }, + "flags": {}, + "id": 260, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "initialize", + "parameters": [], + "type": { + "name": "None", + "type": "literal", + "value": null + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Return a new proxy URL based on provided configuration options and the `sessionId` parameter.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 261, + "module": "proxy_configuration", + "name": "new_url", + "parsedDocstring": { + "text": "Return a new proxy URL based on provided configuration options and the `sessionId` parameter.\n", + "args": { + "session_id": "Represents the identifier of a proxy session (https://docs.apify.com/proxy#sessions).\nAll the HTTP requests going through the proxy with the same session identifier\nwill use the same target proxy server (i.e. the same IP address).\nThe identifier must not be longer than 50 characters and include only the following: `0-9`, `a-z`, `A-Z`, `\".\"`, `\"_\"` and `\"~\"`.\n" + }, + "returns": "str: A string with a proxy URL, including authentication credentials and port number.\nFor example, `http://bob:password123@proxy.example.com:8000`" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/proxy_configuration.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 208 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "str: A string with a proxy URL, including authentication credentials and port number.\nFor example, `http://bob:password123@proxy.example.com:8000`" + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Return a new proxy URL based on provided configuration options and the `sessionId` parameter.\n" + } + ] + }, + "flags": {}, + "id": 262, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "new_url", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Represents the identifier of a proxy session (https://docs.apify.com/proxy#sessions).\nAll the HTTP requests going through the proxy with the same session identifier\nwill use the same target proxy server (i.e. the same IP address).\nThe identifier must not be longer than 50 characters and include only the following: `0-9`, `a-z`, `A-Z`, `\".\"`, `\"_\"` and `\"~\"`.\n" + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": false + }, + "id": 263, + "kind": 32768, + "kindString": "Parameter", + "name": "session_id", + "type": { + "name": "int | str | None", + "type": "union", + "types": [ + { + "type": "union", + "types": [ + { + "type": "reference", + "name": "int" + }, + { + "type": "reference", + "name": "str" + } + ] + }, + { + "type": "literal", + "value": null + } + ] + } + } + ], + "type": { + "name": "str", + "type": "reference" + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Create a new ProxyInfo object.\n\nUse it if you want to work with a rich representation of a proxy URL.\nIf you need the URL string only, use `ProxyConfiguration.new_url`.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 264, + "module": "proxy_configuration", + "name": "new_proxy_info", + "parsedDocstring": { + "text": "Create a new ProxyInfo object.\n\nUse it if you want to work with a rich representation of a proxy URL.\nIf you need the URL string only, use `ProxyConfiguration.new_url`.\n", + "args": { + "session_id": "Represents the identifier of a proxy session (https://docs.apify.com/proxy#sessions).\nAll the HTTP requests going through the proxy with the same session identifier\nwill use the same target proxy server (i.e. the same IP address).\nThe identifier must not be longer than 50 characters and include only the following: `0-9`, `a-z`, `A-Z`, `\".\"`, `\"_\"` and `\"~\"`.\n" + }, + "returns": "ProxyInfo: Dictionary that represents information about the proxy and its configuration." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/proxy_configuration.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 251 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "ProxyInfo: Dictionary that represents information about the proxy and its configuration." + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Create a new ProxyInfo object.\n\nUse it if you want to work with a rich representation of a proxy URL.\nIf you need the URL string only, use `ProxyConfiguration.new_url`.\n" + } + ] + }, + "flags": {}, + "id": 265, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "new_proxy_info", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Represents the identifier of a proxy session (https://docs.apify.com/proxy#sessions).\nAll the HTTP requests going through the proxy with the same session identifier\nwill use the same target proxy server (i.e. the same IP address).\nThe identifier must not be longer than 50 characters and include only the following: `0-9`, `a-z`, `A-Z`, `\".\"`, `\"_\"` and `\"~\"`.\n" + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": false + }, + "id": 266, + "kind": 32768, + "kindString": "Parameter", + "name": "session_id", + "type": { + "name": "int | str | None", + "type": "union", + "types": [ + { + "type": "union", + "types": [ + { + "type": "reference", + "name": "int" + }, + { + "type": "reference", + "name": "str" + } + ] + }, + { + "type": "literal", + "value": null + } + ] + } + } + ], + "type": { + "name": "ProxyInfo", + "type": "reference", + "target": "248" + } + } + ] + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Configures a connection to a proxy server with the provided options.\n\nProxy servers are used to prevent target websites from blocking your crawlers based on IP address rate limits or blacklists.\nThe default servers used by this class are managed by [Apify Proxy](https://docs.apify.com/proxy).\nTo be able to use Apify Proxy, you need an Apify account and access to the selected proxies. If you provide no configuration option,\nthe proxies will be managed automatically using a smart algorithm.\n\nIf you want to use your own proxies, use the `proxy_urls` or `new_url_function` constructor options.\nYour list of proxy URLs will be rotated by the configuration, if this option is provided." + } + ] + }, + "flags": {}, + "groups": [ + { + "children": [ + 259, + 264, + 261 + ], + "title": "Methods" + }, + { + "children": [ + 258 + ], + "title": "Properties" + } + ], + "id": 257, + "module": "proxy_configuration", + "name": "ProxyConfiguration", + "parsedDocstring": { + "text": "Configures a connection to a proxy server with the provided options.\n\nProxy servers are used to prevent target websites from blocking your crawlers based on IP address rate limits or blacklists.\nThe default servers used by this class are managed by [Apify Proxy](https://docs.apify.com/proxy).\nTo be able to use Apify Proxy, you need an Apify account and access to the selected proxies. If you provide no configuration option,\nthe proxies will be managed automatically using a smart algorithm.\n\nIf you want to use your own proxies, use the `proxy_urls` or `new_url_function` constructor options.\nYour list of proxy URLs will be rotated by the configuration, if this option is provided." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/proxy_configuration.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 103 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 267, + "module": "_utils", + "name": "T", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_utils.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 63 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 268, + "module": "_utils", + "name": "logger", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_utils.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 64 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 269, + "module": "_utils", + "name": "get_system_info", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_utils.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 67 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "flags": {}, + "id": 270, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "get_system_info", + "parameters": [], + "type": { + "name": "dict", + "type": "reference" + } + } + ] + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 271, + "module": "_utils", + "name": "DualPropertyType", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_utils.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 83 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 272, + "module": "_utils", + "name": "DualPropertyOwner", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_utils.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 84 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Initialize the dualproperty.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 273, + "module": "_utils", + "name": "__init__", + "parsedDocstring": { + "text": "Initialize the dualproperty.\n", + "args": { + "getter": "The getter of the property.\nIt should accept either an instance or a class as its first argument." + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_utils.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 95 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Initialize the dualproperty.\n" + } + ] + }, + "flags": {}, + "id": 274, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "__init__", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The getter of the property.\nIt should accept either an instance or a class as its first argument." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 275, + "kind": 32768, + "kindString": "Parameter", + "name": "getter", + "type": { + "name": "Callable", + "type": "reference", + "typeArguments": [ + { + "type": "literal", + "value": "..." + }, + { + "type": "reference", + "name": "DualPropertyType", + "target": "271" + } + ] + } + } + ], + "type": { + "name": "None", + "type": "literal", + "value": null + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Call the getter with the right object.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 276, + "module": "_utils", + "name": "__get__", + "parsedDocstring": { + "text": "Call the getter with the right object.\n", + "args": { + "obj": "The instance of class T on which the getter will be called", + "owner": "The class object of class T on which the getter will be called, if obj is None\n" + }, + "returns": "The result of the getter." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_utils.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 104 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "The result of the getter." + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Call the getter with the right object.\n" + } + ] + }, + "flags": {}, + "id": 277, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "__get__", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The instance of class T on which the getter will be called" + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 278, + "kind": 32768, + "kindString": "Parameter", + "name": "obj", + "type": { + "name": "DualPropertyOwner | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "DualPropertyOwner", + "target": "272" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The class object of class T on which the getter will be called, if obj is None\n" + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 279, + "kind": 32768, + "kindString": "Parameter", + "name": "owner", + "type": { + "name": "type", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "DualPropertyOwner", + "target": "272" + } + ] + } + } + ], + "type": { + "name": "DualPropertyType", + "type": "reference", + "target": "271" + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 280, + "module": "_utils", + "name": "fetch_and_parse_env_var", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_utils.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 173 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "flags": {}, + "id": 281, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "fetch_and_parse_env_var", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 282, + "kind": 32768, + "kindString": "Parameter", + "name": "env_var", + "type": { + "name": "Any", + "type": "reference" + } + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": false + }, + "id": 283, + "kind": 32768, + "kindString": "Parameter", + "name": "default", + "type": { + "name": "Any", + "type": "reference" + } + } + ], + "type": { + "name": "Any", + "type": "reference" + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 284, + "module": "_utils", + "name": "get_cpu_usage_percent", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_utils.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 197 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "flags": {}, + "id": 285, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "get_cpu_usage_percent", + "parameters": [], + "type": { + "name": "float", + "type": "reference" + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 286, + "module": "_utils", + "name": "get_memory_usage_bytes", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_utils.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 201 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "flags": {}, + "id": 287, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "get_memory_usage_bytes", + "parameters": [], + "type": { + "name": "int", + "type": "reference" + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 288, + "module": "_utils", + "name": "maybe_parse_bool", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_utils.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 210 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "flags": {}, + "id": 289, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "maybe_parse_bool", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 290, + "kind": 32768, + "kindString": "Parameter", + "name": "val", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + } + ], + "type": { + "name": "bool", + "type": "reference" + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 291, + "module": "_utils", + "name": "maybe_parse_datetime", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_utils.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 216 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "flags": {}, + "id": 292, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "maybe_parse_datetime", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 293, + "kind": 32768, + "kindString": "Parameter", + "name": "val", + "type": { + "name": "str", + "type": "reference" + } + } + ], + "type": { + "name": "datetime | str", + "type": "union", + "types": [ + { + "type": "reference", + "name": "datetime" + }, + { + "type": "reference", + "name": "str" + } + ] + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 294, + "module": "_utils", + "name": "maybe_parse_float", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_utils.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 223 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "flags": {}, + "id": 295, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "maybe_parse_float", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 296, + "kind": 32768, + "kindString": "Parameter", + "name": "val", + "type": { + "name": "str", + "type": "reference" + } + } + ], + "type": { + "name": "float | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "float" + }, + { + "type": "literal", + "value": null + } + ] + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 297, + "module": "_utils", + "name": "maybe_parse_int", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_utils.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 230 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "flags": {}, + "id": 298, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "maybe_parse_int", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 299, + "kind": 32768, + "kindString": "Parameter", + "name": "val", + "type": { + "name": "str", + "type": "reference" + } + } + ], + "type": { + "name": "int | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "int" + }, + { + "type": "literal", + "value": null + } + ] + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 300, + "module": "_utils", + "name": "run_func_at_interval_async", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_utils.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 237 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "flags": {}, + "id": 301, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "run_func_at_interval_async", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 302, + "kind": 32768, + "kindString": "Parameter", + "name": "func", + "type": { + "name": "Callable", + "type": "reference" + } + }, + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 303, + "kind": 32768, + "kindString": "Parameter", + "name": "interval_secs", + "type": { + "name": "float", + "type": "reference" + } + } + ], + "type": { + "name": "None", + "type": "literal", + "value": null + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "JS-like rm(filename, { force: true })." + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 304, + "module": "_utils", + "name": "force_remove", + "parsedDocstring": { + "text": "JS-like rm(filename, { force: true })." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_utils.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 254 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "JS-like rm(filename, { force: true })." + } + ] + }, + "flags": {}, + "id": 305, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "force_remove", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 306, + "kind": 32768, + "kindString": "Parameter", + "name": "filename", + "type": { + "name": "str", + "type": "reference" + } + } + ], + "type": { + "name": "None", + "type": "literal", + "value": null + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 307, + "module": "_utils", + "name": "raise_on_non_existing_storage", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_utils.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 260 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "flags": {}, + "id": 308, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "raise_on_non_existing_storage", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 309, + "kind": 32768, + "kindString": "Parameter", + "name": "client_type", + "type": { + "name": "StorageTypes", + "type": "reference", + "target": "220" + } + }, + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 310, + "kind": 32768, + "kindString": "Parameter", + "name": "id", + "type": { + "name": "str", + "type": "reference" + } + } + ], + "type": { + "name": "NoReturn", + "type": "reference" + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 311, + "module": "_utils", + "name": "raise_on_duplicate_storage", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_utils.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 265 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "flags": {}, + "id": 312, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "raise_on_duplicate_storage", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 313, + "kind": 32768, + "kindString": "Parameter", + "name": "client_type", + "type": { + "name": "StorageTypes", + "type": "reference", + "target": "220" + } + }, + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 314, + "kind": 32768, + "kindString": "Parameter", + "name": "key_name", + "type": { + "name": "str", + "type": "reference" + } + }, + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 315, + "kind": 32768, + "kindString": "Parameter", + "name": "value", + "type": { + "name": "str", + "type": "reference" + } + } + ], + "type": { + "name": "NoReturn", + "type": "reference" + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Guess the file extension based on content type." + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 316, + "module": "_utils", + "name": "guess_file_extension", + "parsedDocstring": { + "text": "Guess the file extension based on content type." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_utils.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 270 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Guess the file extension based on content type." + } + ] + }, + "flags": {}, + "id": 317, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "guess_file_extension", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 318, + "kind": 32768, + "kindString": "Parameter", + "name": "content_type", + "type": { + "name": "str", + "type": "reference" + } + } + ], + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 319, + "module": "_utils", + "name": "maybe_parse_body", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_utils.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 288 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "flags": {}, + "id": 320, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "maybe_parse_body", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 321, + "kind": 32768, + "kindString": "Parameter", + "name": "body", + "type": { + "name": "bytes", + "type": "reference" + } + }, + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 322, + "kind": 32768, + "kindString": "Parameter", + "name": "content_type", + "type": { + "name": "str", + "type": "reference" + } + } + ], + "type": { + "name": "Any", + "type": "reference" + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Generate request ID based on unique key in a deterministic way." + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 323, + "module": "_utils", + "name": "unique_key_to_request_id", + "parsedDocstring": { + "text": "Generate request ID based on unique key in a deterministic way." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_utils.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 296 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Generate request ID based on unique key in a deterministic way." + } + ] + }, + "flags": {}, + "id": 324, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "unique_key_to_request_id", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 325, + "kind": 32768, + "kindString": "Parameter", + "name": "unique_key", + "type": { + "name": "str", + "type": "reference" + } + } + ], + "type": { + "name": "str", + "type": "reference" + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Rename a directory. Checks for existence of soruce directory and removes destination directory if it exists." + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 326, + "module": "_utils", + "name": "force_rename", + "parsedDocstring": { + "text": "Rename a directory. Checks for existence of soruce directory and removes destination directory if it exists." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_utils.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 302 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Rename a directory. Checks for existence of soruce directory and removes destination directory if it exists." + } + ] + }, + "flags": {}, + "id": 327, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "force_rename", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 328, + "kind": 32768, + "kindString": "Parameter", + "name": "src_dir", + "type": { + "name": "str", + "type": "reference" + } + }, + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 329, + "kind": 32768, + "kindString": "Parameter", + "name": "dst_dir", + "type": { + "name": "str", + "type": "reference" + } + } + ], + "type": { + "name": "None", + "type": "literal", + "value": null + } + } + ] + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 330, + "module": "_utils", + "name": "ImplementationType", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_utils.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 312 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 331, + "module": "_utils", + "name": "MetadataType", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_utils.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 313 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 332, + "module": "_utils", + "name": "wrap_internal", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_utils.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 316 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "flags": {}, + "id": 333, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "wrap_internal", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 334, + "kind": 32768, + "kindString": "Parameter", + "name": "implementation", + "type": { + "name": "ImplementationType", + "type": "reference", + "target": "330" + } + }, + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 335, + "kind": 32768, + "kindString": "Parameter", + "name": "metadata_source", + "type": { + "name": "MetadataType", + "type": "reference", + "target": "331" + } + } + ], + "type": { + "name": "MetadataType", + "type": "reference", + "target": "331" + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Create a LRUCache with a specific max_length." + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 336, + "module": "_utils", + "name": "__init__", + "parsedDocstring": { + "text": "Create a LRUCache with a specific max_length." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_utils.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 332 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Create a LRUCache with a specific max_length." + } + ] + }, + "flags": {}, + "id": 337, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "__init__", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 338, + "kind": 32768, + "kindString": "Parameter", + "name": "max_length", + "type": { + "name": "int", + "type": "reference" + } + } + ], + "type": { + "name": "None", + "type": "literal", + "value": null + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Get an item from the cache. Move it to the end if present." + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 339, + "module": "_utils", + "name": "__getitem__", + "parsedDocstring": { + "text": "Get an item from the cache. Move it to the end if present." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_utils.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 337 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Get an item from the cache. Move it to the end if present." + } + ] + }, + "flags": {}, + "id": 340, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "__getitem__", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 341, + "kind": 32768, + "kindString": "Parameter", + "name": "key", + "type": { + "name": "str", + "type": "reference" + } + } + ], + "type": { + "name": "T", + "type": "reference", + "target": "26" + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Add an item to the cache. Remove least used item if max_length exceeded." + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 342, + "module": "_utils", + "name": "__setitem__", + "parsedDocstring": { + "text": "Add an item to the cache. Remove least used item if max_length exceeded." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_utils.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 345 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Add an item to the cache. Remove least used item if max_length exceeded." + } + ] + }, + "flags": {}, + "id": 343, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "__setitem__", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 344, + "kind": 32768, + "kindString": "Parameter", + "name": "key", + "type": { + "name": "str", + "type": "reference" + } + }, + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 345, + "kind": 32768, + "kindString": "Parameter", + "name": "value", + "type": { + "name": "T", + "type": "reference", + "target": "26" + } + } + ], + "type": { + "name": "None", + "type": "literal", + "value": null + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Remove an item from the cache." + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 346, + "module": "_utils", + "name": "__delitem__", + "parsedDocstring": { + "text": "Remove an item from the cache." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_utils.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 351 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Remove an item from the cache." + } + ] + }, + "flags": {}, + "id": 347, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "__delitem__", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 348, + "kind": 32768, + "kindString": "Parameter", + "name": "key", + "type": { + "name": "str", + "type": "reference" + } + } + ], + "type": { + "name": "None", + "type": "literal", + "value": null + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Iterate over the keys of the cache in order of insertion." + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 349, + "module": "_utils", + "name": "__iter__", + "parsedDocstring": { + "text": "Iterate over the keys of the cache in order of insertion." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_utils.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 355 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Iterate over the keys of the cache in order of insertion." + } + ] + }, + "flags": {}, + "id": 350, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "__iter__", + "parameters": [], + "type": { + "name": "Iterator", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "str" + } + ] + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Get the number of items in the cache." + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 351, + "module": "_utils", + "name": "__len__", + "parsedDocstring": { + "text": "Get the number of items in the cache." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_utils.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 359 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Get the number of items in the cache." + } + ] + }, + "flags": {}, + "id": 352, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "__len__", + "parameters": [], + "type": { + "name": "int", + "type": "reference" + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Iterate over the values in the cache in order of insertion." + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 353, + "module": "_utils", + "name": "values", + "parsedDocstring": { + "text": "Iterate over the values in the cache in order of insertion." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_utils.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 363 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Iterate over the values in the cache in order of insertion." + } + ] + }, + "flags": {}, + "id": 354, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "values", + "parameters": [], + "type": { + "name": "ValuesView", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "T", + "target": "26" + } + ] + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Iterate over the pairs of (key, value) in the cache in order of insertion." + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 355, + "module": "_utils", + "name": "items", + "parsedDocstring": { + "text": "Iterate over the pairs of (key, value) in the cache in order of insertion." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_utils.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 367 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Iterate over the pairs of (key, value) in the cache in order of insertion." + } + ] + }, + "flags": {}, + "id": 356, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "items", + "parameters": [], + "type": { + "name": "ItemsView", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "reference", + "name": "T", + "target": "26" + } + ] + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 357, + "module": "_utils", + "name": "is_running_in_ipython", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_utils.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 372 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "flags": {}, + "id": 358, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "is_running_in_ipython", + "parameters": [], + "type": { + "name": "bool", + "type": "reference" + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Budget version of ow." + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 359, + "module": "_utils", + "name": "budget_ow", + "parsedDocstring": { + "text": "Budget version of ow." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_utils.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 386 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Budget version of ow." + } + ] + }, + "flags": {}, + "id": 360, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "budget_ow", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 361, + "kind": 32768, + "kindString": "Parameter", + "name": "value", + "type": { + "name": "dict | str | float | bool", + "type": "union", + "types": [ + { + "type": "union", + "types": [ + { + "type": "union", + "types": [ + { + "type": "reference", + "name": "dict" + }, + { + "type": "reference", + "name": "str" + } + ] + }, + { + "type": "reference", + "name": "float" + } + ] + }, + { + "type": "reference", + "name": "bool" + } + ] + } + }, + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 362, + "kind": 32768, + "kindString": "Parameter", + "name": "predicate", + "type": { + "name": "dict[str, tuple[type, bool]] | tuple[type, bool]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "dict", + "typeArguments": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "reference", + "name": "tuple", + "typeArguments": [ + { + "type": "reference", + "name": "type" + }, + { + "type": "reference", + "name": "bool" + } + ] + } + ] + }, + { + "type": "reference", + "name": "tuple", + "typeArguments": [ + { + "type": "reference", + "name": "type" + }, + { + "type": "reference", + "name": "bool" + } + ] + } + ] + } + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": false + }, + "id": 363, + "kind": 32768, + "kindString": "Parameter", + "name": "value_name", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + } + ], + "type": { + "name": "None", + "type": "literal", + "value": null + } + } + ] + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 364, + "module": "_utils", + "name": "PARSE_DATE_FIELDS_MAX_DEPTH", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_utils.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 412 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 365, + "module": "_utils", + "name": "PARSE_DATE_FIELDS_KEY_SUFFIX", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_utils.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 413 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 366, + "module": "_utils", + "name": "ListOrDictOrAny", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_utils.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 414 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Computes a hexadecimal SHA-256 hash of the provided data and returns a substring (prefix) of it.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 367, + "module": "_utils", + "name": "compute_short_hash", + "parsedDocstring": { + "text": "Computes a hexadecimal SHA-256 hash of the provided data and returns a substring (prefix) of it.\n", + "args": { + "data": "The binary data to be hashed.", + "length": "The length of the hash to be returned.\n" + }, + "returns": "A substring (prefix) of the hexadecimal hash of the data." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_utils.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 417 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "A substring (prefix) of the hexadecimal hash of the data." + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Computes a hexadecimal SHA-256 hash of the provided data and returns a substring (prefix) of it.\n" + } + ] + }, + "flags": {}, + "id": 368, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "compute_short_hash", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The binary data to be hashed." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 369, + "kind": 32768, + "kindString": "Parameter", + "name": "data", + "type": { + "name": "bytes", + "type": "reference" + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The length of the hash to be returned.\n" + } + ] + }, + "defaultValue": "8", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 370, + "kind": 32768, + "kindString": "Parameter", + "name": "length", + "type": { + "name": "int", + "type": "reference" + } + } + ], + "type": { + "name": "str", + "type": "reference" + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Normalizes a URL.\n\nThis function cleans and standardizes a URL by removing leading and trailing whitespaces,\nconverting the scheme and netloc to lower case, stripping unwanted tracking parameters\n(specifically those beginning with 'utm_'), sorting the remaining query parameters alphabetically,\nand optionally retaining the URL fragment. The goal is to ensure that URLs that are functionally\nidentical but differ in trivial ways (such as parameter order or casing) are treated as the same.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 371, + "module": "_utils", + "name": "normalize_url", + "parsedDocstring": { + "text": "Normalizes a URL.\n\nThis function cleans and standardizes a URL by removing leading and trailing whitespaces,\nconverting the scheme and netloc to lower case, stripping unwanted tracking parameters\n(specifically those beginning with 'utm_'), sorting the remaining query parameters alphabetically,\nand optionally retaining the URL fragment. The goal is to ensure that URLs that are functionally\nidentical but differ in trivial ways (such as parameter order or casing) are treated as the same.\n", + "args": { + "url": "The URL to be normalized.", + "keep_url_fragment": "Flag to determine whether the fragment part of the URL should be retained.\n" + }, + "returns": "A string containing the normalized URL." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_utils.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 431 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "A string containing the normalized URL." + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Normalizes a URL.\n\nThis function cleans and standardizes a URL by removing leading and trailing whitespaces,\nconverting the scheme and netloc to lower case, stripping unwanted tracking parameters\n(specifically those beginning with 'utm_'), sorting the remaining query parameters alphabetically,\nand optionally retaining the URL fragment. The goal is to ensure that URLs that are functionally\nidentical but differ in trivial ways (such as parameter order or casing) are treated as the same.\n" + } + ] + }, + "flags": {}, + "id": 372, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "normalize_url", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The URL to be normalized." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 373, + "kind": 32768, + "kindString": "Parameter", + "name": "url", + "type": { + "name": "str", + "type": "reference" + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Flag to determine whether the fragment part of the URL should be retained.\n" + } + ] + }, + "defaultValue": "False", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 374, + "kind": 32768, + "kindString": "Parameter", + "name": "keep_url_fragment", + "type": { + "name": "bool", + "type": "reference" + } + } + ], + "type": { + "name": "str", + "type": "reference" + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Computes a unique key for caching & deduplication of requests.\n\nThis function computes a unique key by normalizing the provided URL and method.\nIf 'use_extended_unique_key' is True and a payload is provided, the payload is hashed and\nincluded in the key. Otherwise, the unique key is just the normalized URL.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 375, + "module": "_utils", + "name": "compute_unique_key", + "parsedDocstring": { + "text": "Computes a unique key for caching & deduplication of requests.\n\nThis function computes a unique key by normalizing the provided URL and method.\nIf 'use_extended_unique_key' is True and a payload is provided, the payload is hashed and\nincluded in the key. Otherwise, the unique key is just the normalized URL.\n", + "args": { + "url": "The request URL.", + "method": "The HTTP method, defaults to 'GET'.", + "payload": "The request payload, defaults to None.", + "keep_url_fragment": "A flag indicating whether to keep the URL fragment, defaults to False.", + "use_extended_unique_key": "A flag indicating whether to include a hashed payload in the key, defaults to False.\n" + }, + "returns": "A string representing the unique key for the request." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_utils.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 477 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "A string representing the unique key for the request." + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Computes a unique key for caching & deduplication of requests.\n\nThis function computes a unique key by normalizing the provided URL and method.\nIf 'use_extended_unique_key' is True and a payload is provided, the payload is hashed and\nincluded in the key. Otherwise, the unique key is just the normalized URL.\n" + } + ] + }, + "flags": {}, + "id": 376, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "compute_unique_key", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The request URL." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 377, + "kind": 32768, + "kindString": "Parameter", + "name": "url", + "type": { + "name": "str", + "type": "reference" + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The HTTP method, defaults to 'GET'." + } + ] + }, + "defaultValue": "'GET'", + "flags": { + "isOptional": true, + "keyword-only": false + }, + "id": 378, + "kind": 32768, + "kindString": "Parameter", + "name": "method", + "type": { + "name": "str", + "type": "reference" + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The request payload, defaults to None." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": false + }, + "id": 379, + "kind": 32768, + "kindString": "Parameter", + "name": "payload", + "type": { + "name": "bytes | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "bytes" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "A flag indicating whether to keep the URL fragment, defaults to False." + } + ] + }, + "defaultValue": "False", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 380, + "kind": 32768, + "kindString": "Parameter", + "name": "keep_url_fragment", + "type": { + "name": "bool", + "type": "reference" + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "A flag indicating whether to include a hashed payload in the key, defaults to False.\n" + } + ] + }, + "defaultValue": "False", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 381, + "kind": 32768, + "kindString": "Parameter", + "name": "use_extended_unique_key", + "type": { + "name": "bool", + "type": "reference" + } + } + ], + "type": { + "name": "str", + "type": "reference" + } + } + ] + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 382, + "module": "scrapy.utils", + "name": "nested_event_loop", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/scrapy/utils.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 19 + } + ], + "type": { + "name": "asyncio.AbstractEventLoop", + "type": "reference" + } + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Generate a basic authentication header for the given username and password." + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 383, + "module": "scrapy.utils", + "name": "get_basic_auth_header", + "parsedDocstring": { + "text": "Generate a basic authentication header for the given username and password." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/scrapy/utils.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 22 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Generate a basic authentication header for the given username and password." + } + ] + }, + "flags": {}, + "id": 384, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "get_basic_auth_header", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 385, + "kind": 32768, + "kindString": "Parameter", + "name": "username", + "type": { + "name": "str", + "type": "reference" + } + }, + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 386, + "kind": 32768, + "kindString": "Parameter", + "name": "password", + "type": { + "name": "str", + "type": "reference" + } + }, + { + "defaultValue": "'latin-1'", + "flags": { + "isOptional": true, + "keyword-only": false + }, + "id": 387, + "kind": 32768, + "kindString": "Parameter", + "name": "auth_encoding", + "type": { + "name": "str", + "type": "reference" + } + } + ], + "type": { + "name": "bytes", + "type": "reference" + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Get the ID of the currently running event loop.\n\nIt could be useful mainly for debugging purposes.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 388, + "module": "scrapy.utils", + "name": "get_running_event_loop_id", + "parsedDocstring": { + "text": "Get the ID of the currently running event loop.\n\nIt could be useful mainly for debugging purposes.\n", + "returns": "The ID of the event loop." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/scrapy/utils.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 29 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "The ID of the event loop." + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Get the ID of the currently running event loop.\n\nIt could be useful mainly for debugging purposes.\n" + } + ] + }, + "flags": {}, + "id": 389, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "get_running_event_loop_id", + "parameters": [], + "type": { + "name": "int", + "type": "reference" + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Integrates Apify configuration into a Scrapy project settings.\n\nNote: The function directly modifies the passed `settings` object and also returns it.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 390, + "module": "scrapy.utils", + "name": "apply_apify_settings", + "parsedDocstring": { + "text": "Integrates Apify configuration into a Scrapy project settings.\n\nNote: The function directly modifies the passed `settings` object and also returns it.\n", + "args": { + "settings": "Scrapy project settings to be modified.", + "proxy_config": "Proxy configuration to be stored in the settings.\n" + }, + "returns": "Scrapy project settings with custom configurations." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/scrapy/utils.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 40 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "Scrapy project settings with custom configurations." + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Integrates Apify configuration into a Scrapy project settings.\n\nNote: The function directly modifies the passed `settings` object and also returns it.\n" + } + ] + }, + "flags": {}, + "id": 391, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "apply_apify_settings", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Scrapy project settings to be modified." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 392, + "kind": 32768, + "kindString": "Parameter", + "name": "settings", + "type": { + "name": "Settings | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "Settings" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Proxy configuration to be stored in the settings.\n" + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 393, + "kind": 32768, + "kindString": "Parameter", + "name": "proxy_config", + "type": { + "name": "dict | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "dict" + }, + { + "type": "literal", + "value": null + } + ] + } + } + ], + "type": { + "name": "Settings", + "type": "reference" + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Open a Request Queue with custom Apify Client.\n\nTODO: add support for custom client to Actor.open_request_queue(), so that\nwe don't have to do this hacky workaround" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 394, + "module": "scrapy.utils", + "name": "open_queue_with_custom_client", + "parsedDocstring": { + "text": "Open a Request Queue with custom Apify Client.\n\nTODO: add support for custom client to Actor.open_request_queue(), so that\nwe don't have to do this hacky workaround" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/scrapy/utils.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 76 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Open a Request Queue with custom Apify Client.\n\nTODO: add support for custom client to Actor.open_request_queue(), so that\nwe don't have to do this hacky workaround" + } + ] + }, + "flags": {}, + "id": 395, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "open_queue_with_custom_client", + "parameters": [], + "type": { + "name": "RequestQueue", + "type": "reference", + "target": "764" + } + } + ] + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Create a new instance." + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 397, + "module": "scrapy.scheduler", + "name": "__init__", + "parsedDocstring": { + "text": "Create a new instance." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/scrapy/scheduler.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 28 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Create a new instance." + } + ] + }, + "flags": {}, + "id": 398, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "__init__", + "parameters": [], + "type": { + "name": "None", + "type": "literal", + "value": null + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Open the scheduler.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 399, + "module": "scrapy.scheduler", + "name": "open", + "parsedDocstring": { + "text": "Open the scheduler.\n", + "args": { + "spider": "The spider that the scheduler is associated with." + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/scrapy/scheduler.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 39 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Open the scheduler.\n" + } + ] + }, + "flags": {}, + "id": 400, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "open", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The spider that the scheduler is associated with." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 401, + "kind": 32768, + "kindString": "Parameter", + "name": "spider", + "type": { + "name": "Spider", + "type": "reference" + } + } + ], + "type": { + "name": "None", + "type": "literal", + "value": null + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Check if the scheduler has any pending requests.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 402, + "module": "scrapy.scheduler", + "name": "has_pending_requests", + "parsedDocstring": { + "text": "Check if the scheduler has any pending requests.\n", + "returns": "True if the scheduler has any pending requests, False otherwise." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/scrapy/scheduler.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 53 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "True if the scheduler has any pending requests, False otherwise." + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Check if the scheduler has any pending requests.\n" + } + ] + }, + "flags": {}, + "id": 403, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "has_pending_requests", + "parameters": [], + "type": { + "name": "bool", + "type": "reference" + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Add a request to the scheduler.\n\nThis could be called from either from a spider or a downloader middleware (e.g. redirect, retry, ...).\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 404, + "module": "scrapy.scheduler", + "name": "enqueue_request", + "parsedDocstring": { + "text": "Add a request to the scheduler.\n\nThis could be called from either from a spider or a downloader middleware (e.g. redirect, retry, ...).\n", + "args": { + "request": "The request to add to the scheduler.\n" + }, + "returns": "True if the request was successfully enqueued, False otherwise." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/scrapy/scheduler.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 70 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "True if the request was successfully enqueued, False otherwise." + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Add a request to the scheduler.\n\nThis could be called from either from a spider or a downloader middleware (e.g. redirect, retry, ...).\n" + } + ] + }, + "flags": {}, + "id": 405, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "enqueue_request", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The request to add to the scheduler.\n" + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 406, + "kind": 32768, + "kindString": "Parameter", + "name": "request", + "type": { + "name": "Request", + "type": "reference" + } + } + ], + "type": { + "name": "bool", + "type": "reference" + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Fetch the next request from the scheduler.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 407, + "module": "scrapy.scheduler", + "name": "next_request", + "parsedDocstring": { + "text": "Fetch the next request from the scheduler.\n", + "returns": "The next request, or None if there are no more requests." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/scrapy/scheduler.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 111 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "The next request, or None if there are no more requests." + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Fetch the next request from the scheduler.\n" + } + ] + }, + "flags": {}, + "id": 408, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "next_request", + "parameters": [], + "type": { + "name": "Request | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "Request" + }, + { + "type": "literal", + "value": null + } + ] + } + } + ] + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "A Scrapy scheduler that uses the Apify Request Queue to manage requests.\n\nThis scheduler requires the asyncio Twisted reactor to be installed." + } + ] + }, + "flags": {}, + "groups": [ + { + "children": [ + 397, + 404, + 402, + 407, + 399 + ], + "title": "Methods" + } + ], + "id": 396, + "module": "scrapy.scheduler", + "name": "ApifyScheduler", + "parsedDocstring": { + "text": "A Scrapy scheduler that uses the Apify Request Queue to manage requests.\n\nThis scheduler requires the asyncio Twisted reactor to be installed." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/scrapy/scheduler.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 22 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Create a new instance.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 410, + "module": "scrapy.middlewares.apify_proxy", + "name": "__init__", + "parsedDocstring": { + "text": "Create a new instance.\n", + "args": { + "proxy_settings": "Dictionary containing proxy settings, provided by the Actor input.", + "auth_encoding": "Encoding for basic authentication (default is 'latin-1')." + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/scrapy/middlewares/apify_proxy.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 31 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Create a new instance.\n" + } + ] + }, + "flags": {}, + "id": 411, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "__init__", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Dictionary containing proxy settings, provided by the Actor input." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 412, + "kind": 32768, + "kindString": "Parameter", + "name": "proxy_settings", + "type": { + "name": "dict", + "type": "reference" + } + } + ], + "type": { + "name": "None", + "type": "literal", + "value": null + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Create an instance of ApifyHttpProxyMiddleware from a Scrapy Crawler.\n" + } + ] + }, + "decorations": [ + { + "name": "classmethod" + } + ], + "flags": {}, + "groups": [], + "id": 413, + "module": "scrapy.middlewares.apify_proxy", + "name": "from_crawler", + "parsedDocstring": { + "text": "Create an instance of ApifyHttpProxyMiddleware from a Scrapy Crawler.\n", + "args": { + "cls": "Class type.", + "crawler": "Scrapy Crawler object.\n" + }, + "returns": "ApifyHttpProxyMiddleware: Instance of the class." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/scrapy/middlewares/apify_proxy.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 42 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "ApifyHttpProxyMiddleware: Instance of the class." + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Create an instance of ApifyHttpProxyMiddleware from a Scrapy Crawler.\n" + } + ] + }, + "flags": {}, + "id": 414, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "from_crawler", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Scrapy Crawler object.\n" + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 415, + "kind": 32768, + "kindString": "Parameter", + "name": "crawler", + "type": { + "name": "Crawler", + "type": "reference" + } + } + ], + "type": { + "name": "ApifyHttpProxyMiddleware", + "type": "reference", + "target": "409" + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Process a Scrapy request by assigning a new proxy.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 416, + "module": "scrapy.middlewares.apify_proxy", + "name": "process_request", + "parsedDocstring": { + "text": "Process a Scrapy request by assigning a new proxy.\n", + "args": { + "request": "Scrapy Request object.", + "spider": "Scrapy Spider object.\n" + }, + "returns": "None: The request is processed and middleware pipeline can continue." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/scrapy/middlewares/apify_proxy.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 68 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "None: The request is processed and middleware pipeline can continue." + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Process a Scrapy request by assigning a new proxy.\n" + } + ] + }, + "flags": {}, + "id": 417, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "process_request", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Scrapy Request object." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 418, + "kind": 32768, + "kindString": "Parameter", + "name": "request", + "type": { + "name": "Request", + "type": "reference" + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Scrapy Spider object.\n" + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 419, + "kind": 32768, + "kindString": "Parameter", + "name": "spider", + "type": { + "name": "Spider", + "type": "reference" + } + } + ], + "type": { + "name": "None", + "type": "literal", + "value": null + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Process an exception that occurs during request processing.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 420, + "module": "scrapy.middlewares.apify_proxy", + "name": "process_exception", + "parsedDocstring": { + "text": "Process an exception that occurs during request processing.\n", + "args": { + "request": "Scrapy Request object.", + "exception": "Exception object.", + "spider": "Scrapy Spider object.\n" + }, + "returns": "If a TunnelError occurs, return the request object to halt its processing in the middleware pipeline.\nReturn None otherwise to allow the continuation of request processing." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/scrapy/middlewares/apify_proxy.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 93 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "If a TunnelError occurs, return the request object to halt its processing in the middleware pipeline.\nReturn None otherwise to allow the continuation of request processing." + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Process an exception that occurs during request processing.\n" + } + ] + }, + "flags": {}, + "id": 421, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "process_exception", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Scrapy Request object." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 422, + "kind": 32768, + "kindString": "Parameter", + "name": "request", + "type": { + "name": "Request", + "type": "reference" + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Exception object." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 423, + "kind": 32768, + "kindString": "Parameter", + "name": "exception", + "type": { + "name": "Exception", + "type": "reference" + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Scrapy Spider object.\n" + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 424, + "kind": 32768, + "kindString": "Parameter", + "name": "spider", + "type": { + "name": "Spider", + "type": "reference" + } + } + ], + "type": { + "name": "None | Request", + "type": "union", + "types": [ + { + "type": "literal", + "value": null + }, + { + "type": "reference", + "name": "Request" + } + ] + } + } + ] + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Apify HTTP proxy middleware for Scrapy.\n\nThis middleware enhances request processing by adding a 'proxy' field to the request's meta and an authentication\nheader. It draws inspiration from the `HttpProxyMiddleware` included by default in Scrapy projects. The proxy URL\nis sourced from the settings under the `APIFY_PROXY_SETTINGS` key. The value of this key, a dictionary, should be\nprovided by the Actor input. An example of the proxy settings:\n\nproxy_settings = {'useApifyProxy': true, 'apifyProxyGroups': []}" + } + ] + }, + "flags": {}, + "groups": [ + { + "children": [ + 410, + 413, + 420, + 416 + ], + "title": "Methods" + } + ], + "id": 409, + "module": "scrapy.middlewares.apify_proxy", + "name": "ApifyHttpProxyMiddleware", + "parsedDocstring": { + "text": "Apify HTTP proxy middleware for Scrapy.\n\nThis middleware enhances request processing by adding a 'proxy' field to the request's meta and an authentication\nheader. It draws inspiration from the `HttpProxyMiddleware` included by default in Scrapy projects. The proxy URL\nis sourced from the settings under the `APIFY_PROXY_SETTINGS` key. The value of this key, a dictionary, should be\nprovided by the Actor input. An example of the proxy settings:\n\nproxy_settings = {'useApifyProxy': true, 'apifyProxyGroups': []}" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/scrapy/middlewares/apify_proxy.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 20 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Convert a Scrapy request to an Apify request.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 425, + "module": "scrapy.requests", + "name": "to_apify_request", + "parsedDocstring": { + "text": "Convert a Scrapy request to an Apify request.\n", + "args": { + "scrapy_request": "The Scrapy request to be converted.", + "spider": "The Scrapy spider that the request is associated with.\n" + }, + "returns": "The converted Apify request if the conversion was successful, otherwise None." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/scrapy/requests.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 28 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "The converted Apify request if the conversion was successful, otherwise None." + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Convert a Scrapy request to an Apify request.\n" + } + ] + }, + "flags": {}, + "id": 426, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "to_apify_request", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The Scrapy request to be converted." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 427, + "kind": 32768, + "kindString": "Parameter", + "name": "scrapy_request", + "type": { + "name": "Request", + "type": "reference" + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The Scrapy spider that the request is associated with.\n" + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 428, + "kind": 32768, + "kindString": "Parameter", + "name": "spider", + "type": { + "name": "Spider", + "type": "reference" + } + } + ], + "type": { + "name": "dict | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "dict" + }, + { + "type": "literal", + "value": null + } + ] + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Convert an Apify request to a Scrapy request.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 429, + "module": "scrapy.requests", + "name": "to_scrapy_request", + "parsedDocstring": { + "text": "Convert an Apify request to a Scrapy request.\n", + "args": { + "apify_request": "The Apify request to be converted.", + "spider": "The Scrapy spider that the request is associated with.\n" + }, + "returns": "The converted Scrapy request." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/scrapy/requests.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 96 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "The converted Scrapy request." + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Convert an Apify request to a Scrapy request.\n" + } + ] + }, + "flags": {}, + "id": 430, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "to_scrapy_request", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The Apify request to be converted." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 431, + "kind": 32768, + "kindString": "Parameter", + "name": "apify_request", + "type": { + "name": "dict", + "type": "reference" + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The Scrapy spider that the request is associated with.\n" + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 432, + "kind": 32768, + "kindString": "Parameter", + "name": "spider", + "type": { + "name": "Spider", + "type": "reference" + } + } + ], + "type": { + "name": "Request", + "type": "reference" + } + } + ] + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Pushes the provided Scrapy item to the Actor's default dataset." + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 434, + "module": "scrapy.pipelines.actor_dataset_push", + "name": "process_item", + "parsedDocstring": { + "text": "Pushes the provided Scrapy item to the Actor's default dataset." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/scrapy/pipelines/actor_dataset_push.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 21 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Pushes the provided Scrapy item to the Actor's default dataset." + } + ] + }, + "flags": {}, + "id": 435, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "process_item", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 436, + "kind": 32768, + "kindString": "Parameter", + "name": "item", + "type": { + "name": "Item", + "type": "reference" + } + }, + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 437, + "kind": 32768, + "kindString": "Parameter", + "name": "spider", + "type": { + "name": "Spider", + "type": "reference" + } + } + ], + "type": { + "name": "Item", + "type": "reference" + } + } + ] + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "A Scrapy pipeline for pushing items to an Actor's default dataset.\n\nThis pipeline is designed to be enabled only when the Scrapy project is run as an Actor." + } + ] + }, + "flags": {}, + "groups": [ + { + "children": [ + 434 + ], + "title": "Methods" + } + ], + "id": 433, + "module": "scrapy.pipelines.actor_dataset_push", + "name": "ActorDatasetPushPipeline", + "parsedDocstring": { + "text": "A Scrapy pipeline for pushing items to an Actor's default dataset.\n\nThis pipeline is designed to be enabled only when the Scrapy project is run as an Actor." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/scrapy/pipelines/actor_dataset_push.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 15 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 438, + "module": "_memory_storage.file_storage_utils", + "name": "update_metadata", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_memory_storage/file_storage_utils.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 12 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "flags": {}, + "id": 439, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "update_metadata", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 440, + "kind": 32768, + "kindString": "Parameter", + "name": "data", + "type": { + "name": "dict", + "type": "reference" + } + }, + { + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 441, + "kind": 32768, + "kindString": "Parameter", + "name": "entity_directory", + "type": { + "name": "str", + "type": "reference" + } + }, + { + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 442, + "kind": 32768, + "kindString": "Parameter", + "name": "write_metadata", + "type": { + "name": "bool", + "type": "reference" + } + } + ], + "type": { + "name": "None", + "type": "literal", + "value": null + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 443, + "module": "_memory_storage.file_storage_utils", + "name": "update_request_queue_item", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_memory_storage/file_storage_utils.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 46 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "flags": {}, + "id": 444, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "update_request_queue_item", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 445, + "kind": 32768, + "kindString": "Parameter", + "name": "request_id", + "type": { + "name": "str", + "type": "reference" + } + }, + { + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 446, + "kind": 32768, + "kindString": "Parameter", + "name": "request", + "type": { + "name": "dict", + "type": "reference" + } + }, + { + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 447, + "kind": 32768, + "kindString": "Parameter", + "name": "entity_directory", + "type": { + "name": "str", + "type": "reference" + } + }, + { + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 448, + "kind": 32768, + "kindString": "Parameter", + "name": "persist_storage", + "type": { + "name": "bool", + "type": "reference" + } + } + ], + "type": { + "name": "None", + "type": "literal", + "value": null + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 449, + "module": "_memory_storage.file_storage_utils", + "name": "delete_request", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_memory_storage/file_storage_utils.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 66 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "flags": {}, + "id": 450, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "delete_request", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 451, + "kind": 32768, + "kindString": "Parameter", + "name": "request_id", + "type": { + "name": "str", + "type": "reference" + } + }, + { + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 452, + "kind": 32768, + "kindString": "Parameter", + "name": "entity_directory", + "type": { + "name": "str", + "type": "reference" + } + } + ], + "type": { + "name": "None", + "type": "literal", + "value": null + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Initialize the MemoryStorageClient.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 453, + "module": "_memory_storage.memory_storage_client", + "name": "__init__", + "parsedDocstring": { + "text": "Initialize the MemoryStorageClient.\n", + "args": { + "local_data_directory": "A local directory where all data will be persisted", + "persist_storage": "Whether to persist the data to the `local_data_directory` or just keep them in memory", + "write_metadata": "Whether to persist metadata of the storages as well" + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_memory_storage/memory_storage_client.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 49 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Initialize the MemoryStorageClient.\n" + } + ] + }, + "flags": {}, + "id": 454, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "__init__", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "A local directory where all data will be persisted" + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 455, + "kind": 32768, + "kindString": "Parameter", + "name": "local_data_directory", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Whether to persist metadata of the storages as well" + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 456, + "kind": 32768, + "kindString": "Parameter", + "name": "write_metadata", + "type": { + "name": "bool | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "bool" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Whether to persist the data to the `local_data_directory` or just keep them in memory" + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 457, + "kind": 32768, + "kindString": "Parameter", + "name": "persist_storage", + "type": { + "name": "bool | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "bool" + }, + { + "type": "literal", + "value": null + } + ] + } + } + ], + "type": { + "name": "None", + "type": "literal", + "value": null + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Retrieve the sub-client for manipulating datasets." + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 458, + "module": "_memory_storage.memory_storage_client", + "name": "datasets", + "parsedDocstring": { + "text": "Retrieve the sub-client for manipulating datasets." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_memory_storage/memory_storage_client.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 74 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Retrieve the sub-client for manipulating datasets." + } + ] + }, + "flags": {}, + "id": 459, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "datasets", + "parameters": [], + "type": { + "name": "DatasetCollectionClient", + "type": "reference" + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Retrieve the sub-client for manipulating a single dataset.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 460, + "module": "_memory_storage.memory_storage_client", + "name": "dataset", + "parsedDocstring": { + "text": "Retrieve the sub-client for manipulating a single dataset.\n", + "args": { + "dataset_id": "ID of the dataset to be manipulated" + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_memory_storage/memory_storage_client.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 78 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Retrieve the sub-client for manipulating a single dataset.\n" + } + ] + }, + "flags": {}, + "id": 461, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "dataset", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "ID of the dataset to be manipulated" + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 462, + "kind": 32768, + "kindString": "Parameter", + "name": "dataset_id", + "type": { + "name": "str", + "type": "reference" + } + } + ], + "type": { + "name": "DatasetClient", + "type": "reference" + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Retrieve the sub-client for manipulating key-value stores." + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 463, + "module": "_memory_storage.memory_storage_client", + "name": "key_value_stores", + "parsedDocstring": { + "text": "Retrieve the sub-client for manipulating key-value stores." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_memory_storage/memory_storage_client.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 86 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Retrieve the sub-client for manipulating key-value stores." + } + ] + }, + "flags": {}, + "id": 464, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "key_value_stores", + "parameters": [], + "type": { + "name": "KeyValueStoreCollectionClient", + "type": "reference" + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Retrieve the sub-client for manipulating a single key-value store.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 465, + "module": "_memory_storage.memory_storage_client", + "name": "key_value_store", + "parsedDocstring": { + "text": "Retrieve the sub-client for manipulating a single key-value store.\n", + "args": { + "key_value_store_id": "ID of the key-value store to be manipulated" + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_memory_storage/memory_storage_client.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 90 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Retrieve the sub-client for manipulating a single key-value store.\n" + } + ] + }, + "flags": {}, + "id": 466, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "key_value_store", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "ID of the key-value store to be manipulated" + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 467, + "kind": 32768, + "kindString": "Parameter", + "name": "key_value_store_id", + "type": { + "name": "str", + "type": "reference" + } + } + ], + "type": { + "name": "KeyValueStoreClient", + "type": "reference" + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Retrieve the sub-client for manipulating request queues." + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 468, + "module": "_memory_storage.memory_storage_client", + "name": "request_queues", + "parsedDocstring": { + "text": "Retrieve the sub-client for manipulating request queues." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_memory_storage/memory_storage_client.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 98 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Retrieve the sub-client for manipulating request queues." + } + ] + }, + "flags": {}, + "id": 469, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "request_queues", + "parameters": [], + "type": { + "name": "RequestQueueCollectionClient", + "type": "reference" + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Retrieve the sub-client for manipulating a single request queue.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 470, + "module": "_memory_storage.memory_storage_client", + "name": "request_queue", + "parsedDocstring": { + "text": "Retrieve the sub-client for manipulating a single request queue.\n", + "args": { + "request_queue_id": "ID of the request queue to be manipulated", + "client_key": "A unique identifier of the client accessing the request queue" + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_memory_storage/memory_storage_client.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 102 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Retrieve the sub-client for manipulating a single request queue.\n" + } + ] + }, + "flags": {}, + "id": 471, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "request_queue", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "ID of the request queue to be manipulated" + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 472, + "kind": 32768, + "kindString": "Parameter", + "name": "request_queue_id", + "type": { + "name": "str", + "type": "reference" + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "A unique identifier of the client accessing the request queue" + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 473, + "kind": 32768, + "kindString": "Parameter", + "name": "client_key", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + } + ], + "type": { + "name": "RequestQueueClient", + "type": "reference" + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "List the available key-value stores.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 474, + "module": "_memory_storage.resource_clients.key_value_store_collection", + "name": "list", + "parsedDocstring": { + "text": "List the available key-value stores.\n", + "returns": "ListPage: The list of available key-value stores matching the specified filters." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_memory_storage/resource_clients/key_value_store_collection.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 24 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "ListPage: The list of available key-value stores matching the specified filters." + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "List the available key-value stores.\n" + } + ] + }, + "flags": {}, + "id": 475, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "list", + "parameters": [], + "type": { + "name": "ListPage", + "type": "reference" + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Retrieve a named key-value store, or create a new one when it doesn't exist.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 476, + "module": "_memory_storage.resource_clients.key_value_store_collection", + "name": "get_or_create", + "parsedDocstring": { + "text": "Retrieve a named key-value store, or create a new one when it doesn't exist.\n", + "args": { + "name": "The name of the key-value store to retrieve or create.", + "schema": "The schema of the key-value store\n" + }, + "returns": "dict: The retrieved or newly-created key-value store." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_memory_storage/resource_clients/key_value_store_collection.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 32 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "dict: The retrieved or newly-created key-value store." + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Retrieve a named key-value store, or create a new one when it doesn't exist.\n" + } + ] + }, + "flags": {}, + "id": 477, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "get_or_create", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The name of the key-value store to retrieve or create." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 478, + "kind": 32768, + "kindString": "Parameter", + "name": "name", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The schema of the key-value store\n" + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 479, + "kind": 32768, + "kindString": "Parameter", + "name": "schema", + "type": { + "name": "dict | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "dict" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 480, + "kind": 32768, + "kindString": "Parameter", + "name": "_id", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + } + ], + "type": { + "name": "dict", + "type": "reference" + } + } + ] + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "noqa: PLC0105" + } + ] + }, + "flags": {}, + "groups": [], + "id": 481, + "module": "_memory_storage.resource_clients.base_resource_collection_client", + "name": "ResourceClientType", + "parsedDocstring": { + "text": "noqa: PLC0105" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_memory_storage/resource_clients/base_resource_collection_client.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 17 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Initialize the DatasetCollectionClient with the passed arguments." + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 482, + "module": "_memory_storage.resource_clients.base_resource_collection_client", + "name": "__init__", + "parsedDocstring": { + "text": "Initialize the DatasetCollectionClient with the passed arguments." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_memory_storage/resource_clients/base_resource_collection_client.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 27 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Initialize the DatasetCollectionClient with the passed arguments." + } + ] + }, + "flags": {}, + "id": 483, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "__init__", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 484, + "kind": 32768, + "kindString": "Parameter", + "name": "base_storage_directory", + "type": { + "name": "str", + "type": "reference" + } + }, + { + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 485, + "kind": 32768, + "kindString": "Parameter", + "name": "memory_storage_client", + "type": { + "name": "MemoryStorageClient", + "type": "reference" + } + } + ], + "type": { + "name": "None", + "type": "literal", + "value": null + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "List the available storages.\n" + } + ] + }, + "decorations": [ + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 486, + "module": "_memory_storage.resource_clients.base_resource_collection_client", + "name": "list", + "parsedDocstring": { + "text": "List the available storages.\n", + "returns": "ListPage: The list of available storages matching the specified filters." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_memory_storage/resource_clients/base_resource_collection_client.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 46 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "ListPage: The list of available storages matching the specified filters." + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "List the available storages.\n" + } + ] + }, + "flags": {}, + "id": 487, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "list", + "parameters": [], + "type": { + "name": "ListPage", + "type": "reference" + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Retrieve a named storage, or create a new one when it doesn't exist.\n" + } + ] + }, + "decorations": [ + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 488, + "module": "_memory_storage.resource_clients.base_resource_collection_client", + "name": "get_or_create", + "parsedDocstring": { + "text": "Retrieve a named storage, or create a new one when it doesn't exist.\n", + "args": { + "name": "The name of the storage to retrieve or create.", + "schema": "The schema of the storage\n" + }, + "returns": "dict: The retrieved or newly-created storage." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_memory_storage/resource_clients/base_resource_collection_client.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 68 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "dict: The retrieved or newly-created storage." + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Retrieve a named storage, or create a new one when it doesn't exist.\n" + } + ] + }, + "flags": {}, + "id": 489, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "get_or_create", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The name of the storage to retrieve or create." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 490, + "kind": 32768, + "kindString": "Parameter", + "name": "name", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The schema of the storage\n" + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 491, + "kind": 32768, + "kindString": "Parameter", + "name": "schema", + "type": { + "name": "dict | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "dict" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 492, + "kind": 32768, + "kindString": "Parameter", + "name": "_id", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + } + ], + "type": { + "name": "dict", + "type": "reference" + } + } + ] + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 493, + "module": "_memory_storage.resource_clients.dataset", + "name": "LIST_ITEMS_LIMIT", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_memory_storage/resource_clients/dataset.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 26 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 494, + "module": "_memory_storage.resource_clients.dataset", + "name": "LOCAL_ENTRY_NAME_DIGITS", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_memory_storage/resource_clients/dataset.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 30 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Initialize the DatasetClient." + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 495, + "module": "_memory_storage.resource_clients.dataset", + "name": "__init__", + "parsedDocstring": { + "text": "Initialize the DatasetClient." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_memory_storage/resource_clients/dataset.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 48 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Initialize the DatasetClient." + } + ] + }, + "flags": {}, + "id": 496, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "__init__", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 497, + "kind": 32768, + "kindString": "Parameter", + "name": "base_storage_directory", + "type": { + "name": "str", + "type": "reference" + } + }, + { + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 498, + "kind": 32768, + "kindString": "Parameter", + "name": "memory_storage_client", + "type": { + "name": "MemoryStorageClient", + "type": "reference" + } + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 499, + "kind": 32768, + "kindString": "Parameter", + "name": "id", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 500, + "kind": 32768, + "kindString": "Parameter", + "name": "name", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + } + ], + "type": { + "name": "None", + "type": "literal", + "value": null + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Retrieve the dataset.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 501, + "module": "_memory_storage.resource_clients.dataset", + "name": "get", + "parsedDocstring": { + "text": "Retrieve the dataset.\n", + "returns": "dict, optional: The retrieved dataset, or None, if it does not exist" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_memory_storage/resource_clients/dataset.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 67 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "dict, optional: The retrieved dataset, or None, if it does not exist" + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Retrieve the dataset.\n" + } + ] + }, + "flags": {}, + "id": 502, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "get", + "parameters": [], + "type": { + "name": "dict | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "dict" + }, + { + "type": "literal", + "value": null + } + ] + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Update the dataset with specified fields.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 503, + "module": "_memory_storage.resource_clients.dataset", + "name": "update", + "parsedDocstring": { + "text": "Update the dataset with specified fields.\n", + "args": { + "name": "The new name for the dataset\n" + }, + "returns": "dict: The updated dataset" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_memory_storage/resource_clients/dataset.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 82 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "dict: The updated dataset" + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Update the dataset with specified fields.\n" + } + ] + }, + "flags": {}, + "id": 504, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "update", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The new name for the dataset\n" + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 505, + "kind": 32768, + "kindString": "Parameter", + "name": "name", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + } + ], + "type": { + "name": "dict", + "type": "reference" + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Delete the dataset." + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 506, + "module": "_memory_storage.resource_clients.dataset", + "name": "delete", + "parsedDocstring": { + "text": "Delete the dataset." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_memory_storage/resource_clients/dataset.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 128 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Delete the dataset." + } + ] + }, + "flags": {}, + "id": 507, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "delete", + "parameters": [], + "type": { + "name": "None", + "type": "literal", + "value": null + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "List the items of the dataset.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 508, + "module": "_memory_storage.resource_clients.dataset", + "name": "list_items", + "parsedDocstring": { + "text": "List the items of the dataset.\n", + "args": { + "offset": "Number of items that should be skipped at the start. The default value is 0", + "limit": "Maximum number of items to return. By default there is no limit.", + "desc": "By default, results are returned in the same order as they were stored.\nTo reverse the order, set this parameter to True.", + "clean": "If True, returns only non-empty items and skips hidden fields (i.e. fields starting with the # character).\nThe clean parameter is just a shortcut for skip_hidden=True and skip_empty=True parameters.\nNote that since some objects might be skipped from the output, that the result might contain less items than the limit value.", + "fields": "A list of fields which should be picked from the items,\nonly these fields will remain in the resulting record objects.\nNote that the fields in the outputted items are sorted the same way as they are specified in the fields parameter.\nYou can use this feature to effectively fix the output format.", + "omit": "A list of fields which should be omitted from the items.", + "unwind": "Name of a field which should be unwound.\nIf the field is an array then every element of the array will become a separate record and merged with parent object.\nIf the unwound field is an object then it is merged with the parent object.\nIf the unwound field is missing or its value is neither an array nor an object and therefore cannot be merged with a parent object,\nthen the item gets preserved as it is. Note that the unwound items ignore the desc parameter.", + "skip_empty": "If True, then empty items are skipped from the output.\nNote that if used, the results might contain less items than the limit value.", + "skip_hidden": "If True, then hidden fields are skipped from the output, i.e. fields starting with the # character.", + "flatten": "A list of fields that should be flattened", + "view": "Name of the dataset view to be used\n" + }, + "returns": "ListPage: A page of the list of dataset items according to the specified filters." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_memory_storage/resource_clients/dataset.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 141 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "ListPage: A page of the list of dataset items according to the specified filters." + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "List the items of the dataset.\n" + } + ] + }, + "flags": {}, + "id": 509, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "list_items", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Number of items that should be skipped at the start. The default value is 0" + } + ] + }, + "defaultValue": "0", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 510, + "kind": 32768, + "kindString": "Parameter", + "name": "offset", + "type": { + "name": "int | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "int" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Maximum number of items to return. By default there is no limit." + } + ] + }, + "defaultValue": "LIST_ITEMS_LIMIT", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 511, + "kind": 32768, + "kindString": "Parameter", + "name": "limit", + "type": { + "name": "int | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "int" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "If True, returns only non-empty items and skips hidden fields (i.e. fields starting with the # character).\nThe clean parameter is just a shortcut for skip_hidden=True and skip_empty=True parameters.\nNote that since some objects might be skipped from the output, that the result might contain less items than the limit value." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 512, + "kind": 32768, + "kindString": "Parameter", + "name": "clean", + "type": { + "name": "bool | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "bool" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "By default, results are returned in the same order as they were stored.\nTo reverse the order, set this parameter to True." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 513, + "kind": 32768, + "kindString": "Parameter", + "name": "desc", + "type": { + "name": "bool | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "bool" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "A list of fields which should be picked from the items,\nonly these fields will remain in the resulting record objects.\nNote that the fields in the outputted items are sorted the same way as they are specified in the fields parameter.\nYou can use this feature to effectively fix the output format." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 514, + "kind": 32768, + "kindString": "Parameter", + "name": "fields", + "type": { + "name": "list[str] | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "list", + "typeArguments": [ + { + "type": "reference", + "name": "str" + } + ], + "target": "474" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "A list of fields which should be omitted from the items." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 515, + "kind": 32768, + "kindString": "Parameter", + "name": "omit", + "type": { + "name": "list[str] | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "list", + "typeArguments": [ + { + "type": "reference", + "name": "str" + } + ], + "target": "474" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Name of a field which should be unwound.\nIf the field is an array then every element of the array will become a separate record and merged with parent object.\nIf the unwound field is an object then it is merged with the parent object.\nIf the unwound field is missing or its value is neither an array nor an object and therefore cannot be merged with a parent object,\nthen the item gets preserved as it is. Note that the unwound items ignore the desc parameter." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 516, + "kind": 32768, + "kindString": "Parameter", + "name": "unwind", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "If True, then empty items are skipped from the output.\nNote that if used, the results might contain less items than the limit value." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 517, + "kind": 32768, + "kindString": "Parameter", + "name": "skip_empty", + "type": { + "name": "bool | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "bool" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "If True, then hidden fields are skipped from the output, i.e. fields starting with the # character." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 518, + "kind": 32768, + "kindString": "Parameter", + "name": "skip_hidden", + "type": { + "name": "bool | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "bool" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "A list of fields that should be flattened" + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 519, + "kind": 32768, + "kindString": "Parameter", + "name": "flatten", + "type": { + "name": "list[str] | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "list", + "typeArguments": [ + { + "type": "reference", + "name": "str" + } + ], + "target": "474" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Name of the dataset view to be used\n" + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 520, + "kind": 32768, + "kindString": "Parameter", + "name": "view", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + } + ], + "type": { + "name": "ListPage", + "type": "reference" + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Iterate over the items in the dataset.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 521, + "module": "_memory_storage.resource_clients.dataset", + "name": "iterate_items", + "parsedDocstring": { + "text": "Iterate over the items in the dataset.\n", + "args": { + "offset": "Number of items that should be skipped at the start. The default value is 0", + "limit": "Maximum number of items to return. By default there is no limit.", + "desc": "By default, results are returned in the same order as they were stored.\nTo reverse the order, set this parameter to True.", + "clean": "If True, returns only non-empty items and skips hidden fields (i.e. fields starting with the # character).\nThe clean parameter is just a shortcut for skip_hidden=True and skip_empty=True parameters.\nNote that since some objects might be skipped from the output, that the result might contain less items than the limit value.", + "fields": "A list of fields which should be picked from the items,\nonly these fields will remain in the resulting record objects.\nNote that the fields in the outputted items are sorted the same way as they are specified in the fields parameter.\nYou can use this feature to effectively fix the output format.", + "omit": "A list of fields which should be omitted from the items.", + "unwind": "Name of a field which should be unwound.\nIf the field is an array then every element of the array will become a separate record and merged with parent object.\nIf the unwound field is an object then it is merged with the parent object.\nIf the unwound field is missing or its value is neither an array nor an object and therefore cannot be merged with a parent object,\nthen the item gets preserved as it is. Note that the unwound items ignore the desc parameter.", + "skip_empty": "If True, then empty items are skipped from the output.\nNote that if used, the results might contain less items than the limit value.", + "skip_hidden": "If True, then hidden fields are skipped from the output, i.e. fields starting with the # character.\n" + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_memory_storage/resource_clients/dataset.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 223 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Iterate over the items in the dataset.\n" + } + ] + }, + "flags": {}, + "id": 522, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "iterate_items", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Number of items that should be skipped at the start. The default value is 0" + } + ] + }, + "defaultValue": "0", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 523, + "kind": 32768, + "kindString": "Parameter", + "name": "offset", + "type": { + "name": "int", + "type": "reference" + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Maximum number of items to return. By default there is no limit." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 524, + "kind": 32768, + "kindString": "Parameter", + "name": "limit", + "type": { + "name": "int | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "int" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "If True, returns only non-empty items and skips hidden fields (i.e. fields starting with the # character).\nThe clean parameter is just a shortcut for skip_hidden=True and skip_empty=True parameters.\nNote that since some objects might be skipped from the output, that the result might contain less items than the limit value." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 525, + "kind": 32768, + "kindString": "Parameter", + "name": "clean", + "type": { + "name": "bool | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "bool" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "By default, results are returned in the same order as they were stored.\nTo reverse the order, set this parameter to True." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 526, + "kind": 32768, + "kindString": "Parameter", + "name": "desc", + "type": { + "name": "bool | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "bool" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "A list of fields which should be picked from the items,\nonly these fields will remain in the resulting record objects.\nNote that the fields in the outputted items are sorted the same way as they are specified in the fields parameter.\nYou can use this feature to effectively fix the output format." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 527, + "kind": 32768, + "kindString": "Parameter", + "name": "fields", + "type": { + "name": "list[str] | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "list", + "typeArguments": [ + { + "type": "reference", + "name": "str" + } + ], + "target": "474" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "A list of fields which should be omitted from the items." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 528, + "kind": 32768, + "kindString": "Parameter", + "name": "omit", + "type": { + "name": "list[str] | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "list", + "typeArguments": [ + { + "type": "reference", + "name": "str" + } + ], + "target": "474" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Name of a field which should be unwound.\nIf the field is an array then every element of the array will become a separate record and merged with parent object.\nIf the unwound field is an object then it is merged with the parent object.\nIf the unwound field is missing or its value is neither an array nor an object and therefore cannot be merged with a parent object,\nthen the item gets preserved as it is. Note that the unwound items ignore the desc parameter." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 529, + "kind": 32768, + "kindString": "Parameter", + "name": "unwind", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "If True, then empty items are skipped from the output.\nNote that if used, the results might contain less items than the limit value." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 530, + "kind": 32768, + "kindString": "Parameter", + "name": "skip_empty", + "type": { + "name": "bool | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "bool" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "If True, then hidden fields are skipped from the output, i.e. fields starting with the # character.\n" + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 531, + "kind": 32768, + "kindString": "Parameter", + "name": "skip_hidden", + "type": { + "name": "bool | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "bool" + }, + { + "type": "literal", + "value": null + } + ] + } + } + ], + "type": { + "name": "AsyncIterator", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "dict" + } + ] + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 532, + "module": "_memory_storage.resource_clients.dataset", + "name": "get_items_as_bytes", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_memory_storage/resource_clients/dataset.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 286 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "flags": {}, + "id": 533, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "get_items_as_bytes", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 534, + "kind": 32768, + "kindString": "Parameter", + "name": "_args", + "type": { + "name": "Any", + "type": "reference" + } + }, + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 535, + "kind": 32768, + "kindString": "Parameter", + "name": "_kwargs", + "type": { + "name": "Any", + "type": "reference" + } + } + ], + "type": { + "name": "bytes", + "type": "reference" + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 536, + "module": "_memory_storage.resource_clients.dataset", + "name": "stream_items", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_memory_storage/resource_clients/dataset.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 289 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "flags": {}, + "id": 537, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "stream_items", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 538, + "kind": 32768, + "kindString": "Parameter", + "name": "_args", + "type": { + "name": "Any", + "type": "reference" + } + }, + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 539, + "kind": 32768, + "kindString": "Parameter", + "name": "_kwargs", + "type": { + "name": "Any", + "type": "reference" + } + } + ], + "type": { + "name": "AsyncIterator", + "type": "reference" + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Push items to the dataset.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 540, + "module": "_memory_storage.resource_clients.dataset", + "name": "push_items", + "parsedDocstring": { + "text": "Push items to the dataset.\n", + "args": { + "items": "The items which to push in the dataset. Either a stringified JSON, a dictionary, or a list of strings or dictionaries." + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_memory_storage/resource_clients/dataset.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 292 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Push items to the dataset.\n" + } + ] + }, + "flags": {}, + "id": 541, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "push_items", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The items which to push in the dataset. Either a stringified JSON, a dictionary, or a list of strings or dictionaries." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 542, + "kind": 32768, + "kindString": "Parameter", + "name": "items", + "type": { + "name": "JSONSerializable", + "type": "reference" + } + } + ], + "type": { + "name": "None", + "type": "literal", + "value": null + } + } + ] + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 544, + "module": "_memory_storage.resource_clients.key_value_store", + "name": "key", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_memory_storage/resource_clients/key_value_store.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 39 + } + ], + "type": { + "name": "str", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 545, + "module": "_memory_storage.resource_clients.key_value_store", + "name": "value", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_memory_storage/resource_clients/key_value_store.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 40 + } + ], + "type": { + "name": "Any", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 546, + "module": "_memory_storage.resource_clients.key_value_store", + "name": "contentType", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_memory_storage/resource_clients/key_value_store.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 41 + } + ], + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 547, + "module": "_memory_storage.resource_clients.key_value_store", + "name": "filename", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_memory_storage/resource_clients/key_value_store.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 42 + } + ], + "type": { + "name": "NotRequired", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "str" + } + ] + } + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [ + { + "children": [ + 546, + 547, + 544, + 545 + ], + "title": "Properties" + } + ], + "id": 543, + "module": "_memory_storage.resource_clients.key_value_store", + "name": "KeyValueStoreRecord", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_memory_storage/resource_clients/key_value_store.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 38 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Initialize the KeyValueStoreClient." + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 548, + "module": "_memory_storage.resource_clients.key_value_store", + "name": "__init__", + "parsedDocstring": { + "text": "Initialize the KeyValueStoreClient." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_memory_storage/resource_clients/key_value_store.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 74 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Initialize the KeyValueStoreClient." + } + ] + }, + "flags": {}, + "id": 549, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "__init__", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 550, + "kind": 32768, + "kindString": "Parameter", + "name": "base_storage_directory", + "type": { + "name": "str", + "type": "reference" + } + }, + { + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 551, + "kind": 32768, + "kindString": "Parameter", + "name": "memory_storage_client", + "type": { + "name": "MemoryStorageClient", + "type": "reference" + } + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 552, + "kind": 32768, + "kindString": "Parameter", + "name": "id", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 553, + "kind": 32768, + "kindString": "Parameter", + "name": "name", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + } + ], + "type": { + "name": "None", + "type": "literal", + "value": null + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Retrieve the key-value store.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 554, + "module": "_memory_storage.resource_clients.key_value_store", + "name": "get", + "parsedDocstring": { + "text": "Retrieve the key-value store.\n", + "returns": "dict, optional: The retrieved key-value store, or None if it does not exist" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_memory_storage/resource_clients/key_value_store.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 93 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "dict, optional: The retrieved key-value store, or None if it does not exist" + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Retrieve the key-value store.\n" + } + ] + }, + "flags": {}, + "id": 555, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "get", + "parameters": [], + "type": { + "name": "dict | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "dict" + }, + { + "type": "literal", + "value": null + } + ] + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Update the key-value store with specified fields.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 556, + "module": "_memory_storage.resource_clients.key_value_store", + "name": "update", + "parsedDocstring": { + "text": "Update the key-value store with specified fields.\n", + "args": { + "name": "The new name for key-value store\n" + }, + "returns": "dict: The updated key-value store" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_memory_storage/resource_clients/key_value_store.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 108 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "dict: The updated key-value store" + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Update the key-value store with specified fields.\n" + } + ] + }, + "flags": {}, + "id": 557, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "update", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The new name for key-value store\n" + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 558, + "kind": 32768, + "kindString": "Parameter", + "name": "name", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + } + ], + "type": { + "name": "dict", + "type": "reference" + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Delete the key-value store." + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 559, + "module": "_memory_storage.resource_clients.key_value_store", + "name": "delete", + "parsedDocstring": { + "text": "Delete the key-value store." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_memory_storage/resource_clients/key_value_store.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 152 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Delete the key-value store." + } + ] + }, + "flags": {}, + "id": 560, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "delete", + "parameters": [], + "type": { + "name": "None", + "type": "literal", + "value": null + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "List the keys in the key-value store.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 561, + "module": "_memory_storage.resource_clients.key_value_store", + "name": "list_keys", + "parsedDocstring": { + "text": "List the keys in the key-value store.\n", + "args": { + "limit": "Number of keys to be returned. Maximum value is 1000", + "exclusive_start_key": "All keys up to this one (including) are skipped from the result\n" + }, + "returns": "dict: The list of keys in the key-value store matching the given arguments" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_memory_storage/resource_clients/key_value_store.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 164 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "dict: The list of keys in the key-value store matching the given arguments" + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "List the keys in the key-value store.\n" + } + ] + }, + "flags": {}, + "id": 562, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "list_keys", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Number of keys to be returned. Maximum value is 1000" + } + ] + }, + "defaultValue": "DEFAULT_API_PARAM_LIMIT", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 563, + "kind": 32768, + "kindString": "Parameter", + "name": "limit", + "type": { + "name": "int", + "type": "reference" + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "All keys up to this one (including) are skipped from the result\n" + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 564, + "kind": 32768, + "kindString": "Parameter", + "name": "exclusive_start_key", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + } + ], + "type": { + "name": "dict", + "type": "reference" + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Retrieve the given record from the key-value store.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 565, + "module": "_memory_storage.resource_clients.key_value_store", + "name": "get_record", + "parsedDocstring": { + "text": "Retrieve the given record from the key-value store.\n", + "args": { + "key": "Key of the record to retrieve\n" + }, + "returns": "dict, optional: The requested record, or None, if the record does not exist" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_memory_storage/resource_clients/key_value_store.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 271 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "dict, optional: The requested record, or None, if the record does not exist" + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Retrieve the given record from the key-value store.\n" + } + ] + }, + "flags": {}, + "id": 566, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "get_record", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Key of the record to retrieve\n" + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 567, + "kind": 32768, + "kindString": "Parameter", + "name": "key", + "type": { + "name": "str", + "type": "reference" + } + } + ], + "type": { + "name": "dict | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "dict" + }, + { + "type": "literal", + "value": null + } + ] + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Retrieve the given record from the key-value store, without parsing it.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 568, + "module": "_memory_storage.resource_clients.key_value_store", + "name": "get_record_as_bytes", + "parsedDocstring": { + "text": "Retrieve the given record from the key-value store, without parsing it.\n", + "args": { + "key": "Key of the record to retrieve\n" + }, + "returns": "dict, optional: The requested record, or None, if the record does not exist" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_memory_storage/resource_clients/key_value_store.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 282 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "dict, optional: The requested record, or None, if the record does not exist" + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Retrieve the given record from the key-value store, without parsing it.\n" + } + ] + }, + "flags": {}, + "id": 569, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "get_record_as_bytes", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Key of the record to retrieve\n" + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 570, + "kind": 32768, + "kindString": "Parameter", + "name": "key", + "type": { + "name": "str", + "type": "reference" + } + } + ], + "type": { + "name": "dict | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "dict" + }, + { + "type": "literal", + "value": null + } + ] + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 571, + "module": "_memory_storage.resource_clients.key_value_store", + "name": "stream_record", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_memory_storage/resource_clients/key_value_store.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 293 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "flags": {}, + "id": 572, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "stream_record", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 573, + "kind": 32768, + "kindString": "Parameter", + "name": "_key", + "type": { + "name": "str", + "type": "reference" + } + } + ], + "type": { + "name": "AsyncIterator", + "type": "reference", + "typeArguments": [ + { + "type": "union", + "types": [ + { + "type": "reference", + "name": "dict" + }, + { + "type": "literal", + "value": null + } + ] + } + ] + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Set a value to the given record in the key-value store.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 574, + "module": "_memory_storage.resource_clients.key_value_store", + "name": "set_record", + "parsedDocstring": { + "text": "Set a value to the given record in the key-value store.\n", + "args": { + "key": "The key of the record to save the value to", + "value": "The value to save into the record", + "content_type": "The content type of the saved value" + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_memory_storage/resource_clients/key_value_store.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 296 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Set a value to the given record in the key-value store.\n" + } + ] + }, + "flags": {}, + "id": 575, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "set_record", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The key of the record to save the value to" + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 576, + "kind": 32768, + "kindString": "Parameter", + "name": "key", + "type": { + "name": "str", + "type": "reference" + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The value to save into the record" + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 577, + "kind": 32768, + "kindString": "Parameter", + "name": "value", + "type": { + "name": "Any", + "type": "reference" + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The content type of the saved value" + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": false + }, + "id": 578, + "kind": 32768, + "kindString": "Parameter", + "name": "content_type", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + } + ], + "type": { + "name": "None", + "type": "literal", + "value": null + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Delete the specified record from the key-value store.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 579, + "module": "_memory_storage.resource_clients.key_value_store", + "name": "delete_record", + "parsedDocstring": { + "text": "Delete the specified record from the key-value store.\n", + "args": { + "key": "The key of the record which to delete" + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_memory_storage/resource_clients/key_value_store.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 373 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Delete the specified record from the key-value store.\n" + } + ] + }, + "flags": {}, + "id": 580, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "delete_record", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The key of the record which to delete" + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 581, + "kind": 32768, + "kindString": "Parameter", + "name": "key", + "type": { + "name": "str", + "type": "reference" + } + } + ], + "type": { + "name": "None", + "type": "literal", + "value": null + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Initialize the RequestQueueClient." + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 582, + "module": "_memory_storage.resource_clients.request_queue", + "name": "__init__", + "parsedDocstring": { + "text": "Initialize the RequestQueueClient." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_memory_storage/resource_clients/request_queue.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 41 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Initialize the RequestQueueClient." + } + ] + }, + "flags": {}, + "id": 583, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "__init__", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 584, + "kind": 32768, + "kindString": "Parameter", + "name": "base_storage_directory", + "type": { + "name": "str", + "type": "reference" + } + }, + { + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 585, + "kind": 32768, + "kindString": "Parameter", + "name": "memory_storage_client", + "type": { + "name": "MemoryStorageClient", + "type": "reference" + } + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 586, + "kind": 32768, + "kindString": "Parameter", + "name": "id", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 587, + "kind": 32768, + "kindString": "Parameter", + "name": "name", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + } + ], + "type": { + "name": "None", + "type": "literal", + "value": null + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Retrieve the request queue.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 588, + "module": "_memory_storage.resource_clients.request_queue", + "name": "get", + "parsedDocstring": { + "text": "Retrieve the request queue.\n", + "returns": "dict, optional: The retrieved request queue, or None, if it does not exist" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_memory_storage/resource_clients/request_queue.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 60 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "dict, optional: The retrieved request queue, or None, if it does not exist" + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Retrieve the request queue.\n" + } + ] + }, + "flags": {}, + "id": 589, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "get", + "parameters": [], + "type": { + "name": "dict | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "dict" + }, + { + "type": "literal", + "value": null + } + ] + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Update the request queue with specified fields.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 590, + "module": "_memory_storage.resource_clients.request_queue", + "name": "update", + "parsedDocstring": { + "text": "Update the request queue with specified fields.\n", + "args": { + "name": "The new name for the request queue\n" + }, + "returns": "dict: The updated request queue" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_memory_storage/resource_clients/request_queue.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 75 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "dict: The updated request queue" + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Update the request queue with specified fields.\n" + } + ] + }, + "flags": {}, + "id": 591, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "update", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The new name for the request queue\n" + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 592, + "kind": 32768, + "kindString": "Parameter", + "name": "name", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + } + ], + "type": { + "name": "dict", + "type": "reference" + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Delete the request queue." + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 593, + "module": "_memory_storage.resource_clients.request_queue", + "name": "delete", + "parsedDocstring": { + "text": "Delete the request queue." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_memory_storage/resource_clients/request_queue.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 118 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Delete the request queue." + } + ] + }, + "flags": {}, + "id": 594, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "delete", + "parameters": [], + "type": { + "name": "None", + "type": "literal", + "value": null + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Retrieve a given number of requests from the beginning of the queue.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 595, + "module": "_memory_storage.resource_clients.request_queue", + "name": "list_head", + "parsedDocstring": { + "text": "Retrieve a given number of requests from the beginning of the queue.\n", + "args": { + "limit": "How many requests to retrieve\n" + }, + "returns": "dict: The desired number of requests from the beginning of the queue." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_memory_storage/resource_clients/request_queue.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 132 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "dict: The desired number of requests from the beginning of the queue." + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Retrieve a given number of requests from the beginning of the queue.\n" + } + ] + }, + "flags": {}, + "id": 596, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "list_head", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "How many requests to retrieve\n" + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 597, + "kind": 32768, + "kindString": "Parameter", + "name": "limit", + "type": { + "name": "int | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "int" + }, + { + "type": "literal", + "value": null + } + ] + } + } + ], + "type": { + "name": "dict", + "type": "reference" + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Add a request to the queue.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 598, + "module": "_memory_storage.resource_clients.request_queue", + "name": "add_request", + "parsedDocstring": { + "text": "Add a request to the queue.\n", + "args": { + "request": "The request to add to the queue", + "forefront": "Whether to add the request to the head or the end of the queue\n" + }, + "returns": "dict: The added request." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_memory_storage/resource_clients/request_queue.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 173 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "dict: The added request." + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Add a request to the queue.\n" + } + ] + }, + "flags": {}, + "id": 599, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "add_request", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The request to add to the queue" + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 600, + "kind": 32768, + "kindString": "Parameter", + "name": "request", + "type": { + "name": "dict", + "type": "reference" + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Whether to add the request to the head or the end of the queue\n" + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 601, + "kind": 32768, + "kindString": "Parameter", + "name": "forefront", + "type": { + "name": "bool | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "bool" + }, + { + "type": "literal", + "value": null + } + ] + } + } + ], + "type": { + "name": "dict", + "type": "reference" + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Retrieve a request from the queue.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 602, + "module": "_memory_storage.resource_clients.request_queue", + "name": "get_request", + "parsedDocstring": { + "text": "Retrieve a request from the queue.\n", + "args": { + "request_id": "ID of the request to retrieve\n" + }, + "returns": "dict, optional: The retrieved request, or None, if it did not exist." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_memory_storage/resource_clients/request_queue.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 226 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "dict, optional: The retrieved request, or None, if it did not exist." + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Retrieve a request from the queue.\n" + } + ] + }, + "flags": {}, + "id": 603, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "get_request", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "ID of the request to retrieve\n" + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 604, + "kind": 32768, + "kindString": "Parameter", + "name": "request_id", + "type": { + "name": "str", + "type": "reference" + } + } + ], + "type": { + "name": "dict | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "dict" + }, + { + "type": "literal", + "value": null + } + ] + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Update a request in the queue.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 605, + "module": "_memory_storage.resource_clients.request_queue", + "name": "update_request", + "parsedDocstring": { + "text": "Update a request in the queue.\n", + "args": { + "request": "The updated request", + "forefront": "Whether to put the updated request in the beginning or the end of the queue\n" + }, + "returns": "dict: The updated request" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_memory_storage/resource_clients/request_queue.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 248 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "dict: The updated request" + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Update a request in the queue.\n" + } + ] + }, + "flags": {}, + "id": 606, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "update_request", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The updated request" + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 607, + "kind": 32768, + "kindString": "Parameter", + "name": "request", + "type": { + "name": "dict", + "type": "reference" + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Whether to put the updated request in the beginning or the end of the queue\n" + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 608, + "kind": 32768, + "kindString": "Parameter", + "name": "forefront", + "type": { + "name": "bool | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "bool" + }, + { + "type": "literal", + "value": null + } + ] + } + } + ], + "type": { + "name": "dict", + "type": "reference" + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Delete a request from the queue.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 609, + "module": "_memory_storage.resource_clients.request_queue", + "name": "delete_request", + "parsedDocstring": { + "text": "Delete a request from the queue.\n", + "args": { + "request_id": "ID of the request to delete." + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_memory_storage/resource_clients/request_queue.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 306 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Delete a request from the queue.\n" + } + ] + }, + "flags": {}, + "id": 610, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "delete_request", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "ID of the request to delete." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 611, + "kind": 32768, + "kindString": "Parameter", + "name": "request_id", + "type": { + "name": "str", + "type": "reference" + } + } + ], + "type": { + "name": "None", + "type": "literal", + "value": null + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Initialize the BaseResourceClient." + } + ] + }, + "decorations": [ + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 612, + "module": "_memory_storage.resource_clients.base_resource_client", + "name": "__init__", + "parsedDocstring": { + "text": "Initialize the BaseResourceClient." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_memory_storage/resource_clients/base_resource_client.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 25 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Initialize the BaseResourceClient." + } + ] + }, + "flags": {}, + "id": 613, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "__init__", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 614, + "kind": 32768, + "kindString": "Parameter", + "name": "base_storage_directory", + "type": { + "name": "str", + "type": "reference" + } + }, + { + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 615, + "kind": 32768, + "kindString": "Parameter", + "name": "memory_storage_client", + "type": { + "name": "MemoryStorageClient", + "type": "reference" + } + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 616, + "kind": 32768, + "kindString": "Parameter", + "name": "id", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 617, + "kind": 32768, + "kindString": "Parameter", + "name": "name", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + } + ], + "type": { + "name": "None", + "type": "literal", + "value": null + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Retrieve the storage.\n" + } + ] + }, + "decorations": [ + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 618, + "module": "_memory_storage.resource_clients.base_resource_client", + "name": "get", + "parsedDocstring": { + "text": "Retrieve the storage.\n", + "returns": "dict, optional: The retrieved storage, or None, if it does not exist" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_memory_storage/resource_clients/base_resource_client.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 37 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "dict, optional: The retrieved storage, or None, if it does not exist" + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Retrieve the storage.\n" + } + ] + }, + "flags": {}, + "id": 619, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "get", + "parameters": [], + "type": { + "name": "dict | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "dict" + }, + { + "type": "literal", + "value": null + } + ] + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "List the available request queues.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 620, + "module": "_memory_storage.resource_clients.request_queue_collection", + "name": "list", + "parsedDocstring": { + "text": "List the available request queues.\n", + "returns": "ListPage: The list of available request queues matching the specified filters." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_memory_storage/resource_clients/request_queue_collection.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 24 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "ListPage: The list of available request queues matching the specified filters." + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "List the available request queues.\n" + } + ] + }, + "flags": {}, + "id": 621, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "list", + "parameters": [], + "type": { + "name": "ListPage", + "type": "reference" + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Retrieve a named request queue, or create a new one when it doesn't exist.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 622, + "module": "_memory_storage.resource_clients.request_queue_collection", + "name": "get_or_create", + "parsedDocstring": { + "text": "Retrieve a named request queue, or create a new one when it doesn't exist.\n", + "args": { + "name": "The name of the request queue to retrieve or create.", + "schema": "The schema of the request queue\n" + }, + "returns": "dict: The retrieved or newly-created request queue." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_memory_storage/resource_clients/request_queue_collection.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 32 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "dict: The retrieved or newly-created request queue." + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Retrieve a named request queue, or create a new one when it doesn't exist.\n" + } + ] + }, + "flags": {}, + "id": 623, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "get_or_create", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The name of the request queue to retrieve or create." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 624, + "kind": 32768, + "kindString": "Parameter", + "name": "name", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The schema of the request queue\n" + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 625, + "kind": 32768, + "kindString": "Parameter", + "name": "schema", + "type": { + "name": "dict | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "dict" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 626, + "kind": 32768, + "kindString": "Parameter", + "name": "_id", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + } + ], + "type": { + "name": "dict", + "type": "reference" + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "List the available datasets.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 627, + "module": "_memory_storage.resource_clients.dataset_collection", + "name": "list", + "parsedDocstring": { + "text": "List the available datasets.\n", + "returns": "ListPage: The list of available datasets matching the specified filters." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_memory_storage/resource_clients/dataset_collection.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 24 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "ListPage: The list of available datasets matching the specified filters." + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "List the available datasets.\n" + } + ] + }, + "flags": {}, + "id": 628, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "list", + "parameters": [], + "type": { + "name": "ListPage", + "type": "reference" + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Retrieve a named dataset, or create a new one when it doesn't exist.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 629, + "module": "_memory_storage.resource_clients.dataset_collection", + "name": "get_or_create", + "parsedDocstring": { + "text": "Retrieve a named dataset, or create a new one when it doesn't exist.\n", + "args": { + "name": "The name of the dataset to retrieve or create.", + "schema": "The schema of the dataset\n" + }, + "returns": "dict: The retrieved or newly-created dataset." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_memory_storage/resource_clients/dataset_collection.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 32 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "dict: The retrieved or newly-created dataset." + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Retrieve a named dataset, or create a new one when it doesn't exist.\n" + } + ] + }, + "flags": {}, + "id": 630, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "get_or_create", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The name of the dataset to retrieve or create." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 631, + "kind": 32768, + "kindString": "Parameter", + "name": "name", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The schema of the dataset\n" + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 632, + "kind": 32768, + "kindString": "Parameter", + "name": "schema", + "type": { + "name": "dict | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "dict" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 633, + "kind": 32768, + "kindString": "Parameter", + "name": "_id", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + } + ], + "type": { + "name": "dict", + "type": "reference" + } + } + ] + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 634, + "module": "storages.dataset", + "name": "SAFETY_BUFFER_PERCENT", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storages/dataset.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 26 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 635, + "module": "storages.dataset", + "name": "EFFECTIVE_LIMIT_BYTES", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storages/dataset.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 27 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Store an object or an array of objects to the dataset.\n\nThe size of the data is limited by the receiving API and therefore `push_data()` will only\nallow objects whose JSON representation is smaller than 9MB. When an array is passed,\nnone of the included objects may be larger than 9MB, but the array itself may be of any size.\n" + } + ] + }, + "decorations": [ + { + "name": "classmethod" + } + ], + "flags": {}, + "groups": [], + "id": 637, + "module": "storages.dataset", + "name": "push_data", + "parsedDocstring": { + "text": "Store an object or an array of objects to the dataset.\n\nThe size of the data is limited by the receiving API and therefore `push_data()` will only\nallow objects whose JSON representation is smaller than 9MB. When an array is passed,\nnone of the included objects may be larger than 9MB, but the array itself may be of any size.\n", + "args": { + "data": "dict or array of dicts containing data to be stored in the default dataset.\nThe JSON representation of each item must be smaller than 9MB." + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storages/dataset.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 153 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Store an object or an array of objects to the dataset.\n\nThe size of the data is limited by the receiving API and therefore `push_data()` will only\nallow objects whose JSON representation is smaller than 9MB. When an array is passed,\nnone of the included objects may be larger than 9MB, but the array itself may be of any size.\n" + } + ] + }, + "flags": {}, + "id": 638, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "push_data", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "dict or array of dicts containing data to be stored in the default dataset.\nThe JSON representation of each item must be smaller than 9MB." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 639, + "kind": 32768, + "kindString": "Parameter", + "name": "data", + "type": { + "name": "JSONSerializable", + "type": "reference" + } + } + ], + "type": { + "name": "None", + "type": "literal", + "value": null + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Get items from the dataset.\n" + } + ] + }, + "decorations": [ + { + "name": "classmethod" + } + ], + "flags": {}, + "groups": [], + "id": 640, + "module": "storages.dataset", + "name": "get_data", + "parsedDocstring": { + "text": "Get items from the dataset.\n", + "args": { + "offset": "Number of items that should be skipped at the start. The default value is 0", + "limit": "Maximum number of items to return. By default there is no limit.", + "desc": "By default, results are returned in the same order as they were stored.\nTo reverse the order, set this parameter to True.", + "clean": "If True, returns only non-empty items and skips hidden fields (i.e. fields starting with the # character).\nThe clean parameter is just a shortcut for skip_hidden=True and skip_empty=True parameters.\nNote that since some objects might be skipped from the output, that the result might contain less items than the limit value.", + "fields": "A list of fields which should be picked from the items,\nonly these fields will remain in the resulting record objects.\nNote that the fields in the outputted items are sorted the same way as they are specified in the fields parameter.\nYou can use this feature to effectively fix the output format.", + "omit": "A list of fields which should be omitted from the items.", + "unwind": "Name of a field which should be unwound.\nIf the field is an array then every element of the array will become a separate record and merged with parent object.\nIf the unwound field is an object then it is merged with the parent object.\nIf the unwound field is missing or its value is neither an array nor an object and therefore cannot be merged with a parent object,\nthen the item gets preserved as it is. Note that the unwound items ignore the desc parameter.", + "skip_empty": "If True, then empty items are skipped from the output.\nNote that if used, the results might contain less items than the limit value.", + "skip_hidden": "If True, then hidden fields are skipped from the output, i.e. fields starting with the # character.", + "flatten": "A list of fields that should be flattened", + "view": "Name of the dataset view to be used\n" + }, + "returns": "ListPage: A page of the list of dataset items according to the specified filters." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storages/dataset.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 182 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "ListPage: A page of the list of dataset items according to the specified filters." + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Get items from the dataset.\n" + } + ] + }, + "flags": {}, + "id": 641, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "get_data", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Number of items that should be skipped at the start. The default value is 0" + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 642, + "kind": 32768, + "kindString": "Parameter", + "name": "offset", + "type": { + "name": "int | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "int" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Maximum number of items to return. By default there is no limit." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 643, + "kind": 32768, + "kindString": "Parameter", + "name": "limit", + "type": { + "name": "int | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "int" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "If True, returns only non-empty items and skips hidden fields (i.e. fields starting with the # character).\nThe clean parameter is just a shortcut for skip_hidden=True and skip_empty=True parameters.\nNote that since some objects might be skipped from the output, that the result might contain less items than the limit value." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 644, + "kind": 32768, + "kindString": "Parameter", + "name": "clean", + "type": { + "name": "bool | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "bool" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "By default, results are returned in the same order as they were stored.\nTo reverse the order, set this parameter to True." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 645, + "kind": 32768, + "kindString": "Parameter", + "name": "desc", + "type": { + "name": "bool | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "bool" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "A list of fields which should be picked from the items,\nonly these fields will remain in the resulting record objects.\nNote that the fields in the outputted items are sorted the same way as they are specified in the fields parameter.\nYou can use this feature to effectively fix the output format." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 646, + "kind": 32768, + "kindString": "Parameter", + "name": "fields", + "type": { + "name": "list[str] | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "list", + "typeArguments": [ + { + "type": "reference", + "name": "str" + } + ], + "target": "474" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "A list of fields which should be omitted from the items." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 647, + "kind": 32768, + "kindString": "Parameter", + "name": "omit", + "type": { + "name": "list[str] | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "list", + "typeArguments": [ + { + "type": "reference", + "name": "str" + } + ], + "target": "474" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Name of a field which should be unwound.\nIf the field is an array then every element of the array will become a separate record and merged with parent object.\nIf the unwound field is an object then it is merged with the parent object.\nIf the unwound field is missing or its value is neither an array nor an object and therefore cannot be merged with a parent object,\nthen the item gets preserved as it is. Note that the unwound items ignore the desc parameter." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 648, + "kind": 32768, + "kindString": "Parameter", + "name": "unwind", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "If True, then empty items are skipped from the output.\nNote that if used, the results might contain less items than the limit value." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 649, + "kind": 32768, + "kindString": "Parameter", + "name": "skip_empty", + "type": { + "name": "bool | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "bool" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "If True, then hidden fields are skipped from the output, i.e. fields starting with the # character." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 650, + "kind": 32768, + "kindString": "Parameter", + "name": "skip_hidden", + "type": { + "name": "bool | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "bool" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "A list of fields that should be flattened" + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 651, + "kind": 32768, + "kindString": "Parameter", + "name": "flatten", + "type": { + "name": "list[str] | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "list", + "typeArguments": [ + { + "type": "reference", + "name": "str" + } + ], + "target": "474" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Name of the dataset view to be used\n" + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 652, + "kind": 32768, + "kindString": "Parameter", + "name": "view", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + } + ], + "type": { + "name": "ListPage", + "type": "reference" + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Save the entirety of the dataset's contents into one file within a key-value store.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 653, + "module": "storages.dataset", + "name": "export_to", + "parsedDocstring": { + "text": "Save the entirety of the dataset's contents into one file within a key-value store.\n", + "args": { + "key": "The key to save the data under.", + "to_key_value_store_id": "The id of the key-value store in which the result will be saved.", + "to_key_value_store_name": "The name of the key-value store in which the result will be saved.\nYou must specify only one of `to_key_value_store_id` and `to_key_value_store_name` arguments.\nIf you omit both, it uses the default key-value store.", + "content_type": "Either 'text/csv' or 'application/json'. Defaults to JSON." + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storages/dataset.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 272 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Save the entirety of the dataset's contents into one file within a key-value store.\n" + } + ] + }, + "flags": {}, + "id": 654, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "export_to", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The key to save the data under." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 655, + "kind": 32768, + "kindString": "Parameter", + "name": "key", + "type": { + "name": "str", + "type": "reference" + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The id of the key-value store in which the result will be saved." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 656, + "kind": 32768, + "kindString": "Parameter", + "name": "to_key_value_store_id", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The name of the key-value store in which the result will be saved.\nYou must specify only one of `to_key_value_store_id` and `to_key_value_store_name` arguments.\nIf you omit both, it uses the default key-value store." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 657, + "kind": 32768, + "kindString": "Parameter", + "name": "to_key_value_store_name", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Either 'text/csv' or 'application/json'. Defaults to JSON." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 658, + "kind": 32768, + "kindString": "Parameter", + "name": "content_type", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + } + ], + "type": { + "name": "None", + "type": "literal", + "value": null + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Save the entirety of the dataset's contents into one JSON file within a key-value store.\n" + } + ] + }, + "decorations": [ + { + "name": "classmethod" + } + ], + "flags": {}, + "groups": [], + "id": 659, + "module": "storages.dataset", + "name": "export_to_json", + "parsedDocstring": { + "text": "Save the entirety of the dataset's contents into one JSON file within a key-value store.\n", + "args": { + "key": "The key to save the data under.", + "from_dataset_id": "The ID of the dataset in case of calling the class method. Uses default dataset if omitted.", + "from_dataset_name": "The name of the dataset in case of calling the class method. Uses default dataset if omitted.\nYou must specify only one of `from_dataset_id` and `from_dataset_name` arguments.\nIf you omit both, it uses the default dataset.", + "to_key_value_store_id": "The id of the key-value store in which the result will be saved.", + "to_key_value_store_name": "The name of the key-value store in which the result will be saved.\nYou must specify only one of `to_key_value_store_id` and `to_key_value_store_name` arguments.\nIf you omit both, it uses the default key-value store." + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storages/dataset.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 317 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Save the entirety of the dataset's contents into one JSON file within a key-value store.\n" + } + ] + }, + "flags": {}, + "id": 660, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "export_to_json", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The key to save the data under." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 661, + "kind": 32768, + "kindString": "Parameter", + "name": "key", + "type": { + "name": "str", + "type": "reference" + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The ID of the dataset in case of calling the class method. Uses default dataset if omitted." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 662, + "kind": 32768, + "kindString": "Parameter", + "name": "from_dataset_id", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The name of the dataset in case of calling the class method. Uses default dataset if omitted.\nYou must specify only one of `from_dataset_id` and `from_dataset_name` arguments.\nIf you omit both, it uses the default dataset." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 663, + "kind": 32768, + "kindString": "Parameter", + "name": "from_dataset_name", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The id of the key-value store in which the result will be saved." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 664, + "kind": 32768, + "kindString": "Parameter", + "name": "to_key_value_store_id", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The name of the key-value store in which the result will be saved.\nYou must specify only one of `to_key_value_store_id` and `to_key_value_store_name` arguments.\nIf you omit both, it uses the default key-value store." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 665, + "kind": 32768, + "kindString": "Parameter", + "name": "to_key_value_store_name", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + } + ], + "type": { + "name": "None", + "type": "literal", + "value": null + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Save the entirety of the dataset's contents into one CSV file within a key-value store.\n" + } + ] + }, + "decorations": [ + { + "name": "classmethod" + } + ], + "flags": {}, + "groups": [], + "id": 666, + "module": "storages.dataset", + "name": "export_to_csv", + "parsedDocstring": { + "text": "Save the entirety of the dataset's contents into one CSV file within a key-value store.\n", + "args": { + "key": "The key to save the data under.", + "from_dataset_id": "The ID of the dataset in case of calling the class method. Uses default dataset if omitted.", + "from_dataset_name": "The name of the dataset in case of calling the class method. Uses default dataset if omitted.\nYou must specify only one of `from_dataset_id` and `from_dataset_name` arguments.\nIf you omit both, it uses the default dataset.", + "to_key_value_store_id": "The id of the key-value store in which the result will be saved.", + "to_key_value_store_name": "The name of the key-value store in which the result will be saved.\nYou must specify only one of `to_key_value_store_id` and `to_key_value_store_name` arguments.\nIf you omit both, it uses the default key-value store." + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storages/dataset.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 359 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Save the entirety of the dataset's contents into one CSV file within a key-value store.\n" + } + ] + }, + "flags": {}, + "id": 667, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "export_to_csv", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The key to save the data under." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 668, + "kind": 32768, + "kindString": "Parameter", + "name": "key", + "type": { + "name": "str", + "type": "reference" + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The ID of the dataset in case of calling the class method. Uses default dataset if omitted." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 669, + "kind": 32768, + "kindString": "Parameter", + "name": "from_dataset_id", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The name of the dataset in case of calling the class method. Uses default dataset if omitted.\nYou must specify only one of `from_dataset_id` and `from_dataset_name` arguments.\nIf you omit both, it uses the default dataset." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 670, + "kind": 32768, + "kindString": "Parameter", + "name": "from_dataset_name", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The id of the key-value store in which the result will be saved." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 671, + "kind": 32768, + "kindString": "Parameter", + "name": "to_key_value_store_id", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The name of the key-value store in which the result will be saved.\nYou must specify only one of `to_key_value_store_id` and `to_key_value_store_name` arguments.\nIf you omit both, it uses the default key-value store." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 672, + "kind": 32768, + "kindString": "Parameter", + "name": "to_key_value_store_name", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + } + ], + "type": { + "name": "None", + "type": "literal", + "value": null + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Get an object containing general information about the dataset.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 673, + "module": "storages.dataset", + "name": "get_info", + "parsedDocstring": { + "text": "Get an object containing general information about the dataset.\n", + "returns": "dict: Object returned by calling the GET dataset API endpoint." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storages/dataset.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 400 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "dict: Object returned by calling the GET dataset API endpoint." + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Get an object containing general information about the dataset.\n" + } + ] + }, + "flags": {}, + "id": 674, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "get_info", + "parameters": [], + "type": { + "name": "dict | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "dict" + }, + { + "type": "literal", + "value": null + } + ] + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Iterate over the items in the dataset.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 675, + "module": "storages.dataset", + "name": "iterate_items", + "parsedDocstring": { + "text": "Iterate over the items in the dataset.\n", + "args": { + "offset": "Number of items that should be skipped at the start. The default value is 0", + "limit": "Maximum number of items to return. By default there is no limit.", + "desc": "By default, results are returned in the same order as they were stored.\nTo reverse the order, set this parameter to True.", + "clean": "If True, returns only non-empty items and skips hidden fields (i.e. fields starting with the # character).\nThe clean parameter is just a shortcut for skip_hidden=True and skip_empty=True parameters.\nNote that since some objects might be skipped from the output, that the result might contain less items than the limit value.", + "fields": "A list of fields which should be picked from the items,\nonly these fields will remain in the resulting record objects.\nNote that the fields in the outputted items are sorted the same way as they are specified in the fields parameter.\nYou can use this feature to effectively fix the output format.", + "omit": "A list of fields which should be omitted from the items.", + "unwind": "Name of a field which should be unwound.\nIf the field is an array then every element of the array will become a separate record and merged with parent object.\nIf the unwound field is an object then it is merged with the parent object.\nIf the unwound field is missing or its value is neither an array nor an object and therefore cannot be merged with a parent object,\nthen the item gets preserved as it is. Note that the unwound items ignore the desc parameter.", + "skip_empty": "If True, then empty items are skipped from the output.\nNote that if used, the results might contain less items than the limit value.", + "skip_hidden": "If True, then hidden fields are skipped from the output, i.e. fields starting with the # character.\n" + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storages/dataset.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 408 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Iterate over the items in the dataset.\n" + } + ] + }, + "flags": {}, + "id": 676, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "iterate_items", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Number of items that should be skipped at the start. The default value is 0" + } + ] + }, + "defaultValue": "0", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 677, + "kind": 32768, + "kindString": "Parameter", + "name": "offset", + "type": { + "name": "int", + "type": "reference" + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Maximum number of items to return. By default there is no limit." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 678, + "kind": 32768, + "kindString": "Parameter", + "name": "limit", + "type": { + "name": "int | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "int" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "If True, returns only non-empty items and skips hidden fields (i.e. fields starting with the # character).\nThe clean parameter is just a shortcut for skip_hidden=True and skip_empty=True parameters.\nNote that since some objects might be skipped from the output, that the result might contain less items than the limit value." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 679, + "kind": 32768, + "kindString": "Parameter", + "name": "clean", + "type": { + "name": "bool | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "bool" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "By default, results are returned in the same order as they were stored.\nTo reverse the order, set this parameter to True." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 680, + "kind": 32768, + "kindString": "Parameter", + "name": "desc", + "type": { + "name": "bool | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "bool" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "A list of fields which should be picked from the items,\nonly these fields will remain in the resulting record objects.\nNote that the fields in the outputted items are sorted the same way as they are specified in the fields parameter.\nYou can use this feature to effectively fix the output format." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 681, + "kind": 32768, + "kindString": "Parameter", + "name": "fields", + "type": { + "name": "list[str] | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "list", + "typeArguments": [ + { + "type": "reference", + "name": "str" + } + ], + "target": "474" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "A list of fields which should be omitted from the items." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 682, + "kind": 32768, + "kindString": "Parameter", + "name": "omit", + "type": { + "name": "list[str] | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "list", + "typeArguments": [ + { + "type": "reference", + "name": "str" + } + ], + "target": "474" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Name of a field which should be unwound.\nIf the field is an array then every element of the array will become a separate record and merged with parent object.\nIf the unwound field is an object then it is merged with the parent object.\nIf the unwound field is missing or its value is neither an array nor an object and therefore cannot be merged with a parent object,\nthen the item gets preserved as it is. Note that the unwound items ignore the desc parameter." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 683, + "kind": 32768, + "kindString": "Parameter", + "name": "unwind", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "If True, then empty items are skipped from the output.\nNote that if used, the results might contain less items than the limit value." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 684, + "kind": 32768, + "kindString": "Parameter", + "name": "skip_empty", + "type": { + "name": "bool | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "bool" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "If True, then hidden fields are skipped from the output, i.e. fields starting with the # character.\n" + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 685, + "kind": 32768, + "kindString": "Parameter", + "name": "skip_hidden", + "type": { + "name": "bool | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "bool" + }, + { + "type": "literal", + "value": null + } + ] + } + } + ], + "type": { + "name": "AsyncIterator", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "dict" + } + ] + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Remove the dataset either from the Apify cloud storage or from the local directory." + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 686, + "module": "storages.dataset", + "name": "drop", + "parsedDocstring": { + "text": "Remove the dataset either from the Apify cloud storage or from the local directory." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storages/dataset.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 460 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Remove the dataset either from the Apify cloud storage or from the local directory." + } + ] + }, + "flags": {}, + "id": 687, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "drop", + "parameters": [], + "type": { + "name": "None", + "type": "literal", + "value": null + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Open a dataset.\n\nDatasets are used to store structured data where each object stored has the same attributes,\nsuch as online store products or real estate offers.\nThe actual data is stored either on the local filesystem or in the Apify cloud.\n" + } + ] + }, + "decorations": [ + { + "name": "classmethod" + } + ], + "flags": {}, + "groups": [], + "id": 688, + "module": "storages.dataset", + "name": "open", + "parsedDocstring": { + "text": "Open a dataset.\n\nDatasets are used to store structured data where each object stored has the same attributes,\nsuch as online store products or real estate offers.\nThe actual data is stored either on the local filesystem or in the Apify cloud.\n", + "args": { + "id": "ID of the dataset to be opened.\nIf neither `id` nor `name` are provided, the method returns the default dataset associated with the actor run.\nIf the dataset with the given ID does not exist, it raises an error.", + "name": "Name of the dataset to be opened.\nIf neither `id` nor `name` are provided, the method returns the default dataset associated with the actor run.\nIf the dataset with the given name does not exist, it is created.", + "force_cloud": "If set to True, it will open a dataset on the Apify Platform even when running the actor locally.\nDefaults to False.", + "config": "A `Configuration` instance, uses global configuration if omitted.\n" + }, + "returns": "Dataset: An instance of the `Dataset` class for the given ID or name." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storages/dataset.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 466 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "Dataset: An instance of the `Dataset` class for the given ID or name." + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Open a dataset.\n\nDatasets are used to store structured data where each object stored has the same attributes,\nsuch as online store products or real estate offers.\nThe actual data is stored either on the local filesystem or in the Apify cloud.\n" + } + ] + }, + "flags": {}, + "id": 689, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "open", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "ID of the dataset to be opened.\nIf neither `id` nor `name` are provided, the method returns the default dataset associated with the actor run.\nIf the dataset with the given ID does not exist, it raises an error." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 690, + "kind": 32768, + "kindString": "Parameter", + "name": "id", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Name of the dataset to be opened.\nIf neither `id` nor `name` are provided, the method returns the default dataset associated with the actor run.\nIf the dataset with the given name does not exist, it is created." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 691, + "kind": 32768, + "kindString": "Parameter", + "name": "name", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "If set to True, it will open a dataset on the Apify Platform even when running the actor locally.\nDefaults to False." + } + ] + }, + "defaultValue": "False", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 692, + "kind": 32768, + "kindString": "Parameter", + "name": "force_cloud", + "type": { + "name": "bool", + "type": "reference" + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "A `Configuration` instance, uses global configuration if omitted.\n" + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 693, + "kind": 32768, + "kindString": "Parameter", + "name": "config", + "type": { + "name": "Configuration | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "Configuration", + "target": "1" + }, + { + "type": "literal", + "value": null + } + ] + } + } + ], + "type": { + "name": "Dataset", + "type": "reference", + "target": "636" + } + } + ] + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "The `Dataset` class represents a store for structured data where each object stored has the same attributes.\n\nYou can imagine it as a table, where each object is a row and its attributes are columns.\nDataset is an append-only storage - you can only add new records to it but you cannot modify or remove existing records.\nTypically it is used to store crawling results.\n\nDo not instantiate this class directly, use the `Actor.open_dataset()` function instead.\n\n`Dataset` stores its data either on local disk or in the Apify cloud,\ndepending on whether the `APIFY_LOCAL_STORAGE_DIR` or `APIFY_TOKEN` environment variables are set.\n\nIf the `APIFY_LOCAL_STORAGE_DIR` environment variable is set, the data is stored in\nthe local directory in the following files:\n```\n{APIFY_LOCAL_STORAGE_DIR}/datasets/{DATASET_ID}/{INDEX}.json\n```\n\nNote that `{DATASET_ID}` is the name or ID of the dataset. The default dataset has ID: `default`,\nunless you override it by setting the `APIFY_DEFAULT_DATASET_ID` environment variable.\nEach dataset item is stored as a separate JSON file, where `{INDEX}` is a zero-based index of the item in the dataset.\n\nIf the `APIFY_TOKEN` environment variable is set but `APIFY_LOCAL_STORAGE_DIR` is not, the data is stored in the\n[Apify Dataset](https://docs.apify.com/storage/dataset) cloud storage." + } + ] + }, + "flags": {}, + "groups": [ + { + "children": [ + 686, + 653, + 666, + 659, + 640, + 673, + 675, + 688, + 637 + ], + "title": "Methods" + } + ], + "id": 636, + "module": "storages.dataset", + "name": "Dataset", + "parsedDocstring": { + "text": "The `Dataset` class represents a store for structured data where each object stored has the same attributes.\n\nYou can imagine it as a table, where each object is a row and its attributes are columns.\nDataset is an append-only storage - you can only add new records to it but you cannot modify or remove existing records.\nTypically it is used to store crawling results.\n\nDo not instantiate this class directly, use the `Actor.open_dataset()` function instead.\n\n`Dataset` stores its data either on local disk or in the Apify cloud,\ndepending on whether the `APIFY_LOCAL_STORAGE_DIR` or `APIFY_TOKEN` environment variables are set.\n\nIf the `APIFY_LOCAL_STORAGE_DIR` environment variable is set, the data is stored in\nthe local directory in the following files:\n```\n{APIFY_LOCAL_STORAGE_DIR}/datasets/{DATASET_ID}/{INDEX}.json\n```\n\nNote that `{DATASET_ID}` is the name or ID of the dataset. The default dataset has ID: `default`,\nunless you override it by setting the `APIFY_DEFAULT_DATASET_ID` environment variable.\nEach dataset item is stored as a separate JSON file, where `{INDEX}` is a zero-based index of the item in the dataset.\n\nIf the `APIFY_TOKEN` environment variable is set but `APIFY_LOCAL_STORAGE_DIR` is not, the data is stored in the\n[Apify Dataset](https://docs.apify.com/storage/dataset) cloud storage." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storages/dataset.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 73 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 694, + "module": "storages.key_value_store", + "name": "T", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storages/key_value_store.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 19 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 696, + "module": "storages.key_value_store", + "name": "size", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storages/key_value_store.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 25 + } + ], + "type": { + "name": "int", + "type": "reference" + } + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Contains information about a key-value store record." + } + ] + }, + "flags": {}, + "groups": [ + { + "children": [ + 696 + ], + "title": "Properties" + } + ], + "id": 695, + "module": "storages.key_value_store", + "name": "IterateKeysInfo", + "parsedDocstring": { + "text": "Contains information about a key-value store record." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storages/key_value_store.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 22 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 698, + "module": "storages.key_value_store", + "name": "key", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storages/key_value_store.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 31 + } + ], + "type": { + "name": "str", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 699, + "module": "storages.key_value_store", + "name": "info", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storages/key_value_store.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 32 + } + ], + "type": { + "name": "IterateKeysInfo", + "type": "reference", + "target": "695" + } + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "A tuple representing a key-value store record." + } + ] + }, + "flags": {}, + "groups": [ + { + "children": [ + 699, + 698 + ], + "title": "Properties" + } + ], + "id": 697, + "module": "storages.key_value_store", + "name": "IterateKeysTuple", + "parsedDocstring": { + "text": "A tuple representing a key-value store record." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storages/key_value_store.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 28 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Get a value from the key-value store.\n" + } + ] + }, + "decorations": [ + { + "name": "classmethod" + } + ], + "flags": {}, + "groups": [], + "id": 701, + "module": "storages.key_value_store", + "name": "get_value", + "parsedDocstring": { + "text": "Get a value from the key-value store.\n", + "args": { + "key": "Key of the record to retrieve.", + "default_value": "Default value returned in case the record does not exist.\n" + }, + "returns": "Any: The value associated with the given key. `default_value` is used in case the record does not exist." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storages/key_value_store.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 137 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "Any: The value associated with the given key. `default_value` is used in case the record does not exist." + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Get a value from the key-value store.\n" + } + ] + }, + "flags": {}, + "id": 702, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "get_value", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Key of the record to retrieve." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 703, + "kind": 32768, + "kindString": "Parameter", + "name": "key", + "type": { + "name": "str", + "type": "reference" + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Default value returned in case the record does not exist.\n" + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": false + }, + "id": 704, + "kind": 32768, + "kindString": "Parameter", + "name": "default_value", + "type": { + "name": "T | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "T", + "target": "26" + }, + { + "type": "literal", + "value": null + } + ] + } + } + ], + "type": { + "name": "T | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "T", + "target": "26" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "Any: The value associated with the given key. `default_value` is used in case the record does not exist." + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Get a value from the key-value store.\n" + } + ] + }, + "flags": {}, + "id": 724, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "get_value", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Key of the record to retrieve." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 725, + "kind": 32768, + "kindString": "Parameter", + "name": "key", + "type": { + "name": "str", + "type": "reference" + } + } + ], + "type": { + "name": "Any", + "type": "reference" + } + }, + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "Any: The value associated with the given key. `default_value` is used in case the record does not exist." + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Get a value from the key-value store.\n" + } + ] + }, + "flags": {}, + "id": 726, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "get_value", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Key of the record to retrieve." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 727, + "kind": 32768, + "kindString": "Parameter", + "name": "key", + "type": { + "name": "str", + "type": "reference" + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Default value returned in case the record does not exist.\n" + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 728, + "kind": 32768, + "kindString": "Parameter", + "name": "default_value", + "type": { + "name": "T", + "type": "reference", + "target": "26" + } + } + ], + "type": { + "name": "T", + "type": "reference", + "target": "26" + } + }, + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "Any: The value associated with the given key. `default_value` is used in case the record does not exist." + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Get a value from the key-value store.\n" + } + ] + }, + "flags": {}, + "id": 729, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "get_value", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Key of the record to retrieve." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 730, + "kind": 32768, + "kindString": "Parameter", + "name": "key", + "type": { + "name": "str", + "type": "reference" + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Default value returned in case the record does not exist.\n" + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": false + }, + "id": 731, + "kind": 32768, + "kindString": "Parameter", + "name": "default_value", + "type": { + "name": "T | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "T", + "target": "26" + }, + { + "type": "literal", + "value": null + } + ] + } + } + ], + "type": { + "name": "T | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "T", + "target": "26" + }, + { + "type": "literal", + "value": null + } + ] + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Iterate over the keys in the key-value store.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 705, + "module": "storages.key_value_store", + "name": "iterate_keys", + "parsedDocstring": { + "text": "Iterate over the keys in the key-value store.\n", + "args": { + "exclusive_start_key": "All keys up to this one (including) are skipped from the result.\n" + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storages/key_value_store.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 154 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Iterate over the keys in the key-value store.\n" + } + ] + }, + "flags": {}, + "id": 706, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "iterate_keys", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "All keys up to this one (including) are skipped from the result.\n" + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": false + }, + "id": 707, + "kind": 32768, + "kindString": "Parameter", + "name": "exclusive_start_key", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + } + ], + "type": { + "name": "AsyncIterator", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "IterateKeysTuple", + "target": "697" + } + ] + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Set or delete a value in the key-value store.\n" + } + ] + }, + "decorations": [ + { + "name": "classmethod" + } + ], + "flags": {}, + "groups": [], + "id": 708, + "module": "storages.key_value_store", + "name": "set_value", + "parsedDocstring": { + "text": "Set or delete a value in the key-value store.\n", + "args": { + "key": "The key under which the value should be saved.", + "value": "The value to save. If the value is `None`, the corresponding key-value pair will be deleted.", + "content_type": "The content type of the saved value." + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storages/key_value_store.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 178 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Set or delete a value in the key-value store.\n" + } + ] + }, + "flags": {}, + "id": 709, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "set_value", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The key under which the value should be saved." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 710, + "kind": 32768, + "kindString": "Parameter", + "name": "key", + "type": { + "name": "str", + "type": "reference" + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The value to save. If the value is `None`, the corresponding key-value pair will be deleted." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 711, + "kind": 32768, + "kindString": "Parameter", + "name": "value", + "type": { + "name": "Any", + "type": "reference" + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The content type of the saved value." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": false + }, + "id": 712, + "kind": 32768, + "kindString": "Parameter", + "name": "content_type", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + } + ], + "type": { + "name": "None", + "type": "literal", + "value": null + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Get a URL for the given key that may be used to publicly access the value in the remote key-value store.\n" + } + ] + }, + "decorations": [ + { + "name": "classmethod" + } + ], + "flags": {}, + "groups": [], + "id": 713, + "module": "storages.key_value_store", + "name": "get_public_url", + "parsedDocstring": { + "text": "Get a URL for the given key that may be used to publicly access the value in the remote key-value store.\n", + "args": { + "key": "The key for which the URL should be generated." + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storages/key_value_store.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 206 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Get a URL for the given key that may be used to publicly access the value in the remote key-value store.\n" + } + ] + }, + "flags": {}, + "id": 714, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "get_public_url", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The key for which the URL should be generated." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 715, + "kind": 32768, + "kindString": "Parameter", + "name": "key", + "type": { + "name": "str", + "type": "reference" + } + } + ], + "type": { + "name": "str", + "type": "reference" + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Remove the key-value store either from the Apify cloud storage or from the local directory." + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 716, + "module": "storages.key_value_store", + "name": "drop", + "parsedDocstring": { + "text": "Remove the key-value store either from the Apify cloud storage or from the local directory." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storages/key_value_store.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 223 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Remove the key-value store either from the Apify cloud storage or from the local directory." + } + ] + }, + "flags": {}, + "id": 717, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "drop", + "parameters": [], + "type": { + "name": "None", + "type": "literal", + "value": null + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Open a key-value store.\n\nKey-value stores are used to store records or files, along with their MIME content type.\nThe records are stored and retrieved using a unique key.\nThe actual data is stored either on a local filesystem or in the Apify cloud.\n" + } + ] + }, + "decorations": [ + { + "name": "classmethod" + } + ], + "flags": {}, + "groups": [], + "id": 718, + "module": "storages.key_value_store", + "name": "open", + "parsedDocstring": { + "text": "Open a key-value store.\n\nKey-value stores are used to store records or files, along with their MIME content type.\nThe records are stored and retrieved using a unique key.\nThe actual data is stored either on a local filesystem or in the Apify cloud.\n", + "args": { + "id": "ID of the key-value store to be opened.\nIf neither `id` nor `name` are provided, the method returns the default key-value store associated with the actor run.\nIf the key-value store with the given ID does not exist, it raises an error.", + "name": "Name of the key-value store to be opened.\nIf neither `id` nor `name` are provided, the method returns the default key-value store associated with the actor run.\nIf the key-value store with the given name does not exist, it is created.", + "force_cloud": "If set to True, it will open a key-value store on the Apify Platform even when running the actor locally.\nDefaults to False.", + "config": "A `Configuration` instance, uses global configuration if omitted.\n" + }, + "returns": "KeyValueStore: An instance of the `KeyValueStore` class for the given ID or name." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storages/key_value_store.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 229 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "KeyValueStore: An instance of the `KeyValueStore` class for the given ID or name." + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Open a key-value store.\n\nKey-value stores are used to store records or files, along with their MIME content type.\nThe records are stored and retrieved using a unique key.\nThe actual data is stored either on a local filesystem or in the Apify cloud.\n" + } + ] + }, + "flags": {}, + "id": 719, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "open", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "ID of the key-value store to be opened.\nIf neither `id` nor `name` are provided, the method returns the default key-value store associated with the actor run.\nIf the key-value store with the given ID does not exist, it raises an error." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 720, + "kind": 32768, + "kindString": "Parameter", + "name": "id", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Name of the key-value store to be opened.\nIf neither `id` nor `name` are provided, the method returns the default key-value store associated with the actor run.\nIf the key-value store with the given name does not exist, it is created." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 721, + "kind": 32768, + "kindString": "Parameter", + "name": "name", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "If set to True, it will open a key-value store on the Apify Platform even when running the actor locally.\nDefaults to False." + } + ] + }, + "defaultValue": "False", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 722, + "kind": 32768, + "kindString": "Parameter", + "name": "force_cloud", + "type": { + "name": "bool", + "type": "reference" + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "A `Configuration` instance, uses global configuration if omitted.\n" + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 723, + "kind": 32768, + "kindString": "Parameter", + "name": "config", + "type": { + "name": "Configuration | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "Configuration", + "target": "1" + }, + { + "type": "literal", + "value": null + } + ] + } + } + ], + "type": { + "name": "KeyValueStore", + "type": "reference", + "target": "700" + } + } + ] + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "The `KeyValueStore` class represents a key-value store.\n\nYou can imagine it as a simple data storage that is used\nfor saving and reading data records or files. Each data record is\nrepresented by a unique key and associated with a MIME content type.\n\nDo not instantiate this class directly, use the `Actor.open_key_value_store()` function instead.\n\nEach crawler run is associated with a default key-value store, which is created exclusively\nfor the run. By convention, the crawler input and output are stored into the\ndefault key-value store under the `INPUT` and `OUTPUT` key, respectively.\nTypically, input and output are JSON files, although it can be any other format.\nTo access the default key-value store directly, you can use the\n`KeyValueStore.get_value` and `KeyValueStore.set_value` convenience functions.\n\n`KeyValueStore` stores its data either on local disk or in the Apify cloud,\ndepending on whether the `APIFY_LOCAL_STORAGE_DIR` or `APIFY_TOKEN` environment variables are set.\n\nIf the `APIFY_LOCAL_STORAGE_DIR` environment variable is set, the data is stored in\nthe local directory in the following files:\n```\n{APIFY_LOCAL_STORAGE_DIR}/key_value_stores/{STORE_ID}/{INDEX}.{EXT}\n```\n\nNote that `{STORE_ID}` is the name or ID of the key-value store. The default key-value store has ID: `default`,\nunless you override it by setting the `APIFY_DEFAULT_KEY_VALUE_STORE_ID` environment variable.\nThe `{KEY}` is the key of the record and `{EXT}` corresponds to the MIME content type of the data value.\n\nIf the `APIFY_TOKEN` environment variable is set but `APIFY_LOCAL_STORAGE_DIR` is not, the data is stored in the\n[Apify Key-value store](https://docs.apify.com/storage/key-value-store) cloud storage." + } + ] + }, + "flags": {}, + "groups": [ + { + "children": [ + 716, + 713, + 701, + 705, + 718, + 708 + ], + "title": "Methods" + } + ], + "id": 700, + "module": "storages.key_value_store", + "name": "KeyValueStore", + "parsedDocstring": { + "text": "The `KeyValueStore` class represents a key-value store.\n\nYou can imagine it as a simple data storage that is used\nfor saving and reading data records or files. Each data record is\nrepresented by a unique key and associated with a MIME content type.\n\nDo not instantiate this class directly, use the `Actor.open_key_value_store()` function instead.\n\nEach crawler run is associated with a default key-value store, which is created exclusively\nfor the run. By convention, the crawler input and output are stored into the\ndefault key-value store under the `INPUT` and `OUTPUT` key, respectively.\nTypically, input and output are JSON files, although it can be any other format.\nTo access the default key-value store directly, you can use the\n`KeyValueStore.get_value` and `KeyValueStore.set_value` convenience functions.\n\n`KeyValueStore` stores its data either on local disk or in the Apify cloud,\ndepending on whether the `APIFY_LOCAL_STORAGE_DIR` or `APIFY_TOKEN` environment variables are set.\n\nIf the `APIFY_LOCAL_STORAGE_DIR` environment variable is set, the data is stored in\nthe local directory in the following files:\n```\n{APIFY_LOCAL_STORAGE_DIR}/key_value_stores/{STORE_ID}/{INDEX}.{EXT}\n```\n\nNote that `{STORE_ID}` is the name or ID of the key-value store. The default key-value store has ID: `default`,\nunless you override it by setting the `APIFY_DEFAULT_KEY_VALUE_STORE_ID` environment variable.\nThe `{KEY}` is the key of the record and `{EXT}` corresponds to the MIME content type of the data value.\n\nIf the `APIFY_TOKEN` environment variable is set but `APIFY_LOCAL_STORAGE_DIR` is not, the data is stored in the\n[Apify Key-value store](https://docs.apify.com/storage/key-value-store) cloud storage." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storages/key_value_store.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 35 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Create a `StorageClientManager` instance." + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 732, + "module": "storages.storage_client_manager", + "name": "__init__", + "parsedDocstring": { + "text": "Create a `StorageClientManager` instance." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storages/storage_client_manager.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 25 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Create a `StorageClientManager` instance." + } + ] + }, + "flags": {}, + "id": 733, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "__init__", + "parameters": [], + "type": { + "name": "None", + "type": "literal", + "value": null + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Set the config for the StorageClientManager.\n" + } + ] + }, + "decorations": [ + { + "name": "classmethod" + } + ], + "flags": {}, + "groups": [], + "id": 734, + "module": "storages.storage_client_manager", + "name": "set_config", + "parsedDocstring": { + "text": "Set the config for the StorageClientManager.\n", + "args": { + "config": "The configuration this StorageClientManager should use." + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storages/storage_client_manager.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 30 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Set the config for the StorageClientManager.\n" + } + ] + }, + "flags": {}, + "id": 735, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "set_config", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The configuration this StorageClientManager should use." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 736, + "kind": 32768, + "kindString": "Parameter", + "name": "config", + "type": { + "name": "Configuration", + "type": "reference", + "target": "1" + } + } + ], + "type": { + "name": "None", + "type": "literal", + "value": null + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Get the current storage client instance.\n" + } + ] + }, + "decorations": [ + { + "name": "classmethod" + } + ], + "flags": {}, + "groups": [], + "id": 737, + "module": "storages.storage_client_manager", + "name": "get_storage_client", + "parsedDocstring": { + "text": "Get the current storage client instance.\n", + "returns": "ApifyClientAsync or MemoryStorageClient: The current storage client instance." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storages/storage_client_manager.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 39 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "ApifyClientAsync or MemoryStorageClient: The current storage client instance." + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Get the current storage client instance.\n" + } + ] + }, + "flags": {}, + "id": 738, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "get_storage_client", + "parameters": [ + { + "defaultValue": "False", + "flags": { + "isOptional": true, + "keyword-only": false + }, + "id": 739, + "kind": 32768, + "kindString": "Parameter", + "name": "force_cloud", + "type": { + "name": "bool", + "type": "reference" + } + } + ], + "type": { + "name": "ApifyClientAsync | MemoryStorageClient", + "type": "union", + "types": [ + { + "type": "reference", + "name": "ApifyClientAsync" + }, + { + "type": "reference", + "name": "MemoryStorageClient" + } + ] + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Set the storage client.\n" + } + ] + }, + "decorations": [ + { + "name": "classmethod" + } + ], + "flags": {}, + "groups": [], + "id": 740, + "module": "storages.storage_client_manager", + "name": "set_cloud_client", + "parsedDocstring": { + "text": "Set the storage client.\n", + "args": { + "client": "The instance of a storage client." + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storages/storage_client_manager.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 59 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Set the storage client.\n" + } + ] + }, + "flags": {}, + "id": 741, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "set_cloud_client", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The instance of a storage client." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 742, + "kind": 32768, + "kindString": "Parameter", + "name": "client", + "type": { + "name": "ApifyClientAsync", + "type": "reference" + } + } + ], + "type": { + "name": "None", + "type": "literal", + "value": null + } + } + ] + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 743, + "module": "storages.base_storage", + "name": "BaseResourceClientType", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storages/base_storage.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 17 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 744, + "module": "storages.base_storage", + "name": "BaseResourceCollectionClientType", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storages/base_storage.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 18 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Initialize the storage.\n\nDo not use this method directly, but use `Actor.open_()` instead.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 745, + "module": "storages.base_storage", + "name": "__init__", + "parsedDocstring": { + "text": "Initialize the storage.\n\nDo not use this method directly, but use `Actor.open_()` instead.\n", + "args": { + "id": "The storage id", + "name": "The storage name", + "client": "The storage client", + "config": "The configuration" + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storages/base_storage.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 34 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Initialize the storage.\n\nDo not use this method directly, but use `Actor.open_()` instead.\n" + } + ] + }, + "flags": {}, + "id": 746, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "__init__", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The storage id" + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 747, + "kind": 32768, + "kindString": "Parameter", + "name": "id", + "type": { + "name": "str", + "type": "reference" + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The storage name" + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 748, + "kind": 32768, + "kindString": "Parameter", + "name": "name", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The storage client" + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 749, + "kind": 32768, + "kindString": "Parameter", + "name": "client", + "type": { + "name": "ApifyClientAsync | MemoryStorageClient", + "type": "union", + "types": [ + { + "type": "reference", + "name": "ApifyClientAsync" + }, + { + "type": "reference", + "name": "MemoryStorageClient" + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The configuration" + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 750, + "kind": 32768, + "kindString": "Parameter", + "name": "config", + "type": { + "name": "Configuration", + "type": "reference", + "target": "1" + } + } + ], + "type": { + "name": "None", + "type": "literal", + "value": null + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Open a storage, or return a cached storage object if it was opened before.\n\nOpens a storage with the given ID or name.\nReturns the cached storage object if the storage was opened before.\n" + } + ] + }, + "decorations": [ + { + "name": "classmethod" + }, + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 751, + "module": "storages.base_storage", + "name": "open", + "parsedDocstring": { + "text": "Open a storage, or return a cached storage object if it was opened before.\n\nOpens a storage with the given ID or name.\nReturns the cached storage object if the storage was opened before.\n", + "args": { + "id": "ID of the storage to be opened.\nIf neither `id` nor `name` are provided, the method returns the default storage associated with the actor run.\nIf the storage with the given ID does not exist, it raises an error.", + "name": "Name of the storage to be opened.\nIf neither `id` nor `name` are provided, the method returns the default storage associated with the actor run.\nIf the storage with the given name does not exist, it is created.", + "force_cloud": "If set to True, it will open a storage on the Apify Platform even when running the actor locally.\nDefaults to False.", + "config": "A `Configuration` instance, uses global configuration if omitted.\n" + }, + "returns": "An instance of the storage." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storages/base_storage.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 94 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "An instance of the storage." + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Open a storage, or return a cached storage object if it was opened before.\n\nOpens a storage with the given ID or name.\nReturns the cached storage object if the storage was opened before.\n" + } + ] + }, + "flags": {}, + "id": 752, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "open", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "ID of the storage to be opened.\nIf neither `id` nor `name` are provided, the method returns the default storage associated with the actor run.\nIf the storage with the given ID does not exist, it raises an error." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 753, + "kind": 32768, + "kindString": "Parameter", + "name": "id", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Name of the storage to be opened.\nIf neither `id` nor `name` are provided, the method returns the default storage associated with the actor run.\nIf the storage with the given name does not exist, it is created." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 754, + "kind": 32768, + "kindString": "Parameter", + "name": "name", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "If set to True, it will open a storage on the Apify Platform even when running the actor locally.\nDefaults to False." + } + ] + }, + "defaultValue": "False", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 755, + "kind": 32768, + "kindString": "Parameter", + "name": "force_cloud", + "type": { + "name": "bool", + "type": "reference" + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "A `Configuration` instance, uses global configuration if omitted.\n" + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 756, + "kind": 32768, + "kindString": "Parameter", + "name": "config", + "type": { + "name": "Configuration | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "Configuration", + "target": "1" + }, + { + "type": "literal", + "value": null + } + ] + } + } + ], + "type": { + "name": "BaseStorage", + "type": "reference" + } + } + ] + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 757, + "module": "storages.request_queue", + "name": "MAX_CACHED_REQUESTS", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storages/request_queue.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 26 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 758, + "module": "storages.request_queue", + "name": "QUERY_HEAD_MIN_LENGTH", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storages/request_queue.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 29 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 759, + "module": "storages.request_queue", + "name": "QUERY_HEAD_BUFFER", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storages/request_queue.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 31 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 760, + "module": "storages.request_queue", + "name": "API_PROCESSED_REQUESTS_DELAY_MILLIS", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storages/request_queue.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 35 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 761, + "module": "storages.request_queue", + "name": "MAX_QUERIES_FOR_CONSISTENCY", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storages/request_queue.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 38 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 762, + "module": "storages.request_queue", + "name": "RECENTLY_HANDLED_CACHE_SIZE", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storages/request_queue.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 42 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 763, + "module": "storages.request_queue", + "name": "STORAGE_CONSISTENCY_DELAY_MILLIS", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storages/request_queue.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 46 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Adds a request to the `RequestQueue` while managing deduplication and positioning within the queue.\n\nThe deduplication of requests relies on the `uniqueKey` field within the request dictionary. If `uniqueKey`\nexists, it remains unchanged; if it does not, it is generated based on the request's `url`, `method`,\nand `payload` fields. The generation of `uniqueKey` can be influenced by the `keep_url_fragment` and\n`use_extended_unique_key` flags, which dictate whether to include the URL fragment and the request's method\nand payload, respectively, in its computation.\n\nThe request can be added to the forefront (beginning) or the back of the queue based on the `forefront`\nparameter. Information about the request's addition to the queue, including whether it was already present or\nhandled, is returned in an output dictionary.\n\n\nReturns: A dictionary containing information about the operation, including:\n- `requestId` (str): The ID of the request.\n- `uniqueKey` (str): The unique key associated with the request.\n- `wasAlreadyPresent` (bool): Indicates whether the request was already in the queue.\n- `wasAlreadyHandled` (bool): Indicates whether the request was already processed." + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 765, + "module": "storages.request_queue", + "name": "add_request", + "parsedDocstring": { + "text": "Adds a request to the `RequestQueue` while managing deduplication and positioning within the queue.\n\nThe deduplication of requests relies on the `uniqueKey` field within the request dictionary. If `uniqueKey`\nexists, it remains unchanged; if it does not, it is generated based on the request's `url`, `method`,\nand `payload` fields. The generation of `uniqueKey` can be influenced by the `keep_url_fragment` and\n`use_extended_unique_key` flags, which dictate whether to include the URL fragment and the request's method\nand payload, respectively, in its computation.\n\nThe request can be added to the forefront (beginning) or the back of the queue based on the `forefront`\nparameter. Information about the request's addition to the queue, including whether it was already present or\nhandled, is returned in an output dictionary.\n\n\nReturns: A dictionary containing information about the operation, including:\n- `requestId` (str): The ID of the request.\n- `uniqueKey` (str): The unique key associated with the request.\n- `wasAlreadyPresent` (bool): Indicates whether the request was already in the queue.\n- `wasAlreadyHandled` (bool): Indicates whether the request was already processed.", + "args": { + "request": "The request object to be added to the queue. Must include at least the `url` key.\nOptionaly it can include the `method`, `payload` and `uniqueKey` keys.\n", + "forefront": "If True, adds the request to the forefront of the queue; otherwise, adds it to the end.\n", + "keep_url_fragment": "Determines whether the URL fragment (the part of the URL after '#') should be retained\nin the unique key computation.\n", + "use_extended_unique_key": "Determines whether to use an extended unique key, incorporating the request's\nmethod and payload into the unique key computation.\n" + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storages/request_queue.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 143 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Adds a request to the `RequestQueue` while managing deduplication and positioning within the queue.\n\nThe deduplication of requests relies on the `uniqueKey` field within the request dictionary. If `uniqueKey`\nexists, it remains unchanged; if it does not, it is generated based on the request's `url`, `method`,\nand `payload` fields. The generation of `uniqueKey` can be influenced by the `keep_url_fragment` and\n`use_extended_unique_key` flags, which dictate whether to include the URL fragment and the request's method\nand payload, respectively, in its computation.\n\nThe request can be added to the forefront (beginning) or the back of the queue based on the `forefront`\nparameter. Information about the request's addition to the queue, including whether it was already present or\nhandled, is returned in an output dictionary.\n\n\nReturns: A dictionary containing information about the operation, including:\n- `requestId` (str): The ID of the request.\n- `uniqueKey` (str): The unique key associated with the request.\n- `wasAlreadyPresent` (bool): Indicates whether the request was already in the queue.\n- `wasAlreadyHandled` (bool): Indicates whether the request was already processed." + } + ] + }, + "flags": {}, + "id": 766, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "add_request", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The request object to be added to the queue. Must include at least the `url` key.\nOptionaly it can include the `method`, `payload` and `uniqueKey` keys.\n" + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 767, + "kind": 32768, + "kindString": "Parameter", + "name": "request", + "type": { + "name": "dict", + "type": "reference" + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "If True, adds the request to the forefront of the queue; otherwise, adds it to the end.\n" + } + ] + }, + "defaultValue": "False", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 768, + "kind": 32768, + "kindString": "Parameter", + "name": "forefront", + "type": { + "name": "bool", + "type": "reference" + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Determines whether the URL fragment (the part of the URL after '#') should be retained\nin the unique key computation.\n" + } + ] + }, + "defaultValue": "False", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 769, + "kind": 32768, + "kindString": "Parameter", + "name": "keep_url_fragment", + "type": { + "name": "bool", + "type": "reference" + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Determines whether to use an extended unique key, incorporating the request's\nmethod and payload into the unique key computation.\n" + } + ] + }, + "defaultValue": "False", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 770, + "kind": 32768, + "kindString": "Parameter", + "name": "use_extended_unique_key", + "type": { + "name": "bool", + "type": "reference" + } + } + ], + "type": { + "name": "dict", + "type": "reference" + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Retrieve a request from the queue.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 771, + "module": "storages.request_queue", + "name": "get_request", + "parsedDocstring": { + "text": "Retrieve a request from the queue.\n", + "args": { + "request_id": "ID of the request to retrieve.\n" + }, + "returns": "dict, optional: The retrieved request, or `None`, if it does not exist." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storages/request_queue.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 226 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "dict, optional: The retrieved request, or `None`, if it does not exist." + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Retrieve a request from the queue.\n" + } + ] + }, + "flags": {}, + "id": 772, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "get_request", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "ID of the request to retrieve.\n" + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 773, + "kind": 32768, + "kindString": "Parameter", + "name": "request_id", + "type": { + "name": "str", + "type": "reference" + } + } + ], + "type": { + "name": "dict | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "dict" + }, + { + "type": "literal", + "value": null + } + ] + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Return the next request in the queue to be processed.\n\nOnce you successfully finish processing of the request, you need to call\n`RequestQueue.mark_request_as_handled` to mark the request as handled in the queue.\nIf there was some error in processing the request, call `RequestQueue.reclaim_request` instead,\nso that the queue will give the request to some other consumer in another call to the `fetch_next_request` method.\n\nNote that the `None` return value does not mean the queue processing finished, it means there are currently no pending requests.\nTo check whether all requests in queue were finished, use `RequestQueue.is_finished` instead.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 774, + "module": "storages.request_queue", + "name": "fetch_next_request", + "parsedDocstring": { + "text": "Return the next request in the queue to be processed.\n\nOnce you successfully finish processing of the request, you need to call\n`RequestQueue.mark_request_as_handled` to mark the request as handled in the queue.\nIf there was some error in processing the request, call `RequestQueue.reclaim_request` instead,\nso that the queue will give the request to some other consumer in another call to the `fetch_next_request` method.\n\nNote that the `None` return value does not mean the queue processing finished, it means there are currently no pending requests.\nTo check whether all requests in queue were finished, use `RequestQueue.is_finished` instead.\n", + "returns": "dict, optional: The request or `None` if there are no more pending requests." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storages/request_queue.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 238 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "dict, optional: The request or `None` if there are no more pending requests." + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Return the next request in the queue to be processed.\n\nOnce you successfully finish processing of the request, you need to call\n`RequestQueue.mark_request_as_handled` to mark the request as handled in the queue.\nIf there was some error in processing the request, call `RequestQueue.reclaim_request` instead,\nso that the queue will give the request to some other consumer in another call to the `fetch_next_request` method.\n\nNote that the `None` return value does not mean the queue processing finished, it means there are currently no pending requests.\nTo check whether all requests in queue were finished, use `RequestQueue.is_finished` instead.\n" + } + ] + }, + "flags": {}, + "id": 775, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "fetch_next_request", + "parameters": [], + "type": { + "name": "dict | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "dict" + }, + { + "type": "literal", + "value": null + } + ] + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Mark a request as handled after successful processing.\n\nHandled requests will never again be returned by the `RequestQueue.fetch_next_request` method.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 776, + "module": "storages.request_queue", + "name": "mark_request_as_handled", + "parsedDocstring": { + "text": "Mark a request as handled after successful processing.\n\nHandled requests will never again be returned by the `RequestQueue.fetch_next_request` method.\n", + "args": { + "request": "The request to mark as handled.\n" + }, + "returns": "dict, optional: Information about the queue operation with keys `requestId`, `uniqueKey`, `wasAlreadyPresent`, `wasAlreadyHandled`.\n`None` if the given request was not in progress." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storages/request_queue.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 306 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "dict, optional: Information about the queue operation with keys `requestId`, `uniqueKey`, `wasAlreadyPresent`, `wasAlreadyHandled`.\n`None` if the given request was not in progress." + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Mark a request as handled after successful processing.\n\nHandled requests will never again be returned by the `RequestQueue.fetch_next_request` method.\n" + } + ] + }, + "flags": {}, + "id": 777, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "mark_request_as_handled", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The request to mark as handled.\n" + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 778, + "kind": 32768, + "kindString": "Parameter", + "name": "request", + "type": { + "name": "dict", + "type": "reference" + } + } + ], + "type": { + "name": "dict | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "dict" + }, + { + "type": "literal", + "value": null + } + ] + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Reclaim a failed request back to the queue.\n\nThe request will be returned for processing later again\nby another call to `RequestQueue.fetchNextRequest`.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 779, + "module": "storages.request_queue", + "name": "reclaim_request", + "parsedDocstring": { + "text": "Reclaim a failed request back to the queue.\n\nThe request will be returned for processing later again\nby another call to `RequestQueue.fetchNextRequest`.\n", + "args": { + "request": "The request to return to the queue.", + "forefront": "Whether to add the request to the head or the end of the queue" + }, + "returns": "dict, optional: Information about the queue operation with keys `requestId`, `uniqueKey`, `wasAlreadyPresent`, `wasAlreadyHandled`.\n`None` if the given request was not in progress." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storages/request_queue.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 345 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "dict, optional: Information about the queue operation with keys `requestId`, `uniqueKey`, `wasAlreadyPresent`, `wasAlreadyHandled`.\n`None` if the given request was not in progress." + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Reclaim a failed request back to the queue.\n\nThe request will be returned for processing later again\nby another call to `RequestQueue.fetchNextRequest`.\n" + } + ] + }, + "flags": {}, + "id": 780, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "reclaim_request", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The request to return to the queue." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 781, + "kind": 32768, + "kindString": "Parameter", + "name": "request", + "type": { + "name": "dict", + "type": "reference" + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Whether to add the request to the head or the end of the queue" + } + ] + }, + "defaultValue": "False", + "flags": { + "isOptional": true, + "keyword-only": false + }, + "id": 782, + "kind": 32768, + "kindString": "Parameter", + "name": "forefront", + "type": { + "name": "bool", + "type": "reference" + } + } + ], + "type": { + "name": "dict | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "dict" + }, + { + "type": "literal", + "value": null + } + ] + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Check whether the queue is empty.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 783, + "module": "storages.request_queue", + "name": "is_empty", + "parsedDocstring": { + "text": "Check whether the queue is empty.\n", + "returns": "bool: `True` if the next call to `RequestQueue.fetchNextRequest` would return `None`, otherwise `False`." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storages/request_queue.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 401 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "bool: `True` if the next call to `RequestQueue.fetchNextRequest` would return `None`, otherwise `False`." + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Check whether the queue is empty.\n" + } + ] + }, + "flags": {}, + "id": 784, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "is_empty", + "parameters": [], + "type": { + "name": "bool", + "type": "reference" + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Check whether the queue is finished.\n\nDue to the nature of distributed storage used by the queue,\nthe function might occasionally return a false negative,\nbut it will never return a false positive.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 785, + "module": "storages.request_queue", + "name": "is_finished", + "parsedDocstring": { + "text": "Check whether the queue is finished.\n\nDue to the nature of distributed storage used by the queue,\nthe function might occasionally return a false negative,\nbut it will never return a false positive.\n", + "returns": "bool: `True` if all requests were already handled and there are no more left. `False` otherwise." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storages/request_queue.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 410 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "bool: `True` if all requests were already handled and there are no more left. `False` otherwise." + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Check whether the queue is finished.\n\nDue to the nature of distributed storage used by the queue,\nthe function might occasionally return a false negative,\nbut it will never return a false positive.\n" + } + ] + }, + "flags": {}, + "id": 786, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "is_finished", + "parameters": [], + "type": { + "name": "bool", + "type": "reference" + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Remove the request queue either from the Apify cloud storage or from the local directory." + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 787, + "module": "storages.request_queue", + "name": "drop", + "parsedDocstring": { + "text": "Remove the request queue either from the Apify cloud storage or from the local directory." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storages/request_queue.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 557 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Remove the request queue either from the Apify cloud storage or from the local directory." + } + ] + }, + "flags": {}, + "id": 788, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "drop", + "parameters": [], + "type": { + "name": "None", + "type": "literal", + "value": null + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Get an object containing general information about the request queue.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 789, + "module": "storages.request_queue", + "name": "get_info", + "parsedDocstring": { + "text": "Get an object containing general information about the request queue.\n", + "returns": "dict: Object returned by calling the GET request queue API endpoint." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storages/request_queue.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 562 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "dict: Object returned by calling the GET request queue API endpoint." + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Get an object containing general information about the request queue.\n" + } + ] + }, + "flags": {}, + "id": 790, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "get_info", + "parameters": [], + "type": { + "name": "dict | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "dict" + }, + { + "type": "literal", + "value": null + } + ] + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Open a request queue.\n\nRequest queue represents a queue of URLs to crawl, which is stored either on local filesystem or in the Apify cloud.\nThe queue is used for deep crawling of websites, where you start with several URLs and then\nrecursively follow links to other pages. The data structure supports both breadth-first\nand depth-first crawling orders.\n" + } + ] + }, + "decorations": [ + { + "name": "classmethod" + } + ], + "flags": {}, + "groups": [], + "id": 791, + "module": "storages.request_queue", + "name": "open", + "parsedDocstring": { + "text": "Open a request queue.\n\nRequest queue represents a queue of URLs to crawl, which is stored either on local filesystem or in the Apify cloud.\nThe queue is used for deep crawling of websites, where you start with several URLs and then\nrecursively follow links to other pages. The data structure supports both breadth-first\nand depth-first crawling orders.\n", + "args": { + "id": "ID of the request queue to be opened.\nIf neither `id` nor `name` are provided, the method returns the default request queue associated with the actor run.\nIf the request queue with the given ID does not exist, it raises an error.", + "name": "Name of the request queue to be opened.\nIf neither `id` nor `name` are provided, the method returns the default request queue associated with the actor run.\nIf the request queue with the given name does not exist, it is created.", + "force_cloud": "If set to True, it will open a request queue on the Apify Platform even when running the actor locally.\nDefaults to False.", + "config": "A `Configuration` instance, uses global configuration if omitted.\n" + }, + "returns": "RequestQueue: An instance of the `RequestQueue` class for the given ID or name." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storages/request_queue.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 571 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "RequestQueue: An instance of the `RequestQueue` class for the given ID or name." + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Open a request queue.\n\nRequest queue represents a queue of URLs to crawl, which is stored either on local filesystem or in the Apify cloud.\nThe queue is used for deep crawling of websites, where you start with several URLs and then\nrecursively follow links to other pages. The data structure supports both breadth-first\nand depth-first crawling orders.\n" + } + ] + }, + "flags": {}, + "id": 792, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "open", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "ID of the request queue to be opened.\nIf neither `id` nor `name` are provided, the method returns the default request queue associated with the actor run.\nIf the request queue with the given ID does not exist, it raises an error." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 793, + "kind": 32768, + "kindString": "Parameter", + "name": "id", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Name of the request queue to be opened.\nIf neither `id` nor `name` are provided, the method returns the default request queue associated with the actor run.\nIf the request queue with the given name does not exist, it is created." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 794, + "kind": 32768, + "kindString": "Parameter", + "name": "name", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "If set to True, it will open a request queue on the Apify Platform even when running the actor locally.\nDefaults to False." + } + ] + }, + "defaultValue": "False", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 795, + "kind": 32768, + "kindString": "Parameter", + "name": "force_cloud", + "type": { + "name": "bool", + "type": "reference" + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "A `Configuration` instance, uses global configuration if omitted.\n" + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 796, + "kind": 32768, + "kindString": "Parameter", + "name": "config", + "type": { + "name": "Configuration | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "Configuration", + "target": "1" + }, + { + "type": "literal", + "value": null + } + ] + } + } + ], + "type": { + "name": "RequestQueue", + "type": "reference", + "target": "764" + } + } + ] + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Represents a queue of URLs to crawl.\n\nCan be used for deep crawling of websites where you start with several URLs and then recursively\nfollow links to other pages. The data structure supports both breadth-first and depth-first crawling orders.\n\nEach URL is represented using an instance of the {@apilink Request} class.\nThe queue can only contain unique URLs. More precisely, it can only contain request dictionaries\nwith distinct `uniqueKey` properties. By default, `uniqueKey` is generated from the URL, but it can also be overridden.\nTo add a single URL multiple times to the queue,\ncorresponding request dictionary will need to have different `uniqueKey` properties.\n\nDo not instantiate this class directly, use the `Actor.open_request_queue()` function instead.\n\n`RequestQueue` stores its data either on local disk or in the Apify cloud,\ndepending on whether the `APIFY_LOCAL_STORAGE_DIR` or `APIFY_TOKEN` environment variables are set.\n\nIf the `APIFY_LOCAL_STORAGE_DIR` environment variable is set, the data is stored in\nthe local directory in the following files:\n```\n{APIFY_LOCAL_STORAGE_DIR}/request_queues/{QUEUE_ID}/{REQUEST_ID}.json\n```\n\nNote that `{QUEUE_ID}` is the name or ID of the request queue. The default request queue has ID: `default`,\nunless you override it by setting the `APIFY_DEFAULT_REQUEST_QUEUE_ID` environment variable.\nThe `{REQUEST_ID}` is the id of the request.\n\nIf the `APIFY_TOKEN` environment variable is set but `APIFY_LOCAL_STORAGE_DIR` is not, the data is stored in the\n[Apify Request Queue](https://docs.apify.com/storage/request-queue)\ncloud storage." + } + ] + }, + "flags": {}, + "groups": [ + { + "children": [ + 765, + 787, + 774, + 789, + 771, + 783, + 785, + 776, + 791, + 779 + ], + "title": "Methods" + } + ], + "id": 764, + "module": "storages.request_queue", + "name": "RequestQueue", + "parsedDocstring": { + "text": "Represents a queue of URLs to crawl.\n\nCan be used for deep crawling of websites where you start with several URLs and then recursively\nfollow links to other pages. The data structure supports both breadth-first and depth-first crawling orders.\n\nEach URL is represented using an instance of the {@apilink Request} class.\nThe queue can only contain unique URLs. More precisely, it can only contain request dictionaries\nwith distinct `uniqueKey` properties. By default, `uniqueKey` is generated from the URL, but it can also be overridden.\nTo add a single URL multiple times to the queue,\ncorresponding request dictionary will need to have different `uniqueKey` properties.\n\nDo not instantiate this class directly, use the `Actor.open_request_queue()` function instead.\n\n`RequestQueue` stores its data either on local disk or in the Apify cloud,\ndepending on whether the `APIFY_LOCAL_STORAGE_DIR` or `APIFY_TOKEN` environment variables are set.\n\nIf the `APIFY_LOCAL_STORAGE_DIR` environment variable is set, the data is stored in\nthe local directory in the following files:\n```\n{APIFY_LOCAL_STORAGE_DIR}/request_queues/{QUEUE_ID}/{REQUEST_ID}.json\n```\n\nNote that `{QUEUE_ID}` is the name or ID of the request queue. The default request queue has ID: `default`,\nunless you override it by setting the `APIFY_DEFAULT_REQUEST_QUEUE_ID` environment variable.\nThe `{REQUEST_ID}` is the id of the request.\n\nIf the `APIFY_TOKEN` environment variable is set but `APIFY_LOCAL_STORAGE_DIR` is not, the data is stored in the\n[Apify Request Queue](https://docs.apify.com/storage/request-queue)\ncloud storage." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storages/request_queue.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 49 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Get the storage ID." + } + ] + }, + "decorations": [ + { + "name": "property" + }, + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 3127, + "module": "storages._base", + "name": "id", + "parsedDocstring": { + "text": "Get the storage ID." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_base.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 20, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storages/_base.py#L20" + } + ], + "type": { + "name": "str", + "type": "reference", + "target": 0, + "ref": {} + }, + "permalink": "/python/api/class/Storage#id", + "parentId": 2330 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Get the storage name." + } + ] + }, + "decorations": [ + { + "name": "property" + }, + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 3128, + "module": "storages._base", + "name": "name", + "parsedDocstring": { + "text": "Get the storage name." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_base.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 25, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storages/_base.py#L25" + } + ], + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "permalink": "/python/api/class/Storage#name", + "parentId": 2330 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Get the storage metadata." + } + ] + }, + "decorations": [ + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 3129, + "module": "storages._base", + "name": "get_metadata", + "parsedDocstring": { + "text": "Get the storage metadata." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_base.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 29, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storages/_base.py#L29" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Get the storage metadata." + } + ] + }, + "flags": {}, + "id": 3130, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "get_metadata", + "parameters": [], + "type": { + "name": "DatasetMetadata | KeyValueStoreMetadata | RequestQueueMetadata", + "type": "union", + "types": [ + { + "type": "union", + "types": [ + { + "type": "reference", + "name": "DatasetMetadata", + "target": "1314", + "ref": { + "id": 2110, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 42 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/DatasetMetadata" + } + }, + { + "type": "reference", + "name": "KeyValueStoreMetadata", + "target": "1317", + "ref": { + "id": 2113, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 52 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/KeyValueStoreMetadata" + } + } + ] + }, + { + "type": "reference", + "name": "RequestQueueMetadata", + "target": "1319", + "ref": { + "id": 2115, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 59 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/RequestQueueMetadata" + } + } + ] + }, + "overwrites": { + "name": "Storage.get_metadata", + "target": 2333, + "type": "reference", + "ref": { + "id": 3129, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_base.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 29 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/Storage#get_metadata" + } + }, + "parentId": 2333 + } + ], + "permalink": "/python/api/class/Storage#get_metadata", + "parentId": 2330 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Open a storage, either restore existing or create a new one.\n" + } + ] + }, + "decorations": [ + { + "name": "classmethod" + }, + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 3131, + "module": "storages._base", + "name": "open", + "parsedDocstring": { + "text": "Open a storage, either restore existing or create a new one.\n", + "args": { + "id": "The storage ID.", + "name": "The storage name (global scope, persists across runs). Name can only contain letters \"a\" through \"z\",\nthe digits \"0\" through \"9\", and the hyphen (\"-\") but only in the middle of the string\n(e.g. \"my-value-1\").", + "alias": "The storage alias (run scope, creates unnamed storage).", + "configuration": "Configuration object used during the storage creation or restoration process.", + "storage_client": "Underlying storage client to use. If not provided, the default global storage client\nfrom the service locator will be used." + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_base.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 34, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storages/_base.py#L34" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Open a storage, either restore existing or create a new one.\n" + } + ] + }, + "flags": {}, + "id": 3132, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "open", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The storage ID." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 3133, + "kind": 32768, + "kindString": "Parameter", + "name": "id", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2336 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The storage name (global scope, persists across runs). Name can only contain letters \"a\" through \"z\",\nthe digits \"0\" through \"9\", and the hyphen (\"-\") but only in the middle of the string\n(e.g. \"my-value-1\")." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 3134, + "kind": 32768, + "kindString": "Parameter", + "name": "name", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2336 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The storage alias (run scope, creates unnamed storage)." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 3135, + "kind": 32768, + "kindString": "Parameter", + "name": "alias", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2336 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Configuration object used during the storage creation or restoration process." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 3136, + "kind": 32768, + "kindString": "Parameter", + "name": "configuration", + "type": { + "name": "Configuration | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "Configuration", + "target": "2", + "ref": { + "id": 798, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/configuration.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 20 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/Configuration" + } + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2336 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Underlying storage client to use. If not provided, the default global storage client\nfrom the service locator will be used." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 3137, + "kind": 32768, + "kindString": "Parameter", + "name": "storage_client", + "type": { + "name": "StorageClient | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "StorageClient", + "target": "1282", + "ref": { + "id": 2078, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 19 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/StorageClient" + } + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2336 + } + ], + "type": { + "name": "Storage", + "type": "reference", + "target": "2330", + "ref": { + "id": 3126, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_base.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 15 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/Storage" + } + }, + "overwrites": { + "name": "Storage.open", + "target": 2335, + "type": "reference", + "ref": { + "id": 3131, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_base.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 34 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/Storage#open" + } + }, + "parentId": 2335 + } + ], + "permalink": "/python/api/class/Storage#open", + "parentId": 2330 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Drop the storage, removing it from the underlying storage client and clearing the cache." + } + ] + }, + "decorations": [ + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 3138, + "module": "storages._base", + "name": "drop", + "parsedDocstring": { + "text": "Drop the storage, removing it from the underlying storage client and clearing the cache." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_base.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 57, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storages/_base.py#L57" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Drop the storage, removing it from the underlying storage client and clearing the cache." + } + ] + }, + "flags": {}, + "id": 3139, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "drop", + "parameters": [], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "overwrites": { + "name": "Storage.drop", + "target": 2342, + "type": "reference", + "ref": { + "id": 3138, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_base.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 57 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/Storage#drop" + } + }, + "parentId": 2342 + } + ], + "permalink": "/python/api/class/Storage#drop", + "parentId": 2330 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Purge the storage, removing all items from the underlying storage client.\n\nThis method does not remove the storage itself, e.g. don't remove the metadata,\nbut clears all items within it." + } + ] + }, + "decorations": [ + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 3140, + "module": "storages._base", + "name": "purge", + "parsedDocstring": { + "text": "Purge the storage, removing all items from the underlying storage client.\n\nThis method does not remove the storage itself, e.g. don't remove the metadata,\nbut clears all items within it." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_base.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 61, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storages/_base.py#L61" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Purge the storage, removing all items from the underlying storage client.\n\nThis method does not remove the storage itself, e.g. don't remove the metadata,\nbut clears all items within it." + } + ] + }, + "flags": {}, + "id": 3141, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "purge", + "parameters": [], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "overwrites": { + "name": "Storage.purge", + "target": 2344, + "type": "reference", + "ref": { + "id": 3140, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_base.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 61 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/Storage#purge" + } + }, + "parentId": 2344 + } + ], + "permalink": "/python/api/class/Storage#purge", + "parentId": 2330 + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Base class for storages." + } + ] + }, + "decorations": [ + { + "args": "('Storages')", + "name": "docs_group" + } + ], + "flags": {}, + "groups": [ + { + "children": [ + 3138, + 3129, + 3131, + 3140 + ], + "title": "Methods" + }, + { + "children": [ + 3127, + 3128 + ], + "title": "Properties" + } + ], + "id": 3126, + "module": "storages._base", + "name": "Storage", + "parsedDocstring": { + "text": "Base class for storages." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_base.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 15, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storages/_base.py#L15" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "extendedBy": [ + { + "name": "KeyValueStore", + "target": "2269", + "type": "reference", + "ref": { + "id": 3065, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_key_value_store.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 40 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/KeyValueStore" + } + }, + { + "name": "RequestQueue", + "target": "2348", + "type": "reference", + "ref": { + "id": 3144, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_request_queue.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 33 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/RequestQueue" + } + }, + { + "name": "Dataset", + "target": "2402", + "type": "reference", + "ref": { + "id": 3198, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_dataset.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 33 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/Dataset" + } + } + ], + "permalink": "/python/api/class/Storage", + "categories": [] + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Initialize a new instance.\n\nPreferably use the `Dataset.open` constructor to create a new instance.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 5601, + "module": "storages._dataset", + "name": "__init__", + "parsedDocstring": { + "text": "Initialize a new instance.\n\nPreferably use the `Dataset.open` constructor to create a new instance.\n", + "args": { + "client": "An instance of a storage client.", + "id": "The unique identifier of the storage.", + "name": "The name of the storage, if available." + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_dataset.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 69, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storages/_dataset.py#L69" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Initialize a new instance.\n\nPreferably use the `Dataset.open` constructor to create a new instance.\n" + } + ] + }, + "flags": {}, + "id": 5602, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "__init__", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "An instance of a storage client." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 5603, + "kind": 32768, + "kindString": "Parameter", + "name": "client", + "type": { + "name": "DatasetClient", + "type": "reference", + "target": "1220", + "ref": { + "id": 4418, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_dataset_client.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 13 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/DatasetClient" + } + }, + "parentId": 2404 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The unique identifier of the storage." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 5604, + "kind": 32768, + "kindString": "Parameter", + "name": "id", + "type": { + "name": "str", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2404 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The name of the storage, if available." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 5605, + "kind": 32768, + "kindString": "Parameter", + "name": "name", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2404 + } + ], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "parentId": 2403 + } + ], + "permalink": "/python/api/class/Dataset#__init__", + "parentId": 2402 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Get the storage ID." + } + ] + }, + "decorations": [ + { + "name": "property" + }, + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 5606, + "module": "storages._base", + "name": "id", + "parsedDocstring": { + "text": "Get the storage ID." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_dataset.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 87, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storages/_dataset.py#L87" + } + ], + "type": { + "name": "str", + "type": "reference", + "target": 0, + "ref": {} + }, + "overwrites": { + "name": "Storage.id", + "target": 2331, + "type": "reference", + "ref": { + "id": 5529, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_base.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 20 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api/class/Storage#id" + } + }, + "permalink": "/python/api/class/Dataset#id", + "parentId": 2402 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Get the storage name." + } + ] + }, + "decorations": [ + { + "name": "property" + }, + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 5607, + "module": "storages._base", + "name": "name", + "parsedDocstring": { + "text": "Get the storage name." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_dataset.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 92, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storages/_dataset.py#L92" + } + ], + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "overwrites": { + "name": "Storage.name", + "target": 2332, + "type": "reference", + "ref": { + "id": 5530, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_base.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 25 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api/class/Storage#name" + } + }, + "permalink": "/python/api/class/Dataset#name", + "parentId": 2402 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Get the storage metadata." + } + ] + }, + "decorations": [ + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 5608, + "module": "storages._base", + "name": "get_metadata", + "parsedDocstring": { + "text": "Get the storage metadata." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_dataset.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 96, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storages/_dataset.py#L96" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Get the storage metadata." + } + ] + }, + "flags": {}, + "id": 5532, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "get_metadata", + "parameters": [], + "type": { + "name": "DatasetMetadata | KeyValueStoreMetadata | RequestQueueMetadata", + "type": "union", + "types": [ + { + "type": "union", + "types": [ + { + "type": "reference", + "name": "DatasetMetadata", + "target": "1314", + "ref": { + "id": 4512, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 42 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/DatasetMetadata" + } + }, + { + "type": "reference", + "name": "KeyValueStoreMetadata", + "target": "1317", + "ref": { + "id": 4515, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 52 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/KeyValueStoreMetadata" + } + } + ] + }, + { + "type": "reference", + "name": "RequestQueueMetadata", + "target": "1319", + "ref": { + "id": 4517, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 59 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/RequestQueueMetadata" + } + } + ] + }, + "overwrites": { + "name": "Storage.get_metadata", + "target": 2333, + "type": "reference", + "ref": { + "id": 5531, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_base.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 29 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/Storage#get_metadata" + } + }, + "parentId": 2410 + } + ], + "overwrites": { + "name": "Storage.get_metadata", + "target": 2333, + "type": "reference", + "ref": { + "id": 5531, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_base.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 29 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/Storage#get_metadata" + } + }, + "permalink": "/python/api/class/Dataset#get_metadata", + "parentId": 2402 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Open a storage, either restore existing or create a new one.\n" + } + ] + }, + "decorations": [ + { + "name": "classmethod" + }, + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 5610, + "module": "storages._base", + "name": "open", + "parsedDocstring": { + "text": "Open a storage, either restore existing or create a new one.\n", + "args": { + "id": "The storage ID.", + "name": "The storage name (global scope, persists across runs). Name can only contain letters \"a\" through \"z\",\nthe digits \"0\" through \"9\", and the hyphen (\"-\") but only in the middle of the string\n(e.g. \"my-value-1\").", + "alias": "The storage alias (run scope, creates unnamed storage).", + "configuration": "Configuration object used during the storage creation or restoration process.", + "storage_client": "Underlying storage client to use. If not provided, the default global storage client\nfrom the service locator will be used." + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_dataset.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 101, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storages/_dataset.py#L101" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Open a storage, either restore existing or create a new one.\n" + } + ] + }, + "flags": {}, + "id": 5534, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "open", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The storage ID." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 5535, + "kind": 32768, + "kindString": "Parameter", + "name": "id", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2336 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The storage name (global scope, persists across runs). Name can only contain letters \"a\" through \"z\",\nthe digits \"0\" through \"9\", and the hyphen (\"-\") but only in the middle of the string\n(e.g. \"my-value-1\")." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 5536, + "kind": 32768, + "kindString": "Parameter", + "name": "name", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2336 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The storage alias (run scope, creates unnamed storage)." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 5537, + "kind": 32768, + "kindString": "Parameter", + "name": "alias", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2336 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Configuration object used during the storage creation or restoration process." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 5538, + "kind": 32768, + "kindString": "Parameter", + "name": "configuration", + "type": { + "name": "Configuration | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "Configuration", + "target": "2", + "ref": { + "id": 3200, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/configuration.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 20 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/Configuration" + } + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2336 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Underlying storage client to use. If not provided, the default global storage client\nfrom the service locator will be used." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 5539, + "kind": 32768, + "kindString": "Parameter", + "name": "storage_client", + "type": { + "name": "StorageClient | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "StorageClient", + "target": "1282", + "ref": { + "id": 4480, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 19 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/StorageClient" + } + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2336 + } + ], + "type": { + "name": "Storage", + "type": "reference", + "target": "2330", + "ref": { + "id": 5528, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_base.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 15 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/Storage" + } + }, + "overwrites": { + "name": "Storage.open", + "target": 2335, + "type": "reference", + "ref": { + "id": 5533, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_base.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 34 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/Storage#open" + } + }, + "parentId": 2412 + } + ], + "overwrites": { + "name": "Storage.open", + "target": 2335, + "type": "reference", + "ref": { + "id": 5533, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_base.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 34 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/Storage#open" + } + }, + "permalink": "/python/api/class/Dataset#open", + "parentId": 2402 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Drop the storage, removing it from the underlying storage client and clearing the cache." + } + ] + }, + "decorations": [ + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 5617, + "module": "storages._base", + "name": "drop", + "parsedDocstring": { + "text": "Drop the storage, removing it from the underlying storage client and clearing the cache." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_dataset.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 128, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storages/_dataset.py#L128" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Drop the storage, removing it from the underlying storage client and clearing the cache." + } + ] + }, + "flags": {}, + "id": 5541, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "drop", + "parameters": [], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "overwrites": { + "name": "Storage.drop", + "target": 2342, + "type": "reference", + "ref": { + "id": 5540, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_base.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 57 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/Storage#drop" + } + }, + "parentId": 2419 + } + ], + "overwrites": { + "name": "Storage.drop", + "target": 2342, + "type": "reference", + "ref": { + "id": 5540, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_base.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 57 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/Storage#drop" + } + }, + "permalink": "/python/api/class/Dataset#drop", + "parentId": 2402 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Purge the storage, removing all items from the underlying storage client.\n\nThis method does not remove the storage itself, e.g. don't remove the metadata,\nbut clears all items within it." + } + ] + }, + "decorations": [ + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 5619, + "module": "storages._base", + "name": "purge", + "parsedDocstring": { + "text": "Purge the storage, removing all items from the underlying storage client.\n\nThis method does not remove the storage itself, e.g. don't remove the metadata,\nbut clears all items within it." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_dataset.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 134, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storages/_dataset.py#L134" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Purge the storage, removing all items from the underlying storage client.\n\nThis method does not remove the storage itself, e.g. don't remove the metadata,\nbut clears all items within it." + } + ] + }, + "flags": {}, + "id": 5543, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "purge", + "parameters": [], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "overwrites": { + "name": "Storage.purge", + "target": 2344, + "type": "reference", + "ref": { + "id": 5542, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_base.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 61 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/Storage#purge" + } + }, + "parentId": 2421 + } + ], + "overwrites": { + "name": "Storage.purge", + "target": 2344, + "type": "reference", + "ref": { + "id": 5542, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_base.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 61 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/Storage#purge" + } + }, + "permalink": "/python/api/class/Dataset#purge", + "parentId": 2402 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Store an object or an array of objects to the dataset.\n\nThe size of the data is limited by the receiving API and therefore `push_data()` will only\nallow objects whose JSON representation is smaller than 9MB. When an array is passed,\nnone of the included objects may be larger than 9MB, but the array itself may be of any size.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 5621, + "module": "storages._dataset", + "name": "push_data", + "parsedDocstring": { + "text": "Store an object or an array of objects to the dataset.\n\nThe size of the data is limited by the receiving API and therefore `push_data()` will only\nallow objects whose JSON representation is smaller than 9MB. When an array is passed,\nnone of the included objects may be larger than 9MB, but the array itself may be of any size.\n", + "args": { + "data": "A JSON serializable data structure to be stored in the dataset. The JSON representation\nof each item must be smaller than 9MB." + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_dataset.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 137, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storages/_dataset.py#L137" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Store an object or an array of objects to the dataset.\n\nThe size of the data is limited by the receiving API and therefore `push_data()` will only\nallow objects whose JSON representation is smaller than 9MB. When an array is passed,\nnone of the included objects may be larger than 9MB, but the array itself may be of any size.\n" + } + ] + }, + "flags": {}, + "id": 5622, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "push_data", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "A JSON serializable data structure to be stored in the dataset. The JSON representation\nof each item must be smaller than 9MB." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 5623, + "kind": 32768, + "kindString": "Parameter", + "name": "data", + "type": { + "name": "list[dict[str, Any]] | dict[str, Any]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "list", + "typeArguments": [ + { + "type": "reference", + "name": "dict", + "typeArguments": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "reference", + "name": "Any", + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + }, + { + "type": "reference", + "name": "dict", + "typeArguments": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "reference", + "name": "Any", + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + } + ] + }, + "parentId": 2424 + } + ], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "parentId": 2423 + } + ], + "permalink": "/python/api/class/Dataset#push_data", + "parentId": 2402 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Retrieve a paginated list of items from a dataset based on various filtering parameters.\n\nThis method provides the flexibility to filter, sort, and modify the appearance of dataset items\nwhen listed. Each parameter modifies the result set according to its purpose. The method also\nsupports pagination through 'offset' and 'limit' parameters.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 5624, + "module": "storages._dataset", + "name": "get_data", + "parsedDocstring": { + "text": "Retrieve a paginated list of items from a dataset based on various filtering parameters.\n\nThis method provides the flexibility to filter, sort, and modify the appearance of dataset items\nwhen listed. Each parameter modifies the result set according to its purpose. The method also\nsupports pagination through 'offset' and 'limit' parameters.\n", + "args": { + "offset": "Skips the specified number of items at the start.", + "limit": "The maximum number of items to retrieve. Unlimited if None.", + "clean": "Return only non-empty items and excludes hidden fields. Shortcut for skip_hidden and skip_empty.", + "desc": "Set to True to sort results in descending order.", + "fields": "Fields to include in each item. Sorts fields as specified if provided.", + "omit": "Fields to exclude from each item.", + "unwind": "Unwinds items by a specified array field, turning each element into a separate item.", + "skip_empty": "Excludes empty items from the results if True.", + "skip_hidden": "Excludes fields starting with '#' if True.", + "flatten": "Fields to be flattened in returned items.", + "view": "Specifies the dataset view to be used.\n" + }, + "returns": "An object with filtered, sorted, and paginated dataset items plus pagination details." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_dataset.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 150, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storages/_dataset.py#L150" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "An object with filtered, sorted, and paginated dataset items plus pagination details." + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Retrieve a paginated list of items from a dataset based on various filtering parameters.\n\nThis method provides the flexibility to filter, sort, and modify the appearance of dataset items\nwhen listed. Each parameter modifies the result set according to its purpose. The method also\nsupports pagination through 'offset' and 'limit' parameters.\n" + } + ] + }, + "flags": {}, + "id": 5625, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "get_data", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Skips the specified number of items at the start." + } + ] + }, + "defaultValue": "0", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 5626, + "kind": 32768, + "kindString": "Parameter", + "name": "offset", + "type": { + "name": "int", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2427 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The maximum number of items to retrieve. Unlimited if None." + } + ] + }, + "defaultValue": "999_999_999_999", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 5627, + "kind": 32768, + "kindString": "Parameter", + "name": "limit", + "type": { + "name": "int | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "int", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2427 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Return only non-empty items and excludes hidden fields. Shortcut for skip_hidden and skip_empty." + } + ] + }, + "defaultValue": "False", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 5628, + "kind": 32768, + "kindString": "Parameter", + "name": "clean", + "type": { + "name": "bool", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2427 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Set to True to sort results in descending order." + } + ] + }, + "defaultValue": "False", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 5629, + "kind": 32768, + "kindString": "Parameter", + "name": "desc", + "type": { + "name": "bool", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2427 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Fields to include in each item. Sorts fields as specified if provided." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 5630, + "kind": 32768, + "kindString": "Parameter", + "name": "fields", + "type": { + "name": "list[str] | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "list", + "typeArguments": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2427 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Fields to exclude from each item." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 5631, + "kind": 32768, + "kindString": "Parameter", + "name": "omit", + "type": { + "name": "list[str] | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "list", + "typeArguments": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2427 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Unwinds items by a specified array field, turning each element into a separate item." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 5632, + "kind": 32768, + "kindString": "Parameter", + "name": "unwind", + "type": { + "name": "list[str] | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "list", + "typeArguments": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2427 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Excludes empty items from the results if True." + } + ] + }, + "defaultValue": "False", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 5633, + "kind": 32768, + "kindString": "Parameter", + "name": "skip_empty", + "type": { + "name": "bool", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2427 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Excludes fields starting with '#' if True." + } + ] + }, + "defaultValue": "False", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 5634, + "kind": 32768, + "kindString": "Parameter", + "name": "skip_hidden", + "type": { + "name": "bool", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2427 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Fields to be flattened in returned items." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 5635, + "kind": 32768, + "kindString": "Parameter", + "name": "flatten", + "type": { + "name": "list[str] | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "list", + "typeArguments": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2427 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Specifies the dataset view to be used.\n" + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 5636, + "kind": 32768, + "kindString": "Parameter", + "name": "view", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2427 + } + ], + "type": { + "name": "DatasetItemsListPage", + "type": "reference", + "target": "1333", + "ref": { + "id": 4531, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 110 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/DatasetItemsListPage" + } + }, + "parentId": 2426 + } + ], + "permalink": "/python/api/class/Dataset#get_data", + "parentId": 2402 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Iterate over items in the dataset according to specified filters and sorting.\n\nThis method allows for asynchronously iterating through dataset items while applying various filters such as\nskipping empty items, hiding specific fields, and sorting. It supports pagination via `offset` and `limit`\nparameters, and can modify the appearance of dataset items using `fields`, `omit`, `unwind`, `skip_empty`, and\n`skip_hidden` parameters.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 5637, + "module": "storages._dataset", + "name": "iterate_items", + "parsedDocstring": { + "text": "Iterate over items in the dataset according to specified filters and sorting.\n\nThis method allows for asynchronously iterating through dataset items while applying various filters such as\nskipping empty items, hiding specific fields, and sorting. It supports pagination via `offset` and `limit`\nparameters, and can modify the appearance of dataset items using `fields`, `omit`, `unwind`, `skip_empty`, and\n`skip_hidden` parameters.\n", + "args": { + "offset": "Skips the specified number of items at the start.", + "limit": "The maximum number of items to retrieve. Unlimited if None.", + "clean": "Return only non-empty items and excludes hidden fields. Shortcut for skip_hidden and skip_empty.", + "desc": "Set to True to sort results in descending order.", + "fields": "Fields to include in each item. Sorts fields as specified if provided.", + "omit": "Fields to exclude from each item.", + "unwind": "Unwinds items by a specified array field, turning each element into a separate item.", + "skip_empty": "Excludes empty items from the results if True.", + "skip_hidden": "Excludes fields starting with '#' if True.\n" + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_dataset.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 201, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storages/_dataset.py#L201" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Iterate over items in the dataset according to specified filters and sorting.\n\nThis method allows for asynchronously iterating through dataset items while applying various filters such as\nskipping empty items, hiding specific fields, and sorting. It supports pagination via `offset` and `limit`\nparameters, and can modify the appearance of dataset items using `fields`, `omit`, `unwind`, `skip_empty`, and\n`skip_hidden` parameters.\n" + } + ] + }, + "flags": {}, + "id": 5638, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "iterate_items", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Skips the specified number of items at the start." + } + ] + }, + "defaultValue": "0", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 5639, + "kind": 32768, + "kindString": "Parameter", + "name": "offset", + "type": { + "name": "int", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2440 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The maximum number of items to retrieve. Unlimited if None." + } + ] + }, + "defaultValue": "999_999_999_999", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 5640, + "kind": 32768, + "kindString": "Parameter", + "name": "limit", + "type": { + "name": "int | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "int", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2440 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Return only non-empty items and excludes hidden fields. Shortcut for skip_hidden and skip_empty." + } + ] + }, + "defaultValue": "False", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 5641, + "kind": 32768, + "kindString": "Parameter", + "name": "clean", + "type": { + "name": "bool", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2440 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Set to True to sort results in descending order." + } + ] + }, + "defaultValue": "False", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 5642, + "kind": 32768, + "kindString": "Parameter", + "name": "desc", + "type": { + "name": "bool", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2440 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Fields to include in each item. Sorts fields as specified if provided." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 5643, + "kind": 32768, + "kindString": "Parameter", + "name": "fields", + "type": { + "name": "list[str] | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "list", + "typeArguments": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2440 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Fields to exclude from each item." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 5644, + "kind": 32768, + "kindString": "Parameter", + "name": "omit", + "type": { + "name": "list[str] | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "list", + "typeArguments": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2440 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Unwinds items by a specified array field, turning each element into a separate item." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 5645, + "kind": 32768, + "kindString": "Parameter", + "name": "unwind", + "type": { + "name": "list[str] | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "list", + "typeArguments": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2440 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Excludes empty items from the results if True." + } + ] + }, + "defaultValue": "False", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 5646, + "kind": 32768, + "kindString": "Parameter", + "name": "skip_empty", + "type": { + "name": "bool", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2440 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Excludes fields starting with '#' if True.\n" + } + ] + }, + "defaultValue": "False", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 5647, + "kind": 32768, + "kindString": "Parameter", + "name": "skip_hidden", + "type": { + "name": "bool", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2440 + } + ], + "type": { + "name": "AsyncIterator", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "dict", + "typeArguments": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "reference", + "name": "Any", + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + }, + "parentId": 2439 + } + ], + "permalink": "/python/api/class/Dataset#iterate_items", + "parentId": 2402 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Retrieve a list of all items from the dataset according to specified filters and sorting.\n\nThis method collects all dataset items into a list while applying various filters such as\nskipping empty items, hiding specific fields, and sorting. It supports pagination via `offset` and `limit`\nparameters, and can modify the appearance of dataset items using `fields`, `omit`, `unwind`, `skip_empty`, and\n`skip_hidden` parameters.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 5648, + "module": "storages._dataset", + "name": "list_items", + "parsedDocstring": { + "text": "Retrieve a list of all items from the dataset according to specified filters and sorting.\n\nThis method collects all dataset items into a list while applying various filters such as\nskipping empty items, hiding specific fields, and sorting. It supports pagination via `offset` and `limit`\nparameters, and can modify the appearance of dataset items using `fields`, `omit`, `unwind`, `skip_empty`, and\n`skip_hidden` parameters.\n", + "args": { + "offset": "Skips the specified number of items at the start.", + "limit": "The maximum number of items to retrieve. Unlimited if None.", + "clean": "Return only non-empty items and excludes hidden fields. Shortcut for skip_hidden and skip_empty.", + "desc": "Set to True to sort results in descending order.", + "fields": "Fields to include in each item. Sorts fields as specified if provided.", + "omit": "Fields to exclude from each item.", + "unwind": "Unwinds items by a specified array field, turning each element into a separate item.", + "skip_empty": "Excludes empty items from the results if True.", + "skip_hidden": "Excludes fields starting with '#' if True.\n" + }, + "returns": "A list of dictionary objects, each representing a dataset item after applying\nthe specified filters and transformations." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_dataset.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 249, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storages/_dataset.py#L249" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "A list of dictionary objects, each representing a dataset item after applying\nthe specified filters and transformations." + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Retrieve a list of all items from the dataset according to specified filters and sorting.\n\nThis method collects all dataset items into a list while applying various filters such as\nskipping empty items, hiding specific fields, and sorting. It supports pagination via `offset` and `limit`\nparameters, and can modify the appearance of dataset items using `fields`, `omit`, `unwind`, `skip_empty`, and\n`skip_hidden` parameters.\n" + } + ] + }, + "flags": {}, + "id": 5649, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "list_items", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Skips the specified number of items at the start." + } + ] + }, + "defaultValue": "0", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 5650, + "kind": 32768, + "kindString": "Parameter", + "name": "offset", + "type": { + "name": "int", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2451 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The maximum number of items to retrieve. Unlimited if None." + } + ] + }, + "defaultValue": "999_999_999_999", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 5651, + "kind": 32768, + "kindString": "Parameter", + "name": "limit", + "type": { + "name": "int | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "int", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2451 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Return only non-empty items and excludes hidden fields. Shortcut for skip_hidden and skip_empty." + } + ] + }, + "defaultValue": "False", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 5652, + "kind": 32768, + "kindString": "Parameter", + "name": "clean", + "type": { + "name": "bool", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2451 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Set to True to sort results in descending order." + } + ] + }, + "defaultValue": "False", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 5653, + "kind": 32768, + "kindString": "Parameter", + "name": "desc", + "type": { + "name": "bool", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2451 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Fields to include in each item. Sorts fields as specified if provided." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 5654, + "kind": 32768, + "kindString": "Parameter", + "name": "fields", + "type": { + "name": "list[str] | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "list", + "typeArguments": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2451 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Fields to exclude from each item." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 5655, + "kind": 32768, + "kindString": "Parameter", + "name": "omit", + "type": { + "name": "list[str] | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "list", + "typeArguments": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2451 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Unwinds items by a specified array field, turning each element into a separate item." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 5656, + "kind": 32768, + "kindString": "Parameter", + "name": "unwind", + "type": { + "name": "list[str] | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "list", + "typeArguments": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2451 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Excludes empty items from the results if True." + } + ] + }, + "defaultValue": "False", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 5657, + "kind": 32768, + "kindString": "Parameter", + "name": "skip_empty", + "type": { + "name": "bool", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2451 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Excludes fields starting with '#' if True.\n" + } + ] + }, + "defaultValue": "False", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 5658, + "kind": 32768, + "kindString": "Parameter", + "name": "skip_hidden", + "type": { + "name": "bool", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2451 + } + ], + "type": { + "name": "list", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "dict", + "typeArguments": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "reference", + "name": "Any", + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + }, + "parentId": 2450 + } + ], + "permalink": "/python/api/class/Dataset#list_items", + "parentId": 2402 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Export the entire dataset into a specified file stored under a key in a key-value store.\n\nThis method consolidates all entries from a specified dataset into one file, which is then saved under a\ngiven key in a key-value store. The format of the exported file is determined by the `content_type` parameter.\nEither the dataset's ID or name should be specified, and similarly, either the target key-value store's ID or\nname should be used.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 5659, + "module": "storages._dataset", + "name": "export_to", + "parsedDocstring": { + "text": "Export the entire dataset into a specified file stored under a key in a key-value store.\n\nThis method consolidates all entries from a specified dataset into one file, which is then saved under a\ngiven key in a key-value store. The format of the exported file is determined by the `content_type` parameter.\nEither the dataset's ID or name should be specified, and similarly, either the target key-value store's ID or\nname should be used.\n", + "args": { + "key": "The key under which to save the data in the key-value store.", + "content_type": "The format in which to export the data.", + "to_kvs_id": "ID of the key-value store to save the exported file.\nSpecify only one of ID or name.", + "to_kvs_name": "Name of the key-value store to save the exported file.\nSpecify only one of ID or name.", + "to_kvs_storage_client": "Storage client to use for the key-value store.", + "to_kvs_configuration": "Configuration for the key-value store.", + "kwargs": "Additional parameters for the export operation, specific to the chosen content type." + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_dataset.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 323, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storages/_dataset.py#L323" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Export the entire dataset into a specified file stored under a key in a key-value store.\n\nThis method consolidates all entries from a specified dataset into one file, which is then saved under a\ngiven key in a key-value store. The format of the exported file is determined by the `content_type` parameter.\nEither the dataset's ID or name should be specified, and similarly, either the target key-value store's ID or\nname should be used.\n" + } + ] + }, + "flags": {}, + "id": 5660, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "export_to", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The key under which to save the data in the key-value store." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 5661, + "kind": 32768, + "kindString": "Parameter", + "name": "key", + "type": { + "name": "str", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2462 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The format in which to export the data." + } + ] + }, + "defaultValue": "'json'", + "flags": { + "isOptional": true, + "keyword-only": false + }, + "id": 5662, + "kind": 32768, + "kindString": "Parameter", + "name": "content_type", + "type": { + "name": "Literal", + "type": "reference", + "typeArguments": [ + { + "type": "literal", + "value": "json" + }, + { + "type": "literal", + "value": "csv" + } + ], + "target": 0, + "ref": {} + }, + "parentId": 2462 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "ID of the key-value store to save the exported file.\nSpecify only one of ID or name." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": false + }, + "id": 5663, + "kind": 32768, + "kindString": "Parameter", + "name": "to_kvs_id", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2462 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Name of the key-value store to save the exported file.\nSpecify only one of ID or name." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": false + }, + "id": 5664, + "kind": 32768, + "kindString": "Parameter", + "name": "to_kvs_name", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2462 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Storage client to use for the key-value store." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": false + }, + "id": 5665, + "kind": 32768, + "kindString": "Parameter", + "name": "to_kvs_storage_client", + "type": { + "name": "StorageClient | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "StorageClient", + "target": "1282", + "ref": { + "id": 4480, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 19 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/StorageClient" + } + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2462 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Configuration for the key-value store." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": false + }, + "id": 5666, + "kind": 32768, + "kindString": "Parameter", + "name": "to_kvs_configuration", + "type": { + "name": "Configuration | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "Configuration", + "target": "2", + "ref": { + "id": 3200, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/configuration.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 20 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/Configuration" + } + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2462 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Additional parameters for the export operation, specific to the chosen content type." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 5667, + "kind": 32768, + "kindString": "Parameter", + "name": "kwargs", + "type": { + "name": "Any", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2462 + } + ], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "parentId": 2461 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Export the entire dataset into a specified file stored under a key in a key-value store.\n\nThis method consolidates all entries from a specified dataset into one file, which is then saved under a\ngiven key in a key-value store. The format of the exported file is determined by the `content_type` parameter.\nEither the dataset's ID or name should be specified, and similarly, either the target key-value store's ID or\nname should be used.\n" + } + ] + }, + "flags": {}, + "id": 5668, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "export_to", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The key under which to save the data in the key-value store." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 5669, + "kind": 32768, + "kindString": "Parameter", + "name": "key", + "type": { + "name": "str", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2470 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The format in which to export the data." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 5670, + "kind": 32768, + "kindString": "Parameter", + "name": "content_type", + "type": { + "name": "Literal", + "type": "reference", + "typeArguments": [ + { + "type": "literal", + "value": "json" + } + ], + "target": 0, + "ref": {} + }, + "parentId": 2470 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "ID of the key-value store to save the exported file.\nSpecify only one of ID or name." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": false + }, + "id": 5671, + "kind": 32768, + "kindString": "Parameter", + "name": "to_kvs_id", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2470 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Name of the key-value store to save the exported file.\nSpecify only one of ID or name." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": false + }, + "id": 5672, + "kind": 32768, + "kindString": "Parameter", + "name": "to_kvs_name", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2470 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Storage client to use for the key-value store." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": false + }, + "id": 5673, + "kind": 32768, + "kindString": "Parameter", + "name": "to_kvs_storage_client", + "type": { + "name": "StorageClient | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "StorageClient", + "target": "1282", + "ref": { + "id": 4480, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 19 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/StorageClient" + } + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2470 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Configuration for the key-value store." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": false + }, + "id": 5674, + "kind": 32768, + "kindString": "Parameter", + "name": "to_kvs_configuration", + "type": { + "name": "Configuration | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "Configuration", + "target": "2", + "ref": { + "id": 3200, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/configuration.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 20 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/Configuration" + } + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2470 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "If True (default: False), dict keys that are not of a basic type (str, int, float, bool, None) will be skipped\ninstead of raising a `TypeError`." + } + ] + }, + "flags": { + "keyword-only": true, + "optional": true + }, + "groups": [], + "id": 3527, + "module": "_types", + "name": "skipkeys", + "parsedDocstring": { + "text": "If True (default: False), dict keys that are not of a basic type (str, int, float, bool, None) will be skipped\ninstead of raising a `TypeError`." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_types.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 759, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/_types.py#L759" + } + ], + "type": { + "name": "NotRequired", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "bool", + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + }, + "parentId": 2470 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Determines if non-ASCII characters should be escaped in the output JSON string." + } + ] + }, + "flags": { + "keyword-only": true, + "optional": true + }, + "groups": [], + "id": 3528, + "module": "_types", + "name": "ensure_ascii", + "parsedDocstring": { + "text": "Determines if non-ASCII characters should be escaped in the output JSON string." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_types.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 763, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/_types.py#L763" + } + ], + "type": { + "name": "NotRequired", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "bool", + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + }, + "parentId": 2470 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "If False (default: True), skips the circular reference check for container types. A circular reference will\nresult in a `RecursionError` or worse if unchecked." + } + ] + }, + "flags": { + "keyword-only": true, + "optional": true + }, + "groups": [], + "id": 3529, + "module": "_types", + "name": "check_circular", + "parsedDocstring": { + "text": "If False (default: True), skips the circular reference check for container types. A circular reference will\nresult in a `RecursionError` or worse if unchecked." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_types.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 766, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/_types.py#L766" + } + ], + "type": { + "name": "NotRequired", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "bool", + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + }, + "parentId": 2470 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "If False (default: True), raises a ValueError for out-of-range float values (nan, inf, -inf) to strictly comply\nwith the JSON specification. If True, uses their JavaScript equivalents (NaN, Infinity, -Infinity)." + } + ] + }, + "flags": { + "keyword-only": true, + "optional": true + }, + "groups": [], + "id": 3530, + "module": "_types", + "name": "allow_nan", + "parsedDocstring": { + "text": "If False (default: True), raises a ValueError for out-of-range float values (nan, inf, -inf) to strictly comply\nwith the JSON specification. If True, uses their JavaScript equivalents (NaN, Infinity, -Infinity)." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_types.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 770, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/_types.py#L770" + } + ], + "type": { + "name": "NotRequired", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "bool", + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + }, + "parentId": 2470 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Allows specifying a custom JSON encoder." + } + ] + }, + "flags": { + "keyword-only": true, + "optional": true + }, + "groups": [], + "id": 3531, + "module": "_types", + "name": "cls", + "parsedDocstring": { + "text": "Allows specifying a custom JSON encoder." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_types.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 774, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/_types.py#L774" + } + ], + "type": { + "name": "NotRequired", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "type", + "typeArguments": [ + { + "type": "reference", + "name": "json.JSONEncoder", + "target": 0, + "ref": {} + } + ], + "target": "3422", + "ref": { + "id": 6620, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_utils/sitemap.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 62 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api/class/SitemapSource#type" + } + } + ], + "target": 0, + "ref": {} + }, + "parentId": 2470 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Specifies the number of spaces for indentation in the pretty-printed JSON output." + } + ] + }, + "flags": { + "keyword-only": true, + "optional": true + }, + "groups": [], + "id": 3532, + "module": "_types", + "name": "indent", + "parsedDocstring": { + "text": "Specifies the number of spaces for indentation in the pretty-printed JSON output." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_types.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 777, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/_types.py#L777" + } + ], + "type": { + "name": "NotRequired", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "int", + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + }, + "parentId": 2470 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "A tuple of (item_separator, key_separator). The default is (', ', ': ') if indent is None and (',', ': ')\notherwise." + } + ] + }, + "flags": { + "keyword-only": true, + "optional": true + }, + "groups": [], + "id": 3533, + "module": "_types", + "name": "separators", + "parsedDocstring": { + "text": "A tuple of (item_separator, key_separator). The default is (', ', ': ') if indent is None and (',', ': ')\notherwise." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_types.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 780, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/_types.py#L780" + } + ], + "type": { + "name": "NotRequired", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "tuple", + "typeArguments": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + }, + "parentId": 2470 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "A function called for objects that can't be serialized otherwise. It should return a JSON-encodable version\nof the object or raise a `TypeError`." + } + ] + }, + "flags": { + "keyword-only": true, + "optional": true + }, + "groups": [], + "id": 3534, + "module": "_types", + "name": "default", + "parsedDocstring": { + "text": "A function called for objects that can't be serialized otherwise. It should return a JSON-encodable version\nof the object or raise a `TypeError`." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_types.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 784, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/_types.py#L784" + } + ], + "type": { + "name": "NotRequired", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "Callable", + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + }, + "parentId": 2470 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Specifies whether the output JSON object should have keys sorted alphabetically." + } + ] + }, + "flags": { + "keyword-only": true, + "optional": true + }, + "groups": [], + "id": 3535, + "module": "_types", + "name": "sort_keys", + "parsedDocstring": { + "text": "Specifies whether the output JSON object should have keys sorted alphabetically." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_types.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 788, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/_types.py#L788" + } + ], + "type": { + "name": "NotRequired", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "bool", + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + }, + "parentId": 2470 + } + ], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "parentId": 2461 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Export the entire dataset into a specified file stored under a key in a key-value store.\n\nThis method consolidates all entries from a specified dataset into one file, which is then saved under a\ngiven key in a key-value store. The format of the exported file is determined by the `content_type` parameter.\nEither the dataset's ID or name should be specified, and similarly, either the target key-value store's ID or\nname should be used.\n" + } + ] + }, + "flags": {}, + "id": 5676, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "export_to", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The key under which to save the data in the key-value store." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 5677, + "kind": 32768, + "kindString": "Parameter", + "name": "key", + "type": { + "name": "str", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2478 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The format in which to export the data." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 5678, + "kind": 32768, + "kindString": "Parameter", + "name": "content_type", + "type": { + "name": "Literal", + "type": "reference", + "typeArguments": [ + { + "type": "literal", + "value": "csv" + } + ], + "target": 0, + "ref": {} + }, + "parentId": 2478 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "ID of the key-value store to save the exported file.\nSpecify only one of ID or name." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": false + }, + "id": 5679, + "kind": 32768, + "kindString": "Parameter", + "name": "to_kvs_id", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2478 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Name of the key-value store to save the exported file.\nSpecify only one of ID or name." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": false + }, + "id": 5680, + "kind": 32768, + "kindString": "Parameter", + "name": "to_kvs_name", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2478 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Storage client to use for the key-value store." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": false + }, + "id": 5681, + "kind": 32768, + "kindString": "Parameter", + "name": "to_kvs_storage_client", + "type": { + "name": "StorageClient | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "StorageClient", + "target": "1282", + "ref": { + "id": 4480, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 19 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/StorageClient" + } + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2478 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Configuration for the key-value store." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": false + }, + "id": 5682, + "kind": 32768, + "kindString": "Parameter", + "name": "to_kvs_configuration", + "type": { + "name": "Configuration | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "Configuration", + "target": "2", + "ref": { + "id": 3200, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/configuration.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 20 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/Configuration" + } + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2478 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Specifies a dialect to be used in CSV parsing and writing." + } + ] + }, + "flags": { + "keyword-only": true, + "optional": true + }, + "groups": [], + "id": 3537, + "module": "_types", + "name": "dialect", + "parsedDocstring": { + "text": "Specifies a dialect to be used in CSV parsing and writing." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_types.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 795, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/_types.py#L795" + } + ], + "type": { + "name": "NotRequired", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + }, + "parentId": 2478 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "A one-character string used to separate fields. Defaults to ','." + } + ] + }, + "flags": { + "keyword-only": true, + "optional": true + }, + "groups": [], + "id": 3538, + "module": "_types", + "name": "delimiter", + "parsedDocstring": { + "text": "A one-character string used to separate fields. Defaults to ','." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_types.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 798, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/_types.py#L798" + } + ], + "type": { + "name": "NotRequired", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + }, + "parentId": 2478 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Controls how instances of `quotechar` inside a field should be quoted. When True, the character is doubled;\nwhen False, the `escapechar` is used as a prefix. Defaults to True." + } + ] + }, + "flags": { + "keyword-only": true, + "optional": true + }, + "groups": [], + "id": 3539, + "module": "_types", + "name": "doublequote", + "parsedDocstring": { + "text": "Controls how instances of `quotechar` inside a field should be quoted. When True, the character is doubled;\nwhen False, the `escapechar` is used as a prefix. Defaults to True." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_types.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 801, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/_types.py#L801" + } + ], + "type": { + "name": "NotRequired", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "bool", + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + }, + "parentId": 2478 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "A one-character string used to escape the delimiter if `quoting` is set to `QUOTE_NONE` and the `quotechar`\nif `doublequote` is False. Defaults to None, disabling escaping." + } + ] + }, + "flags": { + "keyword-only": true, + "optional": true + }, + "groups": [], + "id": 3540, + "module": "_types", + "name": "escapechar", + "parsedDocstring": { + "text": "A one-character string used to escape the delimiter if `quoting` is set to `QUOTE_NONE` and the `quotechar`\nif `doublequote` is False. Defaults to None, disabling escaping." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_types.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 805, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/_types.py#L805" + } + ], + "type": { + "name": "NotRequired", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + }, + "parentId": 2478 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "The string used to terminate lines produced by the writer. Defaults to '\\r\\n'." + } + ] + }, + "flags": { + "keyword-only": true, + "optional": true + }, + "groups": [], + "id": 3541, + "module": "_types", + "name": "lineterminator", + "parsedDocstring": { + "text": "The string used to terminate lines produced by the writer. Defaults to '\\r\\n'." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_types.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 809, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/_types.py#L809" + } + ], + "type": { + "name": "NotRequired", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + }, + "parentId": 2478 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "A one-character string used to quote fields containing special characters, like the delimiter or quotechar,\nor fields containing new-line characters. Defaults to '\"'." + } + ] + }, + "flags": { + "keyword-only": true, + "optional": true + }, + "groups": [], + "id": 3542, + "module": "_types", + "name": "quotechar", + "parsedDocstring": { + "text": "A one-character string used to quote fields containing special characters, like the delimiter or quotechar,\nor fields containing new-line characters. Defaults to '\"'." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_types.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 812, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/_types.py#L812" + } + ], + "type": { + "name": "NotRequired", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + }, + "parentId": 2478 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Controls when quotes should be generated by the writer and recognized by the reader. Can take any of\nthe `QUOTE_*` constants, with a default of `QUOTE_MINIMAL`." + } + ] + }, + "flags": { + "keyword-only": true, + "optional": true + }, + "groups": [], + "id": 3543, + "module": "_types", + "name": "quoting", + "parsedDocstring": { + "text": "Controls when quotes should be generated by the writer and recognized by the reader. Can take any of\nthe `QUOTE_*` constants, with a default of `QUOTE_MINIMAL`." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_types.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 816, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/_types.py#L816" + } + ], + "type": { + "name": "NotRequired", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "Literal", + "typeArguments": [ + { + "type": "literal", + "value": 0 + }, + { + "type": "literal", + "value": 1 + }, + { + "type": "literal", + "value": 2 + }, + { + "type": "literal", + "value": 3 + } + ], + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + }, + "parentId": 2478 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "When True, spaces immediately following the delimiter are ignored. Defaults to False." + } + ] + }, + "flags": { + "keyword-only": true, + "optional": true + }, + "groups": [], + "id": 3544, + "module": "_types", + "name": "skipinitialspace", + "parsedDocstring": { + "text": "When True, spaces immediately following the delimiter are ignored. Defaults to False." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_types.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 820, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/_types.py#L820" + } + ], + "type": { + "name": "NotRequired", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "bool", + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + }, + "parentId": 2478 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "When True, raises an exception on bad CSV input. Defaults to False." + } + ] + }, + "flags": { + "keyword-only": true, + "optional": true + }, + "groups": [], + "id": 3545, + "module": "_types", + "name": "strict", + "parsedDocstring": { + "text": "When True, raises an exception on bad CSV input. Defaults to False." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_types.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 823, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/_types.py#L823" + } + ], + "type": { + "name": "NotRequired", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "bool", + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + }, + "parentId": 2478 + } + ], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "parentId": 2461 + } + ], + "permalink": "/python/api/class/Dataset#export_to", + "parentId": 2402 + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Dataset is a storage for managing structured tabular data.\n\nThe dataset class provides a high-level interface for storing and retrieving structured data\nwith consistent schema, similar to database tables or spreadsheets. It abstracts the underlying\nstorage implementation details, offering a consistent API regardless of where the data is\nphysically stored.\n\nDataset operates in an append-only mode, allowing new records to be added but not modified\nor deleted after creation. This makes it particularly suitable for storing crawling results\nand other data that should be immutable once collected.\n\nThe class provides methods for adding data, retrieving data with various filtering options,\nand exporting data to different formats. You can create a dataset using the `open` class method,\nspecifying either a name or ID. The underlying storage implementation is determined by\nthe configured storage client.\n\n### Usage\n\n```python\nfrom crawlee.storages import Dataset\n\n# Open a dataset\ndataset = await Dataset.open(name='my-dataset')\n\n# Add data\nawait dataset.push_data({'title': 'Example Product', 'price': 99.99})\n\n# Retrieve filtered data\nresults = await dataset.get_data(limit=10, desc=True)\n\n# Export data\nawait dataset.export_to('results.json', content_type='json')\n```" + } + ] + }, + "decorations": [ + { + "args": "('Storages')", + "name": "docs_group" + } + ], + "flags": {}, + "groups": [ + { + "children": [ + 5601, + 5617, + 5659, + 5624, + 5608, + 5637, + 5648, + 5610, + 5619, + 5621 + ], + "title": "Methods" + }, + { + "children": [ + 5606, + 5607 + ], + "title": "Properties" + } + ], + "id": 5600, + "module": "storages._dataset", + "name": "Dataset", + "parsedDocstring": { + "text": "Dataset is a storage for managing structured tabular data.\n\nThe dataset class provides a high-level interface for storing and retrieving structured data\nwith consistent schema, similar to database tables or spreadsheets. It abstracts the underlying\nstorage implementation details, offering a consistent API regardless of where the data is\nphysically stored.\n\nDataset operates in an append-only mode, allowing new records to be added but not modified\nor deleted after creation. This makes it particularly suitable for storing crawling results\nand other data that should be immutable once collected.\n\nThe class provides methods for adding data, retrieving data with various filtering options,\nand exporting data to different formats. You can create a dataset using the `open` class method,\nspecifying either a name or ID. The underlying storage implementation is determined by\nthe configured storage client.\n\n### Usage\n\n```python\nfrom crawlee.storages import Dataset\n\n# Open a dataset\ndataset = await Dataset.open(name='my-dataset')\n\n# Add data\nawait dataset.push_data({'title': 'Example Product', 'price': 99.99})\n\n# Retrieve filtered data\nresults = await dataset.get_data(limit=10, desc=True)\n\n# Export data\nawait dataset.export_to('results.json', content_type='json')\n```" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_dataset.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 33, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storages/_dataset.py#L33" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "extendedTypes": [ + { + "name": "Storage", + "target": "2330", + "type": "reference", + "ref": { + "id": 5528, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_base.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 15 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/Storage" + } + } + ], + "permalink": "/python/api/class/Dataset", + "categories": [] + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Initialize a new instance.\n\nPreferably use the `KeyValueStore.open` constructor to create a new instance.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 8890, + "module": "storages._key_value_store", + "name": "__init__", + "parsedDocstring": { + "text": "Initialize a new instance.\n\nPreferably use the `KeyValueStore.open` constructor to create a new instance.\n", + "args": { + "client": "An instance of a storage client.", + "id": "The unique identifier of the storage.", + "name": "The name of the storage, if available." + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_key_value_store.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 78, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storages/_key_value_store.py#L78" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Initialize a new instance.\n\nPreferably use the `KeyValueStore.open` constructor to create a new instance.\n" + } + ] + }, + "flags": {}, + "id": 8891, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "__init__", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "An instance of a storage client." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 8892, + "kind": 32768, + "kindString": "Parameter", + "name": "client", + "type": { + "name": "KeyValueStoreClient", + "type": "reference", + "target": "1254", + "ref": { + "id": 7874, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_key_value_store_client.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 14 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/KeyValueStoreClient" + } + }, + "parentId": 2271 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The unique identifier of the storage." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 8893, + "kind": 32768, + "kindString": "Parameter", + "name": "id", + "type": { + "name": "str", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2271 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The name of the storage, if available." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 8894, + "kind": 32768, + "kindString": "Parameter", + "name": "name", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2271 + } + ], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "parentId": 2270 + } + ], + "permalink": "/python/api/class/KeyValueStore#__init__", + "parentId": 2269 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Get the storage ID." + } + ] + }, + "decorations": [ + { + "name": "property" + }, + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 8895, + "module": "storages._base", + "name": "id", + "parsedDocstring": { + "text": "Get the storage ID." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_key_value_store.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 99, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storages/_key_value_store.py#L99" + } + ], + "type": { + "name": "str", + "type": "reference", + "target": 0, + "ref": {} + }, + "overwrites": { + "name": "Storage.id", + "target": 2331, + "type": "reference", + "ref": { + "id": 8951, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_base.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 20 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api/class/Storage#id" + } + }, + "permalink": "/python/api/class/KeyValueStore#id", + "parentId": 2269 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Get the storage name." + } + ] + }, + "decorations": [ + { + "name": "property" + }, + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 8896, + "module": "storages._base", + "name": "name", + "parsedDocstring": { + "text": "Get the storage name." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_key_value_store.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 104, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storages/_key_value_store.py#L104" + } + ], + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "overwrites": { + "name": "Storage.name", + "target": 2332, + "type": "reference", + "ref": { + "id": 8952, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_base.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 25 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api/class/Storage#name" + } + }, + "permalink": "/python/api/class/KeyValueStore#name", + "parentId": 2269 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Get the storage metadata." + } + ] + }, + "decorations": [ + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 8897, + "module": "storages._base", + "name": "get_metadata", + "parsedDocstring": { + "text": "Get the storage metadata." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_key_value_store.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 108, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storages/_key_value_store.py#L108" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Get the storage metadata." + } + ] + }, + "flags": {}, + "id": 8954, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "get_metadata", + "parameters": [], + "type": { + "name": "DatasetMetadata | KeyValueStoreMetadata | RequestQueueMetadata", + "type": "union", + "types": [ + { + "type": "union", + "types": [ + { + "type": "reference", + "name": "DatasetMetadata", + "target": "1314", + "ref": { + "id": 7934, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 42 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/DatasetMetadata" + } + }, + { + "type": "reference", + "name": "KeyValueStoreMetadata", + "target": "1317", + "ref": { + "id": 7937, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 52 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/KeyValueStoreMetadata" + } + } + ] + }, + { + "type": "reference", + "name": "RequestQueueMetadata", + "target": "1319", + "ref": { + "id": 7939, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 59 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/RequestQueueMetadata" + } + } + ] + }, + "overwrites": { + "name": "Storage.get_metadata", + "target": 2333, + "type": "reference", + "ref": { + "id": 8953, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_base.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 29 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/Storage#get_metadata" + } + }, + "parentId": 2277 + } + ], + "overwrites": { + "name": "Storage.get_metadata", + "target": 2333, + "type": "reference", + "ref": { + "id": 8953, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_base.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 29 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/Storage#get_metadata" + } + }, + "permalink": "/python/api/class/KeyValueStore#get_metadata", + "parentId": 2269 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Open a storage, either restore existing or create a new one.\n" + } + ] + }, + "decorations": [ + { + "name": "classmethod" + }, + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 8899, + "module": "storages._base", + "name": "open", + "parsedDocstring": { + "text": "Open a storage, either restore existing or create a new one.\n", + "args": { + "id": "The storage ID.", + "name": "The storage name (global scope, persists across runs). Name can only contain letters \"a\" through \"z\",\nthe digits \"0\" through \"9\", and the hyphen (\"-\") but only in the middle of the string\n(e.g. \"my-value-1\").", + "alias": "The storage alias (run scope, creates unnamed storage).", + "configuration": "Configuration object used during the storage creation or restoration process.", + "storage_client": "Underlying storage client to use. If not provided, the default global storage client\nfrom the service locator will be used." + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_key_value_store.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 113, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storages/_key_value_store.py#L113" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Open a storage, either restore existing or create a new one.\n" + } + ] + }, + "flags": {}, + "id": 8956, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "open", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The storage ID." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 8957, + "kind": 32768, + "kindString": "Parameter", + "name": "id", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2336 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The storage name (global scope, persists across runs). Name can only contain letters \"a\" through \"z\",\nthe digits \"0\" through \"9\", and the hyphen (\"-\") but only in the middle of the string\n(e.g. \"my-value-1\")." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 8958, + "kind": 32768, + "kindString": "Parameter", + "name": "name", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2336 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The storage alias (run scope, creates unnamed storage)." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 8959, + "kind": 32768, + "kindString": "Parameter", + "name": "alias", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2336 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Configuration object used during the storage creation or restoration process." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 8960, + "kind": 32768, + "kindString": "Parameter", + "name": "configuration", + "type": { + "name": "Configuration | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "Configuration", + "target": "2", + "ref": { + "id": 6622, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/configuration.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 20 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/Configuration" + } + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2336 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Underlying storage client to use. If not provided, the default global storage client\nfrom the service locator will be used." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 8961, + "kind": 32768, + "kindString": "Parameter", + "name": "storage_client", + "type": { + "name": "StorageClient | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "StorageClient", + "target": "1282", + "ref": { + "id": 7902, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 19 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/StorageClient" + } + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2336 + } + ], + "type": { + "name": "Storage", + "type": "reference", + "target": "2330", + "ref": { + "id": 8950, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_base.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 15 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/Storage" + } + }, + "overwrites": { + "name": "Storage.open", + "target": 2335, + "type": "reference", + "ref": { + "id": 8955, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_base.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 34 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/Storage#open" + } + }, + "parentId": 2279 + } + ], + "overwrites": { + "name": "Storage.open", + "target": 2335, + "type": "reference", + "ref": { + "id": 8955, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_base.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 34 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/Storage#open" + } + }, + "permalink": "/python/api/class/KeyValueStore#open", + "parentId": 2269 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Drop the storage, removing it from the underlying storage client and clearing the cache." + } + ] + }, + "decorations": [ + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 8906, + "module": "storages._base", + "name": "drop", + "parsedDocstring": { + "text": "Drop the storage, removing it from the underlying storage client and clearing the cache." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_key_value_store.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 140, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storages/_key_value_store.py#L140" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Drop the storage, removing it from the underlying storage client and clearing the cache." + } + ] + }, + "flags": {}, + "id": 8963, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "drop", + "parameters": [], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "overwrites": { + "name": "Storage.drop", + "target": 2342, + "type": "reference", + "ref": { + "id": 8962, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_base.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 57 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/Storage#drop" + } + }, + "parentId": 2286 + } + ], + "overwrites": { + "name": "Storage.drop", + "target": 2342, + "type": "reference", + "ref": { + "id": 8962, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_base.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 57 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/Storage#drop" + } + }, + "permalink": "/python/api/class/KeyValueStore#drop", + "parentId": 2269 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Purge the storage, removing all items from the underlying storage client.\n\nThis method does not remove the storage itself, e.g. don't remove the metadata,\nbut clears all items within it." + } + ] + }, + "decorations": [ + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 8908, + "module": "storages._base", + "name": "purge", + "parsedDocstring": { + "text": "Purge the storage, removing all items from the underlying storage client.\n\nThis method does not remove the storage itself, e.g. don't remove the metadata,\nbut clears all items within it." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_key_value_store.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 148, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storages/_key_value_store.py#L148" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Purge the storage, removing all items from the underlying storage client.\n\nThis method does not remove the storage itself, e.g. don't remove the metadata,\nbut clears all items within it." + } + ] + }, + "flags": {}, + "id": 8965, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "purge", + "parameters": [], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "overwrites": { + "name": "Storage.purge", + "target": 2344, + "type": "reference", + "ref": { + "id": 8964, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_base.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 61 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/Storage#purge" + } + }, + "parentId": 2288 + } + ], + "overwrites": { + "name": "Storage.purge", + "target": 2344, + "type": "reference", + "ref": { + "id": 8964, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_base.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 61 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/Storage#purge" + } + }, + "permalink": "/python/api/class/KeyValueStore#purge", + "parentId": 2269 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Get a value from the KVS.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 8910, + "module": "storages._key_value_store", + "name": "get_value", + "parsedDocstring": { + "text": "Get a value from the KVS.\n", + "args": { + "key": "Key of the record to retrieve.", + "default_value": "Default value returned in case the record does not exist.\n" + }, + "returns": "The value associated with the given key. `default_value` is used in case the record does not exist." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_key_value_store.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 160, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storages/_key_value_store.py#L160" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "The value associated with the given key. `default_value` is used in case the record does not exist." + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Get a value from the KVS.\n" + } + ] + }, + "flags": {}, + "id": 8911, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "get_value", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Key of the record to retrieve." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 8912, + "kind": 32768, + "kindString": "Parameter", + "name": "key", + "type": { + "name": "str", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2291 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Default value returned in case the record does not exist.\n" + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": false + }, + "id": 8913, + "kind": 32768, + "kindString": "Parameter", + "name": "default_value", + "type": { + "name": "T | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "T", + "target": "83", + "ref": { + "id": 6703, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_types.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 36 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api#T" + } + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2291 + } + ], + "type": { + "name": "T | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "T", + "target": "83", + "ref": { + "id": 6703, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_types.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 36 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api#T" + } + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2290 + }, + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "The value associated with the given key. `default_value` is used in case the record does not exist." + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Get a value from the KVS.\n" + } + ] + }, + "flags": {}, + "id": 8942, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "get_value", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Key of the record to retrieve." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 8943, + "kind": 32768, + "kindString": "Parameter", + "name": "key", + "type": { + "name": "str", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2322 + } + ], + "type": { + "name": "Any", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2290 + }, + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "The value associated with the given key. `default_value` is used in case the record does not exist." + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Get a value from the KVS.\n" + } + ] + }, + "flags": {}, + "id": 8944, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "get_value", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Key of the record to retrieve." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 8945, + "kind": 32768, + "kindString": "Parameter", + "name": "key", + "type": { + "name": "str", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2324 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Default value returned in case the record does not exist.\n" + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 8946, + "kind": 32768, + "kindString": "Parameter", + "name": "default_value", + "type": { + "name": "T", + "type": "reference", + "target": "83", + "ref": { + "id": 6703, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_types.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 36 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api#T" + } + }, + "parentId": 2324 + } + ], + "type": { + "name": "T", + "type": "reference", + "target": "83", + "ref": { + "id": 6703, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_types.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 36 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api#T" + } + }, + "parentId": 2290 + }, + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "The value associated with the given key. `default_value` is used in case the record does not exist." + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Get a value from the KVS.\n" + } + ] + }, + "flags": {}, + "id": 8947, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "get_value", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Key of the record to retrieve." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 8948, + "kind": 32768, + "kindString": "Parameter", + "name": "key", + "type": { + "name": "str", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2327 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Default value returned in case the record does not exist.\n" + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": false + }, + "id": 8949, + "kind": 32768, + "kindString": "Parameter", + "name": "default_value", + "type": { + "name": "T | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "T", + "target": "83", + "ref": { + "id": 6703, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_types.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 36 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api#T" + } + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2327 + } + ], + "type": { + "name": "T | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "T", + "target": "83", + "ref": { + "id": 6703, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_types.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 36 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api#T" + } + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2290 + } + ], + "permalink": "/python/api/class/KeyValueStore#get_value", + "parentId": 2269 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Set a value in the KVS.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 8914, + "module": "storages._key_value_store", + "name": "set_value", + "parsedDocstring": { + "text": "Set a value in the KVS.\n", + "args": { + "key": "Key of the record to set.", + "value": "Value to set.", + "content_type": "The MIME content type string." + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_key_value_store.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 173, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storages/_key_value_store.py#L173" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Set a value in the KVS.\n" + } + ] + }, + "flags": {}, + "id": 8915, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "set_value", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Key of the record to set." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 8916, + "kind": 32768, + "kindString": "Parameter", + "name": "key", + "type": { + "name": "str", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2295 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Value to set." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 8917, + "kind": 32768, + "kindString": "Parameter", + "name": "value", + "type": { + "name": "Any", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2295 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The MIME content type string." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": false + }, + "id": 8918, + "kind": 32768, + "kindString": "Parameter", + "name": "content_type", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2295 + } + ], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "parentId": 2294 + } + ], + "permalink": "/python/api/class/KeyValueStore#set_value", + "parentId": 2269 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Delete a value from the KVS.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 8919, + "module": "storages._key_value_store", + "name": "delete_value", + "parsedDocstring": { + "text": "Delete a value from the KVS.\n", + "args": { + "key": "Key of the record to delete." + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_key_value_store.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 188, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storages/_key_value_store.py#L188" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Delete a value from the KVS.\n" + } + ] + }, + "flags": {}, + "id": 8920, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "delete_value", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Key of the record to delete." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 8921, + "kind": 32768, + "kindString": "Parameter", + "name": "key", + "type": { + "name": "str", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2300 + } + ], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "parentId": 2299 + } + ], + "permalink": "/python/api/class/KeyValueStore#delete_value", + "parentId": 2269 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Iterate over the existing keys in the KVS.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 8922, + "module": "storages._key_value_store", + "name": "iterate_keys", + "parsedDocstring": { + "text": "Iterate over the existing keys in the KVS.\n", + "args": { + "exclusive_start_key": "Key to start the iteration from.", + "limit": "Maximum number of keys to return. None means no limit.\n" + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_key_value_store.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 196, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storages/_key_value_store.py#L196" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Iterate over the existing keys in the KVS.\n" + } + ] + }, + "flags": {}, + "id": 8923, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "iterate_keys", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Key to start the iteration from." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": false + }, + "id": 8924, + "kind": 32768, + "kindString": "Parameter", + "name": "exclusive_start_key", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2303 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Maximum number of keys to return. None means no limit.\n" + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": false + }, + "id": 8925, + "kind": 32768, + "kindString": "Parameter", + "name": "limit", + "type": { + "name": "int | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "int", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2303 + } + ], + "type": { + "name": "AsyncIterator", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "KeyValueStoreRecordMetadata", + "target": "1325", + "ref": { + "id": 7945, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 78 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/KeyValueStoreRecordMetadata" + } + } + ], + "target": 0, + "ref": {} + }, + "parentId": 2302 + } + ], + "permalink": "/python/api/class/KeyValueStore#iterate_keys", + "parentId": 2269 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "List all the existing keys in the KVS.\n\nIt uses client's `iterate_keys` method to get the keys.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 8926, + "module": "storages._key_value_store", + "name": "list_keys", + "parsedDocstring": { + "text": "List all the existing keys in the KVS.\n\nIt uses client's `iterate_keys` method to get the keys.\n", + "args": { + "exclusive_start_key": "Key to start the iteration from.", + "limit": "Maximum number of keys to return.\n" + }, + "returns": "A list of keys in the KVS." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_key_value_store.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 216, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storages/_key_value_store.py#L216" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "A list of keys in the KVS." + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "List all the existing keys in the KVS.\n\nIt uses client's `iterate_keys` method to get the keys.\n" + } + ] + }, + "flags": {}, + "id": 8927, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "list_keys", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Key to start the iteration from." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": false + }, + "id": 8928, + "kind": 32768, + "kindString": "Parameter", + "name": "exclusive_start_key", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2307 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Maximum number of keys to return.\n" + } + ] + }, + "defaultValue": "1000", + "flags": { + "isOptional": true, + "keyword-only": false + }, + "id": 8929, + "kind": 32768, + "kindString": "Parameter", + "name": "limit", + "type": { + "name": "int", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2307 + } + ], + "type": { + "name": "list", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "KeyValueStoreRecordMetadata", + "target": "1325", + "ref": { + "id": 7945, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 78 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/KeyValueStoreRecordMetadata" + } + } + ], + "target": 0, + "ref": {} + }, + "parentId": 2306 + } + ], + "permalink": "/python/api/class/KeyValueStore#list_keys", + "parentId": 2269 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Check if a record with the given key exists in the key-value store.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 8930, + "module": "storages._key_value_store", + "name": "record_exists", + "parsedDocstring": { + "text": "Check if a record with the given key exists in the key-value store.\n", + "args": { + "key": "Key of the record to check for existence.\n" + }, + "returns": "True if a record with the given key exists, False otherwise." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_key_value_store.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 240, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storages/_key_value_store.py#L240" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "True if a record with the given key exists, False otherwise." + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Check if a record with the given key exists in the key-value store.\n" + } + ] + }, + "flags": {}, + "id": 8931, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "record_exists", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Key of the record to check for existence.\n" + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 8932, + "kind": 32768, + "kindString": "Parameter", + "name": "key", + "type": { + "name": "str", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2311 + } + ], + "type": { + "name": "bool", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2310 + } + ], + "permalink": "/python/api/class/KeyValueStore#record_exists", + "parentId": 2269 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Get the public URL for the given key.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 8933, + "module": "storages._key_value_store", + "name": "get_public_url", + "parsedDocstring": { + "text": "Get the public URL for the given key.\n", + "args": { + "key": "Key of the record for which URL is required.\n" + }, + "returns": "The public URL for the given key." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_key_value_store.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 251, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storages/_key_value_store.py#L251" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "The public URL for the given key." + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Get the public URL for the given key.\n" + } + ] + }, + "flags": {}, + "id": 8934, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "get_public_url", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Key of the record for which URL is required.\n" + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 8935, + "kind": 32768, + "kindString": "Parameter", + "name": "key", + "type": { + "name": "str", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2314 + } + ], + "type": { + "name": "str", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2313 + } + ], + "permalink": "/python/api/class/KeyValueStore#get_public_url", + "parentId": 2269 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Get a value from KVS that will be automatically saved on changes.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 8936, + "module": "storages._key_value_store", + "name": "get_auto_saved_value", + "parsedDocstring": { + "text": "Get a value from KVS that will be automatically saved on changes.\n", + "args": { + "key": "Key of the record, to store the value.", + "default_value": "Value to be used if the record does not exist yet. Should be a dictionary.\n" + }, + "returns": "Return the value of the key." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_key_value_store.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 262, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storages/_key_value_store.py#L262" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "Return the value of the key." + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Get a value from KVS that will be automatically saved on changes.\n" + } + ] + }, + "flags": {}, + "id": 8937, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "get_auto_saved_value", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Key of the record, to store the value." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 8938, + "kind": 32768, + "kindString": "Parameter", + "name": "key", + "type": { + "name": "str", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2317 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Value to be used if the record does not exist yet. Should be a dictionary.\n" + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": false + }, + "id": 8939, + "kind": 32768, + "kindString": "Parameter", + "name": "default_value", + "type": { + "name": "dict[str, JsonSerializable] | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "dict", + "typeArguments": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "reference", + "name": "JsonSerializable", + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2317 + } + ], + "type": { + "name": "dict", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "reference", + "name": "JsonSerializable", + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + }, + "parentId": 2316 + } + ], + "permalink": "/python/api/class/KeyValueStore#get_auto_saved_value", + "parentId": 2269 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Force autosaved values to be saved without waiting for an event in Event Manager." + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 8940, + "module": "storages._key_value_store", + "name": "persist_autosaved_values", + "parsedDocstring": { + "text": "Force autosaved values to be saved without waiting for an event in Event Manager." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_key_value_store.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 299, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storages/_key_value_store.py#L299" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Force autosaved values to be saved without waiting for an event in Event Manager." + } + ] + }, + "flags": {}, + "id": 8941, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "persist_autosaved_values", + "parameters": [], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "parentId": 2320 + } + ], + "permalink": "/python/api/class/KeyValueStore#persist_autosaved_values", + "parentId": 2269 + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Key-value store is a storage for reading and writing data records with unique key identifiers.\n\nThe key-value store class acts as a high-level interface for storing, retrieving, and managing data records\nidentified by unique string keys. It abstracts away the underlying storage implementation details,\nallowing you to work with the same API regardless of whether data is stored in memory, on disk,\nor in the cloud.\n\nEach data record is associated with a specific MIME content type, allowing storage of various\ndata formats such as JSON, text, images, HTML snapshots or any binary data. This class is\ncommonly used to store inputs, outputs, and other artifacts of crawler operations.\n\nYou can instantiate a key-value store using the `open` class method, which will create a store\nwith the specified name or id. The underlying storage implementation is determined by the configured\nstorage client.\n\n### Usage\n\n```python\nfrom crawlee.storages import KeyValueStore\n\n# Open a named key-value store\nkvs = await KeyValueStore.open(name='my-store')\n\n# Store and retrieve data\nawait kvs.set_value('product-1234.json', [{'name': 'Smartphone', 'price': 799.99}])\nproduct = await kvs.get_value('product-1234')\n```" + } + ] + }, + "decorations": [ + { + "args": "('Storages')", + "name": "docs_group" + } + ], + "flags": {}, + "groups": [ + { + "children": [ + 8890, + 8919, + 8906, + 8936, + 8897, + 8933, + 8910, + 8922, + 8926, + 8899, + 8940, + 8908, + 8930, + 8914 + ], + "title": "Methods" + }, + { + "children": [ + 8895, + 8896 + ], + "title": "Properties" + } + ], + "id": 8889, + "module": "storages._key_value_store", + "name": "KeyValueStore", + "parsedDocstring": { + "text": "Key-value store is a storage for reading and writing data records with unique key identifiers.\n\nThe key-value store class acts as a high-level interface for storing, retrieving, and managing data records\nidentified by unique string keys. It abstracts away the underlying storage implementation details,\nallowing you to work with the same API regardless of whether data is stored in memory, on disk,\nor in the cloud.\n\nEach data record is associated with a specific MIME content type, allowing storage of various\ndata formats such as JSON, text, images, HTML snapshots or any binary data. This class is\ncommonly used to store inputs, outputs, and other artifacts of crawler operations.\n\nYou can instantiate a key-value store using the `open` class method, which will create a store\nwith the specified name or id. The underlying storage implementation is determined by the configured\nstorage client.\n\n### Usage\n\n```python\nfrom crawlee.storages import KeyValueStore\n\n# Open a named key-value store\nkvs = await KeyValueStore.open(name='my-store')\n\n# Store and retrieve data\nawait kvs.set_value('product-1234.json', [{'name': 'Smartphone', 'price': 799.99}])\nproduct = await kvs.get_value('product-1234')\n```" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_key_value_store.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 40, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storages/_key_value_store.py#L40" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "extendedTypes": [ + { + "name": "Storage", + "target": "2330", + "type": "reference", + "ref": { + "id": 8950, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_base.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 15 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/Storage" + } + } + ], + "permalink": "/python/api/class/KeyValueStore", + "categories": [] + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Initialize a new instance.\n\nPreferably use the `RequestQueue.open` constructor to create a new instance.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 11314, + "module": "storages._request_queue", + "name": "__init__", + "parsedDocstring": { + "text": "Initialize a new instance.\n\nPreferably use the `RequestQueue.open` constructor to create a new instance.\n", + "args": { + "client": "An instance of a storage client.", + "id": "The unique identifier of the storage.", + "name": "The name of the storage, if available." + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_request_queue.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 74, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storages/_request_queue.py#L74" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Initialize a new instance.\n\nPreferably use the `RequestQueue.open` constructor to create a new instance.\n" + } + ] + }, + "flags": {}, + "id": 11315, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "__init__", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "An instance of a storage client." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 11316, + "kind": 32768, + "kindString": "Parameter", + "name": "client", + "type": { + "name": "RequestQueueClient", + "type": "reference", + "target": "1195", + "ref": { + "id": 10160, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_request_queue_client.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 13 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/RequestQueueClient" + } + }, + "parentId": 2350 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The unique identifier of the storage." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 11317, + "kind": 32768, + "kindString": "Parameter", + "name": "id", + "type": { + "name": "str", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2350 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The name of the storage, if available." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 11318, + "kind": 32768, + "kindString": "Parameter", + "name": "name", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2350 + } + ], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "parentId": 2349 + } + ], + "permalink": "/python/api/class/RequestQueue#__init__", + "parentId": 2348 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Get the storage ID." + } + ] + }, + "decorations": [ + { + "name": "property" + }, + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 11319, + "module": "storages._base", + "name": "id", + "parsedDocstring": { + "text": "Get the storage ID." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_request_queue.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 95, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storages/_request_queue.py#L95" + } + ], + "type": { + "name": "str", + "type": "reference", + "target": 0, + "ref": {} + }, + "overwrites": { + "name": "Storage.id", + "target": 2331, + "type": "reference", + "ref": { + "id": 11296, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_base.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 20 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api/class/Storage#id" + } + }, + "permalink": "/python/api/class/RequestQueue#id", + "parentId": 2348 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Get the storage name." + } + ] + }, + "decorations": [ + { + "name": "property" + }, + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 11320, + "module": "storages._base", + "name": "name", + "parsedDocstring": { + "text": "Get the storage name." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_request_queue.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 100, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storages/_request_queue.py#L100" + } + ], + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "overwrites": { + "name": "Storage.name", + "target": 2332, + "type": "reference", + "ref": { + "id": 11297, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_base.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 25 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api/class/Storage#name" + } + }, + "permalink": "/python/api/class/RequestQueue#name", + "parentId": 2348 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Get the storage metadata." + } + ] + }, + "decorations": [ + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 11321, + "module": "storages._base", + "name": "get_metadata", + "parsedDocstring": { + "text": "Get the storage metadata." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_request_queue.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 104, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storages/_request_queue.py#L104" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Get the storage metadata." + } + ] + }, + "flags": {}, + "id": 11299, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "get_metadata", + "parameters": [], + "type": { + "name": "DatasetMetadata | KeyValueStoreMetadata | RequestQueueMetadata", + "type": "union", + "types": [ + { + "type": "union", + "types": [ + { + "type": "reference", + "name": "DatasetMetadata", + "target": "1314", + "ref": { + "id": 10279, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 42 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/DatasetMetadata" + } + }, + { + "type": "reference", + "name": "KeyValueStoreMetadata", + "target": "1317", + "ref": { + "id": 10282, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 52 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/KeyValueStoreMetadata" + } + } + ] + }, + { + "type": "reference", + "name": "RequestQueueMetadata", + "target": "1319", + "ref": { + "id": 10284, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 59 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/RequestQueueMetadata" + } + } + ] + }, + "overwrites": { + "name": "Storage.get_metadata", + "target": 2333, + "type": "reference", + "ref": { + "id": 11298, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_base.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 29 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/Storage#get_metadata" + } + }, + "parentId": 2356 + } + ], + "overwrites": { + "name": "Storage.get_metadata", + "target": 2333, + "type": "reference", + "ref": { + "id": 11298, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_base.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 29 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/Storage#get_metadata" + } + }, + "permalink": "/python/api/class/RequestQueue#get_metadata", + "parentId": 2348 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Get the number of requests in the loader that have been handled." + } + ] + }, + "decorations": [ + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 11323, + "module": "request_loaders._request_loader", + "name": "get_handled_count", + "parsedDocstring": { + "text": "Get the number of requests in the loader that have been handled." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_request_queue.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 108, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storages/_request_queue.py#L108" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Get the number of requests in the loader that have been handled." + } + ] + }, + "flags": {}, + "id": 11901, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "get_handled_count", + "parameters": [], + "type": { + "name": "int", + "type": "reference", + "target": 0, + "ref": {} + }, + "overwrites": { + "name": "RequestManager.get_handled_count", + "target": 4143, + "type": "reference", + "ref": { + "id": 13108, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 29 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestManager#get_handled_count" + } + }, + "parentId": 2358 + } + ], + "overwrites": { + "name": "RequestManager.get_handled_count", + "target": 4143, + "type": "reference", + "ref": { + "id": 13108, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 29 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestManager#get_handled_count" + } + }, + "permalink": "/python/api/class/RequestQueue#get_handled_count", + "parentId": 2348 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Get an offline approximation of the total number of requests in the loader (i.e. pending + handled)." + } + ] + }, + "decorations": [ + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 11325, + "module": "request_loaders._request_loader", + "name": "get_total_count", + "parsedDocstring": { + "text": "Get an offline approximation of the total number of requests in the loader (i.e. pending + handled)." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_request_queue.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 113, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storages/_request_queue.py#L113" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Get an offline approximation of the total number of requests in the loader (i.e. pending + handled)." + } + ] + }, + "flags": {}, + "id": 11903, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "get_total_count", + "parameters": [], + "type": { + "name": "int", + "type": "reference", + "target": 0, + "ref": {} + }, + "overwrites": { + "name": "RequestManager.get_total_count", + "target": 4144, + "type": "reference", + "ref": { + "id": 13109, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 33 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestManager#get_total_count" + } + }, + "parentId": 2360 + } + ], + "overwrites": { + "name": "RequestManager.get_total_count", + "target": 4144, + "type": "reference", + "ref": { + "id": 13109, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 33 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestManager#get_total_count" + } + }, + "permalink": "/python/api/class/RequestQueue#get_total_count", + "parentId": 2348 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Open a storage, either restore existing or create a new one.\n" + } + ] + }, + "decorations": [ + { + "name": "classmethod" + }, + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 11327, + "module": "storages._base", + "name": "open", + "parsedDocstring": { + "text": "Open a storage, either restore existing or create a new one.\n", + "args": { + "id": "The storage ID.", + "name": "The storage name (global scope, persists across runs). Name can only contain letters \"a\" through \"z\",\nthe digits \"0\" through \"9\", and the hyphen (\"-\") but only in the middle of the string\n(e.g. \"my-value-1\").", + "alias": "The storage alias (run scope, creates unnamed storage).", + "configuration": "Configuration object used during the storage creation or restoration process.", + "storage_client": "Underlying storage client to use. If not provided, the default global storage client\nfrom the service locator will be used." + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_request_queue.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 119, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storages/_request_queue.py#L119" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Open a storage, either restore existing or create a new one.\n" + } + ] + }, + "flags": {}, + "id": 11301, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "open", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The storage ID." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 11302, + "kind": 32768, + "kindString": "Parameter", + "name": "id", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2336 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The storage name (global scope, persists across runs). Name can only contain letters \"a\" through \"z\",\nthe digits \"0\" through \"9\", and the hyphen (\"-\") but only in the middle of the string\n(e.g. \"my-value-1\")." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 11303, + "kind": 32768, + "kindString": "Parameter", + "name": "name", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2336 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The storage alias (run scope, creates unnamed storage)." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 11304, + "kind": 32768, + "kindString": "Parameter", + "name": "alias", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2336 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Configuration object used during the storage creation or restoration process." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 11305, + "kind": 32768, + "kindString": "Parameter", + "name": "configuration", + "type": { + "name": "Configuration | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "Configuration", + "target": "2", + "ref": { + "id": 8967, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/configuration.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 20 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/Configuration" + } + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2336 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Underlying storage client to use. If not provided, the default global storage client\nfrom the service locator will be used." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 11306, + "kind": 32768, + "kindString": "Parameter", + "name": "storage_client", + "type": { + "name": "StorageClient | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "StorageClient", + "target": "1282", + "ref": { + "id": 10247, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 19 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/StorageClient" + } + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2336 + } + ], + "type": { + "name": "Storage", + "type": "reference", + "target": "2330", + "ref": { + "id": 11295, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_base.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 15 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/Storage" + } + }, + "overwrites": { + "name": "Storage.open", + "target": 2335, + "type": "reference", + "ref": { + "id": 11300, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_base.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 34 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/Storage#open" + } + }, + "parentId": 2362 + } + ], + "overwrites": { + "name": "Storage.open", + "target": 2335, + "type": "reference", + "ref": { + "id": 11300, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_base.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 34 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/Storage#open" + } + }, + "permalink": "/python/api/class/RequestQueue#open", + "parentId": 2348 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Remove persistent state either from the Apify Cloud storage or from the local database." + } + ] + }, + "decorations": [ + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 11334, + "module": "request_loaders._request_manager", + "name": "drop", + "parsedDocstring": { + "text": "Remove persistent state either from the Apify Cloud storage or from the local database." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_request_queue.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 144, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storages/_request_queue.py#L144" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Remove persistent state either from the Apify Cloud storage or from the local database." + } + ] + }, + "flags": {}, + "id": 11968, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "drop", + "parameters": [], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "overwrites": { + "name": "Storage.drop", + "target": 2342, + "type": "reference", + "ref": { + "id": 11307, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_base.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 57 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/Storage#drop" + } + }, + "parentId": 2369 + } + ], + "overwrites": { + "name": "Storage.drop", + "target": 2342, + "type": "reference", + "ref": { + "id": 11307, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_base.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 57 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/Storage#drop" + } + }, + "permalink": "/python/api/class/RequestQueue#drop", + "parentId": 2348 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Remove all queued requests from this manager while keeping it usable.\n\nDefault implementation is a no-op. Subclasses with persistent state should override to clear it." + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 11336, + "module": "request_loaders._request_manager", + "name": "purge", + "parsedDocstring": { + "text": "Remove all queued requests from this manager while keeping it usable.\n\nDefault implementation is a no-op. Subclasses with persistent state should override to clear it." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_request_queue.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 152, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storages/_request_queue.py#L152" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Remove all queued requests from this manager while keeping it usable.\n\nDefault implementation is a no-op. Subclasses with persistent state should override to clear it." + } + ] + }, + "flags": {}, + "id": 11970, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "purge", + "parameters": [], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "overwrites": { + "name": "Storage.purge", + "target": 2344, + "type": "reference", + "ref": { + "id": 11309, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_base.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 61 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/Storage#purge" + } + }, + "parentId": 2371 + } + ], + "overwrites": { + "name": "Storage.purge", + "target": 2344, + "type": "reference", + "ref": { + "id": 11309, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_base.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 61 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/Storage#purge" + } + }, + "permalink": "/python/api/class/RequestQueue#purge", + "parentId": 2348 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Add a single request to the manager and store it in underlying resource client.\n" + } + ] + }, + "decorations": [ + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 11338, + "module": "request_loaders._request_manager", + "name": "add_request", + "parsedDocstring": { + "text": "Add a single request to the manager and store it in underlying resource client.\n", + "args": { + "request": "The request object (or its string representation) to be added to the manager.", + "forefront": "Determines whether the request should be added to the beginning (if True) or the end (if False)\nof the manager.\n" + }, + "returns": "Information about the request addition to the manager or None if the request was not added." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_request_queue.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 177, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storages/_request_queue.py#L177" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "Information about the request addition to the manager or None if the request was not added." + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Add a single request to the manager and store it in underlying resource client.\n" + } + ] + }, + "flags": {}, + "id": 11972, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "add_request", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The request object (or its string representation) to be added to the manager." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 11973, + "kind": 32768, + "kindString": "Parameter", + "name": "request", + "type": { + "name": "str | Request", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "reference", + "name": "Request", + "target": "409", + "ref": { + "id": 9374, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_request.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 145 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/Request" + } + } + ] + }, + "parentId": 3007 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Determines whether the request should be added to the beginning (if True) or the end (if False)\nof the manager.\n" + } + ] + }, + "defaultValue": "False", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 11974, + "kind": 32768, + "kindString": "Parameter", + "name": "forefront", + "type": { + "name": "bool", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 3007 + } + ], + "type": { + "name": "ProcessedRequest | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "ProcessedRequest", + "target": "1340", + "ref": { + "id": 10305, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 140 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/ProcessedRequest" + } + }, + { + "type": "literal", + "value": {} + } + ] + }, + "overwrites": { + "name": "RequestManager.add_request", + "target": 3006, + "type": "reference", + "ref": { + "id": 11971, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_manager.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 32 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestManager#add_request" + } + }, + "parentId": 2373 + } + ], + "overwrites": { + "name": "RequestManager.add_request", + "target": 3006, + "type": "reference", + "ref": { + "id": 11971, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_manager.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 32 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestManager#add_request" + } + }, + "permalink": "/python/api/class/RequestQueue#add_request", + "parentId": 2348 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Add requests to the manager in batches.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 11342, + "module": "request_loaders._request_manager", + "name": "add_requests", + "parsedDocstring": { + "text": "Add requests to the manager in batches.\n", + "args": { + "requests": "Requests to enqueue.", + "forefront": "If True, add requests to the beginning of the queue.", + "batch_size": "The number of requests to add in one batch.", + "wait_time_between_batches": "Time to wait between adding batches.", + "wait_for_all_requests_to_be_added": "If True, wait for all requests to be added before returning.", + "wait_for_all_requests_to_be_added_timeout": "Timeout for waiting for all requests to be added." + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_request_queue.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 201, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storages/_request_queue.py#L201" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Add requests to the manager in batches.\n" + } + ] + }, + "flags": {}, + "id": 11976, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "add_requests", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Requests to enqueue." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 11977, + "kind": 32768, + "kindString": "Parameter", + "name": "requests", + "type": { + "name": "Sequence", + "type": "reference", + "typeArguments": [ + { + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "reference", + "name": "Request", + "target": "409", + "ref": { + "id": 9374, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_request.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 145 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/Request" + } + } + ] + } + ], + "target": 0, + "ref": {} + }, + "parentId": 3011 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "If True, add requests to the beginning of the queue." + } + ] + }, + "defaultValue": "False", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 11978, + "kind": 32768, + "kindString": "Parameter", + "name": "forefront", + "type": { + "name": "bool", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 3011 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The number of requests to add in one batch." + } + ] + }, + "defaultValue": "1000", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 11979, + "kind": 32768, + "kindString": "Parameter", + "name": "batch_size", + "type": { + "name": "int", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 3011 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Time to wait between adding batches." + } + ] + }, + "defaultValue": "timedelta(seconds=1)", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 11980, + "kind": 32768, + "kindString": "Parameter", + "name": "wait_time_between_batches", + "type": { + "name": "timedelta", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 3011 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "If True, wait for all requests to be added before returning." + } + ] + }, + "defaultValue": "False", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 11981, + "kind": 32768, + "kindString": "Parameter", + "name": "wait_for_all_requests_to_be_added", + "type": { + "name": "bool", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 3011 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Timeout for waiting for all requests to be added." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 11982, + "kind": 32768, + "kindString": "Parameter", + "name": "wait_for_all_requests_to_be_added_timeout", + "type": { + "name": "timedelta | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "timedelta", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 3011 + } + ], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "overwrites": { + "name": "RequestManager.add_requests", + "target": 3010, + "type": "reference", + "ref": { + "id": 11975, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_manager.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 49 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestManager#add_requests" + } + }, + "parentId": 2377 + } + ], + "overwrites": { + "name": "RequestManager.add_requests", + "target": 3010, + "type": "reference", + "ref": { + "id": 11975, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_manager.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 49 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestManager#add_requests" + } + }, + "permalink": "/python/api/class/RequestQueue#add_requests", + "parentId": 2348 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Return the next request in the queue to be processed.\n\nOnce you successfully finish processing of the request, you need to call `RequestQueue.mark_request_as_handled`\nto mark the request as handled in the queue. If there was some error in processing the request, call\n`RequestQueue.reclaim_request` instead, so that the queue will give the request to some other consumer\nin another call to the `fetch_next_request` method.\n\nNote that the `None` return value does not mean the queue processing finished, it means there are currently\nno pending requests. To check whether all requests in queue were finished, use `RequestQueue.is_finished`\ninstead.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 11350, + "module": "storages._request_queue", + "name": "fetch_next_request", + "parsedDocstring": { + "text": "Return the next request in the queue to be processed.\n\nOnce you successfully finish processing of the request, you need to call `RequestQueue.mark_request_as_handled`\nto mark the request as handled in the queue. If there was some error in processing the request, call\n`RequestQueue.reclaim_request` instead, so that the queue will give the request to some other consumer\nin another call to the `fetch_next_request` method.\n\nNote that the `None` return value does not mean the queue processing finished, it means there are currently\nno pending requests. To check whether all requests in queue were finished, use `RequestQueue.is_finished`\ninstead.\n", + "returns": "The next request to process, or `None` if there are no more pending requests." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_request_queue.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 251, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storages/_request_queue.py#L251" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "The next request to process, or `None` if there are no more pending requests." + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Return the next request in the queue to be processed.\n\nOnce you successfully finish processing of the request, you need to call `RequestQueue.mark_request_as_handled`\nto mark the request as handled in the queue. If there was some error in processing the request, call\n`RequestQueue.reclaim_request` instead, so that the queue will give the request to some other consumer\nin another call to the `fetch_next_request` method.\n\nNote that the `None` return value does not mean the queue processing finished, it means there are currently\nno pending requests. To check whether all requests in queue were finished, use `RequestQueue.is_finished`\ninstead.\n" + } + ] + }, + "flags": {}, + "id": 11351, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "fetch_next_request", + "parameters": [], + "type": { + "name": "Request | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "Request", + "target": "409", + "ref": { + "id": 9374, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_request.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 145 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/Request" + } + }, + { + "type": "literal", + "value": {} + } + ] + }, + "overwrites": { + "name": "RequestManager.fetch_next_request", + "target": 4147, + "type": "reference", + "ref": { + "id": 13112, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 45 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestManager#fetch_next_request" + } + }, + "parentId": 2385 + } + ], + "overwrites": { + "name": "RequestManager.fetch_next_request", + "target": 4147, + "type": "reference", + "ref": { + "id": 13112, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 45 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestManager#fetch_next_request" + } + }, + "permalink": "/python/api/class/RequestQueue#fetch_next_request", + "parentId": 2348 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Retrieve a specific request from the queue by its ID.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 11352, + "module": "storages._request_queue", + "name": "get_request", + "parsedDocstring": { + "text": "Retrieve a specific request from the queue by its ID.\n", + "args": { + "unique_key": "Unique key of the request to retrieve.\n" + }, + "returns": "The request with the specified ID, or `None` if no such request exists." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_request_queue.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 268, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storages/_request_queue.py#L268" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "The request with the specified ID, or `None` if no such request exists." + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Retrieve a specific request from the queue by its ID.\n" + } + ] + }, + "flags": {}, + "id": 11353, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "get_request", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Unique key of the request to retrieve.\n" + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 11354, + "kind": 32768, + "kindString": "Parameter", + "name": "unique_key", + "type": { + "name": "str", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2388 + } + ], + "type": { + "name": "Request | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "Request", + "target": "409", + "ref": { + "id": 9374, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_request.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 145 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/Request" + } + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2387 + } + ], + "permalink": "/python/api/class/RequestQueue#get_request", + "parentId": 2348 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Mark a request as handled after successful processing.\n\nThis method should be called after a request has been successfully processed.\nOnce marked as handled, the request will be removed from the queue and will\nnot be returned in subsequent calls to `fetch_next_request` method.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 11355, + "module": "storages._request_queue", + "name": "mark_request_as_handled", + "parsedDocstring": { + "text": "Mark a request as handled after successful processing.\n\nThis method should be called after a request has been successfully processed.\nOnce marked as handled, the request will be removed from the queue and will\nnot be returned in subsequent calls to `fetch_next_request` method.\n", + "args": { + "request": "The request to mark as handled.\n" + }, + "returns": "Information about the queue operation." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_request_queue.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 279, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storages/_request_queue.py#L279" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "Information about the queue operation." + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Mark a request as handled after successful processing.\n\nThis method should be called after a request has been successfully processed.\nOnce marked as handled, the request will be removed from the queue and will\nnot be returned in subsequent calls to `fetch_next_request` method.\n" + } + ] + }, + "flags": {}, + "id": 11356, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "mark_request_as_handled", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The request to mark as handled.\n" + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 11357, + "kind": 32768, + "kindString": "Parameter", + "name": "request", + "type": { + "name": "Request", + "type": "reference", + "target": "409", + "ref": { + "id": 9374, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_request.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 145 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/Request" + } + }, + "parentId": 2391 + } + ], + "type": { + "name": "ProcessedRequest | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "ProcessedRequest", + "target": "1340", + "ref": { + "id": 10305, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 140 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/ProcessedRequest" + } + }, + { + "type": "literal", + "value": {} + } + ] + }, + "overwrites": { + "name": "RequestManager.mark_request_as_handled", + "target": 4148, + "type": "reference", + "ref": { + "id": 13113, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 53 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestManager#mark_request_as_handled" + } + }, + "parentId": 2390 + } + ], + "overwrites": { + "name": "RequestManager.mark_request_as_handled", + "target": 4148, + "type": "reference", + "ref": { + "id": 13113, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 53 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestManager#mark_request_as_handled" + } + }, + "permalink": "/python/api/class/RequestQueue#mark_request_as_handled", + "parentId": 2348 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Reclaim a failed request back to the queue for later processing.\n\nIf a request fails during processing, this method can be used to return it to the queue.\nThe request will be returned for processing again in a subsequent call\nto `RequestQueue.fetch_next_request`.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 11358, + "module": "storages._request_queue", + "name": "reclaim_request", + "parsedDocstring": { + "text": "Reclaim a failed request back to the queue for later processing.\n\nIf a request fails during processing, this method can be used to return it to the queue.\nThe request will be returned for processing again in a subsequent call\nto `RequestQueue.fetch_next_request`.\n", + "args": { + "request": "The request to return to the queue.", + "forefront": "If true, the request will be added to the beginning of the queue.\nOtherwise, it will be added to the end.\n" + }, + "returns": "Information about the queue operation." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_request_queue.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 294, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storages/_request_queue.py#L294" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "Information about the queue operation." + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Reclaim a failed request back to the queue for later processing.\n\nIf a request fails during processing, this method can be used to return it to the queue.\nThe request will be returned for processing again in a subsequent call\nto `RequestQueue.fetch_next_request`.\n" + } + ] + }, + "flags": {}, + "id": 11359, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "reclaim_request", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The request to return to the queue." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 11360, + "kind": 32768, + "kindString": "Parameter", + "name": "request", + "type": { + "name": "Request", + "type": "reference", + "target": "409", + "ref": { + "id": 9374, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_request.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 145 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/Request" + } + }, + "parentId": 2394 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "If true, the request will be added to the beginning of the queue.\nOtherwise, it will be added to the end.\n" + } + ] + }, + "defaultValue": "False", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 11361, + "kind": 32768, + "kindString": "Parameter", + "name": "forefront", + "type": { + "name": "bool", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2394 + } + ], + "type": { + "name": "ProcessedRequest | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "ProcessedRequest", + "target": "1340", + "ref": { + "id": 10305, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 140 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/ProcessedRequest" + } + }, + { + "type": "literal", + "value": {} + } + ] + }, + "overwrites": { + "name": "RequestManager.reclaim_request", + "target": 3018, + "type": "reference", + "ref": { + "id": 11983, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_manager.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 77 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestManager#reclaim_request" + } + }, + "parentId": 2393 + } + ], + "overwrites": { + "name": "RequestManager.reclaim_request", + "target": 3018, + "type": "reference", + "ref": { + "id": 11983, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_manager.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 77 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestManager#reclaim_request" + } + }, + "permalink": "/python/api/class/RequestQueue#reclaim_request", + "parentId": 2348 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Check if the request queue is empty.\n\nAn empty queue means that there are no requests currently in the queue, either pending or being processed.\nHowever, this does not necessarily mean that the crawling operation is finished, as there still might be\ntasks that could add additional requests to the queue.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 11362, + "module": "storages._request_queue", + "name": "is_empty", + "parsedDocstring": { + "text": "Check if the request queue is empty.\n\nAn empty queue means that there are no requests currently in the queue, either pending or being processed.\nHowever, this does not necessarily mean that the crawling operation is finished, as there still might be\ntasks that could add additional requests to the queue.\n", + "returns": "True if the request queue is empty, False otherwise." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_request_queue.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 316, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storages/_request_queue.py#L316" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "True if the request queue is empty, False otherwise." + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Check if the request queue is empty.\n\nAn empty queue means that there are no requests currently in the queue, either pending or being processed.\nHowever, this does not necessarily mean that the crawling operation is finished, as there still might be\ntasks that could add additional requests to the queue.\n" + } + ] + }, + "flags": {}, + "id": 11363, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "is_empty", + "parameters": [], + "type": { + "name": "bool", + "type": "reference", + "target": 0, + "ref": {} + }, + "overwrites": { + "name": "RequestManager.is_empty", + "target": 4145, + "type": "reference", + "ref": { + "id": 13110, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 37 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestManager#is_empty" + } + }, + "parentId": 2397 + } + ], + "overwrites": { + "name": "RequestManager.is_empty", + "target": 4145, + "type": "reference", + "ref": { + "id": 13110, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 37 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestManager#is_empty" + } + }, + "permalink": "/python/api/class/RequestQueue#is_empty", + "parentId": 2348 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Check if the request queue is finished.\n\nA finished queue means that all requests in the queue have been processed (the queue is empty) and there\nare no more tasks that could add additional requests to the queue. This is the definitive way to check\nif a crawling operation is complete.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 11364, + "module": "storages._request_queue", + "name": "is_finished", + "parsedDocstring": { + "text": "Check if the request queue is finished.\n\nA finished queue means that all requests in the queue have been processed (the queue is empty) and there\nare no more tasks that could add additional requests to the queue. This is the definitive way to check\nif a crawling operation is complete.\n", + "returns": "True if the request queue is finished (empty and no pending add operations), False otherwise." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_request_queue.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 328, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storages/_request_queue.py#L328" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "True if the request queue is finished (empty and no pending add operations), False otherwise." + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Check if the request queue is finished.\n\nA finished queue means that all requests in the queue have been processed (the queue is empty) and there\nare no more tasks that could add additional requests to the queue. This is the definitive way to check\nif a crawling operation is complete.\n" + } + ] + }, + "flags": {}, + "id": 11365, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "is_finished", + "parameters": [], + "type": { + "name": "bool", + "type": "reference", + "target": 0, + "ref": {} + }, + "overwrites": { + "name": "RequestManager.is_finished", + "target": 4146, + "type": "reference", + "ref": { + "id": 13111, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 41 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestManager#is_finished" + } + }, + "parentId": 2399 + } + ], + "overwrites": { + "name": "RequestManager.is_finished", + "target": 4146, + "type": "reference", + "ref": { + "id": 13111, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 41 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestManager#is_finished" + } + }, + "permalink": "/python/api/class/RequestQueue#is_finished", + "parentId": 2348 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Combine the loader with a request manager to support adding and reclaiming requests.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 13115, + "module": "request_loaders._request_loader", + "name": "to_tandem", + "parsedDocstring": { + "text": "Combine the loader with a request manager to support adding and reclaiming requests.\n", + "args": { + "request_manager": "Request manager to combine the loader with.\nIf None is given, the default request queue is used." + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 56, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/request_loaders/_request_loader.py#L56" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Combine the loader with a request manager to support adding and reclaiming requests.\n" + } + ] + }, + "flags": {}, + "id": 11914, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "to_tandem", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Request manager to combine the loader with.\nIf None is given, the default request queue is used." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": false + }, + "id": 11915, + "kind": 32768, + "kindString": "Parameter", + "name": "request_manager", + "type": { + "name": "RequestManager | None", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2949 + } + ], + "type": { + "name": "RequestManagerTandem", + "type": "reference", + "target": "2803", + "ref": { + "id": 11768, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_manager_tandem.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 24 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/RequestManagerTandem" + } + }, + "inheritedFrom": { + "name": "RequestLoader.to_tandem", + "target": 2948, + "type": "reference", + "ref": { + "id": 11913, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 56 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestLoader#to_tandem" + } + }, + "parentId": 4150 + } + ], + "inheritedFrom": { + "name": "RequestLoader.to_tandem", + "target": 2948, + "type": "reference", + "ref": { + "id": 11913, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 56 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestLoader#to_tandem" + } + }, + "permalink": "/python/api/class/RequestQueue#to_tandem", + "parentId": 2348 + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Request queue is a storage for managing HTTP requests.\n\nThe request queue class serves as a high-level interface for organizing and managing HTTP requests\nduring web crawling. It provides methods for adding, retrieving, and manipulating requests throughout\nthe crawling lifecycle, abstracting away the underlying storage implementation details.\n\nRequest queue maintains the state of each URL to be crawled, tracking whether it has been processed,\nis currently being handled, or is waiting in the queue. Each URL in the queue is uniquely identified\nby a `unique_key` property, which prevents duplicate processing unless explicitly configured otherwise.\n\nThe class supports both breadth-first and depth-first crawling strategies through its `forefront` parameter\nwhen adding requests. It also provides mechanisms for error handling and request reclamation when\nprocessing fails.\n\nYou can open a request queue using the `open` class method, specifying either a name or ID to identify\nthe queue. The underlying storage implementation is determined by the configured storage client.\n\n### Usage\n\n```python\nfrom crawlee.storages import RequestQueue\n\n# Open a request queue\nrq = await RequestQueue.open(name='my-queue')\n\n# Add a request\nawait rq.add_request('https://example.com')\n\n# Process requests\nrequest = await rq.fetch_next_request()\nif request:\n try:\n # Process the request\n # ...\n await rq.mark_request_as_handled(request)\n except Exception:\n await rq.reclaim_request(request)\n```" + } + ] + }, + "decorations": [ + { + "args": "('Storages')", + "name": "docs_group" + } + ], + "flags": {}, + "groups": [ + { + "children": [ + 11314, + 11338, + 11342, + 11334, + 11350, + 11323, + 11321, + 11352, + 11325, + 11362, + 11364, + 11355, + 11327, + 11336, + 11358, + 13115 + ], + "title": "Methods" + }, + { + "children": [ + 11319, + 11320 + ], + "title": "Properties" + } + ], + "id": 11313, + "module": "storages._request_queue", + "name": "RequestQueue", + "parsedDocstring": { + "text": "Request queue is a storage for managing HTTP requests.\n\nThe request queue class serves as a high-level interface for organizing and managing HTTP requests\nduring web crawling. It provides methods for adding, retrieving, and manipulating requests throughout\nthe crawling lifecycle, abstracting away the underlying storage implementation details.\n\nRequest queue maintains the state of each URL to be crawled, tracking whether it has been processed,\nis currently being handled, or is waiting in the queue. Each URL in the queue is uniquely identified\nby a `unique_key` property, which prevents duplicate processing unless explicitly configured otherwise.\n\nThe class supports both breadth-first and depth-first crawling strategies through its `forefront` parameter\nwhen adding requests. It also provides mechanisms for error handling and request reclamation when\nprocessing fails.\n\nYou can open a request queue using the `open` class method, specifying either a name or ID to identify\nthe queue. The underlying storage implementation is determined by the configured storage client.\n\n### Usage\n\n```python\nfrom crawlee.storages import RequestQueue\n\n# Open a request queue\nrq = await RequestQueue.open(name='my-queue')\n\n# Add a request\nawait rq.add_request('https://example.com')\n\n# Process requests\nrequest = await rq.fetch_next_request()\nif request:\n try:\n # Process the request\n # ...\n await rq.mark_request_as_handled(request)\n except Exception:\n await rq.reclaim_request(request)\n```" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_request_queue.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 33, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storages/_request_queue.py#L33" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "extendedTypes": [ + { + "name": "RequestManager", + "target": "3001", + "type": "reference", + "ref": { + "id": 11966, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_manager.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 18 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/RequestManager" + } + }, + { + "name": "Storage", + "target": "2330", + "type": "reference", + "ref": { + "id": 11295, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_base.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 15 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/Storage" + } + } + ], + "permalink": "/python/api/class/RequestQueue", + "categories": [] + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 14467, + "module": "storage_clients.models", + "name": "model_config", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 173, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/models.py#L173" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "permalink": "/python/api/class/AddRequestsResponse#model_config", + "parentId": 1351 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Successfully processed requests, including information about whether they were\nalready present in the queue and whether they had been handled previously." + } + ] + }, + "flags": {}, + "groups": [], + "id": 14468, + "module": "storage_clients.models", + "name": "processed_requests", + "parsedDocstring": { + "text": "Successfully processed requests, including information about whether they were\nalready present in the queue and whether they had been handled previously." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 175, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/models.py#L175" + } + ], + "type": { + "name": "list", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "ProcessedRequest", + "target": "1340", + "ref": { + "id": 14455, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 140 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/ProcessedRequest" + } + } + ], + "target": 0, + "ref": {} + }, + "permalink": "/python/api/class/AddRequestsResponse#processed_requests", + "parentId": 1351 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Requests that could not be processed, typically due to validation errors or other issues." + } + ] + }, + "flags": {}, + "groups": [], + "id": 14469, + "module": "storage_clients.models", + "name": "unprocessed_requests", + "parsedDocstring": { + "text": "Requests that could not be processed, typically due to validation errors or other issues." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 179, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/models.py#L179" + } + ], + "type": { + "name": "list", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "UnprocessedRequest", + "target": "1346", + "ref": { + "id": 14461, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 154 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/UnprocessedRequest" + } + } + ], + "target": 0, + "ref": {} + }, + "permalink": "/python/api/class/AddRequestsResponse#unprocessed_requests", + "parentId": 1351 + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Model for a response to add requests to a queue.\n\nContains detailed information about the processing results when adding multiple requests\nto a queue. This includes which requests were successfully processed and which ones\nencountered issues during processing." + } + ] + }, + "decorations": [ + { + "args": "('Storage data')", + "name": "docs_group" + } + ], + "flags": {}, + "groups": [ + { + "children": [ + 14467, + 14468, + 14469 + ], + "title": "Properties" + } + ], + "id": 14466, + "module": "storage_clients.models", + "name": "AddRequestsResponse", + "parsedDocstring": { + "text": "Model for a response to add requests to a queue.\n\nContains detailed information about the processing results when adding multiple requests\nto a queue. This includes which requests were successfully processed and which ones\nencountered issues during processing." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 165, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/models.py#L165" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "permalink": "/python/api/class/AddRequestsResponse", + "categories": [] + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 15803, + "module": "storage_clients.models", + "name": "model_config", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 113, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/models.py#L113" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "permalink": "/python/api/class/DatasetItemsListPage#model_config", + "parentId": 1333 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "The number of objects returned on this page." + } + ] + }, + "flags": {}, + "groups": [], + "id": 15804, + "module": "storage_clients.models", + "name": "count", + "parsedDocstring": { + "text": "The number of objects returned on this page." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 115, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/models.py#L115" + } + ], + "type": { + "name": "int", + "type": "reference", + "target": 0, + "ref": {} + }, + "permalink": "/python/api/class/DatasetItemsListPage#count", + "parentId": 1333 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "The starting position of the first object returned, as specified in the API call." + } + ] + }, + "flags": {}, + "groups": [], + "id": 15805, + "module": "storage_clients.models", + "name": "offset", + "parsedDocstring": { + "text": "The starting position of the first object returned, as specified in the API call." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 118, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/models.py#L118" + } + ], + "type": { + "name": "int", + "type": "reference", + "target": 0, + "ref": {} + }, + "permalink": "/python/api/class/DatasetItemsListPage#offset", + "parentId": 1333 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "The maximum number of objects to return, as specified in the API call." + } + ] + }, + "flags": {}, + "groups": [], + "id": 15806, + "module": "storage_clients.models", + "name": "limit", + "parsedDocstring": { + "text": "The maximum number of objects to return, as specified in the API call." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 121, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/models.py#L121" + } + ], + "type": { + "name": "int", + "type": "reference", + "target": 0, + "ref": {} + }, + "permalink": "/python/api/class/DatasetItemsListPage#limit", + "parentId": 1333 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "The total number of objects that match the criteria of the API call." + } + ] + }, + "flags": {}, + "groups": [], + "id": 15807, + "module": "storage_clients.models", + "name": "total", + "parsedDocstring": { + "text": "The total number of objects that match the criteria of the API call." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 124, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/models.py#L124" + } + ], + "type": { + "name": "int", + "type": "reference", + "target": 0, + "ref": {} + }, + "permalink": "/python/api/class/DatasetItemsListPage#total", + "parentId": 1333 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Indicates if the returned list is in descending order." + } + ] + }, + "flags": {}, + "groups": [], + "id": 15808, + "module": "storage_clients.models", + "name": "desc", + "parsedDocstring": { + "text": "Indicates if the returned list is in descending order." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 127, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/models.py#L127" + } + ], + "type": { + "name": "bool", + "type": "reference", + "target": 0, + "ref": {} + }, + "permalink": "/python/api/class/DatasetItemsListPage#desc", + "parentId": 1333 + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Model for a single page of dataset items returned from a collection list method." + } + ] + }, + "decorations": [ + { + "args": "('Storage data')", + "name": "docs_group" + } + ], + "flags": {}, + "groups": [ + { + "children": [ + 15804, + 15808, + 15806, + 15803, + 15805, + 15807 + ], + "title": "Properties" + } + ], + "id": 15802, + "module": "storage_clients.models", + "name": "DatasetItemsListPage", + "parsedDocstring": { + "text": "Model for a single page of dataset items returned from a collection list method." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 110, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/models.py#L110" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "permalink": "/python/api/class/DatasetItemsListPage", + "categories": [] + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 17123, + "module": "storage_clients.models", + "name": "model_config", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 45, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/models.py#L45" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "overwrites": { + "name": "StorageMetadata.model_config", + "target": 1308, + "type": "reference", + "ref": { + "id": 17116, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 23 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api/class/StorageMetadata#model_config" + } + }, + "permalink": "/python/api/class/DatasetMetadata#model_config", + "parentId": 1314 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "The number of items in the dataset." + } + ] + }, + "flags": {}, + "groups": [], + "id": 17124, + "module": "storage_clients.models", + "name": "item_count", + "parsedDocstring": { + "text": "The number of items in the dataset." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 47, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/models.py#L47" + } + ], + "type": { + "name": "int", + "type": "reference", + "target": 0, + "ref": {} + }, + "permalink": "/python/api/class/DatasetMetadata#item_count", + "parentId": 1314 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "The unique identifier of the storage." + } + ] + }, + "flags": {}, + "groups": [], + "id": 20017, + "module": "storage_clients.models", + "name": "id", + "parsedDocstring": { + "text": "The unique identifier of the storage." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 25, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/models.py#L25" + } + ], + "type": { + "name": "Annotated[str, Field(alias='id')]", + "type": "reference", + "target": 0, + "ref": {} + }, + "inheritedFrom": { + "name": "StorageMetadata.id", + "target": 1309, + "type": "reference", + "ref": { + "id": 17117, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 25 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api/class/StorageMetadata#id" + } + }, + "permalink": "/python/api/class/DatasetMetadata#id", + "parentId": 1314 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "The name of the storage." + } + ] + }, + "flags": {}, + "groups": [], + "id": 20018, + "module": "storage_clients.models", + "name": "name", + "parsedDocstring": { + "text": "The name of the storage." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 28, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/models.py#L28" + } + ], + "type": { + "name": "Annotated[str | None, Field(alias='name', default=None)]", + "type": "reference", + "target": 0, + "ref": {} + }, + "inheritedFrom": { + "name": "StorageMetadata.name", + "target": 1310, + "type": "reference", + "ref": { + "id": 17118, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 28 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api/class/StorageMetadata#name" + } + }, + "permalink": "/python/api/class/DatasetMetadata#name", + "parentId": 1314 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "The timestamp when the storage was last accessed." + } + ] + }, + "flags": {}, + "groups": [], + "id": 20019, + "module": "storage_clients.models", + "name": "accessed_at", + "parsedDocstring": { + "text": "The timestamp when the storage was last accessed." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 31, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/models.py#L31" + } + ], + "type": { + "name": "Annotated[datetime, Field(alias='accessedAt')]", + "type": "reference", + "target": 0, + "ref": {} + }, + "inheritedFrom": { + "name": "StorageMetadata.accessed_at", + "target": 1311, + "type": "reference", + "ref": { + "id": 17119, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 31 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api/class/StorageMetadata#accessed_at" + } + }, + "permalink": "/python/api/class/DatasetMetadata#accessed_at", + "parentId": 1314 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "The timestamp when the storage was created." + } + ] + }, + "flags": {}, + "groups": [], + "id": 20020, + "module": "storage_clients.models", + "name": "created_at", + "parsedDocstring": { + "text": "The timestamp when the storage was created." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 34, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/models.py#L34" + } + ], + "type": { + "name": "Annotated[datetime, Field(alias='createdAt')]", + "type": "reference", + "target": 0, + "ref": {} + }, + "inheritedFrom": { + "name": "StorageMetadata.created_at", + "target": 1312, + "type": "reference", + "ref": { + "id": 17120, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 34 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api/class/StorageMetadata#created_at" + } + }, + "permalink": "/python/api/class/DatasetMetadata#created_at", + "parentId": 1314 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "The timestamp when the storage was last modified." + } + ] + }, + "flags": {}, + "groups": [], + "id": 20021, + "module": "storage_clients.models", + "name": "modified_at", + "parsedDocstring": { + "text": "The timestamp when the storage was last modified." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 37, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/models.py#L37" + } + ], + "type": { + "name": "Annotated[datetime, Field(alias='modifiedAt')]", + "type": "reference", + "target": 0, + "ref": {} + }, + "inheritedFrom": { + "name": "StorageMetadata.modified_at", + "target": 1313, + "type": "reference", + "ref": { + "id": 17121, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 37 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api/class/StorageMetadata#modified_at" + } + }, + "permalink": "/python/api/class/DatasetMetadata#modified_at", + "parentId": 1314 + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Model for a dataset metadata." + } + ] + }, + "decorations": [ + { + "args": "('Storage data')", + "name": "docs_group" + } + ], + "flags": {}, + "groups": [ + { + "children": [ + 20019, + 20020, + 20017, + 17124, + 17123, + 20021, + 20018 + ], + "title": "Properties" + } + ], + "id": 17122, + "module": "storage_clients.models", + "name": "DatasetMetadata", + "parsedDocstring": { + "text": "Model for a dataset metadata." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 42, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/models.py#L42" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "extendedTypes": [ + { + "name": "StorageMetadata", + "target": "1307", + "type": "reference", + "ref": { + "id": 17115, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 17 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/StorageMetadata" + } + } + ], + "permalink": "/python/api/class/DatasetMetadata", + "categories": [] + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 21339, + "module": "storage_clients.models", + "name": "model_config", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 55, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/models.py#L55" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "overwrites": { + "name": "StorageMetadata.model_config", + "target": 1308, + "type": "reference", + "ref": { + "id": 21329, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 23 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api/class/StorageMetadata#model_config" + } + }, + "permalink": "/python/api/class/KeyValueStoreMetadata#model_config", + "parentId": 1317 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "The unique identifier of the storage." + } + ] + }, + "flags": {}, + "groups": [], + "id": 24235, + "module": "storage_clients.models", + "name": "id", + "parsedDocstring": { + "text": "The unique identifier of the storage." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 25, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/models.py#L25" + } + ], + "type": { + "name": "Annotated[str, Field(alias='id')]", + "type": "reference", + "target": 0, + "ref": {} + }, + "inheritedFrom": { + "name": "StorageMetadata.id", + "target": 1309, + "type": "reference", + "ref": { + "id": 21330, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 25 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api/class/StorageMetadata#id" + } + }, + "permalink": "/python/api/class/KeyValueStoreMetadata#id", + "parentId": 1317 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "The name of the storage." + } + ] + }, + "flags": {}, + "groups": [], + "id": 24236, + "module": "storage_clients.models", + "name": "name", + "parsedDocstring": { + "text": "The name of the storage." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 28, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/models.py#L28" + } + ], + "type": { + "name": "Annotated[str | None, Field(alias='name', default=None)]", + "type": "reference", + "target": 0, + "ref": {} + }, + "inheritedFrom": { + "name": "StorageMetadata.name", + "target": 1310, + "type": "reference", + "ref": { + "id": 21331, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 28 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api/class/StorageMetadata#name" + } + }, + "permalink": "/python/api/class/KeyValueStoreMetadata#name", + "parentId": 1317 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "The timestamp when the storage was last accessed." + } + ] + }, + "flags": {}, + "groups": [], + "id": 24237, + "module": "storage_clients.models", + "name": "accessed_at", + "parsedDocstring": { + "text": "The timestamp when the storage was last accessed." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 31, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/models.py#L31" + } + ], + "type": { + "name": "Annotated[datetime, Field(alias='accessedAt')]", + "type": "reference", + "target": 0, + "ref": {} + }, + "inheritedFrom": { + "name": "StorageMetadata.accessed_at", + "target": 1311, + "type": "reference", + "ref": { + "id": 21332, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 31 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api/class/StorageMetadata#accessed_at" + } + }, + "permalink": "/python/api/class/KeyValueStoreMetadata#accessed_at", + "parentId": 1317 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "The timestamp when the storage was created." + } + ] + }, + "flags": {}, + "groups": [], + "id": 24238, + "module": "storage_clients.models", + "name": "created_at", + "parsedDocstring": { + "text": "The timestamp when the storage was created." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 34, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/models.py#L34" + } + ], + "type": { + "name": "Annotated[datetime, Field(alias='createdAt')]", + "type": "reference", + "target": 0, + "ref": {} + }, + "inheritedFrom": { + "name": "StorageMetadata.created_at", + "target": 1312, + "type": "reference", + "ref": { + "id": 21333, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 34 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api/class/StorageMetadata#created_at" + } + }, + "permalink": "/python/api/class/KeyValueStoreMetadata#created_at", + "parentId": 1317 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "The timestamp when the storage was last modified." + } + ] + }, + "flags": {}, + "groups": [], + "id": 24239, + "module": "storage_clients.models", + "name": "modified_at", + "parsedDocstring": { + "text": "The timestamp when the storage was last modified." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 37, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/models.py#L37" + } + ], + "type": { + "name": "Annotated[datetime, Field(alias='modifiedAt')]", + "type": "reference", + "target": 0, + "ref": {} + }, + "inheritedFrom": { + "name": "StorageMetadata.modified_at", + "target": 1313, + "type": "reference", + "ref": { + "id": 21334, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 37 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api/class/StorageMetadata#modified_at" + } + }, + "permalink": "/python/api/class/KeyValueStoreMetadata#modified_at", + "parentId": 1317 + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Model for a key-value store metadata." + } + ] + }, + "decorations": [ + { + "args": "('Storage data')", + "name": "docs_group" + } + ], + "flags": {}, + "groups": [ + { + "children": [ + 24237, + 24238, + 24235, + 21339, + 24239, + 24236 + ], + "title": "Properties" + } + ], + "id": 21338, + "module": "storage_clients.models", + "name": "KeyValueStoreMetadata", + "parsedDocstring": { + "text": "Model for a key-value store metadata." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 52, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/models.py#L52" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "extendedTypes": [ + { + "name": "StorageMetadata", + "target": "1307", + "type": "reference", + "ref": { + "id": 21328, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 17 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/StorageMetadata" + } + } + ], + "permalink": "/python/api/class/KeyValueStoreMetadata", + "categories": [] + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 25570, + "module": "storage_clients.models", + "name": "model_config", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 103, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/models.py#L103" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "overwrites": { + "name": "KeyValueStoreRecordMetadata.model_config", + "target": 1326, + "type": "reference", + "ref": { + "id": 25565, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 81 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api/class/KeyValueStoreRecordMetadata#model_config" + } + }, + "permalink": "/python/api/class/KeyValueStoreRecord#model_config", + "parentId": 1330 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "The value of the record." + } + ] + }, + "flags": {}, + "groups": [], + "id": 25571, + "module": "storage_clients.models", + "name": "value", + "parsedDocstring": { + "text": "The value of the record." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 105, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/models.py#L105" + } + ], + "type": { + "name": "KvsValueType", + "type": "reference", + "target": "1306", + "ref": { + "id": 25545, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 13 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api#KvsValueType" + } + }, + "permalink": "/python/api/class/KeyValueStoreRecord#value", + "parentId": 1330 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "The key of the record.\n\nA unique identifier for the record in the key-value store." + } + ] + }, + "flags": {}, + "groups": [], + "id": 28445, + "module": "storage_clients.models", + "name": "key", + "parsedDocstring": { + "text": "The key of the record.\n\nA unique identifier for the record in the key-value store." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 83, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/models.py#L83" + } + ], + "type": { + "name": "Annotated[str, Field(alias='key')]", + "type": "reference", + "target": 0, + "ref": {} + }, + "inheritedFrom": { + "name": "KeyValueStoreRecordMetadata.key", + "target": 1327, + "type": "reference", + "ref": { + "id": 25566, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 83 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api/class/KeyValueStoreRecordMetadata#key" + } + }, + "permalink": "/python/api/class/KeyValueStoreRecord#key", + "parentId": 1330 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "The MIME type of the record.\n\nDescribe the format and type of data stored in the record, following the MIME specification." + } + ] + }, + "flags": {}, + "groups": [], + "id": 28446, + "module": "storage_clients.models", + "name": "content_type", + "parsedDocstring": { + "text": "The MIME type of the record.\n\nDescribe the format and type of data stored in the record, following the MIME specification." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 89, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/models.py#L89" + } + ], + "type": { + "name": "Annotated[str, Field(alias='contentType')]", + "type": "reference", + "target": 0, + "ref": {} + }, + "inheritedFrom": { + "name": "KeyValueStoreRecordMetadata.content_type", + "target": 1328, + "type": "reference", + "ref": { + "id": 25567, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 89 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api/class/KeyValueStoreRecordMetadata#content_type" + } + }, + "permalink": "/python/api/class/KeyValueStoreRecord#content_type", + "parentId": 1330 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "The size of the record in bytes." + } + ] + }, + "flags": {}, + "groups": [], + "id": 28447, + "module": "storage_clients.models", + "name": "size", + "parsedDocstring": { + "text": "The size of the record in bytes." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 95, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/models.py#L95" + } + ], + "type": { + "name": "Annotated[int | None, Field(alias='size', default=None)]", + "type": "reference", + "target": 0, + "ref": {} + }, + "inheritedFrom": { + "name": "KeyValueStoreRecordMetadata.size", + "target": 1329, + "type": "reference", + "ref": { + "id": 25568, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 95 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api/class/KeyValueStoreRecordMetadata#size" + } + }, + "permalink": "/python/api/class/KeyValueStoreRecord#size", + "parentId": 1330 + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Model for a key-value store record." + } + ] + }, + "decorations": [ + { + "args": "('Storage data')", + "name": "docs_group" + } + ], + "flags": {}, + "groups": [ + { + "children": [ + 28446, + 28445, + 25570, + 28447, + 25571 + ], + "title": "Properties" + } + ], + "id": 25569, + "module": "storage_clients.models", + "name": "KeyValueStoreRecord", + "parsedDocstring": { + "text": "Model for a key-value store record." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 100, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/models.py#L100" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "extendedTypes": [ + { + "name": "KeyValueStoreRecordMetadata", + "target": "1325", + "type": "reference", + "ref": { + "id": 25564, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 78 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/KeyValueStoreRecordMetadata" + } + } + ], + "permalink": "/python/api/class/KeyValueStoreRecord", + "categories": [] + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 29773, + "module": "storage_clients.models", + "name": "model_config", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 81, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/models.py#L81" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "permalink": "/python/api/class/KeyValueStoreRecordMetadata#model_config", + "parentId": 1325 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "The key of the record.\n\nA unique identifier for the record in the key-value store." + } + ] + }, + "flags": {}, + "groups": [], + "id": 29774, + "module": "storage_clients.models", + "name": "key", + "parsedDocstring": { + "text": "The key of the record.\n\nA unique identifier for the record in the key-value store." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 83, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/models.py#L83" + } + ], + "type": { + "name": "str", + "type": "reference", + "target": 0, + "ref": {} + }, + "permalink": "/python/api/class/KeyValueStoreRecordMetadata#key", + "parentId": 1325 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "The MIME type of the record.\n\nDescribe the format and type of data stored in the record, following the MIME specification." + } + ] + }, + "flags": {}, + "groups": [], + "id": 29775, + "module": "storage_clients.models", + "name": "content_type", + "parsedDocstring": { + "text": "The MIME type of the record.\n\nDescribe the format and type of data stored in the record, following the MIME specification." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 89, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/models.py#L89" + } + ], + "type": { + "name": "str", + "type": "reference", + "target": 0, + "ref": {} + }, + "permalink": "/python/api/class/KeyValueStoreRecordMetadata#content_type", + "parentId": 1325 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "The size of the record in bytes." + } + ] + }, + "flags": {}, + "groups": [], + "id": 29776, + "module": "storage_clients.models", + "name": "size", + "parsedDocstring": { + "text": "The size of the record in bytes." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 95, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/models.py#L95" + } + ], + "type": { + "name": "Annotated[int | None, Field(alias='size', default=None)]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "int", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "permalink": "/python/api/class/KeyValueStoreRecordMetadata#size", + "parentId": 1325 + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Model for a key-value store record metadata." + } + ] + }, + "decorations": [ + { + "args": "('Storage data')", + "name": "docs_group" + } + ], + "flags": {}, + "groups": [ + { + "children": [ + 29775, + 29774, + 29773, + 29776 + ], + "title": "Properties" + } + ], + "id": 29772, + "module": "storage_clients.models", + "name": "KeyValueStoreRecordMetadata", + "parsedDocstring": { + "text": "Model for a key-value store record metadata." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 78, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/models.py#L78" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "extendedBy": [ + { + "name": "KeyValueStoreRecord", + "target": "1330", + "type": "reference", + "ref": { + "id": 29777, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 100 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/KeyValueStoreRecord" + } + } + ], + "permalink": "/python/api/class/KeyValueStoreRecordMetadata", + "categories": [] + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 31118, + "module": "storage_clients.models", + "name": "model_config", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 143, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/models.py#L143" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "permalink": "/python/api/class/ProcessedRequest#model_config", + "parentId": 1340 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Internal representation of the request by the storage client. Only some clients use id." + } + ] + }, + "flags": {}, + "groups": [], + "id": 31119, + "module": "storage_clients.models", + "name": "id", + "parsedDocstring": { + "text": "Internal representation of the request by the storage client. Only some clients use id." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 145, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/models.py#L145" + } + ], + "type": { + "name": "Annotated[str | None, Field(alias='requestId', default=None)]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "permalink": "/python/api/class/ProcessedRequest#id", + "parentId": 1340 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 31120, + "module": "storage_clients.models", + "name": "unique_key", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 148, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/models.py#L148" + } + ], + "type": { + "name": "str", + "type": "reference", + "target": 0, + "ref": {} + }, + "permalink": "/python/api/class/ProcessedRequest#unique_key", + "parentId": 1340 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 31121, + "module": "storage_clients.models", + "name": "was_already_present", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 149, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/models.py#L149" + } + ], + "type": { + "name": "bool", + "type": "reference", + "target": 0, + "ref": {} + }, + "permalink": "/python/api/class/ProcessedRequest#was_already_present", + "parentId": 1340 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 31122, + "module": "storage_clients.models", + "name": "was_already_handled", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 150, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/models.py#L150" + } + ], + "type": { + "name": "bool", + "type": "reference", + "target": 0, + "ref": {} + }, + "permalink": "/python/api/class/ProcessedRequest#was_already_handled", + "parentId": 1340 + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Represents a processed request." + } + ] + }, + "decorations": [ + { + "args": "('Storage data')", + "name": "docs_group" + } + ], + "flags": {}, + "groups": [ + { + "children": [ + 31119, + 31118, + 31120, + 31122, + 31121 + ], + "title": "Properties" + } + ], + "id": 31117, + "module": "storage_clients.models", + "name": "ProcessedRequest", + "parsedDocstring": { + "text": "Represents a processed request." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 140, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/models.py#L140" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "permalink": "/python/api/class/ProcessedRequest", + "categories": [] + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 31532, + "module": "_request", + "name": "model_config", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_request.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 169, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/_request.py#L169" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "permalink": "/python/api/class/Request#model_config", + "parentId": 409 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "A unique key identifying the request. Two requests with the same `unique_key` are considered as pointing\nto the same URL.\n\nIf `unique_key` is not provided, then it is automatically generated by normalizing the URL.\nFor example, the URL of `HTTP://www.EXAMPLE.com/something/` will produce the `unique_key`\nof `http://www.example.com/something`.\n\nPass an arbitrary non-empty text value to the `unique_key` property to override the default behavior\nand specify which URLs shall be considered equal." + } + ] + }, + "flags": {}, + "groups": [], + "id": 31533, + "module": "_request", + "name": "unique_key", + "parsedDocstring": { + "text": "A unique key identifying the request. Two requests with the same `unique_key` are considered as pointing\nto the same URL.\n\nIf `unique_key` is not provided, then it is automatically generated by normalizing the URL.\nFor example, the URL of `HTTP://www.EXAMPLE.com/something/` will produce the `unique_key`\nof `http://www.example.com/something`.\n\nPass an arbitrary non-empty text value to the `unique_key` property to override the default behavior\nand specify which URLs shall be considered equal." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_request.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 171, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/_request.py#L171" + } + ], + "type": { + "name": "str", + "type": "reference", + "target": 0, + "ref": {} + }, + "permalink": "/python/api/class/Request#unique_key", + "parentId": 409 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "The URL of the web page to crawl. Must be a valid HTTP or HTTPS URL, and may include query parameters\nand fragments." + } + ] + }, + "flags": {}, + "groups": [], + "id": 31534, + "module": "_request", + "name": "url", + "parsedDocstring": { + "text": "The URL of the web page to crawl. Must be a valid HTTP or HTTPS URL, and may include query parameters\nand fragments." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_request.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 183, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/_request.py#L183" + } + ], + "type": { + "name": "str", + "type": "reference", + "target": 0, + "ref": {} + }, + "permalink": "/python/api/class/Request#url", + "parentId": 409 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "HTTP request method." + } + ] + }, + "flags": {}, + "groups": [], + "id": 31535, + "module": "_request", + "name": "method", + "parsedDocstring": { + "text": "HTTP request method." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_request.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 187, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/_request.py#L187" + } + ], + "type": { + "name": "HttpMethod", + "type": "reference", + "target": "84", + "ref": { + "id": 31206, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_types.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 38 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api#HttpMethod" + } + }, + "permalink": "/python/api/class/Request#method", + "parentId": 409 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "HTTP request payload." + } + ] + }, + "flags": {}, + "groups": [], + "id": 31536, + "module": "_request", + "name": "payload", + "parsedDocstring": { + "text": "HTTP request payload." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_request.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 190, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/_request.py#L190" + } + ], + "type": { + "name": "Annotated[ HttpPayload | None, BeforeValidator(lambda v: v.encode() if isinstance(v, str) else v), PlainSerializer(lambda v: v.decode() if isinstance(v, bytes) else v), Field(frozen=True), ]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "HttpPayload", + "target": "85", + "ref": { + "id": 31207, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_types.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 40 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api#HttpPayload" + } + }, + { + "type": "literal", + "value": {} + } + ] + }, + "permalink": "/python/api/class/Request#payload", + "parentId": 409 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Number of times the request has been retried." + } + ] + }, + "flags": {}, + "groups": [], + "id": 31537, + "module": "_request", + "name": "retry_count", + "parsedDocstring": { + "text": "Number of times the request has been retried." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_request.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 230, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/_request.py#L230" + } + ], + "type": { + "name": "int", + "type": "reference", + "target": 0, + "ref": {} + }, + "permalink": "/python/api/class/Request#retry_count", + "parentId": 409 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "If set to `True`, the request will not be retried in case of failure." + } + ] + }, + "flags": {}, + "groups": [], + "id": 31538, + "module": "_request", + "name": "no_retry", + "parsedDocstring": { + "text": "If set to `True`, the request will not be retried in case of failure." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_request.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 233, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/_request.py#L233" + } + ], + "type": { + "name": "bool", + "type": "reference", + "target": 0, + "ref": {} + }, + "permalink": "/python/api/class/Request#no_retry", + "parentId": 409 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "URL of the web page that was loaded. This can differ from the original URL in case of redirects." + } + ] + }, + "flags": {}, + "groups": [], + "id": 31539, + "module": "_request", + "name": "loaded_url", + "parsedDocstring": { + "text": "URL of the web page that was loaded. This can differ from the original URL in case of redirects." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_request.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 236, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/_request.py#L236" + } + ], + "type": { + "name": "Annotated[str | None, BeforeValidator(validate_http_url), Field(alias='loadedUrl')]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "permalink": "/python/api/class/Request#loaded_url", + "parentId": 409 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Timestamp when the request was handled." + } + ] + }, + "flags": {}, + "groups": [], + "id": 31540, + "module": "_request", + "name": "handled_at", + "parsedDocstring": { + "text": "Timestamp when the request was handled." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_request.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 239, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/_request.py#L239" + } + ], + "type": { + "name": "Annotated[datetime | None, Field(alias='handledAt')]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "datetime", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "permalink": "/python/api/class/Request#handled_at", + "parentId": 409 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Create a new `Request` instance from a URL.\n\nThis is recommended constructor for creating new `Request` instances. It generates a `Request` object from\na given URL with additional options to customize HTTP method, payload, unique key, and other request\nproperties. If no `unique_key` or `id` is provided, they are computed automatically based on the URL,\nmethod and payload. It depends on the `keep_url_fragment` and `use_extended_unique_key` flags.\n" + } + ] + }, + "decorations": [ + { + "name": "classmethod" + } + ], + "flags": {}, + "groups": [], + "id": 31541, + "module": "_request", + "name": "from_url", + "parsedDocstring": { + "text": "Create a new `Request` instance from a URL.\n\nThis is recommended constructor for creating new `Request` instances. It generates a `Request` object from\na given URL with additional options to customize HTTP method, payload, unique key, and other request\nproperties. If no `unique_key` or `id` is provided, they are computed automatically based on the URL,\nmethod and payload. It depends on the `keep_url_fragment` and `use_extended_unique_key` flags.\n", + "args": { + "url": "The URL of the request.", + "method": "The HTTP method of the request.", + "headers": "The HTTP headers of the request.", + "payload": "The data to be sent as the request body. Typically used with 'POST' or 'PUT' requests.", + "label": "A custom label to differentiate between request types. This is stored in `user_data`, and it is\nused for request routing (different requests go to different handlers).", + "session_id": "ID of a specific `Session` to which the request will be strictly bound.\nIf the session becomes unavailable when the request is processed, a `RequestCollisionError` will be\nraised.", + "unique_key": "A unique key identifying the request. If not provided, it is automatically computed based on\nthe URL and other parameters. Requests with the same `unique_key` are treated as identical.", + "keep_url_fragment": "Determines whether the URL fragment (e.g., ``section``) should be included in\nthe `unique_key` computation. This is only relevant when `unique_key` is not provided.", + "use_extended_unique_key": "Determines whether to include the HTTP method, ID Session and payload in the\n`unique_key` computation. This is only relevant when `unique_key` is not provided.", + "always_enqueue": "If set to `True`, the request will be enqueued even if it is already present in the queue.\nUsing this is not allowed when a custom `unique_key` is also provided and will result in a `ValueError`.", + "enqueue_strategy": "The strategy that will be used for enqueuing the request.", + "max_retries": "Maximum number of retries for this request. Allows to override the global `max_request_retries`\noption of `BasicCrawler`.", + "**kwargs": "Additional request properties." + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_request.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 243, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/_request.py#L243" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Create a new `Request` instance from a URL.\n\nThis is recommended constructor for creating new `Request` instances. It generates a `Request` object from\na given URL with additional options to customize HTTP method, payload, unique key, and other request\nproperties. If no `unique_key` or `id` is provided, they are computed automatically based on the URL,\nmethod and payload. It depends on the `keep_url_fragment` and `use_extended_unique_key` flags.\n" + } + ] + }, + "flags": {}, + "id": 31542, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "from_url", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The URL of the request." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 31543, + "kind": 32768, + "kindString": "Parameter", + "name": "url", + "type": { + "name": "str", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 420 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The HTTP method of the request." + } + ] + }, + "defaultValue": "'GET'", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 31544, + "kind": 32768, + "kindString": "Parameter", + "name": "method", + "type": { + "name": "HttpMethod", + "type": "reference", + "target": "84", + "ref": { + "id": 31206, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_types.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 38 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api#HttpMethod" + } + }, + "parentId": 420 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The HTTP headers of the request." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 31545, + "kind": 32768, + "kindString": "Parameter", + "name": "headers", + "type": { + "name": "HttpHeaders | dict[str, str] | None", + "type": "union", + "types": [ + { + "type": "union", + "types": [ + { + "type": "reference", + "name": "HttpHeaders", + "target": "91", + "ref": { + "id": 31213, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_types.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 63 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/HttpHeaders" + } + }, + { + "type": "reference", + "name": "dict", + "typeArguments": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + } + ] + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 420 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The data to be sent as the request body. Typically used with 'POST' or 'PUT' requests." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 31546, + "kind": 32768, + "kindString": "Parameter", + "name": "payload", + "type": { + "name": "HttpPayload | str | None", + "type": "union", + "types": [ + { + "type": "union", + "types": [ + { + "type": "reference", + "name": "HttpPayload", + "target": "85", + "ref": { + "id": 31207, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_types.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 40 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api#HttpPayload" + } + }, + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + } + ] + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 420 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "A custom label to differentiate between request types. This is stored in `user_data`, and it is\nused for request routing (different requests go to different handlers)." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 31547, + "kind": 32768, + "kindString": "Parameter", + "name": "label", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 420 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "ID of a specific `Session` to which the request will be strictly bound.\nIf the session becomes unavailable when the request is processed, a `RequestCollisionError` will be\nraised." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 31548, + "kind": 32768, + "kindString": "Parameter", + "name": "session_id", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 420 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "A unique key identifying the request. If not provided, it is automatically computed based on\nthe URL and other parameters. Requests with the same `unique_key` are treated as identical." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 31549, + "kind": 32768, + "kindString": "Parameter", + "name": "unique_key", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 420 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Determines whether the URL fragment (e.g., ``section``) should be included in\nthe `unique_key` computation. This is only relevant when `unique_key` is not provided." + } + ] + }, + "defaultValue": "False", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 31550, + "kind": 32768, + "kindString": "Parameter", + "name": "keep_url_fragment", + "type": { + "name": "bool", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 420 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Determines whether to include the HTTP method, ID Session and payload in the\n`unique_key` computation. This is only relevant when `unique_key` is not provided." + } + ] + }, + "defaultValue": "False", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 31551, + "kind": 32768, + "kindString": "Parameter", + "name": "use_extended_unique_key", + "type": { + "name": "bool", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 420 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "If set to `True`, the request will be enqueued even if it is already present in the queue.\nUsing this is not allowed when a custom `unique_key` is also provided and will result in a `ValueError`." + } + ] + }, + "defaultValue": "False", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 31552, + "kind": 32768, + "kindString": "Parameter", + "name": "always_enqueue", + "type": { + "name": "bool", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 420 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The strategy that will be used for enqueuing the request." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 31553, + "kind": 32768, + "kindString": "Parameter", + "name": "enqueue_strategy", + "type": { + "name": "EnqueueStrategy | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "EnqueueStrategy", + "target": "88", + "ref": { + "id": 31210, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_types.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 47 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api#EnqueueStrategy" + } + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 420 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Maximum number of retries for this request. Allows to override the global `max_request_retries`\noption of `BasicCrawler`." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 31554, + "kind": 32768, + "kindString": "Parameter", + "name": "max_retries", + "type": { + "name": "int | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "int", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 420 + }, + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 31555, + "kind": 32768, + "kindString": "Parameter", + "name": "kwargs", + "type": { + "name": "Any", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 420 + } + ], + "type": { + "name": "Self", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 419 + } + ], + "permalink": "/python/api/class/Request#from_url", + "parentId": 409 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Get the value of a specific query parameter from the URL." + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 31556, + "module": "_request", + "name": "get_query_param_from_url", + "parsedDocstring": { + "text": "Get the value of a specific query parameter from the URL." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_request.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 344, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/_request.py#L344" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Get the value of a specific query parameter from the URL." + } + ] + }, + "flags": {}, + "id": 31557, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "get_query_param_from_url", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 31558, + "kind": 32768, + "kindString": "Parameter", + "name": "param", + "type": { + "name": "str", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 435 + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 31559, + "kind": 32768, + "kindString": "Parameter", + "name": "default", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 435 + } + ], + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 434 + } + ], + "permalink": "/python/api/class/Request#get_query_param_from_url", + "parentId": 409 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "A string used to differentiate between arbitrary request types." + } + ] + }, + "decorations": [ + { + "name": "property" + } + ], + "flags": {}, + "groups": [], + "id": 31560, + "module": "_request", + "name": "label", + "parsedDocstring": { + "text": "A string used to differentiate between arbitrary request types." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_request.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 350, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/_request.py#L350" + } + ], + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "permalink": "/python/api/class/Request#label", + "parentId": 409 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "The ID of the bound session, if there is any." + } + ] + }, + "decorations": [ + { + "name": "property" + } + ], + "flags": {}, + "groups": [], + "id": 31561, + "module": "_request", + "name": "session_id", + "parsedDocstring": { + "text": "The ID of the bound session, if there is any." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_request.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 355, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/_request.py#L355" + } + ], + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "permalink": "/python/api/class/Request#session_id", + "parentId": 409 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Crawlee-specific configuration stored in the `user_data`." + } + ] + }, + "decorations": [ + { + "name": "property" + } + ], + "flags": {}, + "groups": [], + "id": 31562, + "module": "_request", + "name": "crawlee_data", + "parsedDocstring": { + "text": "Crawlee-specific configuration stored in the `user_data`." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_request.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 360, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/_request.py#L360" + } + ], + "type": { + "name": "CrawleeRequestData", + "type": "reference", + "target": "358", + "ref": { + "id": 31480, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_request.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 34 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/CrawleeRequestData" + } + }, + "permalink": "/python/api/class/Request#crawlee_data", + "parentId": 409 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "The depth of the request in the crawl tree." + } + ] + }, + "decorations": [ + { + "name": "property" + } + ], + "flags": {}, + "groups": [], + "id": 31563, + "module": "_request", + "name": "crawl_depth", + "parsedDocstring": { + "text": "The depth of the request in the crawl tree." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_request.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 369, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/_request.py#L369" + } + ], + "type": { + "name": "int", + "type": "reference", + "target": 0, + "ref": {} + }, + "permalink": "/python/api/class/Request#crawl_depth", + "parentId": 409 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "decorations": [ + { + "args": ".setter", + "name": "crawl_depth" + } + ], + "flags": {}, + "groups": [], + "id": 31564, + "module": "_request", + "name": "crawl_depth", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_request.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 374, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/_request.py#L374" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "flags": {}, + "id": 31565, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "crawl_depth", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 31566, + "kind": 32768, + "kindString": "Parameter", + "name": "new_value", + "type": { + "name": "int", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 443 + } + ], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "parentId": 442 + } + ], + "permalink": "/python/api/class/Request#crawl_depth", + "parentId": 409 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Crawlee-specific request handling state." + } + ] + }, + "decorations": [ + { + "name": "property" + } + ], + "flags": {}, + "groups": [], + "id": 31567, + "module": "_request", + "name": "state", + "parsedDocstring": { + "text": "Crawlee-specific request handling state." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_request.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 378, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/_request.py#L378" + } + ], + "type": { + "name": "RequestState", + "type": "reference", + "target": "349", + "ref": { + "id": 31471, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_request.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 21 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/RequestState" + } + }, + "permalink": "/python/api/class/Request#state", + "parentId": 409 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "decorations": [ + { + "args": ".setter", + "name": "state" + } + ], + "flags": {}, + "groups": [], + "id": 31568, + "module": "_request", + "name": "state", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_request.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 383, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/_request.py#L383" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "flags": {}, + "id": 31569, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "state", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 31570, + "kind": 32768, + "kindString": "Parameter", + "name": "new_state", + "type": { + "name": "RequestState", + "type": "reference", + "target": "349", + "ref": { + "id": 31471, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_request.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 21 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/RequestState" + } + }, + "parentId": 447 + } + ], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "parentId": 446 + } + ], + "permalink": "/python/api/class/Request#state", + "parentId": 409 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Crawlee-specific limit on the number of retries of the request." + } + ] + }, + "decorations": [ + { + "name": "property" + } + ], + "flags": {}, + "groups": [], + "id": 31571, + "module": "_request", + "name": "max_retries", + "parsedDocstring": { + "text": "Crawlee-specific limit on the number of retries of the request." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_request.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 387, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/_request.py#L387" + } + ], + "type": { + "name": "int | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "int", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "permalink": "/python/api/class/Request#max_retries", + "parentId": 409 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Crawlee-specific number of finished session rotations for the request." + } + ] + }, + "decorations": [ + { + "name": "property" + } + ], + "flags": {}, + "groups": [], + "id": 31572, + "module": "_request", + "name": "session_rotation_count", + "parsedDocstring": { + "text": "Crawlee-specific number of finished session rotations for the request." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_request.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 392, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/_request.py#L392" + } + ], + "type": { + "name": "int | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "int", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "permalink": "/python/api/class/Request#session_rotation_count", + "parentId": 409 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "decorations": [ + { + "args": ".setter", + "name": "session_rotation_count" + } + ], + "flags": {}, + "groups": [], + "id": 31573, + "module": "_request", + "name": "session_rotation_count", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_request.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 397, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/_request.py#L397" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "flags": {}, + "id": 31574, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "session_rotation_count", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 31575, + "kind": 32768, + "kindString": "Parameter", + "name": "new_session_rotation_count", + "type": { + "name": "int", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 452 + } + ], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "parentId": 451 + } + ], + "permalink": "/python/api/class/Request#session_rotation_count", + "parentId": 409 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "The strategy that was used for enqueuing the request." + } + ] + }, + "decorations": [ + { + "name": "property" + } + ], + "flags": {}, + "groups": [], + "id": 31576, + "module": "_request", + "name": "enqueue_strategy", + "parsedDocstring": { + "text": "The strategy that was used for enqueuing the request." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_request.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 401, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/_request.py#L401" + } + ], + "type": { + "name": "EnqueueStrategy", + "type": "reference", + "target": "88", + "ref": { + "id": 31210, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_types.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 47 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api#EnqueueStrategy" + } + }, + "permalink": "/python/api/class/Request#enqueue_strategy", + "parentId": 409 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "decorations": [ + { + "args": ".setter", + "name": "enqueue_strategy" + } + ], + "flags": {}, + "groups": [], + "id": 31577, + "module": "_request", + "name": "enqueue_strategy", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_request.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 406, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/_request.py#L406" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "flags": {}, + "id": 31578, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "enqueue_strategy", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 31579, + "kind": 32768, + "kindString": "Parameter", + "name": "new_enqueue_strategy", + "type": { + "name": "EnqueueStrategy", + "type": "reference", + "target": "88", + "ref": { + "id": 31210, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_types.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 47 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api#EnqueueStrategy" + } + }, + "parentId": 456 + } + ], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "parentId": 455 + } + ], + "permalink": "/python/api/class/Request#enqueue_strategy", + "parentId": 409 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "The last proxy tier used to process the request." + } + ] + }, + "decorations": [ + { + "name": "property" + } + ], + "flags": {}, + "groups": [], + "id": 31580, + "module": "_request", + "name": "last_proxy_tier", + "parsedDocstring": { + "text": "The last proxy tier used to process the request." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_request.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 410, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/_request.py#L410" + } + ], + "type": { + "name": "int | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "int", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "permalink": "/python/api/class/Request#last_proxy_tier", + "parentId": 409 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "decorations": [ + { + "args": ".setter", + "name": "last_proxy_tier" + } + ], + "flags": {}, + "groups": [], + "id": 31581, + "module": "_request", + "name": "last_proxy_tier", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_request.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 415, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/_request.py#L415" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "flags": {}, + "id": 31582, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "last_proxy_tier", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 31583, + "kind": 32768, + "kindString": "Parameter", + "name": "new_value", + "type": { + "name": "int", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 460 + } + ], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "parentId": 459 + } + ], + "permalink": "/python/api/class/Request#last_proxy_tier", + "parentId": 409 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Indicate whether the request should be enqueued at the front of the queue." + } + ] + }, + "decorations": [ + { + "name": "property" + } + ], + "flags": {}, + "groups": [], + "id": 31584, + "module": "_request", + "name": "forefront", + "parsedDocstring": { + "text": "Indicate whether the request should be enqueued at the front of the queue." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_request.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 419, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/_request.py#L419" + } + ], + "type": { + "name": "bool", + "type": "reference", + "target": 0, + "ref": {} + }, + "permalink": "/python/api/class/Request#forefront", + "parentId": 409 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "decorations": [ + { + "args": ".setter", + "name": "forefront" + } + ], + "flags": {}, + "groups": [], + "id": 31585, + "module": "_request", + "name": "forefront", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_request.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 424, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/_request.py#L424" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "flags": {}, + "id": 31586, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "forefront", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 31587, + "kind": 32768, + "kindString": "Parameter", + "name": "new_value", + "type": { + "name": "bool", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 464 + } + ], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "parentId": 463 + } + ], + "permalink": "/python/api/class/Request#forefront", + "parentId": 409 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Indicates whether the request was handled." + } + ] + }, + "decorations": [ + { + "name": "property" + } + ], + "flags": {}, + "groups": [], + "id": 31588, + "module": "_request", + "name": "was_already_handled", + "parsedDocstring": { + "text": "Indicates whether the request was handled." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_request.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 428, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/_request.py#L428" + } + ], + "type": { + "name": "bool", + "type": "reference", + "target": 0, + "ref": {} + }, + "permalink": "/python/api/class/Request#was_already_handled", + "parentId": 409 + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Represents a request in the Crawlee framework, containing the necessary information for crawling operations.\n\nThe `Request` class is one of the core components in Crawlee, utilized by various components such as request\nproviders, HTTP clients, crawlers, and more. It encapsulates the essential data for executing web requests,\nincluding the URL, HTTP method, headers, payload, and user data. The user data allows custom information\nto be stored and persisted throughout the request lifecycle, including its retries.\n\nKey functionalities include managing the request's identifier (`id`), unique key (`unique_key`) that is used\nfor request deduplication, controlling retries, handling state management, and enabling configuration for session\nrotation and proxy handling.\n\nThe recommended way to create a new instance is by using the `Request.from_url` constructor, which automatically\ngenerates a unique key and identifier based on the URL and request parameters.\n\n### Usage\n\n```python\nfrom crawlee import Request\n\nrequest = Request.from_url('https://crawlee.dev')\n```" + } + ] + }, + "decorations": [ + { + "args": "('Storage data')", + "name": "docs_group" + } + ], + "flags": {}, + "groups": [ + { + "children": [ + 31564, + 31577, + 31585, + 31541, + 31556, + 31581, + 31573, + 31568 + ], + "title": "Methods" + }, + { + "children": [ + 31563, + 31562, + 31576, + 31584, + 31540, + 31560, + 31580, + 31539, + 31571, + 31535, + 31532, + 31538, + 31536, + 31537, + 31561, + 31572, + 31567, + 31533, + 31534, + 31588 + ], + "title": "Properties" + } + ], + "id": 31531, + "module": "_request", + "name": "Request", + "parsedDocstring": { + "text": "Represents a request in the Crawlee framework, containing the necessary information for crawling operations.\n\nThe `Request` class is one of the core components in Crawlee, utilized by various components such as request\nproviders, HTTP clients, crawlers, and more. It encapsulates the essential data for executing web requests,\nincluding the URL, HTTP method, headers, payload, and user data. The user data allows custom information\nto be stored and persisted throughout the request lifecycle, including its retries.\n\nKey functionalities include managing the request's identifier (`id`), unique key (`unique_key`) that is used\nfor request deduplication, controlling retries, handling state management, and enabling configuration for session\nrotation and proxy handling.\n\nThe recommended way to create a new instance is by using the `Request.from_url` constructor, which automatically\ngenerates a unique key and identifier based on the URL and request parameters.\n\n### Usage\n\n```python\nfrom crawlee import Request\n\nrequest = Request.from_url('https://crawlee.dev')\n```" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_request.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 145, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/_request.py#L145" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "extendedBy": [ + { + "name": "RequestWithLock", + "target": "467", + "type": "reference", + "ref": { + "id": 31589, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_request.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 433 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/RequestWithLock" + } + } + ], + "permalink": "/python/api/class/Request", + "categories": [] + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 32909, + "module": "storage_clients.models", + "name": "model_config", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 62, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/models.py#L62" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "overwrites": { + "name": "StorageMetadata.model_config", + "target": 1308, + "type": "reference", + "ref": { + "id": 32897, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 23 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api/class/StorageMetadata#model_config" + } + }, + "permalink": "/python/api/class/RequestQueueMetadata#model_config", + "parentId": 1319 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Indicates whether the queue has been accessed by multiple clients (consumers)." + } + ] + }, + "flags": {}, + "groups": [], + "id": 32910, + "module": "storage_clients.models", + "name": "had_multiple_clients", + "parsedDocstring": { + "text": "Indicates whether the queue has been accessed by multiple clients (consumers)." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 64, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/models.py#L64" + } + ], + "type": { + "name": "bool", + "type": "reference", + "target": 0, + "ref": {} + }, + "permalink": "/python/api/class/RequestQueueMetadata#had_multiple_clients", + "parentId": 1319 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "The number of requests that have been handled from the queue." + } + ] + }, + "flags": {}, + "groups": [], + "id": 32911, + "module": "storage_clients.models", + "name": "handled_request_count", + "parsedDocstring": { + "text": "The number of requests that have been handled from the queue." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 67, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/models.py#L67" + } + ], + "type": { + "name": "int", + "type": "reference", + "target": 0, + "ref": {} + }, + "permalink": "/python/api/class/RequestQueueMetadata#handled_request_count", + "parentId": 1319 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "The number of requests that are still pending in the queue." + } + ] + }, + "flags": {}, + "groups": [], + "id": 32912, + "module": "storage_clients.models", + "name": "pending_request_count", + "parsedDocstring": { + "text": "The number of requests that are still pending in the queue." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 70, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/models.py#L70" + } + ], + "type": { + "name": "int", + "type": "reference", + "target": 0, + "ref": {} + }, + "permalink": "/python/api/class/RequestQueueMetadata#pending_request_count", + "parentId": 1319 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "The total number of requests that have been added to the queue." + } + ] + }, + "flags": {}, + "groups": [], + "id": 32913, + "module": "storage_clients.models", + "name": "total_request_count", + "parsedDocstring": { + "text": "The total number of requests that have been added to the queue." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 73, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/models.py#L73" + } + ], + "type": { + "name": "int", + "type": "reference", + "target": 0, + "ref": {} + }, + "permalink": "/python/api/class/RequestQueueMetadata#total_request_count", + "parentId": 1319 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "The unique identifier of the storage." + } + ] + }, + "flags": {}, + "groups": [], + "id": 35808, + "module": "storage_clients.models", + "name": "id", + "parsedDocstring": { + "text": "The unique identifier of the storage." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 25, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/models.py#L25" + } + ], + "type": { + "name": "Annotated[str, Field(alias='id')]", + "type": "reference", + "target": 0, + "ref": {} + }, + "inheritedFrom": { + "name": "StorageMetadata.id", + "target": 1309, + "type": "reference", + "ref": { + "id": 32898, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 25 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api/class/StorageMetadata#id" + } + }, + "permalink": "/python/api/class/RequestQueueMetadata#id", + "parentId": 1319 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "The name of the storage." + } + ] + }, + "flags": {}, + "groups": [], + "id": 35809, + "module": "storage_clients.models", + "name": "name", + "parsedDocstring": { + "text": "The name of the storage." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 28, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/models.py#L28" + } + ], + "type": { + "name": "Annotated[str | None, Field(alias='name', default=None)]", + "type": "reference", + "target": 0, + "ref": {} + }, + "inheritedFrom": { + "name": "StorageMetadata.name", + "target": 1310, + "type": "reference", + "ref": { + "id": 32899, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 28 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api/class/StorageMetadata#name" + } + }, + "permalink": "/python/api/class/RequestQueueMetadata#name", + "parentId": 1319 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "The timestamp when the storage was last accessed." + } + ] + }, + "flags": {}, + "groups": [], + "id": 35810, + "module": "storage_clients.models", + "name": "accessed_at", + "parsedDocstring": { + "text": "The timestamp when the storage was last accessed." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 31, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/models.py#L31" + } + ], + "type": { + "name": "Annotated[datetime, Field(alias='accessedAt')]", + "type": "reference", + "target": 0, + "ref": {} + }, + "inheritedFrom": { + "name": "StorageMetadata.accessed_at", + "target": 1311, + "type": "reference", + "ref": { + "id": 32900, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 31 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api/class/StorageMetadata#accessed_at" + } + }, + "permalink": "/python/api/class/RequestQueueMetadata#accessed_at", + "parentId": 1319 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "The timestamp when the storage was created." + } + ] + }, + "flags": {}, + "groups": [], + "id": 35811, + "module": "storage_clients.models", + "name": "created_at", + "parsedDocstring": { + "text": "The timestamp when the storage was created." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 34, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/models.py#L34" + } + ], + "type": { + "name": "Annotated[datetime, Field(alias='createdAt')]", + "type": "reference", + "target": 0, + "ref": {} + }, + "inheritedFrom": { + "name": "StorageMetadata.created_at", + "target": 1312, + "type": "reference", + "ref": { + "id": 32901, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 34 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api/class/StorageMetadata#created_at" + } + }, + "permalink": "/python/api/class/RequestQueueMetadata#created_at", + "parentId": 1319 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "The timestamp when the storage was last modified." + } + ] + }, + "flags": {}, + "groups": [], + "id": 35812, + "module": "storage_clients.models", + "name": "modified_at", + "parsedDocstring": { + "text": "The timestamp when the storage was last modified." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 37, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/models.py#L37" + } + ], + "type": { + "name": "Annotated[datetime, Field(alias='modifiedAt')]", + "type": "reference", + "target": 0, + "ref": {} + }, + "inheritedFrom": { + "name": "StorageMetadata.modified_at", + "target": 1313, + "type": "reference", + "ref": { + "id": 32902, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 37 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api/class/StorageMetadata#modified_at" + } + }, + "permalink": "/python/api/class/RequestQueueMetadata#modified_at", + "parentId": 1319 + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Model for a request queue metadata." + } + ] + }, + "decorations": [ + { + "args": "('Storage data')", + "name": "docs_group" + } + ], + "flags": {}, + "groups": [ + { + "children": [ + 35810, + 35811, + 32910, + 32911, + 35808, + 32909, + 35812, + 35809, + 32912, + 32913 + ], + "title": "Properties" + } + ], + "id": 32908, + "module": "storage_clients.models", + "name": "RequestQueueMetadata", + "parsedDocstring": { + "text": "Model for a request queue metadata." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 59, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/models.py#L59" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "extendedTypes": [ + { + "name": "StorageMetadata", + "target": "1307", + "type": "reference", + "ref": { + "id": 32896, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 17 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/StorageMetadata" + } + } + ], + "permalink": "/python/api/class/RequestQueueMetadata", + "categories": [] + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 37120, + "module": "storage_clients.models", + "name": "model_config", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 23, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/models.py#L23" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "permalink": "/python/api/class/StorageMetadata#model_config", + "parentId": 1307 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "The unique identifier of the storage." + } + ] + }, + "flags": {}, + "groups": [], + "id": 37121, + "module": "storage_clients.models", + "name": "id", + "parsedDocstring": { + "text": "The unique identifier of the storage." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 25, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/models.py#L25" + } + ], + "type": { + "name": "str", + "type": "reference", + "target": 0, + "ref": {} + }, + "permalink": "/python/api/class/StorageMetadata#id", + "parentId": 1307 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "The name of the storage." + } + ] + }, + "flags": {}, + "groups": [], + "id": 37122, + "module": "storage_clients.models", + "name": "name", + "parsedDocstring": { + "text": "The name of the storage." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 28, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/models.py#L28" + } + ], + "type": { + "name": "Annotated[str | None, Field(alias='name', default=None)]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "permalink": "/python/api/class/StorageMetadata#name", + "parentId": 1307 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "The timestamp when the storage was last accessed." + } + ] + }, + "flags": {}, + "groups": [], + "id": 37123, + "module": "storage_clients.models", + "name": "accessed_at", + "parsedDocstring": { + "text": "The timestamp when the storage was last accessed." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 31, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/models.py#L31" + } + ], + "type": { + "name": "datetime", + "type": "reference", + "target": 0, + "ref": {} + }, + "permalink": "/python/api/class/StorageMetadata#accessed_at", + "parentId": 1307 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "The timestamp when the storage was created." + } + ] + }, + "flags": {}, + "groups": [], + "id": 37124, + "module": "storage_clients.models", + "name": "created_at", + "parsedDocstring": { + "text": "The timestamp when the storage was created." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 34, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/models.py#L34" + } + ], + "type": { + "name": "datetime", + "type": "reference", + "target": 0, + "ref": {} + }, + "permalink": "/python/api/class/StorageMetadata#created_at", + "parentId": 1307 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "The timestamp when the storage was last modified." + } + ] + }, + "flags": {}, + "groups": [], + "id": 37125, + "module": "storage_clients.models", + "name": "modified_at", + "parsedDocstring": { + "text": "The timestamp when the storage was last modified." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 37, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/models.py#L37" + } + ], + "type": { + "name": "datetime", + "type": "reference", + "target": 0, + "ref": {} + }, + "permalink": "/python/api/class/StorageMetadata#modified_at", + "parentId": 1307 + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Represents the base model for storage metadata.\n\nIt contains common fields shared across all specific storage types." + } + ] + }, + "decorations": [ + { + "args": "('Storage data')", + "name": "docs_group" + } + ], + "flags": {}, + "groups": [ + { + "children": [ + 37123, + 37124, + 37121, + 37120, + 37125, + 37122 + ], + "title": "Properties" + } + ], + "id": 37119, + "module": "storage_clients.models", + "name": "StorageMetadata", + "parsedDocstring": { + "text": "Represents the base model for storage metadata.\n\nIt contains common fields shared across all specific storage types." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 17, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/models.py#L17" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "extendedBy": [ + { + "name": "DatasetMetadata", + "target": "1314", + "type": "reference", + "ref": { + "id": 37126, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 42 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/DatasetMetadata" + } + }, + { + "name": "KeyValueStoreMetadata", + "target": "1317", + "type": "reference", + "ref": { + "id": 37129, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 52 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/KeyValueStoreMetadata" + } + }, + { + "name": "RequestQueueMetadata", + "target": "1319", + "type": "reference", + "ref": { + "id": 37131, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 59 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/RequestQueueMetadata" + } + } + ], + "permalink": "/python/api/class/StorageMetadata", + "categories": [] + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Initialize a new instance.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 39287, + "module": "events._event_manager", + "name": "__init__", + "parsedDocstring": { + "text": "Initialize a new instance.\n", + "args": { + "persist_state_interval": "Interval between emitted `PersistState` events to maintain state persistence.", + "close_timeout": "Optional timeout for canceling pending event listeners if they exceed this duration." + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_event_manager.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 63, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/events/_event_manager.py#L63" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Initialize a new instance.\n" + } + ] + }, + "flags": {}, + "id": 39288, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "__init__", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Interval between emitted `PersistState` events to maintain state persistence." + } + ] + }, + "defaultValue": "timedelta(minutes=1)", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 39289, + "kind": 32768, + "kindString": "Parameter", + "name": "persist_state_interval", + "type": { + "name": "timedelta", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2157 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Optional timeout for canceling pending event listeners if they exceed this duration." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 39290, + "kind": 32768, + "kindString": "Parameter", + "name": "close_timeout", + "type": { + "name": "timedelta | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "timedelta", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2157 + } + ], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "parentId": 2156 + } + ], + "permalink": "/python/api/class/EventManager#__init__", + "parentId": 2155 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Indicate whether the context is active." + } + ] + }, + "decorations": [ + { + "name": "property" + } + ], + "flags": {}, + "groups": [], + "id": 39291, + "module": "events._event_manager", + "name": "active", + "parsedDocstring": { + "text": "Indicate whether the context is active." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_event_manager.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 100, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/events/_event_manager.py#L100" + } + ], + "type": { + "name": "bool", + "type": "reference", + "target": 0, + "ref": {} + }, + "permalink": "/python/api/class/EventManager#active", + "parentId": 2155 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Initialize the event manager upon entering the async context." + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 39292, + "module": "events._event_manager", + "name": "__aenter__", + "parsedDocstring": { + "text": "Initialize the event manager upon entering the async context." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_event_manager.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 104, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/events/_event_manager.py#L104" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Initialize the event manager upon entering the async context." + } + ] + }, + "flags": {}, + "id": 39293, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "__aenter__", + "parameters": [], + "type": { + "name": "EventManager", + "type": "reference", + "target": "2155", + "ref": { + "id": 39286, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_event_manager.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 55 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/EventManager" + } + }, + "parentId": 2161 + } + ], + "permalink": "/python/api/class/EventManager#__aenter__", + "parentId": 2155 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Close the local event manager upon exiting the async context.\n\nThis will stop listening for the events, and it will wait for all the event listeners to finish.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 39294, + "module": "events._event_manager", + "name": "__aexit__", + "parsedDocstring": { + "text": "Close the local event manager upon exiting the async context.\n\nThis will stop listening for the events, and it will wait for all the event listeners to finish.\n" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_event_manager.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 113, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/events/_event_manager.py#L113" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Close the local event manager upon exiting the async context.\n\nThis will stop listening for the events, and it will wait for all the event listeners to finish.\n" + } + ] + }, + "flags": {}, + "id": 39295, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "__aexit__", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 39296, + "kind": 32768, + "kindString": "Parameter", + "name": "exc_type", + "type": { + "name": "type[BaseException] | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "type", + "typeArguments": [ + { + "type": "reference", + "name": "BaseException", + "target": 0, + "ref": {} + } + ], + "target": "3422", + "ref": { + "id": 40553, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_utils/sitemap.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 62 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api/class/SitemapSource#type" + } + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2164 + }, + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 39297, + "kind": 32768, + "kindString": "Parameter", + "name": "exc_value", + "type": { + "name": "BaseException | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "BaseException", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2164 + }, + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 39298, + "kind": 32768, + "kindString": "Parameter", + "name": "exc_traceback", + "type": { + "name": "TracebackType | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "TracebackType", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2164 + } + ], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "parentId": 2163 + } + ], + "permalink": "/python/api/class/EventManager#__aexit__", + "parentId": 2155 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Register an event listener for a specific event.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 39299, + "module": "events._event_manager", + "name": "on", + "parsedDocstring": { + "text": "Register an event listener for a specific event.\n", + "args": { + "event": "The event for which to listen to.", + "listener": "The function (sync or async) which is to be called when the event is emitted." + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_event_manager.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 160, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/events/_event_manager.py#L160" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Register an event listener for a specific event.\n" + } + ] + }, + "flags": {}, + "id": 39300, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "on", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The event for which to listen to." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 39301, + "kind": 32768, + "kindString": "Parameter", + "name": "event", + "type": { + "name": "Event", + "type": "reference", + "target": "2116", + "ref": { + "id": 39247, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 15 + } + ], + "kind": 8, + "permalink": "https://crawlee.dev/python/api/enum/Event" + } + }, + "parentId": 2169 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The function (sync or async) which is to be called when the event is emitted." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 39302, + "kind": 32768, + "kindString": "Parameter", + "name": "listener", + "type": { + "name": "EventListener", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "Any", + "target": 0, + "ref": {} + } + ], + "target": "2150", + "ref": { + "id": 39281, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 112 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api#EventListener" + } + }, + "parentId": 2169 + } + ], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "parentId": 2168 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Register an event listener for a specific event.\n" + } + ] + }, + "flags": {}, + "id": 39314, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "on", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The event for which to listen to." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 39315, + "kind": 32768, + "kindString": "Parameter", + "name": "event", + "type": { + "name": "Literal", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "Event.PERSIST_STATE", + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + }, + "parentId": 2183 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The function (sync or async) which is to be called when the event is emitted." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 39316, + "kind": 32768, + "kindString": "Parameter", + "name": "listener", + "type": { + "name": "EventListener", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "EventPersistStateData", + "target": "2129", + "ref": { + "id": 39260, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 40 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/EventPersistStateData" + } + } + ], + "target": "2150", + "ref": { + "id": 39281, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 112 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api#EventListener" + } + }, + "parentId": 2183 + } + ], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "parentId": 2168 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Register an event listener for a specific event.\n" + } + ] + }, + "flags": {}, + "id": 39317, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "on", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The event for which to listen to." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 39318, + "kind": 32768, + "kindString": "Parameter", + "name": "event", + "type": { + "name": "Literal", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "Event.SYSTEM_INFO", + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + }, + "parentId": 2186 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The function (sync or async) which is to be called when the event is emitted." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 39319, + "kind": 32768, + "kindString": "Parameter", + "name": "listener", + "type": { + "name": "EventListener", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "EventSystemInfoData", + "target": "2132", + "ref": { + "id": 39263, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 49 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/EventSystemInfoData" + } + } + ], + "target": "2150", + "ref": { + "id": 39281, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 112 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api#EventListener" + } + }, + "parentId": 2186 + } + ], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "parentId": 2168 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Register an event listener for a specific event.\n" + } + ] + }, + "flags": {}, + "id": 39320, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "on", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The event for which to listen to." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 39321, + "kind": 32768, + "kindString": "Parameter", + "name": "event", + "type": { + "name": "Literal", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "Event.MIGRATING", + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + }, + "parentId": 2189 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The function (sync or async) which is to be called when the event is emitted." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 39322, + "kind": 32768, + "kindString": "Parameter", + "name": "listener", + "type": { + "name": "EventListener", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "EventMigratingData", + "target": "2136", + "ref": { + "id": 39267, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 62 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/EventMigratingData" + } + } + ], + "target": "2150", + "ref": { + "id": 39281, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 112 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api#EventListener" + } + }, + "parentId": 2189 + } + ], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "parentId": 2168 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Register an event listener for a specific event.\n" + } + ] + }, + "flags": {}, + "id": 39323, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "on", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The event for which to listen to." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 39324, + "kind": 32768, + "kindString": "Parameter", + "name": "event", + "type": { + "name": "Literal", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "Event.ABORTING", + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + }, + "parentId": 2192 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The function (sync or async) which is to be called when the event is emitted." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 39325, + "kind": 32768, + "kindString": "Parameter", + "name": "listener", + "type": { + "name": "EventListener", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "EventAbortingData", + "target": "2139", + "ref": { + "id": 39270, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 73 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/EventAbortingData" + } + } + ], + "target": "2150", + "ref": { + "id": 39281, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 112 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api#EventListener" + } + }, + "parentId": 2192 + } + ], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "parentId": 2168 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Register an event listener for a specific event.\n" + } + ] + }, + "flags": {}, + "id": 39326, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "on", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The event for which to listen to." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 39327, + "kind": 32768, + "kindString": "Parameter", + "name": "event", + "type": { + "name": "Literal", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "Event.EXIT", + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + }, + "parentId": 2195 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The function (sync or async) which is to be called when the event is emitted." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 39328, + "kind": 32768, + "kindString": "Parameter", + "name": "listener", + "type": { + "name": "EventListener", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "EventExitData", + "target": "2141", + "ref": { + "id": 39272, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 80 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/EventExitData" + } + } + ], + "target": "2150", + "ref": { + "id": 39281, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 112 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api#EventListener" + } + }, + "parentId": 2195 + } + ], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "parentId": 2168 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Register an event listener for a specific event.\n" + } + ] + }, + "flags": {}, + "id": 39329, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "on", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The event for which to listen to." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 39330, + "kind": 32768, + "kindString": "Parameter", + "name": "event", + "type": { + "name": "Literal", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "Event.CRAWLER_STATUS", + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + }, + "parentId": 2198 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The function (sync or async) which is to be called when the event is emitted." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 39331, + "kind": 32768, + "kindString": "Parameter", + "name": "listener", + "type": { + "name": "EventListener", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "EventCrawlerStatusData", + "target": "2143", + "ref": { + "id": 39274, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 87 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/EventCrawlerStatusData" + } + } + ], + "target": "2150", + "ref": { + "id": 39281, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 112 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api#EventListener" + } + }, + "parentId": 2198 + } + ], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "parentId": 2168 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Register an event listener for a specific event.\n" + } + ] + }, + "flags": {}, + "id": 39332, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "on", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The event for which to listen to." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 39333, + "kind": 32768, + "kindString": "Parameter", + "name": "event", + "type": { + "name": "Event", + "type": "reference", + "target": "2116", + "ref": { + "id": 39247, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 15 + } + ], + "kind": 8, + "permalink": "https://crawlee.dev/python/api/enum/Event" + } + }, + "parentId": 2201 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The function (sync or async) which is to be called when the event is emitted." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 39334, + "kind": 32768, + "kindString": "Parameter", + "name": "listener", + "type": { + "name": "EventListener", + "type": "reference", + "typeArguments": [ + { + "type": "literal", + "value": {} + } + ], + "target": "2150", + "ref": { + "id": 39281, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 112 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api#EventListener" + } + }, + "parentId": 2201 + } + ], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "parentId": 2168 + } + ], + "permalink": "/python/api/class/EventManager#on", + "parentId": 2155 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Remove a specific listener or all listeners for an event.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 39303, + "module": "events._event_manager", + "name": "off", + "parsedDocstring": { + "text": "Remove a specific listener or all listeners for an event.\n", + "args": { + "event": "The Actor event for which to remove listeners.", + "listener": "The listener which is supposed to be removed. If not passed, all listeners of this event\nare removed." + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_event_manager.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 210, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/events/_event_manager.py#L210" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Remove a specific listener or all listeners for an event.\n" + } + ] + }, + "flags": {}, + "id": 39304, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "off", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The Actor event for which to remove listeners." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 39305, + "kind": 32768, + "kindString": "Parameter", + "name": "event", + "type": { + "name": "Event", + "type": "reference", + "target": "2116", + "ref": { + "id": 39247, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 15 + } + ], + "kind": 8, + "permalink": "https://crawlee.dev/python/api/enum/Event" + } + }, + "parentId": 2173 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The listener which is supposed to be removed. If not passed, all listeners of this event\nare removed." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 39306, + "kind": 32768, + "kindString": "Parameter", + "name": "listener", + "type": { + "name": "EventListener[Any] | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "EventListener", + "typeArguments": [ + { + "type": "reference", + "name": "Any", + "target": 0, + "ref": {} + } + ], + "target": "2150", + "ref": { + "id": 39281, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 112 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api#EventListener" + } + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2173 + } + ], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "parentId": 2172 + } + ], + "permalink": "/python/api/class/EventManager#off", + "parentId": 2155 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Emit an event with the associated data to all registered listeners.\n" + } + ] + }, + "decorations": [ + { + "name": "ensure_context" + } + ], + "flags": {}, + "groups": [], + "id": 39307, + "module": "events._event_manager", + "name": "emit", + "parsedDocstring": { + "text": "Emit an event with the associated data to all registered listeners.\n", + "args": { + "event": "The event which will be emitted.", + "event_data": "The data which will be passed to the event listeners." + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_event_manager.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 242, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/events/_event_manager.py#L242" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Emit an event with the associated data to all registered listeners.\n" + } + ] + }, + "flags": {}, + "id": 39308, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "emit", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The event which will be emitted." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 39309, + "kind": 32768, + "kindString": "Parameter", + "name": "event", + "type": { + "name": "Event", + "type": "reference", + "target": "2116", + "ref": { + "id": 39247, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 15 + } + ], + "kind": 8, + "permalink": "https://crawlee.dev/python/api/enum/Event" + } + }, + "parentId": 2177 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The data which will be passed to the event listeners." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 39310, + "kind": 32768, + "kindString": "Parameter", + "name": "event_data", + "type": { + "name": "EventData", + "type": "reference", + "target": "2147", + "ref": { + "id": 39278, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 99 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api#EventData" + } + }, + "parentId": 2177 + } + ], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "parentId": 2176 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Emit an event with the associated data to all registered listeners.\n" + } + ] + }, + "flags": {}, + "id": 39335, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "emit", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The event which will be emitted." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 39336, + "kind": 32768, + "kindString": "Parameter", + "name": "event", + "type": { + "name": "Literal", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "Event.PERSIST_STATE", + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + }, + "parentId": 2204 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The data which will be passed to the event listeners." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 39337, + "kind": 32768, + "kindString": "Parameter", + "name": "event_data", + "type": { + "name": "EventPersistStateData", + "type": "reference", + "target": "2129", + "ref": { + "id": 39260, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 40 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/EventPersistStateData" + } + }, + "parentId": 2204 + } + ], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "parentId": 2176 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Emit an event with the associated data to all registered listeners.\n" + } + ] + }, + "flags": {}, + "id": 39338, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "emit", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The event which will be emitted." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 39339, + "kind": 32768, + "kindString": "Parameter", + "name": "event", + "type": { + "name": "Literal", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "Event.SYSTEM_INFO", + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + }, + "parentId": 2207 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The data which will be passed to the event listeners." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 39340, + "kind": 32768, + "kindString": "Parameter", + "name": "event_data", + "type": { + "name": "EventSystemInfoData", + "type": "reference", + "target": "2132", + "ref": { + "id": 39263, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 49 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/EventSystemInfoData" + } + }, + "parentId": 2207 + } + ], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "parentId": 2176 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Emit an event with the associated data to all registered listeners.\n" + } + ] + }, + "flags": {}, + "id": 39341, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "emit", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The event which will be emitted." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 39342, + "kind": 32768, + "kindString": "Parameter", + "name": "event", + "type": { + "name": "Literal", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "Event.MIGRATING", + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + }, + "parentId": 2210 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The data which will be passed to the event listeners." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 39343, + "kind": 32768, + "kindString": "Parameter", + "name": "event_data", + "type": { + "name": "EventMigratingData", + "type": "reference", + "target": "2136", + "ref": { + "id": 39267, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 62 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/EventMigratingData" + } + }, + "parentId": 2210 + } + ], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "parentId": 2176 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Emit an event with the associated data to all registered listeners.\n" + } + ] + }, + "flags": {}, + "id": 39344, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "emit", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The event which will be emitted." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 39345, + "kind": 32768, + "kindString": "Parameter", + "name": "event", + "type": { + "name": "Literal", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "Event.ABORTING", + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + }, + "parentId": 2213 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The data which will be passed to the event listeners." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 39346, + "kind": 32768, + "kindString": "Parameter", + "name": "event_data", + "type": { + "name": "EventAbortingData", + "type": "reference", + "target": "2139", + "ref": { + "id": 39270, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 73 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/EventAbortingData" + } + }, + "parentId": 2213 + } + ], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "parentId": 2176 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Emit an event with the associated data to all registered listeners.\n" + } + ] + }, + "flags": {}, + "id": 39347, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "emit", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The event which will be emitted." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 39348, + "kind": 32768, + "kindString": "Parameter", + "name": "event", + "type": { + "name": "Literal", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "Event.EXIT", + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + }, + "parentId": 2216 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The data which will be passed to the event listeners." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 39349, + "kind": 32768, + "kindString": "Parameter", + "name": "event_data", + "type": { + "name": "EventExitData", + "type": "reference", + "target": "2141", + "ref": { + "id": 39272, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 80 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/EventExitData" + } + }, + "parentId": 2216 + } + ], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "parentId": 2176 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Emit an event with the associated data to all registered listeners.\n" + } + ] + }, + "flags": {}, + "id": 39350, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "emit", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The event which will be emitted." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 39351, + "kind": 32768, + "kindString": "Parameter", + "name": "event", + "type": { + "name": "Literal", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "Event.CRAWLER_STATUS", + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + }, + "parentId": 2219 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The data which will be passed to the event listeners." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 39352, + "kind": 32768, + "kindString": "Parameter", + "name": "event_data", + "type": { + "name": "EventCrawlerStatusData", + "type": "reference", + "target": "2143", + "ref": { + "id": 39274, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 87 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/EventCrawlerStatusData" + } + }, + "parentId": 2219 + } + ], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "parentId": 2176 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Emit an event with the associated data to all registered listeners.\n" + } + ] + }, + "flags": {}, + "id": 39353, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "emit", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The event which will be emitted." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 39354, + "kind": 32768, + "kindString": "Parameter", + "name": "event", + "type": { + "name": "Event", + "type": "reference", + "target": "2116", + "ref": { + "id": 39247, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 15 + } + ], + "kind": 8, + "permalink": "https://crawlee.dev/python/api/enum/Event" + } + }, + "parentId": 2222 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The data which will be passed to the event listeners." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 39355, + "kind": 32768, + "kindString": "Parameter", + "name": "event_data", + "type": { + "name": "Any", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2222 + } + ], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "parentId": 2176 + } + ], + "permalink": "/python/api/class/EventManager#emit", + "parentId": 2155 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Wait for all currently executing event listeners to complete.\n" + } + ] + }, + "decorations": [ + { + "name": "ensure_context" + } + ], + "flags": {}, + "groups": [], + "id": 39311, + "module": "events._event_manager", + "name": "wait_for_all_listeners_to_complete", + "parsedDocstring": { + "text": "Wait for all currently executing event listeners to complete.\n", + "args": { + "timeout": "The maximum time to wait for the event listeners to finish. If they do not complete within\nthe specified timeout, they will be canceled." + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_event_manager.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 252, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/events/_event_manager.py#L252" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Wait for all currently executing event listeners to complete.\n" + } + ] + }, + "flags": {}, + "id": 39312, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "wait_for_all_listeners_to_complete", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The maximum time to wait for the event listeners to finish. If they do not complete within\nthe specified timeout, they will be canceled." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 39313, + "kind": 32768, + "kindString": "Parameter", + "name": "timeout", + "type": { + "name": "timedelta | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "timedelta", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2181 + } + ], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "parentId": 2180 + } + ], + "permalink": "/python/api/class/EventManager#wait_for_all_listeners_to_complete", + "parentId": 2155 + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Manage events and their listeners, enabling registration, emission, and execution control.\n\nIt allows for registering event listeners, emitting events, and ensuring all listeners complete their execution.\nBuilt on top of `pyee.asyncio.AsyncIOEventEmitter`. It implements additional features such as waiting for all\nlisteners to complete and emitting `PersistState` events at regular intervals." + } + ] + }, + "decorations": [ + { + "args": "('Event managers')", + "name": "docs_group" + } + ], + "flags": {}, + "groups": [ + { + "children": [ + 39292, + 39294, + 39287, + 39307, + 39303, + 39299, + 39311 + ], + "title": "Methods" + }, + { + "children": [ + 39291 + ], + "title": "Properties" + } + ], + "id": 39286, + "module": "events._event_manager", + "name": "EventManager", + "parsedDocstring": { + "text": "Manage events and their listeners, enabling registration, emission, and execution control.\n\nIt allows for registering event listeners, emitting events, and ensuring all listeners complete their execution.\nBuilt on top of `pyee.asyncio.AsyncIOEventEmitter`. It implements additional features such as waiting for all\nlisteners to complete and emitting `PersistState` events at regular intervals." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_event_manager.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 55, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/events/_event_manager.py#L55" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "extendedBy": [ + { + "name": "LocalEventManager", + "target": "2226", + "type": "reference", + "ref": { + "id": 39357, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_local_event_manager.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 26 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/LocalEventManager" + } + } + ], + "permalink": "/python/api/class/EventManager", + "categories": [] + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Initialize a new instance.\n\nIn most cases, you should use the `from_config` constructor to create a new instance based on\nthe provided configuration.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 42780, + "module": "events._local_event_manager", + "name": "__init__", + "parsedDocstring": { + "text": "Initialize a new instance.\n\nIn most cases, you should use the `from_config` constructor to create a new instance based on\nthe provided configuration.\n", + "args": { + "system_info_interval": "Interval at which `SystemInfo` events are emitted.", + "event_manager_options": "Additional options for the parent class." + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_local_event_manager.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 34, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/events/_local_event_manager.py#L34" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Initialize a new instance.\n\nIn most cases, you should use the `from_config` constructor to create a new instance based on\nthe provided configuration.\n" + } + ] + }, + "flags": {}, + "id": 42781, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "__init__", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Interval at which `SystemInfo` events are emitted." + } + ] + }, + "defaultValue": "timedelta(seconds=1)", + "flags": { + "isOptional": true, + "keyword-only": false + }, + "id": 42782, + "kind": 32768, + "kindString": "Parameter", + "name": "system_info_interval", + "type": { + "name": "timedelta", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2228 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Interval between emitted `PersistState` events to maintain state persistence." + } + ] + }, + "flags": { + "keyword-only": true, + "optional": true + }, + "groups": [], + "id": 42706, + "module": "events._event_manager", + "name": "persist_state_interval", + "parsedDocstring": { + "text": "Interval between emitted `PersistState` events to maintain state persistence." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_event_manager.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 47, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/events/_event_manager.py#L47" + } + ], + "type": { + "name": "NotRequired", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "timedelta", + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + }, + "parentId": 2228 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Optional timeout for canceling pending event listeners if they exceed this duration." + } + ] + }, + "flags": { + "keyword-only": true, + "optional": true + }, + "groups": [], + "id": 42707, + "module": "events._event_manager", + "name": "close_timeout", + "parsedDocstring": { + "text": "Optional timeout for canceling pending event listeners if they exceed this duration." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_event_manager.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 50, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/events/_event_manager.py#L50" + } + ], + "type": { + "name": "NotRequired", + "type": "reference", + "typeArguments": [ + { + "type": "union", + "types": [ + { + "type": "reference", + "name": "timedelta", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + } + ], + "target": 0, + "ref": {} + }, + "parentId": 2228 + } + ], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "overwrites": { + "name": "EventManager.__init__", + "target": 2156, + "type": "reference", + "ref": { + "id": 42709, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_event_manager.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 63 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/EventManager#__init__" + } + }, + "parentId": 2227 + } + ], + "overwrites": { + "name": "EventManager.__init__", + "target": 2156, + "type": "reference", + "ref": { + "id": 42709, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_event_manager.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 63 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/EventManager#__init__" + } + }, + "permalink": "/python/api/class/LocalEventManager#__init__", + "parentId": 2226 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Initialize a new instance based on the provided `Configuration`.\n" + } + ] + }, + "decorations": [ + { + "name": "classmethod" + } + ], + "flags": {}, + "groups": [], + "id": 42784, + "module": "events._local_event_manager", + "name": "from_config", + "parsedDocstring": { + "text": "Initialize a new instance based on the provided `Configuration`.\n", + "args": { + "config": "The `Configuration` instance. Uses the global (default) one if not provided." + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_local_event_manager.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 59, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/events/_local_event_manager.py#L59" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Initialize a new instance based on the provided `Configuration`.\n" + } + ] + }, + "flags": {}, + "id": 42785, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "from_config", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The `Configuration` instance. Uses the global (default) one if not provided." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": false + }, + "id": 42786, + "kind": 32768, + "kindString": "Parameter", + "name": "config", + "type": { + "name": "Configuration | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "Configuration", + "target": "2", + "ref": { + "id": 40555, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/configuration.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 20 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/Configuration" + } + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2232 + } + ], + "type": { + "name": "LocalEventManager", + "type": "reference", + "target": "2226", + "ref": { + "id": 42779, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_local_event_manager.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 26 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/LocalEventManager" + } + }, + "parentId": 2231 + } + ], + "permalink": "/python/api/class/LocalEventManager#from_config", + "parentId": 2226 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Initialize the local event manager upon entering the async context.\n\nIt starts emitting system info events at regular intervals." + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 42787, + "module": "events._local_event_manager", + "name": "__aenter__", + "parsedDocstring": { + "text": "Initialize the local event manager upon entering the async context.\n\nIt starts emitting system info events at regular intervals." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_local_event_manager.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 72, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/events/_local_event_manager.py#L72" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Initialize the local event manager upon entering the async context.\n\nIt starts emitting system info events at regular intervals." + } + ] + }, + "flags": {}, + "id": 42788, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "__aenter__", + "parameters": [], + "type": { + "name": "LocalEventManager", + "type": "reference", + "target": "2226", + "ref": { + "id": 42779, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_local_event_manager.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 26 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/LocalEventManager" + } + }, + "overwrites": { + "name": "EventManager.__aenter__", + "target": 2161, + "type": "reference", + "ref": { + "id": 42714, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_event_manager.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 104 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/EventManager#__aenter__" + } + }, + "parentId": 2234 + } + ], + "overwrites": { + "name": "EventManager.__aenter__", + "target": 2161, + "type": "reference", + "ref": { + "id": 42714, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_event_manager.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 104 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/EventManager#__aenter__" + } + }, + "permalink": "/python/api/class/LocalEventManager#__aenter__", + "parentId": 2226 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Close the local event manager upon exiting the async context.\n\nIt stops emitting system info events and closes the event manager." + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 42789, + "module": "events._local_event_manager", + "name": "__aexit__", + "parsedDocstring": { + "text": "Close the local event manager upon exiting the async context.\n\nIt stops emitting system info events and closes the event manager." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_local_event_manager.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 84, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/events/_local_event_manager.py#L84" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Close the local event manager upon exiting the async context.\n\nIt stops emitting system info events and closes the event manager." + } + ] + }, + "flags": {}, + "id": 42790, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "__aexit__", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 42791, + "kind": 32768, + "kindString": "Parameter", + "name": "exc_type", + "type": { + "name": "type[BaseException] | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "type", + "typeArguments": [ + { + "type": "reference", + "name": "BaseException", + "target": 0, + "ref": {} + } + ], + "target": "3422", + "ref": { + "id": 43975, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_utils/sitemap.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 62 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api/class/SitemapSource#type" + } + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2237 + }, + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 42792, + "kind": 32768, + "kindString": "Parameter", + "name": "exc_value", + "type": { + "name": "BaseException | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "BaseException", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2237 + }, + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 42793, + "kind": 32768, + "kindString": "Parameter", + "name": "exc_traceback", + "type": { + "name": "TracebackType | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "TracebackType", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2237 + } + ], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "overwrites": { + "name": "EventManager.__aexit__", + "target": 2163, + "type": "reference", + "ref": { + "id": 42716, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_event_manager.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 113 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/EventManager#__aexit__" + } + }, + "parentId": 2236 + } + ], + "overwrites": { + "name": "EventManager.__aexit__", + "target": 2163, + "type": "reference", + "ref": { + "id": 42716, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_event_manager.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 113 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/EventManager#__aexit__" + } + }, + "permalink": "/python/api/class/LocalEventManager#__aexit__", + "parentId": 2226 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Indicate whether the context is active." + } + ] + }, + "decorations": [ + { + "name": "property" + } + ], + "flags": {}, + "groups": [], + "id": 44707, + "module": "events._event_manager", + "name": "active", + "parsedDocstring": { + "text": "Indicate whether the context is active." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_event_manager.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 100, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/events/_event_manager.py#L100" + } + ], + "type": { + "name": "bool", + "type": "reference", + "target": 0, + "ref": {} + }, + "inheritedFrom": { + "name": "EventManager.active", + "target": 2160, + "type": "reference", + "ref": { + "id": 42713, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_event_manager.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 100 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api/class/EventManager#active" + } + }, + "permalink": "/python/api/class/LocalEventManager#active", + "parentId": 2226 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Register an event listener for a specific event.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 44708, + "module": "events._event_manager", + "name": "on", + "parsedDocstring": { + "text": "Register an event listener for a specific event.\n", + "args": { + "event": "The event for which to listen to.", + "listener": "The function (sync or async) which is to be called when the event is emitted." + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_event_manager.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 160, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/events/_event_manager.py#L160" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Register an event listener for a specific event.\n" + } + ] + }, + "flags": {}, + "id": 42722, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "on", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The event for which to listen to." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 42723, + "kind": 32768, + "kindString": "Parameter", + "name": "event", + "type": { + "name": "Event", + "type": "reference", + "target": "2116", + "ref": { + "id": 42669, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 15 + } + ], + "kind": 8, + "permalink": "https://crawlee.dev/python/api/enum/Event" + } + }, + "parentId": 2169 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The function (sync or async) which is to be called when the event is emitted." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 42724, + "kind": 32768, + "kindString": "Parameter", + "name": "listener", + "type": { + "name": "EventListener[Any]", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2169 + } + ], + "type": { + "name": "None", + "type": "reference", + "target": 0, + "ref": {} + }, + "inheritedFrom": { + "name": "EventManager.on", + "target": 2168, + "type": "reference", + "ref": { + "id": 42721, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_event_manager.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 160 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/EventManager#on" + } + }, + "parentId": 4155 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Register an event listener for a specific event.\n" + } + ] + }, + "flags": {}, + "id": 42736, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "on", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The event for which to listen to." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 42737, + "kind": 32768, + "kindString": "Parameter", + "name": "event", + "type": { + "name": "Literal[Event.PERSIST_STATE]", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2183 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The function (sync or async) which is to be called when the event is emitted." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 42738, + "kind": 32768, + "kindString": "Parameter", + "name": "listener", + "type": { + "name": "EventListener[EventPersistStateData]", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2183 + } + ], + "type": { + "name": "None", + "type": "reference", + "target": 0, + "ref": {} + }, + "inheritedFrom": { + "name": "EventManager.on", + "target": 2168, + "type": "reference", + "ref": { + "id": 42721, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_event_manager.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 160 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/EventManager#on" + } + }, + "parentId": 4155 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Register an event listener for a specific event.\n" + } + ] + }, + "flags": {}, + "id": 42739, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "on", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The event for which to listen to." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 42740, + "kind": 32768, + "kindString": "Parameter", + "name": "event", + "type": { + "name": "Literal[Event.SYSTEM_INFO]", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2186 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The function (sync or async) which is to be called when the event is emitted." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 42741, + "kind": 32768, + "kindString": "Parameter", + "name": "listener", + "type": { + "name": "EventListener[EventSystemInfoData]", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2186 + } + ], + "type": { + "name": "None", + "type": "reference", + "target": 0, + "ref": {} + }, + "inheritedFrom": { + "name": "EventManager.on", + "target": 2168, + "type": "reference", + "ref": { + "id": 42721, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_event_manager.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 160 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/EventManager#on" + } + }, + "parentId": 4155 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Register an event listener for a specific event.\n" + } + ] + }, + "flags": {}, + "id": 42742, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "on", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The event for which to listen to." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 42743, + "kind": 32768, + "kindString": "Parameter", + "name": "event", + "type": { + "name": "Literal[Event.MIGRATING]", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2189 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The function (sync or async) which is to be called when the event is emitted." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 42744, + "kind": 32768, + "kindString": "Parameter", + "name": "listener", + "type": { + "name": "EventListener[EventMigratingData]", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2189 + } + ], + "type": { + "name": "None", + "type": "reference", + "target": 0, + "ref": {} + }, + "inheritedFrom": { + "name": "EventManager.on", + "target": 2168, + "type": "reference", + "ref": { + "id": 42721, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_event_manager.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 160 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/EventManager#on" + } + }, + "parentId": 4155 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Register an event listener for a specific event.\n" + } + ] + }, + "flags": {}, + "id": 42745, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "on", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The event for which to listen to." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 42746, + "kind": 32768, + "kindString": "Parameter", + "name": "event", + "type": { + "name": "Literal[Event.ABORTING]", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2192 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The function (sync or async) which is to be called when the event is emitted." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 42747, + "kind": 32768, + "kindString": "Parameter", + "name": "listener", + "type": { + "name": "EventListener[EventAbortingData]", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2192 + } + ], + "type": { + "name": "None", + "type": "reference", + "target": 0, + "ref": {} + }, + "inheritedFrom": { + "name": "EventManager.on", + "target": 2168, + "type": "reference", + "ref": { + "id": 42721, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_event_manager.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 160 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/EventManager#on" + } + }, + "parentId": 4155 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Register an event listener for a specific event.\n" + } + ] + }, + "flags": {}, + "id": 42748, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "on", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The event for which to listen to." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 42749, + "kind": 32768, + "kindString": "Parameter", + "name": "event", + "type": { + "name": "Literal[Event.EXIT]", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2195 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The function (sync or async) which is to be called when the event is emitted." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 42750, + "kind": 32768, + "kindString": "Parameter", + "name": "listener", + "type": { + "name": "EventListener[EventExitData]", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2195 + } + ], + "type": { + "name": "None", + "type": "reference", + "target": 0, + "ref": {} + }, + "inheritedFrom": { + "name": "EventManager.on", + "target": 2168, + "type": "reference", + "ref": { + "id": 42721, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_event_manager.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 160 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/EventManager#on" + } + }, + "parentId": 4155 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Register an event listener for a specific event.\n" + } + ] + }, + "flags": {}, + "id": 42751, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "on", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The event for which to listen to." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 42752, + "kind": 32768, + "kindString": "Parameter", + "name": "event", + "type": { + "name": "Literal[Event.CRAWLER_STATUS]", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2198 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The function (sync or async) which is to be called when the event is emitted." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 42753, + "kind": 32768, + "kindString": "Parameter", + "name": "listener", + "type": { + "name": "EventListener[EventCrawlerStatusData]", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2198 + } + ], + "type": { + "name": "None", + "type": "reference", + "target": 0, + "ref": {} + }, + "inheritedFrom": { + "name": "EventManager.on", + "target": 2168, + "type": "reference", + "ref": { + "id": 42721, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_event_manager.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 160 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/EventManager#on" + } + }, + "parentId": 4155 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Register an event listener for a specific event.\n" + } + ] + }, + "flags": {}, + "id": 42754, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "on", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The event for which to listen to." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 42755, + "kind": 32768, + "kindString": "Parameter", + "name": "event", + "type": { + "name": "Event", + "type": "reference", + "target": "2116", + "ref": { + "id": 42669, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 15 + } + ], + "kind": 8, + "permalink": "https://crawlee.dev/python/api/enum/Event" + } + }, + "parentId": 2201 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The function (sync or async) which is to be called when the event is emitted." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 42756, + "kind": 32768, + "kindString": "Parameter", + "name": "listener", + "type": { + "name": "EventListener[None]", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2201 + } + ], + "type": { + "name": "None", + "type": "reference", + "target": 0, + "ref": {} + }, + "inheritedFrom": { + "name": "EventManager.on", + "target": 2168, + "type": "reference", + "ref": { + "id": 42721, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_event_manager.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 160 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/EventManager#on" + } + }, + "parentId": 4155 + } + ], + "inheritedFrom": { + "name": "EventManager.on", + "target": 2168, + "type": "reference", + "ref": { + "id": 42721, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_event_manager.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 160 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/EventManager#on" + } + }, + "permalink": "/python/api/class/LocalEventManager#on", + "parentId": 2226 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Remove a specific listener or all listeners for an event.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 44709, + "module": "events._event_manager", + "name": "off", + "parsedDocstring": { + "text": "Remove a specific listener or all listeners for an event.\n", + "args": { + "event": "The Actor event for which to remove listeners.", + "listener": "The listener which is supposed to be removed. If not passed, all listeners of this event\nare removed." + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_event_manager.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 210, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/events/_event_manager.py#L210" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Remove a specific listener or all listeners for an event.\n" + } + ] + }, + "flags": {}, + "id": 42726, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "off", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The Actor event for which to remove listeners." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 42727, + "kind": 32768, + "kindString": "Parameter", + "name": "event", + "type": { + "name": "Event", + "type": "reference", + "target": "2116", + "ref": { + "id": 42669, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 15 + } + ], + "kind": 8, + "permalink": "https://crawlee.dev/python/api/enum/Event" + } + }, + "parentId": 2173 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The listener which is supposed to be removed. If not passed, all listeners of this event\nare removed." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 42728, + "kind": 32768, + "kindString": "Parameter", + "name": "listener", + "type": { + "name": "EventListener[Any] | None", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2173 + } + ], + "type": { + "name": "None", + "type": "reference", + "target": 0, + "ref": {} + }, + "inheritedFrom": { + "name": "EventManager.off", + "target": 2172, + "type": "reference", + "ref": { + "id": 42725, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_event_manager.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 210 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/EventManager#off" + } + }, + "parentId": 4156 + } + ], + "inheritedFrom": { + "name": "EventManager.off", + "target": 2172, + "type": "reference", + "ref": { + "id": 42725, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_event_manager.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 210 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/EventManager#off" + } + }, + "permalink": "/python/api/class/LocalEventManager#off", + "parentId": 2226 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Emit an event with the associated data to all registered listeners.\n" + } + ] + }, + "decorations": [ + { + "name": "ensure_context" + } + ], + "flags": {}, + "groups": [], + "id": 44710, + "module": "events._event_manager", + "name": "emit", + "parsedDocstring": { + "text": "Emit an event with the associated data to all registered listeners.\n", + "args": { + "event": "The event which will be emitted.", + "event_data": "The data which will be passed to the event listeners." + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_event_manager.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 242, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/events/_event_manager.py#L242" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Emit an event with the associated data to all registered listeners.\n" + } + ] + }, + "flags": {}, + "id": 42730, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "emit", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The event which will be emitted." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 42731, + "kind": 32768, + "kindString": "Parameter", + "name": "event", + "type": { + "name": "Event", + "type": "reference", + "target": "2116", + "ref": { + "id": 42669, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 15 + } + ], + "kind": 8, + "permalink": "https://crawlee.dev/python/api/enum/Event" + } + }, + "parentId": 2177 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The data which will be passed to the event listeners." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 42732, + "kind": 32768, + "kindString": "Parameter", + "name": "event_data", + "type": { + "name": "EventData", + "type": "reference", + "target": "2147", + "ref": { + "id": 42700, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 99 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api#EventData" + } + }, + "parentId": 2177 + } + ], + "type": { + "name": "None", + "type": "reference", + "target": 0, + "ref": {} + }, + "inheritedFrom": { + "name": "EventManager.emit", + "target": 2176, + "type": "reference", + "ref": { + "id": 42729, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_event_manager.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 242 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/EventManager#emit" + } + }, + "parentId": 4157 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Emit an event with the associated data to all registered listeners.\n" + } + ] + }, + "flags": {}, + "id": 42757, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "emit", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The event which will be emitted." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 42758, + "kind": 32768, + "kindString": "Parameter", + "name": "event", + "type": { + "name": "Literal[Event.PERSIST_STATE]", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2204 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The data which will be passed to the event listeners." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 42759, + "kind": 32768, + "kindString": "Parameter", + "name": "event_data", + "type": { + "name": "EventPersistStateData", + "type": "reference", + "target": "2129", + "ref": { + "id": 42682, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 40 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/EventPersistStateData" + } + }, + "parentId": 2204 + } + ], + "type": { + "name": "None", + "type": "reference", + "target": 0, + "ref": {} + }, + "inheritedFrom": { + "name": "EventManager.emit", + "target": 2176, + "type": "reference", + "ref": { + "id": 42729, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_event_manager.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 242 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/EventManager#emit" + } + }, + "parentId": 4157 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Emit an event with the associated data to all registered listeners.\n" + } + ] + }, + "flags": {}, + "id": 42760, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "emit", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The event which will be emitted." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 42761, + "kind": 32768, + "kindString": "Parameter", + "name": "event", + "type": { + "name": "Literal[Event.SYSTEM_INFO]", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2207 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The data which will be passed to the event listeners." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 42762, + "kind": 32768, + "kindString": "Parameter", + "name": "event_data", + "type": { + "name": "EventSystemInfoData", + "type": "reference", + "target": "2132", + "ref": { + "id": 42685, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 49 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/EventSystemInfoData" + } + }, + "parentId": 2207 + } + ], + "type": { + "name": "None", + "type": "reference", + "target": 0, + "ref": {} + }, + "inheritedFrom": { + "name": "EventManager.emit", + "target": 2176, + "type": "reference", + "ref": { + "id": 42729, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_event_manager.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 242 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/EventManager#emit" + } + }, + "parentId": 4157 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Emit an event with the associated data to all registered listeners.\n" + } + ] + }, + "flags": {}, + "id": 42763, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "emit", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The event which will be emitted." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 42764, + "kind": 32768, + "kindString": "Parameter", + "name": "event", + "type": { + "name": "Literal[Event.MIGRATING]", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2210 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The data which will be passed to the event listeners." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 42765, + "kind": 32768, + "kindString": "Parameter", + "name": "event_data", + "type": { + "name": "EventMigratingData", + "type": "reference", + "target": "2136", + "ref": { + "id": 42689, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 62 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/EventMigratingData" + } + }, + "parentId": 2210 + } + ], + "type": { + "name": "None", + "type": "reference", + "target": 0, + "ref": {} + }, + "inheritedFrom": { + "name": "EventManager.emit", + "target": 2176, + "type": "reference", + "ref": { + "id": 42729, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_event_manager.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 242 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/EventManager#emit" + } + }, + "parentId": 4157 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Emit an event with the associated data to all registered listeners.\n" + } + ] + }, + "flags": {}, + "id": 42766, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "emit", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The event which will be emitted." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 42767, + "kind": 32768, + "kindString": "Parameter", + "name": "event", + "type": { + "name": "Literal[Event.ABORTING]", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2213 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The data which will be passed to the event listeners." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 42768, + "kind": 32768, + "kindString": "Parameter", + "name": "event_data", + "type": { + "name": "EventAbortingData", + "type": "reference", + "target": "2139", + "ref": { + "id": 42692, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 73 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/EventAbortingData" + } + }, + "parentId": 2213 + } + ], + "type": { + "name": "None", + "type": "reference", + "target": 0, + "ref": {} + }, + "inheritedFrom": { + "name": "EventManager.emit", + "target": 2176, + "type": "reference", + "ref": { + "id": 42729, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_event_manager.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 242 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/EventManager#emit" + } + }, + "parentId": 4157 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Emit an event with the associated data to all registered listeners.\n" + } + ] + }, + "flags": {}, + "id": 42769, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "emit", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The event which will be emitted." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 42770, + "kind": 32768, + "kindString": "Parameter", + "name": "event", + "type": { + "name": "Literal[Event.EXIT]", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2216 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The data which will be passed to the event listeners." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 42771, + "kind": 32768, + "kindString": "Parameter", + "name": "event_data", + "type": { + "name": "EventExitData", + "type": "reference", + "target": "2141", + "ref": { + "id": 42694, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 80 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/EventExitData" + } + }, + "parentId": 2216 + } + ], + "type": { + "name": "None", + "type": "reference", + "target": 0, + "ref": {} + }, + "inheritedFrom": { + "name": "EventManager.emit", + "target": 2176, + "type": "reference", + "ref": { + "id": 42729, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_event_manager.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 242 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/EventManager#emit" + } + }, + "parentId": 4157 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Emit an event with the associated data to all registered listeners.\n" + } + ] + }, + "flags": {}, + "id": 42772, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "emit", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The event which will be emitted." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 42773, + "kind": 32768, + "kindString": "Parameter", + "name": "event", + "type": { + "name": "Literal[Event.CRAWLER_STATUS]", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2219 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The data which will be passed to the event listeners." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 42774, + "kind": 32768, + "kindString": "Parameter", + "name": "event_data", + "type": { + "name": "EventCrawlerStatusData", + "type": "reference", + "target": "2143", + "ref": { + "id": 42696, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 87 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/EventCrawlerStatusData" + } + }, + "parentId": 2219 + } + ], + "type": { + "name": "None", + "type": "reference", + "target": 0, + "ref": {} + }, + "inheritedFrom": { + "name": "EventManager.emit", + "target": 2176, + "type": "reference", + "ref": { + "id": 42729, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_event_manager.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 242 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/EventManager#emit" + } + }, + "parentId": 4157 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Emit an event with the associated data to all registered listeners.\n" + } + ] + }, + "flags": {}, + "id": 42775, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "emit", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The event which will be emitted." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 42776, + "kind": 32768, + "kindString": "Parameter", + "name": "event", + "type": { + "name": "Event", + "type": "reference", + "target": "2116", + "ref": { + "id": 42669, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 15 + } + ], + "kind": 8, + "permalink": "https://crawlee.dev/python/api/enum/Event" + } + }, + "parentId": 2222 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The data which will be passed to the event listeners." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 42777, + "kind": 32768, + "kindString": "Parameter", + "name": "event_data", + "type": { + "name": "Any", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2222 + } + ], + "type": { + "name": "None", + "type": "reference", + "target": 0, + "ref": {} + }, + "inheritedFrom": { + "name": "EventManager.emit", + "target": 2176, + "type": "reference", + "ref": { + "id": 42729, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_event_manager.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 242 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/EventManager#emit" + } + }, + "parentId": 4157 + } + ], + "inheritedFrom": { + "name": "EventManager.emit", + "target": 2176, + "type": "reference", + "ref": { + "id": 42729, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_event_manager.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 242 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/EventManager#emit" + } + }, + "permalink": "/python/api/class/LocalEventManager#emit", + "parentId": 2226 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Wait for all currently executing event listeners to complete.\n" + } ] }, + "decorations": [ + { + "name": "ensure_context" + } + ], + "flags": {}, + "groups": [], + "id": 44711, + "module": "events._event_manager", + "name": "wait_for_all_listeners_to_complete", + "parsedDocstring": { + "text": "Wait for all currently executing event listeners to complete.\n", + "args": { + "timeout": "The maximum time to wait for the event listeners to finish. If they do not complete within\nthe specified timeout, they will be canceled." + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_event_manager.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 252, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/events/_event_manager.py#L252" + } + ], "type": { - "type": "intrinsic", - "name": "str" + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} }, - "parameters": [ + "signatures": [ { - "id": 16, - "name": "unique_key", - "kind": 32768, - "kindString": "Parameter", + "comment": { + "summary": [ + { + "kind": "text", + "text": "Wait for all currently executing event listeners to complete.\n" + } + ] + }, "flags": {}, + "id": 42734, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "wait_for_all_listeners_to_complete", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The maximum time to wait for the event listeners to finish. If they do not complete within\nthe specified timeout, they will be canceled." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 42735, + "kind": 32768, + "kindString": "Parameter", + "name": "timeout", + "type": { + "name": "timedelta | None", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2181 + } + ], "type": { - "type": "intrinsic", - "name": "str" - } + "name": "None", + "type": "reference", + "target": 0, + "ref": {} + }, + "inheritedFrom": { + "name": "EventManager.wait_for_all_listeners_to_complete", + "target": 2180, + "type": "reference", + "ref": { + "id": 42733, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_event_manager.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 252 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/EventManager#wait_for_all_listeners_to_complete" + } + }, + "parentId": 4158 } - ] + ], + "inheritedFrom": { + "name": "EventManager.wait_for_all_listeners_to_complete", + "target": 2180, + "type": "reference", + "ref": { + "id": 42733, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_event_manager.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 252 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/EventManager#wait_for_all_listeners_to_complete" + } + }, + "permalink": "/python/api/class/LocalEventManager#wait_for_all_listeners_to_complete", + "parentId": 2226 } - ] - }, - { - "id": 17, - "name": "force_rename", - "module": "_utils", - "kind": 2048, - "kindString": "Method", - "flags": {}, + ], "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Rename a directory. Checks for existence of source directory and removes destination directory if it exists.\"]}" + "text": "Event manager for local environments.\n\nIt extends the `EventManager` to emit `SystemInfo` events at regular intervals. The `LocalEventManager`\nis intended to be used in local environments, where the system metrics are required managing the `Snapshotter`\nand `AutoscaledPool`." } ] }, - "children": [], - "groups": [], + "decorations": [ + { + "args": "('Event managers')", + "name": "docs_group" + } + ], + "flags": {}, + "groups": [ + { + "children": [ + 42787, + 42789, + 42780, + 44710, + 42784, + 44709, + 44708, + 44711 + ], + "title": "Methods" + }, + { + "children": [ + 44707 + ], + "title": "Properties" + } + ], + "id": 42779, + "module": "events._local_event_manager", + "name": "LocalEventManager", + "parsedDocstring": { + "text": "Event manager for local environments.\n\nIt extends the `EventManager` to emit `SystemInfo` events at regular intervals. The `LocalEventManager`\nis intended to be used in local environments, where the system metrics are required managing the `Snapshotter`\nand `AutoscaledPool`." + }, "sources": [ { - "filename": "/tmp-old-source-v17/src/apify/_utils.py", - "line": 302, "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/_utils.py#L302" + "fileName": "/src/crawlee/events/_local_event_manager.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 26, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/events/_local_event_manager.py#L26" } ], - "signatures": [ + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "extendedTypes": [ { - "id": 18, - "name": "force_rename", - "modifiers": [ - "async" + "name": "EventManager", + "target": "2155", + "type": "reference", + "ref": { + "id": 42708, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_event_manager.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 55 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/EventManager" + } + } + ], + "permalink": "/python/api/class/LocalEventManager", + "categories": [] + }, + { + "kind": 8, + "kindString": "Enumeration", + "children": [ + { + "kind": 16, + "kindString": "Enumeration Member", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 46828, + "module": "events._types", + "name": "PERSIST_STATE", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 19, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/events/_types.py#L19" + } ], - "kind": 4096, - "kindString": "Call signature", + "type": { + "type": "literal", + "value": "'persistState'" + }, + "permalink": "/python/api/enum/Event#PERSIST_STATE", + "parentId": 2116 + }, + { + "kind": 16, + "kindString": "Enumeration Member", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 46829, + "module": "events._types", + "name": "SYSTEM_INFO", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 20, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/events/_types.py#L20" + } + ], + "type": { + "type": "literal", + "value": "'systemInfo'" + }, + "permalink": "/python/api/enum/Event#SYSTEM_INFO", + "parentId": 2116 + }, + { + "kind": 16, + "kindString": "Enumeration Member", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 46830, + "module": "events._types", + "name": "MIGRATING", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 21, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/events/_types.py#L21" + } + ], + "type": { + "type": "literal", + "value": "'migrating'" + }, + "permalink": "/python/api/enum/Event#MIGRATING", + "parentId": 2116 + }, + { + "kind": 16, + "kindString": "Enumeration Member", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 46831, + "module": "events._types", + "name": "ABORTING", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 22, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/events/_types.py#L22" + } + ], + "type": { + "type": "literal", + "value": "'aborting'" + }, + "permalink": "/python/api/enum/Event#ABORTING", + "parentId": 2116 + }, + { + "kind": 16, + "kindString": "Enumeration Member", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 46832, + "module": "events._types", + "name": "EXIT", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 23, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/events/_types.py#L23" + } + ], + "type": { + "type": "literal", + "value": "'exit'" + }, + "permalink": "/python/api/enum/Event#EXIT", + "parentId": 2116 + }, + { + "kind": 16, + "kindString": "Enumeration Member", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 46833, + "module": "events._types", + "name": "SESSION_RETIRED", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 26, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/events/_types.py#L26" + } + ], + "type": { + "type": "literal", + "value": "'sessionRetired'" + }, + "permalink": "/python/api/enum/Event#SESSION_RETIRED", + "parentId": 2116 + }, + { + "kind": 16, + "kindString": "Enumeration Member", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 46834, + "module": "events._types", + "name": "BROWSER_LAUNCHED", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 29, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/events/_types.py#L29" + } + ], + "type": { + "type": "literal", + "value": "'browserLaunched'" + }, + "permalink": "/python/api/enum/Event#BROWSER_LAUNCHED", + "parentId": 2116 + }, + { + "kind": 16, + "kindString": "Enumeration Member", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 46835, + "module": "events._types", + "name": "BROWSER_RETIRED", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 30, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/events/_types.py#L30" + } + ], + "type": { + "type": "literal", + "value": "'browserRetired'" + }, + "permalink": "/python/api/enum/Event#BROWSER_RETIRED", + "parentId": 2116 + }, + { + "kind": 16, + "kindString": "Enumeration Member", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 46836, + "module": "events._types", + "name": "BROWSER_CLOSED", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 31, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/events/_types.py#L31" + } + ], + "type": { + "type": "literal", + "value": "'browserClosed'" + }, + "permalink": "/python/api/enum/Event#BROWSER_CLOSED", + "parentId": 2116 + }, + { + "kind": 16, + "kindString": "Enumeration Member", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, "flags": {}, + "groups": [], + "id": 46837, + "module": "events._types", + "name": "PAGE_CREATED", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 32, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/events/_types.py#L32" + } + ], + "type": { + "type": "literal", + "value": "'pageCreated'" + }, + "permalink": "/python/api/enum/Event#PAGE_CREATED", + "parentId": 2116 + }, + { + "kind": 16, + "kindString": "Enumeration Member", + "children": [], "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Rename a directory. Checks for existence of source directory and removes destination directory if it exists.\"]}" + "text": "" } ] }, - "type": { - "type": "reference", - "name": "None" + "flags": {}, + "groups": [], + "id": 46838, + "module": "events._types", + "name": "PAGE_CLOSED", + "parsedDocstring": { + "text": "" }, - "parameters": [ - { - "id": 19, - "name": "src_dir", - "kind": 32768, - "kindString": "Parameter", - "flags": {}, - "type": { - "type": "intrinsic", - "name": "str" - } - }, + "sources": [ { - "id": 20, - "name": "dst_dir", - "kind": 32768, - "kindString": "Parameter", - "flags": {}, - "type": { - "type": "intrinsic", - "name": "str" - } + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 33, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/events/_types.py#L33" } - ] - } - ] - }, - { - "id": 21, - "name": "budget_ow", - "module": "_utils", - "kind": 2048, - "kindString": "Method", - "flags": {}, - "comment": { - "summary": [ - { - "kind": "text", - "text": "{\"content\": [\"Budget version of ow.\"]}" - } - ] - }, - "children": [], - "groups": [], - "sources": [ - { - "filename": "/tmp-old-source-v17/src/apify/_utils.py", - "line": 386, - "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/_utils.py#L386" - } - ], - "signatures": [ + ], + "type": { + "type": "literal", + "value": "'pageClosed'" + }, + "permalink": "/python/api/enum/Event#PAGE_CLOSED", + "parentId": 2116 + }, { - "id": 22, - "name": "budget_ow", - "modifiers": [], - "kind": 4096, - "kindString": "Call signature", - "flags": {}, + "kind": 16, + "kindString": "Enumeration Member", + "children": [], "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Budget version of ow.\"]}" + "text": "" } ] }, - "type": { - "type": "reference", - "name": "None" + "flags": {}, + "groups": [], + "id": 46839, + "module": "events._types", + "name": "CRAWLER_STATUS", + "parsedDocstring": { + "text": "" }, - "parameters": [ - { - "id": 23, - "name": "value", - "kind": 32768, - "kindString": "Parameter", - "flags": {}, - "type": { - "type": "reference", - "name": "dict | str | float | bool" - } - }, - { - "id": 24, - "name": "predicate", - "kind": 32768, - "kindString": "Parameter", - "flags": {}, - "type": { - "type": "reference", - "name": "dict[str, tuple[type, bool]] | tuple[type, bool]" - } - }, + "sources": [ { - "id": 25, - "name": "value_name", - "kind": 32768, - "kindString": "Parameter", - "flags": {}, - "type": { - "type": "reference", - "name": "str | None" - }, - "defaultValue": "None" + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 36, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/events/_types.py#L36" } - ] + ], + "type": { + "type": "literal", + "value": "'crawlerStatus'" + }, + "permalink": "/python/api/enum/Event#CRAWLER_STATUS", + "parentId": 2116 } - ] - }, - { - "id": 26, - "name": "compute_short_hash", - "module": "_utils", - "kind": 2048, - "kindString": "Method", - "flags": {}, + ], "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Computes a hexadecimal SHA-256 hash of the provided data and returns a substring (prefix) of it.\\n\", {\"Arguments\": [{\"param\": \"data\", \"desc\": \"The binary data to be hashed.\"}, {\"param\": \"length\", \"desc\": \"The length of the hash to be returned.\\n\"}]}, {\"Returns\": [\"A substring (prefix) of the hexadecimal hash of the data.\"]}]}" + "text": "Names of all possible events that can be emitted using an `EventManager`." } ] }, - "children": [], - "groups": [], + "decorations": [ + { + "args": "('Event data')", + "name": "docs_group" + } + ], + "flags": {}, + "groups": [ + { + "children": [ + 46831, + 46836, + 46834, + 46835, + 46839, + 46832, + 46830, + 46838, + 46837, + 46828, + 46833, + 46829 + ], + "title": "Enumeration members" + } + ], + "id": 46827, + "module": "events._types", + "name": "Event", + "parsedDocstring": { + "text": "Names of all possible events that can be emitted using an `EventManager`." + }, "sources": [ { - "filename": "/tmp-old-source-v17/src/apify/_utils.py", - "line": 417, "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/_utils.py#L417" + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 15, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/events/_types.py#L15" } ], - "signatures": [ + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "permalink": "/python/api/enum/Event", + "categories": [] + }, + { + "kind": 128, + "kindString": "Class", + "children": [ { - "id": 27, - "name": "compute_short_hash", - "modifiers": [], - "kind": 4096, - "kindString": "Call signature", - "flags": {}, + "kind": 1024, + "kindString": "Property", + "children": [], "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Computes a hexadecimal SHA-256 hash of the provided data and returns a substring (prefix) of it.\\n\", {\"" + "text": "" } ] }, - "type": { - "type": "intrinsic", - "name": "str" + "flags": {}, + "groups": [], + "id": 48979, + "module": "events._types", + "name": "model_config", + "parsedDocstring": { + "text": "" }, - "parameters": [ - { - "id": 28, - "name": "data", - "kind": 32768, - "kindString": "Parameter", - "flags": {}, - "type": { - "type": "reference", - "name": "bytes" - } - }, + "sources": [ { - "id": 29, - "name": "length", - "kind": 32768, - "kindString": "Parameter", - "flags": { - "keyword-only": "true" - }, - "type": { - "type": "intrinsic", - "name": "int" - }, - "defaultValue": "8" + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 76, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/events/_types.py#L76" } - ] + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "permalink": "/python/api/class/EventAbortingData#model_config", + "parentId": 2139 } - ] - }, - { - "id": 30, - "name": "normalize_url", - "module": "_utils", - "kind": 2048, - "kindString": "Method", - "flags": {}, + ], "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Normalizes a URL.\\n\\nThis function cleans and standardizes a URL by removing leading and trailing whitespaces,\\nconverting the scheme and netloc to lower case, stripping unwanted tracking parameters\\n(specifically those beginning with 'utm_'), sorting the remaining query parameters alphabetically,\\nand optionally retaining the URL fragment. The goal is to ensure that URLs that are functionally\\nidentical but differ in trivial ways (such as parameter order or casing) are treated as the same.\\n\", {\"Arguments\": [{\"param\": \"url\", \"desc\": \"The URL to be normalized.\"}, {\"param\": \"keep_url_fragment\", \"desc\": \"Flag to determine whether the fragment part of the URL should be retained.\\n\"}]}, {\"Returns\": [\"A string containing the normalized URL.\"]}]}" + "text": "Data for the aborting event." } ] }, - "children": [], - "groups": [], + "decorations": [ + { + "args": "('Event data')", + "name": "docs_group" + } + ], + "flags": {}, + "groups": [ + { + "children": [ + 48979 + ], + "title": "Properties" + } + ], + "id": 48978, + "module": "events._types", + "name": "EventAbortingData", + "parsedDocstring": { + "text": "Data for the aborting event." + }, "sources": [ { - "filename": "/tmp-old-source-v17/src/apify/_utils.py", - "line": 431, "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/_utils.py#L431" + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 73, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/events/_types.py#L73" } ], - "signatures": [ + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "permalink": "/python/api/class/EventAbortingData", + "categories": [] + }, + { + "kind": 128, + "kindString": "Class", + "children": [ { - "id": 31, - "name": "normalize_url", - "modifiers": [], - "kind": 4096, - "kindString": "Call signature", - "flags": {}, + "kind": 1024, + "kindString": "Property", + "children": [], "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Normalizes a URL.\\n\\nThis function cleans and standardizes a URL by removing leading and trailing whitespaces,\\nconverting the scheme and netloc to lower case, stripping unwanted tracking parameters\\n(specifically those beginning with 'utm_'), sorting the remaining query parameters alphabetically,\\nand optionally retaining the URL fragment. The goal is to ensure that URLs that are functionally\\nidentical but differ in trivial ways (such as parameter order or casing) are treated as the same.\\n\", {\"" + "text": "" } ] }, - "type": { - "type": "intrinsic", - "name": "str" + "flags": {}, + "groups": [], + "id": 51121, + "module": "events._types", + "name": "model_config", + "parsedDocstring": { + "text": "" }, - "parameters": [ - { - "id": 32, - "name": "url", - "kind": 32768, - "kindString": "Parameter", - "flags": {}, - "type": { - "type": "intrinsic", - "name": "str" - } - }, + "sources": [ { - "id": 33, - "name": "keep_url_fragment", - "kind": 32768, - "kindString": "Parameter", - "flags": { - "keyword-only": "true" - }, - "type": { - "type": "intrinsic", - "name": "bool" - }, - "defaultValue": "False" + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 83, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/events/_types.py#L83" } - ] + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "permalink": "/python/api/class/EventExitData#model_config", + "parentId": 2141 } - ] - }, - { - "id": 34, - "name": "compute_unique_key", - "module": "_utils", - "kind": 2048, - "kindString": "Method", - "flags": {}, + ], "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Computes a unique key for caching & deduplication of requests.\\n\\nThis function computes a unique key by normalizing the provided URL and method.\\nIf 'use_extended_unique_key' is True and a payload is provided, the payload is hashed and\\nincluded in the key. Otherwise, the unique key is just the normalized URL.\\n\", {\"Arguments\": [{\"param\": \"url\", \"desc\": \"The request URL.\"}, {\"param\": \"method\", \"desc\": \"The HTTP method, defaults to 'GET'.\"}, {\"param\": \"payload\", \"desc\": \"The request payload, defaults to None.\"}, {\"param\": \"keep_url_fragment\", \"desc\": \"A flag indicating whether to keep the URL fragment, defaults to False.\"}, {\"param\": \"use_extended_unique_key\", \"desc\": \"A flag indicating whether to include a hashed payload in the key, defaults to False.\\n\"}]}, {\"Returns\": [\"A string representing the unique key for the request.\"]}]}" + "text": "Data for the exit event." } ] }, - "children": [], - "groups": [], + "decorations": [ + { + "args": "('Event data')", + "name": "docs_group" + } + ], + "flags": {}, + "groups": [ + { + "children": [ + 51121 + ], + "title": "Properties" + } + ], + "id": 51120, + "module": "events._types", + "name": "EventExitData", + "parsedDocstring": { + "text": "Data for the exit event." + }, "sources": [ { - "filename": "/tmp-old-source-v17/src/apify/_utils.py", - "line": 477, "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/_utils.py#L477" + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 80, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/events/_types.py#L80" } ], - "signatures": [ + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "permalink": "/python/api/class/EventExitData", + "categories": [] + }, + { + "kind": 128, + "kindString": "Class", + "children": [ { - "id": 35, - "name": "compute_unique_key", - "modifiers": [], - "kind": 4096, - "kindString": "Call signature", - "flags": {}, + "kind": 1024, + "kindString": "Property", + "children": [], "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Computes a unique key for caching & deduplication of requests.\\n\\nThis function computes a unique key by normalizing the provided URL and method.\\nIf 'use_extended_unique_key' is True and a payload is provided, the payload is hashed and\\nincluded in the key. Otherwise, the unique key is just the normalized URL.\\n\", {\"" + "text": "" } ] }, - "type": { - "type": "intrinsic", - "name": "str" + "flags": {}, + "groups": [], + "id": 53258, + "module": "events._types", + "name": "model_config", + "parsedDocstring": { + "text": "" }, - "parameters": [ + "sources": [ { - "id": 36, - "name": "url", - "kind": 32768, - "kindString": "Parameter", - "flags": {}, - "type": { - "type": "intrinsic", - "name": "str" + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 65, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/events/_types.py#L65" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "permalink": "/python/api/class/EventMigratingData#model_config", + "parentId": 2136 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" } - }, - { - "id": 37, - "name": "method", - "kind": 32768, - "kindString": "Parameter", - "flags": {}, - "type": { - "type": "intrinsic", - "name": "str" - }, - "defaultValue": "'GET'" - }, + ] + }, + "flags": {}, + "groups": [], + "id": 53259, + "module": "events._types", + "name": "time_remaining", + "parsedDocstring": { + "text": "" + }, + "sources": [ { - "id": 38, - "name": "payload", - "kind": 32768, - "kindString": "Parameter", - "flags": {}, - "type": { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 69, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/events/_types.py#L69" + } + ], + "type": { + "name": "Annotated[timedelta_secs | None, Field(alias='timeRemainingSecs')]", + "type": "union", + "types": [ + { "type": "reference", - "name": "bytes | None" - }, - "defaultValue": "None" - }, - { - "id": 39, - "name": "keep_url_fragment", - "kind": 32768, - "kindString": "Parameter", - "flags": { - "keyword-only": "true" - }, - "type": { - "type": "intrinsic", - "name": "bool" - }, - "defaultValue": "False" - }, - { - "id": 40, - "name": "use_extended_unique_key", - "kind": 32768, - "kindString": "Parameter", - "flags": { - "keyword-only": "true" - }, - "type": { - "type": "intrinsic", - "name": "bool" + "name": "timedelta_secs", + "target": "3321", + "ref": { + "id": 54442, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_utils/models.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 68 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api#timedelta_secs" + } }, - "defaultValue": "False" - } - ] + { + "type": "literal", + "value": {} + } + ] + }, + "permalink": "/python/api/class/EventMigratingData#time_remaining", + "parentId": 2136 } - ] - }, - { - "id": 41, - "name": "Actor", - "module": "actor", - "kind": 128, - "kindString": "Class", - "flags": {}, + ], "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"The main class of the SDK, through which all the actor operations should be done.\"]}" + "text": "Data for the migrating event." } ] }, + "decorations": [ + { + "args": "('Event data')", + "name": "docs_group" + } + ], + "flags": {}, + "groups": [ + { + "children": [ + 53258, + 53259 + ], + "title": "Properties" + } + ], + "id": 53257, + "module": "events._types", + "name": "EventMigratingData", + "parsedDocstring": { + "text": "Data for the migrating event." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 62, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/events/_types.py#L62" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "permalink": "/python/api/class/EventMigratingData", + "categories": [] + }, + { + "kind": 128, + "kindString": "Class", "children": [ { - "id": 42, - "name": "__init__", - "module": "actor", - "kind": 512, - "kindString": "Constructor", - "flags": {}, + "kind": 1024, + "kindString": "Property", + "children": [], "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Create an Actor instance.\\n\\nNote that you don't have to do this, all the methods on this class function as classmethods too,\\nand that is their preferred usage.\\n\", {\"Arguments\": [{\"param\": \"config\", \"type\": \"Configuration, optional\", \"desc\": \"The actor configuration to be used. If not passed, a new Configuration instance will be created.\"}]}]}" + "text": "" } ] }, - "children": [], + "flags": {}, "groups": [], + "id": 56572, + "module": "events._types", + "name": "model_config", + "parsedDocstring": { + "text": "" + }, "sources": [ { - "filename": "/tmp-old-source-v17/src/apify/actor.py", - "line": 81, "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/actor.py#L81" + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 43, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/events/_types.py#L43" } ], - "signatures": [ - { - "id": 43, - "name": "__init__", - "modifiers": [], - "kind": 4096, - "kindString": "Call signature", - "flags": {}, - "comment": { - "summary": [ - { - "kind": "text", - "text": "{\"content\": [\"Create an Actor instance.\\n\\nNote that you don't have to do this, all the methods on this class function as classmethods too,\\nand that is their preferred usage.\\n\", {\"" - } - ] - }, - "type": { - "type": "reference", - "name": "None" - }, - "parameters": [ - { - "id": 44, - "name": "config", - "kind": 32768, - "kindString": "Parameter", - "flags": {}, - "type": { - "type": "reference", - "name": "Configuration | None" - }, - "defaultValue": "None" - } - ] - } - ] + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "permalink": "/python/api/class/EventPersistStateData#model_config", + "parentId": 2129 }, { - "id": 45, - "name": "apify_client", - "module": "actor", "kind": 1024, "kindString": "Property", - "flags": {}, + "children": [], "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"The ApifyClientAsync instance the Actor instance uses.\"]}" + "text": "" } ] }, - "children": [], + "flags": {}, "groups": [], + "id": 56573, + "module": "events._types", + "name": "is_migrating", + "parsedDocstring": { + "text": "" + }, "sources": [ { - "filename": "/tmp-old-source-v17/src/apify/actor.py", - "line": 174, "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/actor.py#L174" + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 45, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/events/_types.py#L45" } - ] - }, + ], + "type": { + "name": "bool", + "type": "reference", + "target": 0, + "ref": {} + }, + "permalink": "/python/api/class/EventPersistStateData#is_migrating", + "parentId": 2129 + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Data for the persist state event." + } + ] + }, + "decorations": [ + { + "args": "('Event data')", + "name": "docs_group" + } + ], + "flags": {}, + "groups": [ + { + "children": [ + 56573, + 56572 + ], + "title": "Properties" + } + ], + "id": 56571, + "module": "events._types", + "name": "EventPersistStateData", + "parsedDocstring": { + "text": "Data for the persist state event." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 40, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/events/_types.py#L40" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "permalink": "/python/api/class/EventPersistStateData", + "categories": [] + }, + { + "kind": 128, + "kindString": "Class", + "children": [ { - "id": 46, - "name": "config", - "module": "actor", "kind": 1024, "kindString": "Property", - "flags": {}, + "children": [], "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"The Configuration instance the Actor instance uses.\"]}" + "text": "" } ] }, - "children": [], + "flags": {}, "groups": [], + "id": 58706, + "module": "events._types", + "name": "model_config", + "parsedDocstring": { + "text": "" + }, "sources": [ { - "filename": "/tmp-old-source-v17/src/apify/actor.py", - "line": 181, "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/actor.py#L181" + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 52, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/events/_types.py#L52" } - ] + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "permalink": "/python/api/class/EventSystemInfoData#model_config", + "parentId": 2132 }, { - "id": 47, - "name": "event_manager", - "module": "actor", "kind": 1024, "kindString": "Property", - "flags": {}, + "children": [], "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"The EventManager instance the Actor instance uses.\"]}" + "text": "" } ] }, - "children": [], + "flags": {}, "groups": [], + "id": 58707, + "module": "events._types", + "name": "cpu_info", + "parsedDocstring": { + "text": "" + }, "sources": [ { - "filename": "/tmp-old-source-v17/src/apify/actor.py", - "line": 188, "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/actor.py#L188" + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 54, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/events/_types.py#L54" } - ] + ], + "type": { + "name": "CpuInfo", + "type": "reference", + "target": "3342", + "ref": { + "id": 59915, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_utils/system.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 36 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/CpuInfo" + } + }, + "permalink": "/python/api/class/EventSystemInfoData#cpu_info", + "parentId": 2132 }, { - "id": 48, - "name": "log", - "module": "actor", "kind": 1024, "kindString": "Property", - "flags": {}, + "children": [], "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"The logging.Logger instance the Actor uses.\"]}" + "text": "" } ] }, - "children": [], + "flags": {}, "groups": [], + "id": 58708, + "module": "events._types", + "name": "memory_info", + "parsedDocstring": { + "text": "" + }, "sources": [ { - "filename": "/tmp-old-source-v17/src/apify/actor.py", - "line": 196, "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/actor.py#L196" + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 55, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/events/_types.py#L55" } - ] - }, + ], + "type": { + "name": "MemoryUsageInfo", + "type": "reference", + "target": "3345", + "ref": { + "id": 59918, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_utils/system.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 59 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/MemoryUsageInfo" + } + }, + "permalink": "/python/api/class/EventSystemInfoData#memory_info", + "parentId": 2132 + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Data for the system info event." + } + ] + }, + "decorations": [ + { + "args": "('Event data')", + "name": "docs_group" + } + ], + "flags": {}, + "groups": [ + { + "children": [ + 58707, + 58708, + 58706 + ], + "title": "Properties" + } + ], + "id": 58705, + "module": "events._types", + "name": "EventSystemInfoData", + "parsedDocstring": { + "text": "Data for the system info event." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 49, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/events/_types.py#L49" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "permalink": "/python/api/class/EventSystemInfoData", + "categories": [] + }, + { + "kind": 128, + "kindString": "Class", + "children": [ { - "id": 49, - "name": "init", - "module": "actor", "kind": 2048, "kindString": "Method", - "flags": {}, + "children": [], "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Initialize the actor instance.\\n\\nThis initializes the Actor instance.\\nIt configures the right storage client based on whether the actor is running locally or on the Apify platform,\\nit initializes the event manager for processing actor events,\\nand starts an interval for regularly sending `PERSIST_STATE` events,\\nso that the actor can regularly persist its state in response to these events.\\n\\nThis method should be called immediately before performing any additional actor actions,\\nand it should be called only once.\"]}" + "text": "Return a cache key that can differentiate between different storages of this and other clients.\n\nCan be based on configuration or on the client itself. By default, returns a module and name of the client\nclass." } ] }, - "children": [], + "decorations": [], + "flags": {}, "groups": [], + "id": 61201, + "module": "storage_clients._base._storage_client", + "name": "get_storage_client_cache_key", + "parsedDocstring": { + "text": "Return a cache key that can differentiate between different storages of this and other clients.\n\nCan be based on configuration or on the client itself. By default, returns a module and name of the client\nclass." + }, "sources": [ { - "filename": "/tmp-old-source-v17/src/apify/actor.py", - "line": 205, "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/actor.py#L205" + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 33, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/_base/_storage_client.py#L33" } ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, "signatures": [ { - "id": 50, - "name": "init", - "modifiers": [ - "async" - ], - "kind": 4096, - "kindString": "Call signature", - "flags": {}, "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Initialize the actor instance.\\n\\nThis initializes the Actor instance.\\nIt configures the right storage client based on whether the actor is running locally or on the Apify platform,\\nit initializes the event manager for processing actor events,\\nand starts an interval for regularly sending `PERSIST_STATE` events,\\nso that the actor can regularly persist its state in response to these events.\\n\\nThis method should be called immediately before performing any additional actor actions,\\nand it should be called only once.\"]}" + "text": "Return a cache key that can differentiate between different storages of this and other clients.\n\nCan be based on configuration or on the client itself. By default, returns a module and name of the client\nclass." } ] }, + "flags": {}, + "id": 61202, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "get_storage_client_cache_key", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 61203, + "kind": 32768, + "kindString": "Parameter", + "name": "configuration", + "type": { + "name": "Configuration", + "type": "reference", + "target": "2", + "ref": { + "id": 59920, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/configuration.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 20 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/Configuration" + } + }, + "parentId": 1284 + } + ], "type": { + "name": "Hashable", "type": "reference", - "name": "None" + "target": 0, + "ref": {} }, - "parameters": [] + "overwrites": { + "name": "StorageClient.get_storage_client_cache_key", + "target": 1283, + "type": "reference", + "ref": { + "id": 61201, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 33 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/StorageClient#get_storage_client_cache_key" + } + }, + "parentId": 1283 } - ] + ], + "permalink": "/python/api/class/StorageClient#get_storage_client_cache_key", + "parentId": 1282 }, { - "id": 51, - "name": "get_system_info", - "module": "actor", "kind": 2048, "kindString": "Method", - "flags": {}, + "children": [], "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Get the current system info.\"]}" + "text": "Create a dataset client." } ] }, - "children": [], + "decorations": [ + { + "name": "abstractmethod" + } + ], + "flags": {}, "groups": [], + "id": 61204, + "module": "storage_clients._base._storage_client", + "name": "create_dataset_client", + "parsedDocstring": { + "text": "Create a dataset client." + }, "sources": [ { - "filename": "/tmp-old-source-v17/src/apify/actor.py", - "line": 261, "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/actor.py#L261" + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 42, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/_base/_storage_client.py#L42" } ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, "signatures": [ { - "id": 52, - "name": "get_system_info", - "modifiers": [], - "kind": 4096, - "kindString": "Call signature", - "flags": {}, "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Get the current system info.\"]}" + "text": "Create a dataset client." } ] }, + "flags": {}, + "id": 61205, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "create_dataset_client", + "parameters": [ + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 61206, + "kind": 32768, + "kindString": "Parameter", + "name": "id", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 1287 + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 61207, + "kind": 32768, + "kindString": "Parameter", + "name": "name", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 1287 + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 61208, + "kind": 32768, + "kindString": "Parameter", + "name": "alias", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 1287 + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 61209, + "kind": 32768, + "kindString": "Parameter", + "name": "configuration", + "type": { + "name": "Configuration | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "Configuration", + "target": "2", + "ref": { + "id": 59920, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/configuration.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 20 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/Configuration" + } + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 1287 + } + ], "type": { - "type": "intrinsic", - "name": "dict" + "name": "DatasetClient", + "type": "reference", + "target": "1220", + "ref": { + "id": 61138, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_dataset_client.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 13 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/DatasetClient" + } + }, + "overwrites": { + "name": "StorageClient.create_dataset_client", + "target": 1286, + "type": "reference", + "ref": { + "id": 61204, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 42 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/StorageClient#create_dataset_client" + } }, - "parameters": [] + "parentId": 1286 } - ] + ], + "permalink": "/python/api/class/StorageClient#create_dataset_client", + "parentId": 1282 }, { - "id": 53, - "name": "exit", - "module": "actor", "kind": 2048, "kindString": "Method", - "flags": {}, + "children": [], "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Exit the actor instance.\\n\\nThis stops the Actor instance.\\nIt cancels all the intervals for regularly sending `PERSIST_STATE` events,\\nsends a final `PERSIST_STATE` event,\\nwaits for all the event listeners to finish,\\nand stops the event manager.\\n\", {\"Arguments\": [{\"param\": \"exit_code\", \"type\": \"int, optional\", \"desc\": \"The exit code with which the actor should fail (defaults to `0`).\"}, {\"param\": \"event_listeners_timeout_secs\", \"type\": \"float, optional\", \"desc\": \"How long should the actor wait for actor event listeners to finish before exiting.\"}, {\"param\": \"status_message\", \"type\": \"str, optional\", \"desc\": \"The final status message that the actor should display.\"}, {\"param\": \"cleanup_timeout\", \"type\": \"timedelta, optional\", \"desc\": \"How long we should wait for event listeners.\"}]}]}" + "text": "Create a key-value store client." } ] }, - "children": [], + "decorations": [ + { + "name": "abstractmethod" + } + ], + "flags": {}, "groups": [], + "id": 61210, + "module": "storage_clients._base._storage_client", + "name": "create_kvs_client", + "parsedDocstring": { + "text": "Create a key-value store client." + }, "sources": [ { - "filename": "/tmp-old-source-v17/src/apify/actor.py", - "line": 298, "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/actor.py#L298" + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 53, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/_base/_storage_client.py#L53" } ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, "signatures": [ { - "id": 54, - "name": "exit", - "modifiers": [ - "async" - ], - "kind": 4096, - "kindString": "Call signature", - "flags": {}, "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Exit the actor instance.\\n\\nThis stops the Actor instance.\\nIt cancels all the intervals for regularly sending `PERSIST_STATE` events,\\nsends a final `PERSIST_STATE` event,\\nwaits for all the event listeners to finish,\\nand stops the event manager.\\n\", {\"" + "text": "Create a key-value store client." } ] }, - "type": { - "type": "reference", - "name": "None" - }, + "flags": {}, + "id": 61211, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "create_kvs_client", "parameters": [ { - "id": 55, - "name": "exit_code", - "kind": 32768, - "kindString": "Parameter", + "defaultValue": "None", "flags": { - "keyword-only": "true" + "isOptional": true, + "keyword-only": true }, + "id": 61212, + "kind": 32768, + "kindString": "Parameter", + "name": "id", "type": { - "type": "intrinsic", - "name": "int" + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] }, - "defaultValue": "0" + "parentId": 1293 }, { - "id": 56, - "name": "event_listeners_timeout_secs", - "kind": 32768, - "kindString": "Parameter", + "defaultValue": "None", "flags": { - "keyword-only": "true" + "isOptional": true, + "keyword-only": true }, + "id": 61213, + "kind": 32768, + "kindString": "Parameter", + "name": "name", "type": { - "type": "reference", - "name": "float | None" + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] }, - "defaultValue": "EVENT_LISTENERS_TIMEOUT_SECS" + "parentId": 1293 }, { - "id": 57, - "name": "status_message", - "kind": 32768, - "kindString": "Parameter", + "defaultValue": "None", "flags": { - "keyword-only": "true" + "isOptional": true, + "keyword-only": true }, + "id": 61214, + "kind": 32768, + "kindString": "Parameter", + "name": "alias", "type": { - "type": "reference", - "name": "str | None" + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] }, - "defaultValue": "None" + "parentId": 1293 }, { - "id": 58, - "name": "cleanup_timeout", - "kind": 32768, - "kindString": "Parameter", + "defaultValue": "None", "flags": { - "keyword-only": "true" + "isOptional": true, + "keyword-only": true }, + "id": 61215, + "kind": 32768, + "kindString": "Parameter", + "name": "configuration", "type": { - "type": "reference", - "name": "timedelta" + "name": "Configuration | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "Configuration", + "target": "2", + "ref": { + "id": 59920, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/configuration.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 20 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/Configuration" + } + }, + { + "type": "literal", + "value": {} + } + ] }, - "defaultValue": "timedelta(seconds=30)" + "parentId": 1293 + } + ], + "type": { + "name": "KeyValueStoreClient", + "type": "reference", + "target": "1254", + "ref": { + "id": 61172, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_key_value_store_client.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 14 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/KeyValueStoreClient" + } + }, + "overwrites": { + "name": "StorageClient.create_kvs_client", + "target": 1292, + "type": "reference", + "ref": { + "id": 61210, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 53 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/StorageClient#create_kvs_client" } - ] + }, + "parentId": 1292 } - ] + ], + "permalink": "/python/api/class/StorageClient#create_kvs_client", + "parentId": 1282 }, { - "id": 59, - "name": "fail", - "module": "actor", "kind": 2048, "kindString": "Method", - "flags": {}, + "children": [], "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Fail the actor instance.\\n\\nThis performs all the same steps as Actor.exit(),\\nbut it additionally sets the exit code to `1` (by default).\\n\", {\"Arguments\": [{\"param\": \"exit_code\", \"type\": \"int, optional\", \"desc\": \"The exit code with which the actor should fail (defaults to `1`).\"}, {\"param\": \"exception\", \"type\": \"BaseException, optional\", \"desc\": \"The exception with which the actor failed.\"}, {\"param\": \"status_message\", \"type\": \"str, optional\", \"desc\": \"The final status message that the actor should display.\"}]}]}" + "text": "Create a request queue client." } ] }, - "children": [], + "decorations": [ + { + "name": "abstractmethod" + } + ], + "flags": {}, "groups": [], + "id": 61216, + "module": "storage_clients._base._storage_client", + "name": "create_rq_client", + "parsedDocstring": { + "text": "Create a request queue client." + }, "sources": [ { - "filename": "/tmp-old-source-v17/src/apify/actor.py", - "line": 372, "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/actor.py#L372" + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 64, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/_base/_storage_client.py#L64" } ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, "signatures": [ { - "id": 60, - "name": "fail", - "modifiers": [ - "async" - ], - "kind": 4096, - "kindString": "Call signature", - "flags": {}, "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Fail the actor instance.\\n\\nThis performs all the same steps as Actor.exit(),\\nbut it additionally sets the exit code to `1` (by default).\\n\", {\"" + "text": "Create a request queue client." } ] }, - "type": { - "type": "reference", - "name": "None" - }, + "flags": {}, + "id": 61217, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "create_rq_client", "parameters": [ { - "id": 61, - "name": "exit_code", - "kind": 32768, - "kindString": "Parameter", + "defaultValue": "None", "flags": { - "keyword-only": "true" + "isOptional": true, + "keyword-only": true }, + "id": 61218, + "kind": 32768, + "kindString": "Parameter", + "name": "id", "type": { - "type": "intrinsic", - "name": "int" + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] }, - "defaultValue": "1" + "parentId": 1299 }, { - "id": 62, - "name": "exception", - "kind": 32768, - "kindString": "Parameter", + "defaultValue": "None", "flags": { - "keyword-only": "true" + "isOptional": true, + "keyword-only": true }, + "id": 61219, + "kind": 32768, + "kindString": "Parameter", + "name": "name", "type": { - "type": "reference", - "name": "BaseException | None" + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] }, - "defaultValue": "None" + "parentId": 1299 }, { - "id": 63, - "name": "status_message", + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 61220, "kind": 32768, "kindString": "Parameter", + "name": "alias", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 1299 + }, + { + "defaultValue": "None", "flags": { - "keyword-only": "true" + "isOptional": true, + "keyword-only": true }, + "id": 61221, + "kind": 32768, + "kindString": "Parameter", + "name": "configuration", "type": { - "type": "reference", - "name": "str | None" + "name": "Configuration | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "Configuration", + "target": "2", + "ref": { + "id": 59920, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/configuration.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 20 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/Configuration" + } + }, + { + "type": "literal", + "value": {} + } + ] }, - "defaultValue": "None" + "parentId": 1299 + } + ], + "type": { + "name": "RequestQueueClient", + "type": "reference", + "target": "1195", + "ref": { + "id": 61113, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_request_queue_client.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 13 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/RequestQueueClient" + } + }, + "overwrites": { + "name": "StorageClient.create_rq_client", + "target": 1298, + "type": "reference", + "ref": { + "id": 61216, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 64 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/StorageClient#create_rq_client" } - ] + }, + "parentId": 1298 } - ] + ], + "permalink": "/python/api/class/StorageClient#create_rq_client", + "parentId": 1282 }, { - "id": 64, - "name": "main", - "module": "actor", "kind": 2048, "kindString": "Method", - "flags": {}, + "children": [], "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Initialize the actor, run the passed function and finish the actor cleanly.\\n\\n**The `Actor.main()` function is optional** and is provided merely for your convenience.\\nIt is mainly useful when you're running your code as an actor on the [Apify platform](https://apify.com/actors).\\n\\nThe `Actor.main()` function performs the following actions:\\n\\n- When running on the Apify platform (i.e. `APIFY_IS_AT_HOME` environment variable is set),\\nit sets up a connection to listen for platform events.\\nFor example, to get a notification about an imminent migration to another server.\\n- It invokes the user function passed as the `main_actor_function` parameter.\\n- If the user function was an async function, it awaits it.\\n- If the user function throws an exception or some other error is encountered,\\nit prints error details to console so that they are stored to the log,\\nand finishes the actor cleanly.\\n- Finally, it exits the Python process, with zero exit code on success and non-zero on errors.\\n\", {\"Arguments\": [{\"param\": \"main_actor_function\", \"type\": \"Callable\", \"desc\": \"The user function which should be run in the actor\"}]}]}" + "text": "Return statistics about rate limit errors encountered by the HTTP client in storage client." } ] }, - "children": [], + "decorations": [], + "flags": {}, "groups": [], + "id": 61222, + "module": "storage_clients._base._storage_client", + "name": "get_rate_limit_errors", + "parsedDocstring": { + "text": "Return statistics about rate limit errors encountered by the HTTP client in storage client." + }, "sources": [ { - "filename": "/tmp-old-source-v17/src/apify/actor.py", - "line": 412, "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/actor.py#L412" + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 74, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/_base/_storage_client.py#L74" } ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, "signatures": [ { - "id": 65, - "name": "main", - "modifiers": [ - "async" - ], - "kind": 4096, - "kindString": "Call signature", - "flags": {}, "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Initialize the actor, run the passed function and finish the actor cleanly.\\n\\n**The `Actor.main()` function is optional** and is provided merely for your convenience.\\nIt is mainly useful when you're running your code as an actor on the [Apify platform](https://apify.com/actors).\\n\\nThe `Actor.main()` function performs the following actions:\\n\\n- When running on the Apify platform (i.e. `APIFY_IS_AT_HOME` environment variable is set),\\nit sets up a connection to listen for platform events.\\nFor example, to get a notification about an imminent migration to another server.\\n- It invokes the user function passed as the `main_actor_function` parameter.\\n- If the user function was an async function, it awaits it.\\n- If the user function throws an exception or some other error is encountered,\\nit prints error details to console so that they are stored to the log,\\nand finishes the actor cleanly.\\n- Finally, it exits the Python process, with zero exit code on success and non-zero on errors.\\n\", {\"" + "text": "Return statistics about rate limit errors encountered by the HTTP client in storage client." } ] }, + "flags": {}, + "id": 61223, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "get_rate_limit_errors", + "parameters": [], "type": { + "name": "dict", "type": "reference", - "name": "MainReturnType | None" - }, - "parameters": [ - { - "id": 66, - "name": "main_actor_function", - "kind": 32768, - "kindString": "Parameter", - "flags": {}, - "type": { + "typeArguments": [ + { "type": "reference", - "name": "Callable[[], MainReturnType]" + "name": "int", + "target": 0, + "ref": {} + }, + { + "type": "reference", + "name": "int", + "target": 0, + "ref": {} } - } - ] + ], + "target": 0, + "ref": {} + }, + "parentId": 1304 } - ] + ], + "permalink": "/python/api/class/StorageClient#get_rate_limit_errors", + "parentId": 1282 + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Base class for storage clients.\n\nThe `StorageClient` serves as an abstract base class that defines the interface for accessing Crawlee's\nstorage types: datasets, key-value stores, and request queues. It provides methods to open clients for\neach of these storage types and handles common functionality.\n\nStorage clients implementations can be provided for various backends (file system, memory, databases,\nvarious cloud providers, etc.) to support different use cases from development to production environments.\n\nEach storage client implementation is responsible for ensuring proper initialization, data persistence\n(where applicable), and consistent access patterns across all storage types it supports." + } + ] + }, + "decorations": [ + { + "args": "('Storage clients')", + "name": "docs_group" + } + ], + "flags": {}, + "groups": [ + { + "children": [ + 61204, + 61210, + 61216, + 61222, + 61201 + ], + "title": "Methods" + } + ], + "id": 61200, + "module": "storage_clients._base._storage_client", + "name": "StorageClient", + "parsedDocstring": { + "text": "Base class for storage clients.\n\nThe `StorageClient` serves as an abstract base class that defines the interface for accessing Crawlee's\nstorage types: datasets, key-value stores, and request queues. It provides methods to open clients for\neach of these storage types and handles common functionality.\n\nStorage clients implementations can be provided for various backends (file system, memory, databases,\nvarious cloud providers, etc.) to support different use cases from development to production environments.\n\nEach storage client implementation is responsible for ensuring proper initialization, data persistence\n(where applicable), and consistent access patterns across all storage types it supports." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 19, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/_base/_storage_client.py#L19" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "extendedBy": [ + { + "name": "MemoryStorageClient", + "target": "1468", + "type": "reference", + "ref": { + "id": 61386, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_memory/_storage_client.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 15 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/MemoryStorageClient" + } }, { - "id": 67, - "name": "new_client", - "module": "actor", + "name": "FileSystemStorageClient", + "target": "1624", + "type": "reference", + "ref": { + "id": 61542, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_file_system/_storage_client.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 20 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/FileSystemStorageClient" + } + }, + { + "name": "SqlStorageClient", + "target": "1893", + "type": "reference", + "ref": { + "id": 61811, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_sql/_storage_client.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 35 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/SqlStorageClient" + } + }, + { + "name": "RedisStorageClient", + "target": "2091", + "type": "reference", + "ref": { + "id": 62009, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_redis/_storage_client.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 19 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/RedisStorageClient" + } + } + ], + "permalink": "/python/api/class/StorageClient", + "categories": [] + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { "kind": 2048, "kindString": "Method", - "flags": {}, + "children": [], "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Return a new instance of the Apify API client.\\n\\nThe `ApifyClientAsync` class is provided by the [apify-client](https://github.com/apify/apify-client-python) package,\\nand it is automatically configured using the `APIFY_API_BASE_URL` and `APIFY_TOKEN` environment variables.\\n\\nYou can override the token via the available options.\\nThat's useful if you want to use the client as a different Apify user than the SDK internals are using.\\n\", {\"Arguments\": [{\"param\": \"token\", \"type\": \"str, optional\", \"desc\": \"The Apify API token\"}, {\"param\": \"api_url\", \"type\": \"str, optional\", \"desc\": \"The URL of the Apify API server to which to connect to. Defaults to https://api.apify.com\"}, {\"param\": \"max_retries\", \"type\": \"int, optional\", \"desc\": \"How many times to retry a failed request at most\"}, {\"param\": \"min_delay_between_retries_millis\", \"type\": \"int, optional\", \"desc\": \"How long will the client wait between retrying requests\\n(increases exponentially from this value)\"}, {\"param\": \"timeout_secs\", \"type\": \"int, optional\", \"desc\": \"The socket timeout of the HTTP requests sent to the Apify API\"}]}]}" + "text": "Create a dataset client." } ] }, - "children": [], + "decorations": [ + { + "name": "abstractmethod" + } + ], + "flags": {}, "groups": [], + "id": 63478, + "module": "storage_clients._base._storage_client", + "name": "create_dataset_client", + "parsedDocstring": { + "text": "Create a dataset client." + }, "sources": [ { - "filename": "/tmp-old-source-v17/src/apify/actor.py", - "line": 457, "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/actor.py#L457" + "fileName": "/src/crawlee/storage_clients/_memory/_storage_client.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 31, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/_memory/_storage_client.py#L31" } ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, "signatures": [ { - "id": 68, - "name": "new_client", - "modifiers": [], - "kind": 4096, - "kindString": "Call signature", - "flags": {}, "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Return a new instance of the Apify API client.\\n\\nThe `ApifyClientAsync` class is provided by the [apify-client](https://github.com/apify/apify-client-python) package,\\nand it is automatically configured using the `APIFY_API_BASE_URL` and `APIFY_TOKEN` environment variables.\\n\\nYou can override the token via the available options.\\nThat's useful if you want to use the client as a different Apify user than the SDK internals are using.\\n\", {\"" + "text": "Create a dataset client." } ] }, - "type": { - "type": "reference", - "name": "ApifyClientAsync" - }, + "flags": {}, + "id": 63296, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "create_dataset_client", "parameters": [ { - "id": 69, - "name": "token", - "kind": 32768, - "kindString": "Parameter", + "defaultValue": "None", "flags": { - "keyword-only": "true" - }, - "type": { - "type": "reference", - "name": "str | None" + "isOptional": true, + "keyword-only": true }, - "defaultValue": "None" - }, - { - "id": 70, - "name": "api_url", + "id": 63297, "kind": 32768, "kindString": "Parameter", - "flags": { - "keyword-only": "true" - }, + "name": "id", "type": { - "type": "reference", - "name": "str | None" + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] }, - "defaultValue": "None" + "parentId": 1287 }, { - "id": 71, - "name": "max_retries", - "kind": 32768, - "kindString": "Parameter", + "defaultValue": "None", "flags": { - "keyword-only": "true" + "isOptional": true, + "keyword-only": true }, + "id": 63298, + "kind": 32768, + "kindString": "Parameter", + "name": "name", "type": { - "type": "reference", - "name": "int | None" + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] }, - "defaultValue": "None" + "parentId": 1287 }, { - "id": 72, - "name": "min_delay_between_retries_millis", - "kind": 32768, - "kindString": "Parameter", + "defaultValue": "None", "flags": { - "keyword-only": "true" + "isOptional": true, + "keyword-only": true }, + "id": 63299, + "kind": 32768, + "kindString": "Parameter", + "name": "alias", "type": { - "type": "reference", - "name": "int | None" + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] }, - "defaultValue": "None" + "parentId": 1287 }, { - "id": 73, - "name": "timeout_secs", - "kind": 32768, - "kindString": "Parameter", + "defaultValue": "None", "flags": { - "keyword-only": "true" + "isOptional": true, + "keyword-only": true }, + "id": 63300, + "kind": 32768, + "kindString": "Parameter", + "name": "configuration", "type": { - "type": "reference", - "name": "int | None" + "name": "Configuration | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "Configuration", + "target": "2", + "ref": { + "id": 62011, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/configuration.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 20 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/Configuration" + } + }, + { + "type": "literal", + "value": {} + } + ] }, - "defaultValue": "None" + "parentId": 1287 + } + ], + "type": { + "name": "DatasetClient", + "type": "reference", + "target": "1220", + "ref": { + "id": 63229, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_dataset_client.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 13 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/DatasetClient" + } + }, + "overwrites": { + "name": "StorageClient.create_dataset_client", + "target": 1286, + "type": "reference", + "ref": { + "id": 63295, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 42 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/StorageClient#create_dataset_client" + } + }, + "parentId": 1469 + } + ], + "overwrites": { + "name": "StorageClient.create_dataset_client", + "target": 1286, + "type": "reference", + "ref": { + "id": 63295, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 42 } - ] + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/StorageClient#create_dataset_client" } - ] + }, + "permalink": "/python/api/class/MemoryStorageClient#create_dataset_client", + "parentId": 1468 }, { - "id": 74, - "name": "open_dataset", - "module": "actor", "kind": 2048, "kindString": "Method", - "flags": {}, + "children": [], "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Open a dataset.\\n\\nDatasets are used to store structured data where each object stored has the same attributes,\\nsuch as online store products or real estate offers.\\nThe actual data is stored either on the local filesystem or in the Apify cloud.\\n\", {\"Arguments\": [{\"param\": \"id\", \"type\": \"str, optional\", \"desc\": \"ID of the dataset to be opened.\\nIf neither `id` nor `name` are provided, the method returns the default dataset associated with the actor run.\"}, {\"param\": \"name\", \"type\": \"str, optional\", \"desc\": \"Name of the dataset to be opened.\\nIf neither `id` nor `name` are provided, the method returns the default dataset associated with the actor run.\"}, {\"param\": \"force_cloud\", \"type\": \"bool, optional\", \"desc\": \"If set to `True` then the Apify cloud storage is always used.\\nThis way it is possible to combine local and cloud storage.\\n\"}]}, {\"Returns\": [{\"param\": \"Dataset\", \"desc\": \"An instance of the `Dataset` class for the given ID or name.\"}]}]}" + "text": "Create a key-value store client." } ] }, - "children": [], + "decorations": [ + { + "name": "abstractmethod" + } + ], + "flags": {}, "groups": [], + "id": 63484, + "module": "storage_clients._base._storage_client", + "name": "create_kvs_client", + "parsedDocstring": { + "text": "Create a key-value store client." + }, "sources": [ { - "filename": "/tmp-old-source-v17/src/apify/actor.py", - "line": 513, "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/actor.py#L513" + "fileName": "/src/crawlee/storage_clients/_memory/_storage_client.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 45, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/_memory/_storage_client.py#L45" } ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, "signatures": [ { - "id": 75, - "name": "open_dataset", - "modifiers": [ - "async" - ], - "kind": 4096, - "kindString": "Call signature", - "flags": {}, "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Open a dataset.\\n\\nDatasets are used to store structured data where each object stored has the same attributes,\\nsuch as online store products or real estate offers.\\nThe actual data is stored either on the local filesystem or in the Apify cloud.\\n\", {\"" + "text": "Create a key-value store client." } ] }, - "type": { - "type": "reference", - "name": "Dataset", - "id": 292 - }, + "flags": {}, + "id": 63302, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "create_kvs_client", "parameters": [ { - "id": 76, - "name": "id", - "kind": 32768, - "kindString": "Parameter", + "defaultValue": "None", "flags": { - "keyword-only": "true" + "isOptional": true, + "keyword-only": true }, + "id": 63303, + "kind": 32768, + "kindString": "Parameter", + "name": "id", "type": { - "type": "reference", - "name": "str | None" + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] }, - "defaultValue": "None" + "parentId": 1293 }, { - "id": 77, - "name": "name", - "kind": 32768, - "kindString": "Parameter", + "defaultValue": "None", "flags": { - "keyword-only": "true" + "isOptional": true, + "keyword-only": true }, + "id": 63304, + "kind": 32768, + "kindString": "Parameter", + "name": "name", "type": { - "type": "reference", - "name": "str | None" + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] }, - "defaultValue": "None" + "parentId": 1293 }, { - "id": 78, - "name": "force_cloud", + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 63305, "kind": 32768, "kindString": "Parameter", + "name": "alias", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 1293 + }, + { + "defaultValue": "None", "flags": { - "keyword-only": "true" + "isOptional": true, + "keyword-only": true }, + "id": 63306, + "kind": 32768, + "kindString": "Parameter", + "name": "configuration", "type": { - "type": "intrinsic", - "name": "bool" + "name": "Configuration | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "Configuration", + "target": "2", + "ref": { + "id": 62011, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/configuration.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 20 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/Configuration" + } + }, + { + "type": "literal", + "value": {} + } + ] }, - "defaultValue": "False" + "parentId": 1293 + } + ], + "type": { + "name": "KeyValueStoreClient", + "type": "reference", + "target": "1254", + "ref": { + "id": 63263, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_key_value_store_client.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 14 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/KeyValueStoreClient" + } + }, + "overwrites": { + "name": "StorageClient.create_kvs_client", + "target": 1292, + "type": "reference", + "ref": { + "id": 63301, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 53 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/StorageClient#create_kvs_client" } - ] + }, + "parentId": 1475 } - ] + ], + "overwrites": { + "name": "StorageClient.create_kvs_client", + "target": 1292, + "type": "reference", + "ref": { + "id": 63301, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 53 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/StorageClient#create_kvs_client" + } + }, + "permalink": "/python/api/class/MemoryStorageClient#create_kvs_client", + "parentId": 1468 }, { - "id": 79, - "name": "open_key_value_store", - "module": "actor", "kind": 2048, "kindString": "Method", - "flags": {}, + "children": [], "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Open a key-value store.\\n\\nKey-value stores are used to store records or files, along with their MIME content type.\\nThe records are stored and retrieved using a unique key.\\nThe actual data is stored either on a local filesystem or in the Apify cloud.\\n\", {\"Arguments\": [{\"param\": \"id\", \"type\": \"str, optional\", \"desc\": \"ID of the key-value store to be opened.\\nIf neither `id` nor `name` are provided, the method returns the default key-value store associated with the actor run.\"}, {\"param\": \"name\", \"type\": \"str, optional\", \"desc\": \"Name of the key-value store to be opened.\\nIf neither `id` nor `name` are provided, the method returns the default key-value store associated with the actor run.\"}, {\"param\": \"force_cloud\", \"type\": \"bool, optional\", \"desc\": \"If set to `True` then the Apify cloud storage is always used.\\nThis way it is possible to combine local and cloud storage.\\n\"}]}, {\"Returns\": [{\"param\": \"KeyValueStore\", \"desc\": \"An instance of the `KeyValueStore` class for the given ID or name.\"}]}]}" + "text": "Create a request queue client." } ] }, - "children": [], + "decorations": [ + { + "name": "abstractmethod" + } + ], + "flags": {}, "groups": [], + "id": 63490, + "module": "storage_clients._base._storage_client", + "name": "create_rq_client", + "parsedDocstring": { + "text": "Create a request queue client." + }, "sources": [ { - "filename": "/tmp-old-source-v17/src/apify/actor.py", - "line": 552, "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/actor.py#L552" + "fileName": "/src/crawlee/storage_clients/_memory/_storage_client.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 59, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/_memory/_storage_client.py#L59" } ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, "signatures": [ { - "id": 80, - "name": "open_key_value_store", - "modifiers": [ - "async" - ], - "kind": 4096, - "kindString": "Call signature", - "flags": {}, "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Open a key-value store.\\n\\nKey-value stores are used to store records or files, along with their MIME content type.\\nThe records are stored and retrieved using a unique key.\\nThe actual data is stored either on a local filesystem or in the Apify cloud.\\n\", {\"" + "text": "Create a request queue client." } ] }, - "type": { - "type": "reference", - "name": "KeyValueStore", - "id": 352 - }, + "flags": {}, + "id": 63308, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "create_rq_client", "parameters": [ { - "id": 81, - "name": "id", - "kind": 32768, - "kindString": "Parameter", + "defaultValue": "None", "flags": { - "keyword-only": "true" + "isOptional": true, + "keyword-only": true }, + "id": 63309, + "kind": 32768, + "kindString": "Parameter", + "name": "id", "type": { - "type": "reference", - "name": "str | None" + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] }, - "defaultValue": "None" + "parentId": 1299 }, { - "id": 82, - "name": "name", - "kind": 32768, - "kindString": "Parameter", + "defaultValue": "None", "flags": { - "keyword-only": "true" + "isOptional": true, + "keyword-only": true }, + "id": 63310, + "kind": 32768, + "kindString": "Parameter", + "name": "name", "type": { - "type": "reference", - "name": "str | None" + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] }, - "defaultValue": "None" + "parentId": 1299 }, { - "id": 83, - "name": "force_cloud", + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 63311, "kind": 32768, "kindString": "Parameter", + "name": "alias", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 1299 + }, + { + "defaultValue": "None", "flags": { - "keyword-only": "true" + "isOptional": true, + "keyword-only": true }, + "id": 63312, + "kind": 32768, + "kindString": "Parameter", + "name": "configuration", "type": { - "type": "intrinsic", - "name": "bool" + "name": "Configuration | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "Configuration", + "target": "2", + "ref": { + "id": 62011, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/configuration.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 20 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/Configuration" + } + }, + { + "type": "literal", + "value": {} + } + ] }, - "defaultValue": "False" + "parentId": 1299 + } + ], + "type": { + "name": "RequestQueueClient", + "type": "reference", + "target": "1195", + "ref": { + "id": 63204, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_request_queue_client.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 13 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/RequestQueueClient" + } + }, + "overwrites": { + "name": "StorageClient.create_rq_client", + "target": 1298, + "type": "reference", + "ref": { + "id": 63307, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 64 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/StorageClient#create_rq_client" } - ] + }, + "parentId": 1481 } - ] + ], + "overwrites": { + "name": "StorageClient.create_rq_client", + "target": 1298, + "type": "reference", + "ref": { + "id": 63307, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 64 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/StorageClient#create_rq_client" + } + }, + "permalink": "/python/api/class/MemoryStorageClient#create_rq_client", + "parentId": 1468 }, { - "id": 84, - "name": "open_request_queue", - "module": "actor", "kind": 2048, "kindString": "Method", - "flags": {}, + "children": [], "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Open a request queue.\\n\\nRequest queue represents a queue of URLs to crawl, which is stored either on local filesystem or in the Apify cloud.\\nThe queue is used for deep crawling of websites, where you start with several URLs and then\\nrecursively follow links to other pages. The data structure supports both breadth-first\\nand depth-first crawling orders.\\n\", {\"Arguments\": [{\"param\": \"id\", \"type\": \"str, optional\", \"desc\": \"ID of the request queue to be opened.\\nIf neither `id` nor `name` are provided, the method returns the default request queue associated with the actor run.\"}, {\"param\": \"name\", \"type\": \"str, optional\", \"desc\": \"Name of the request queue to be opened.\\nIf neither `id` nor `name` are provided, the method returns the default request queue associated with the actor run.\"}, {\"param\": \"force_cloud\", \"type\": \"bool, optional\", \"desc\": \"If set to `True` then the Apify cloud storage is always used.\\nThis way it is possible to combine local and cloud storage.\\n\"}]}, {\"Returns\": [{\"param\": \"RequestQueue\", \"desc\": \"An instance of the `RequestQueue` class for the given ID or name.\"}]}]}" + "text": "Return a cache key that can differentiate between different storages of this and other clients.\n\nCan be based on configuration or on the client itself. By default, returns a module and name of the client\nclass." } ] }, - "children": [], + "decorations": [], + "flags": {}, "groups": [], + "id": 66208, + "module": "storage_clients._base._storage_client", + "name": "get_storage_client_cache_key", + "parsedDocstring": { + "text": "Return a cache key that can differentiate between different storages of this and other clients.\n\nCan be based on configuration or on the client itself. By default, returns a module and name of the client\nclass." + }, "sources": [ { - "filename": "/tmp-old-source-v17/src/apify/actor.py", - "line": 590, "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/actor.py#L590" + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 33, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/_base/_storage_client.py#L33" } ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, "signatures": [ { - "id": 85, - "name": "open_request_queue", - "modifiers": [ - "async" - ], - "kind": 4096, - "kindString": "Call signature", - "flags": {}, "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Open a request queue.\\n\\nRequest queue represents a queue of URLs to crawl, which is stored either on local filesystem or in the Apify cloud.\\nThe queue is used for deep crawling of websites, where you start with several URLs and then\\nrecursively follow links to other pages. The data structure supports both breadth-first\\nand depth-first crawling orders.\\n\", {\"" + "text": "Return a cache key that can differentiate between different storages of this and other clients.\n\nCan be based on configuration or on the client itself. By default, returns a module and name of the client\nclass." } ] }, - "type": { - "type": "reference", - "name": "RequestQueue", - "id": 376 - }, + "flags": {}, + "id": 63293, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "get_storage_client_cache_key", "parameters": [ { - "id": 86, - "name": "id", - "kind": 32768, - "kindString": "Parameter", "flags": { - "keyword-only": "true" - }, - "type": { - "type": "reference", - "name": "str | None" + "isOptional": false, + "keyword-only": false }, - "defaultValue": "None" - }, - { - "id": 87, - "name": "name", + "id": 63294, "kind": 32768, "kindString": "Parameter", - "flags": { - "keyword-only": "true" - }, + "name": "configuration", "type": { + "name": "Configuration", "type": "reference", - "name": "str | None" - }, - "defaultValue": "None" - }, + "target": "2", + "ref": { + "id": 62011, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/configuration.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 20 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/Configuration" + } + }, + "parentId": 1284 + } + ], + "type": { + "name": "Hashable", + "type": "reference", + "target": 0, + "ref": {} + }, + "inheritedFrom": { + "name": "StorageClient.get_storage_client_cache_key", + "target": 1283, + "type": "reference", + "ref": { + "id": 63292, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 33 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/StorageClient#get_storage_client_cache_key" + } + }, + "parentId": 4199 + } + ], + "inheritedFrom": { + "name": "StorageClient.get_storage_client_cache_key", + "target": 1283, + "type": "reference", + "ref": { + "id": 63292, + "sources": [ { - "id": 88, - "name": "force_cloud", - "kind": 32768, - "kindString": "Parameter", - "flags": { - "keyword-only": "true" - }, - "type": { - "type": "intrinsic", - "name": "bool" - }, - "defaultValue": "False" + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 33 } - ] + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/StorageClient#get_storage_client_cache_key" } - ] + }, + "permalink": "/python/api/class/MemoryStorageClient#get_storage_client_cache_key", + "parentId": 1468 }, { - "id": 89, - "name": "push_data", - "module": "actor", "kind": 2048, "kindString": "Method", - "flags": {}, + "children": [], "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Store an object or a list of objects to the default dataset of the current actor run.\\n\", {\"Arguments\": [{\"param\": \"data\", \"type\": \"object or list of objects, optional\", \"desc\": \"The data to push to the default dataset.\"}]}]}" + "text": "Return statistics about rate limit errors encountered by the HTTP client in storage client." } ] }, - "children": [], + "decorations": [], + "flags": {}, "groups": [], + "id": 66209, + "module": "storage_clients._base._storage_client", + "name": "get_rate_limit_errors", + "parsedDocstring": { + "text": "Return statistics about rate limit errors encountered by the HTTP client in storage client." + }, "sources": [ { - "filename": "/tmp-old-source-v17/src/apify/actor.py", - "line": 629, "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/actor.py#L629" + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 74, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/_base/_storage_client.py#L74" } ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, "signatures": [ { - "id": 90, - "name": "push_data", - "modifiers": [ - "async" - ], - "kind": 4096, - "kindString": "Call signature", - "flags": {}, "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Store an object or a list of objects to the default dataset of the current actor run.\\n\", {\"" + "text": "Return statistics about rate limit errors encountered by the HTTP client in storage client." } ] }, + "flags": {}, + "id": 63314, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "get_rate_limit_errors", + "parameters": [], "type": { + "name": "dict[int, int]", "type": "reference", - "name": "None" + "target": 0, + "ref": {} }, - "parameters": [ + "inheritedFrom": { + "name": "StorageClient.get_rate_limit_errors", + "target": 1304, + "type": "reference", + "ref": { + "id": 63313, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 74 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/StorageClient#get_rate_limit_errors" + } + }, + "parentId": 4200 + } + ], + "inheritedFrom": { + "name": "StorageClient.get_rate_limit_errors", + "target": 1304, + "type": "reference", + "ref": { + "id": 63313, + "sources": [ { - "id": 91, - "name": "data", - "kind": 32768, - "kindString": "Parameter", - "flags": {}, - "type": { - "type": "reference", - "name": "Any" - } + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 74 } - ] + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/StorageClient#get_rate_limit_errors" } - ] - }, + }, + "permalink": "/python/api/class/MemoryStorageClient#get_rate_limit_errors", + "parentId": 1468 + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Memory implementation of the storage client.\n\nThis storage client provides access to datasets, key-value stores, and request queues that store all data\nin memory using Python data structures (lists and dictionaries). No data is persisted between process runs,\nmeaning all stored data is lost when the program terminates.\n\nThe memory implementation provides fast access to data but is limited by available memory and does not\nsupport data sharing across different processes. All storage operations happen entirely in memory with\nno disk operations.\n\nThe memory storage client is useful for testing and development environments, or short-lived crawler\noperations where persistence is not required." + } + ] + }, + "decorations": [ + { + "args": "('Storage clients')", + "name": "docs_group" + } + ], + "flags": {}, + "groups": [ + { + "children": [ + 63478, + 63484, + 63490, + 66209, + 66208 + ], + "title": "Methods" + } + ], + "id": 63477, + "module": "storage_clients._memory._storage_client", + "name": "MemoryStorageClient", + "parsedDocstring": { + "text": "Memory implementation of the storage client.\n\nThis storage client provides access to datasets, key-value stores, and request queues that store all data\nin memory using Python data structures (lists and dictionaries). No data is persisted between process runs,\nmeaning all stored data is lost when the program terminates.\n\nThe memory implementation provides fast access to data but is limited by available memory and does not\nsupport data sharing across different processes. All storage operations happen entirely in memory with\nno disk operations.\n\nThe memory storage client is useful for testing and development environments, or short-lived crawler\noperations where persistence is not required." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_memory/_storage_client.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 15, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/_memory/_storage_client.py#L15" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "extendedTypes": [ + { + "name": "StorageClient", + "target": "1282", + "type": "reference", + "ref": { + "id": 63291, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 19 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/StorageClient" + } + } + ], + "permalink": "/python/api/class/MemoryStorageClient", + "categories": [] + }, + { + "kind": 128, + "kindString": "Class", + "children": [ { - "id": 92, - "name": "get_input", - "module": "actor", "kind": 2048, "kindString": "Method", - "flags": {}, + "children": [], "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Get the actor input value from the default key-value store associated with the current actor run.\"]}" + "text": "Return a cache key that can differentiate between different storages of this and other clients.\n\nCan be based on configuration or on the client itself. By default, returns a module and name of the client\nclass." } ] }, - "children": [], + "decorations": [], + "flags": {}, "groups": [], + "id": 67834, + "module": "storage_clients._base._storage_client", + "name": "get_storage_client_cache_key", + "parsedDocstring": { + "text": "Return a cache key that can differentiate between different storages of this and other clients.\n\nCan be based on configuration or on the client itself. By default, returns a module and name of the client\nclass." + }, "sources": [ { - "filename": "/tmp-old-source-v17/src/apify/actor.py", - "line": 647, "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/actor.py#L647" + "fileName": "/src/crawlee/storage_clients/_file_system/_storage_client.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 38, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/_file_system/_storage_client.py#L38" } ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, "signatures": [ { - "id": 93, - "name": "get_input", - "modifiers": [ - "async" - ], - "kind": 4096, - "kindString": "Call signature", - "flags": {}, "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Get the actor input value from the default key-value store associated with the current actor run.\"]}" + "text": "Return a cache key that can differentiate between different storages of this and other clients.\n\nCan be based on configuration or on the client itself. By default, returns a module and name of the client\nclass." } ] }, + "flags": {}, + "id": 67493, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "get_storage_client_cache_key", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 67494, + "kind": 32768, + "kindString": "Parameter", + "name": "configuration", + "type": { + "name": "Configuration", + "type": "reference", + "target": "2", + "ref": { + "id": 66211, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/configuration.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 20 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/Configuration" + } + }, + "parentId": 1284 + } + ], "type": { + "name": "Hashable", "type": "reference", - "name": "Any" + "target": 0, + "ref": {} }, - "parameters": [] + "overwrites": { + "name": "StorageClient.get_storage_client_cache_key", + "target": 1283, + "type": "reference", + "ref": { + "id": 67492, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 33 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/StorageClient#get_storage_client_cache_key" + } + }, + "parentId": 1625 } - ] + ], + "overwrites": { + "name": "StorageClient.get_storage_client_cache_key", + "target": 1283, + "type": "reference", + "ref": { + "id": 67492, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 33 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/StorageClient#get_storage_client_cache_key" + } + }, + "permalink": "/python/api/class/FileSystemStorageClient#get_storage_client_cache_key", + "parentId": 1624 }, { - "id": 94, - "name": "get_value", - "module": "actor", "kind": 2048, "kindString": "Method", - "flags": {}, + "children": [], "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Get a value from the default key-value store associated with the current actor run.\\n\", {\"Arguments\": [{\"param\": \"key\", \"type\": \"str\", \"desc\": \"The key of the record which to retrieve.\"}, {\"param\": \"default_value\", \"type\": \"Any, optional\", \"desc\": \"Default value returned in case the record does not exist.\"}]}]}" + "text": "Create a dataset client." } ] }, - "children": [], + "decorations": [ + { + "name": "abstractmethod" + } + ], + "flags": {}, "groups": [], + "id": 67837, + "module": "storage_clients._base._storage_client", + "name": "create_dataset_client", + "parsedDocstring": { + "text": "Create a dataset client." + }, "sources": [ { - "filename": "/tmp-old-source-v17/src/apify/actor.py", - "line": 667, "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/actor.py#L667" + "fileName": "/src/crawlee/storage_clients/_file_system/_storage_client.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 43, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/_file_system/_storage_client.py#L43" } ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, "signatures": [ { - "id": 95, - "name": "get_value", - "modifiers": [ - "async" - ], - "kind": 4096, - "kindString": "Call signature", - "flags": {}, "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Get a value from the default key-value store associated with the current actor run.\\n\", {\"" + "text": "Create a dataset client." } ] }, - "type": { - "type": "reference", - "name": "Any" - }, + "flags": {}, + "id": 67496, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "create_dataset_client", "parameters": [ { - "id": 96, - "name": "key", + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 67497, "kind": 32768, "kindString": "Parameter", - "flags": {}, + "name": "id", "type": { - "type": "intrinsic", - "name": "str" - } + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 1287 }, { - "id": 97, - "name": "default_value", + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 67498, "kind": 32768, "kindString": "Parameter", - "flags": {}, + "name": "name", "type": { - "type": "reference", - "name": "Any" + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 1287 + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 67499, + "kind": 32768, + "kindString": "Parameter", + "name": "alias", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 1287 + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 67500, + "kind": 32768, + "kindString": "Parameter", + "name": "configuration", + "type": { + "name": "Configuration | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "Configuration", + "target": "2", + "ref": { + "id": 66211, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/configuration.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 20 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/Configuration" + } + }, + { + "type": "literal", + "value": {} + } + ] }, - "defaultValue": "None" + "parentId": 1287 + } + ], + "type": { + "name": "DatasetClient", + "type": "reference", + "target": "1220", + "ref": { + "id": 67429, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_dataset_client.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 13 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/DatasetClient" + } + }, + "overwrites": { + "name": "StorageClient.create_dataset_client", + "target": 1286, + "type": "reference", + "ref": { + "id": 67495, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 42 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/StorageClient#create_dataset_client" } - ] + }, + "parentId": 1628 } - ] + ], + "overwrites": { + "name": "StorageClient.create_dataset_client", + "target": 1286, + "type": "reference", + "ref": { + "id": 67495, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 42 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/StorageClient#create_dataset_client" + } + }, + "permalink": "/python/api/class/FileSystemStorageClient#create_dataset_client", + "parentId": 1624 }, { - "id": 98, - "name": "set_value", - "module": "actor", "kind": 2048, "kindString": "Method", - "flags": {}, + "children": [], "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Set or delete a value in the default key-value store associated with the current actor run.\\n\", {\"Arguments\": [{\"param\": \"key\", \"type\": \"str\", \"desc\": \"The key of the record which to set.\"}, {\"param\": \"value\", \"type\": \"any\", \"desc\": \"The value of the record which to set, or None, if the record should be deleted.\"}, {\"param\": \"content_type\", \"type\": \"str, optional\", \"desc\": \"The content type which should be set to the value.\"}]}]}" + "text": "Create a key-value store client." } ] }, - "children": [], + "decorations": [ + { + "name": "abstractmethod" + } + ], + "flags": {}, "groups": [], + "id": 67843, + "module": "storage_clients._base._storage_client", + "name": "create_kvs_client", + "parsedDocstring": { + "text": "Create a key-value store client." + }, "sources": [ { - "filename": "/tmp-old-source-v17/src/apify/actor.py", - "line": 683, "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/actor.py#L683" + "fileName": "/src/crawlee/storage_clients/_file_system/_storage_client.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 57, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/_file_system/_storage_client.py#L57" } ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, "signatures": [ { - "id": 99, - "name": "set_value", - "modifiers": [ - "async" - ], - "kind": 4096, - "kindString": "Call signature", - "flags": {}, "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Set or delete a value in the default key-value store associated with the current actor run.\\n\", {\"" + "text": "Create a key-value store client." } ] }, - "type": { - "type": "reference", - "name": "None" - }, + "flags": {}, + "id": 67502, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "create_kvs_client", "parameters": [ { - "id": 100, - "name": "key", + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 67503, + "kind": 32768, + "kindString": "Parameter", + "name": "id", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 1293 + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 67504, "kind": 32768, "kindString": "Parameter", - "flags": {}, + "name": "name", "type": { - "type": "intrinsic", - "name": "str" - } + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 1293 }, { - "id": 101, - "name": "value", + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 67505, "kind": 32768, "kindString": "Parameter", - "flags": {}, + "name": "alias", "type": { - "type": "reference", - "name": "Any" - } + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 1293 }, { - "id": 102, - "name": "content_type", - "kind": 32768, - "kindString": "Parameter", + "defaultValue": "None", "flags": { - "keyword-only": "true" + "isOptional": true, + "keyword-only": true }, + "id": 67506, + "kind": 32768, + "kindString": "Parameter", + "name": "configuration", "type": { - "type": "reference", - "name": "str | None" + "name": "Configuration | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "Configuration", + "target": "2", + "ref": { + "id": 66211, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/configuration.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 20 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/Configuration" + } + }, + { + "type": "literal", + "value": {} + } + ] }, - "defaultValue": "None" + "parentId": 1293 + } + ], + "type": { + "name": "KeyValueStoreClient", + "type": "reference", + "target": "1254", + "ref": { + "id": 67463, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_key_value_store_client.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 14 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/KeyValueStoreClient" + } + }, + "overwrites": { + "name": "StorageClient.create_kvs_client", + "target": 1292, + "type": "reference", + "ref": { + "id": 67501, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 53 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/StorageClient#create_kvs_client" } - ] + }, + "parentId": 1634 } - ] + ], + "overwrites": { + "name": "StorageClient.create_kvs_client", + "target": 1292, + "type": "reference", + "ref": { + "id": 67501, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 53 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/StorageClient#create_kvs_client" + } + }, + "permalink": "/python/api/class/FileSystemStorageClient#create_kvs_client", + "parentId": 1624 }, { - "id": 103, - "name": "on", - "module": "actor", "kind": 2048, "kindString": "Method", - "flags": {}, + "children": [], "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Add an event listener to the actor's event manager.\\n\\nThe following events can be emitted:\\n- `ActorEventTypes.SYSTEM_INFO`:\\nEmitted every minute, the event data contains info about the resource usage of the actor.\\n- `ActorEventTypes.MIGRATING`:\\nEmitted when the actor running on the Apify platform is going to be migrated to another worker server soon.\\nYou can use it to persist the state of the actor and gracefully stop your in-progress tasks,\\nso that they are not interrupted by the migration..\\n- `ActorEventTypes.PERSIST_STATE`:\\nEmitted in regular intervals (by default 60 seconds) to notify the actor that it should persist its state,\\nin order to avoid repeating all work when the actor restarts.\\nThis event is automatically emitted together with the migrating event,\\nin which case the `isMigrating` flag in the event data is set to True, otherwise the flag is False.\\nNote that this event is provided merely for your convenience,\\nyou can achieve the same effect using an interval and listening for the migrating event.\\n- `ActorEventTypes.ABORTING`:\\nWhen a user aborts an actor run on the Apify platform,\\nthey can choose to abort it gracefully, to allow the actor some time before getting terminated.\\nThis graceful abort emits the aborting event, which you can use to clean up the actor state.\\n\", {\"Arguments\": [{\"param\": \"event_name\", \"type\": \"ActorEventTypes\", \"desc\": \"The actor event for which to listen to.\"}, {\"param\": \"listener\", \"type\": \"Callable\", \"desc\": \"The function which is to be called when the event is emitted (can be async).\"}]}]}" + "text": "Create a request queue client." } ] }, - "children": [], + "decorations": [ + { + "name": "abstractmethod" + } + ], + "flags": {}, "groups": [], + "id": 67849, + "module": "storage_clients._base._storage_client", + "name": "create_rq_client", + "parsedDocstring": { + "text": "Create a request queue client." + }, "sources": [ { - "filename": "/tmp-old-source-v17/src/apify/actor.py", - "line": 716, "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/actor.py#L716" + "fileName": "/src/crawlee/storage_clients/_file_system/_storage_client.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 71, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/_file_system/_storage_client.py#L71" } ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, "signatures": [ { - "id": 104, - "name": "on", - "modifiers": [], - "kind": 4096, - "kindString": "Call signature", - "flags": {}, "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Add an event listener to the actor's event manager.\\n\\nThe following events can be emitted:\\n- `ActorEventTypes.SYSTEM_INFO`:\\nEmitted every minute, the event data contains info about the resource usage of the actor.\\n- `ActorEventTypes.MIGRATING`:\\nEmitted when the actor running on the Apify platform is going to be migrated to another worker server soon.\\nYou can use it to persist the state of the actor and gracefully stop your in-progress tasks,\\nso that they are not interrupted by the migration..\\n- `ActorEventTypes.PERSIST_STATE`:\\nEmitted in regular intervals (by default 60 seconds) to notify the actor that it should persist its state,\\nin order to avoid repeating all work when the actor restarts.\\nThis event is automatically emitted together with the migrating event,\\nin which case the `isMigrating` flag in the event data is set to True, otherwise the flag is False.\\nNote that this event is provided merely for your convenience,\\nyou can achieve the same effect using an interval and listening for the migrating event.\\n- `ActorEventTypes.ABORTING`:\\nWhen a user aborts an actor run on the Apify platform,\\nthey can choose to abort it gracefully, to allow the actor some time before getting terminated.\\nThis graceful abort emits the aborting event, which you can use to clean up the actor state.\\n\", {\"" + "text": "Create a request queue client." } ] }, - "type": { - "type": "reference", - "name": "Callable" - }, + "flags": {}, + "id": 67508, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "create_rq_client", "parameters": [ { - "id": 105, - "name": "event_name", + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 67509, "kind": 32768, "kindString": "Parameter", - "flags": {}, + "name": "id", "type": { - "type": "reference", - "name": "ActorEventTypes" - } + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 1299 }, { - "id": 106, - "name": "listener", + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 67510, "kind": 32768, "kindString": "Parameter", - "flags": {}, + "name": "name", "type": { - "type": "reference", - "name": "Callable" - } + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 1299 + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 67511, + "kind": 32768, + "kindString": "Parameter", + "name": "alias", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 1299 + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 67512, + "kind": 32768, + "kindString": "Parameter", + "name": "configuration", + "type": { + "name": "Configuration | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "Configuration", + "target": "2", + "ref": { + "id": 66211, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/configuration.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 20 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/Configuration" + } + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 1299 + } + ], + "type": { + "name": "RequestQueueClient", + "type": "reference", + "target": "1195", + "ref": { + "id": 67404, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_request_queue_client.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 13 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/RequestQueueClient" + } + }, + "overwrites": { + "name": "StorageClient.create_rq_client", + "target": 1298, + "type": "reference", + "ref": { + "id": 67507, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 64 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/StorageClient#create_rq_client" } - ] + }, + "parentId": 1640 } - ] + ], + "overwrites": { + "name": "StorageClient.create_rq_client", + "target": 1298, + "type": "reference", + "ref": { + "id": 67507, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 64 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/StorageClient#create_rq_client" + } + }, + "permalink": "/python/api/class/FileSystemStorageClient#create_rq_client", + "parentId": 1624 }, { - "id": 107, - "name": "off", - "module": "actor", "kind": 2048, "kindString": "Method", - "flags": {}, + "children": [], "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Remove a listener, or all listeners, from an actor event.\\n\", {\"Arguments\": [{\"param\": \"event_name\", \"type\": \"ActorEventTypes\", \"desc\": \"The actor event for which to remove listeners.\"}, {\"param\": \"listener\", \"type\": \"Callable, optional\", \"desc\": \"The listener which is supposed to be removed. If not passed, all listeners of this event are removed.\"}]}]}" + "text": "Return statistics about rate limit errors encountered by the HTTP client in storage client." } ] }, - "children": [], + "decorations": [], + "flags": {}, "groups": [], + "id": 70410, + "module": "storage_clients._base._storage_client", + "name": "get_rate_limit_errors", + "parsedDocstring": { + "text": "Return statistics about rate limit errors encountered by the HTTP client in storage client." + }, "sources": [ { - "filename": "/tmp-old-source-v17/src/apify/actor.py", - "line": 750, "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/actor.py#L750" + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 74, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/_base/_storage_client.py#L74" } ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, "signatures": [ { - "id": 108, - "name": "off", - "modifiers": [], - "kind": 4096, - "kindString": "Call signature", - "flags": {}, "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Remove a listener, or all listeners, from an actor event.\\n\", {\"" + "text": "Return statistics about rate limit errors encountered by the HTTP client in storage client." } ] }, + "flags": {}, + "id": 67514, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "get_rate_limit_errors", + "parameters": [], "type": { + "name": "dict[int, int]", "type": "reference", - "name": "None" + "target": 0, + "ref": {} }, - "parameters": [ - { - "id": 109, - "name": "event_name", - "kind": 32768, - "kindString": "Parameter", - "flags": {}, - "type": { - "type": "reference", - "name": "ActorEventTypes" - } - }, + "inheritedFrom": { + "name": "StorageClient.get_rate_limit_errors", + "target": 1304, + "type": "reference", + "ref": { + "id": 67513, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 74 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/StorageClient#get_rate_limit_errors" + } + }, + "parentId": 4201 + } + ], + "inheritedFrom": { + "name": "StorageClient.get_rate_limit_errors", + "target": 1304, + "type": "reference", + "ref": { + "id": 67513, + "sources": [ { - "id": 110, - "name": "listener", - "kind": 32768, - "kindString": "Parameter", - "flags": {}, - "type": { - "type": "reference", - "name": "Callable | None" - }, - "defaultValue": "None" + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 74 } - ] + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/StorageClient#get_rate_limit_errors" } - ] - }, + }, + "permalink": "/python/api/class/FileSystemStorageClient#get_rate_limit_errors", + "parentId": 1624 + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "File system implementation of the storage client.\n\nThis storage client provides access to datasets, key-value stores, and request queues that persist data\nto the local file system. Each storage type is implemented with its own specific file system client\nthat stores data in a structured directory hierarchy.\n\nData is stored in JSON format in predictable file paths, making it easy to inspect and manipulate\nthe stored data outside of the Crawlee application if needed.\n\nAll data persists between program runs but is limited to access from the local machine\nwhere the files are stored.\n\nWarning: This storage client is not safe for concurrent access from multiple crawler processes.\nUse it only when running a single crawler process at a time." + } + ] + }, + "decorations": [ + { + "args": "('Storage clients')", + "name": "docs_group" + } + ], + "flags": {}, + "groups": [ + { + "children": [ + 67837, + 67843, + 67849, + 70410, + 67834 + ], + "title": "Methods" + } + ], + "id": 67833, + "module": "storage_clients._file_system._storage_client", + "name": "FileSystemStorageClient", + "parsedDocstring": { + "text": "File system implementation of the storage client.\n\nThis storage client provides access to datasets, key-value stores, and request queues that persist data\nto the local file system. Each storage type is implemented with its own specific file system client\nthat stores data in a structured directory hierarchy.\n\nData is stored in JSON format in predictable file paths, making it easy to inspect and manipulate\nthe stored data outside of the Crawlee application if needed.\n\nAll data persists between program runs but is limited to access from the local machine\nwhere the files are stored.\n\nWarning: This storage client is not safe for concurrent access from multiple crawler processes.\nUse it only when running a single crawler process at a time." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_file_system/_storage_client.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 20, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/_file_system/_storage_client.py#L20" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "extendedTypes": [ + { + "name": "StorageClient", + "target": "1282", + "type": "reference", + "ref": { + "id": 67491, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 19 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/StorageClient" + } + } + ], + "permalink": "/python/api/class/FileSystemStorageClient", + "categories": [] + }, + { + "kind": 128, + "kindString": "Class", + "children": [ { - "id": 111, - "name": "is_at_home", - "module": "actor", "kind": 2048, "kindString": "Method", - "flags": {}, + "children": [], "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Return `True` when the actor is running on the Apify platform, and `False` otherwise (for example when running locally).\"]}" + "text": "Initialize the SQL storage client.\n" } ] }, - "children": [], + "decorations": [], + "flags": {}, "groups": [], + "id": 72304, + "module": "storage_clients._sql._storage_client", + "name": "__init__", + "parsedDocstring": { + "text": "Initialize the SQL storage client.\n", + "args": { + "connection_string": "Database connection string (e.g., \"sqlite+aiosqlite:///crawlee.db\").\nIf not provided, defaults to SQLite database in the storage directory.", + "engine": "Pre-configured AsyncEngine instance. If provided, connection_string is ignored." + } + }, "sources": [ { - "filename": "/tmp-old-source-v17/src/apify/actor.py", - "line": 765, "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/actor.py#L765" + "fileName": "/src/crawlee/storage_clients/_sql/_storage_client.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 57, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/_sql/_storage_client.py#L57" } ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, "signatures": [ { - "id": 112, - "name": "is_at_home", - "modifiers": [], - "kind": 4096, - "kindString": "Call signature", - "flags": {}, "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Return `True` when the actor is running on the Apify platform, and `False` otherwise (for example when running locally).\"]}" + "text": "Initialize the SQL storage client.\n" } ] }, + "flags": {}, + "id": 72305, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "__init__", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Database connection string (e.g., \"sqlite+aiosqlite:///crawlee.db\").\nIf not provided, defaults to SQLite database in the storage directory." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 72306, + "kind": 32768, + "kindString": "Parameter", + "name": "connection_string", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 1895 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Pre-configured AsyncEngine instance. If provided, connection_string is ignored." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 72307, + "kind": 32768, + "kindString": "Parameter", + "name": "engine", + "type": { + "name": "AsyncEngine | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "AsyncEngine", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 1895 + } + ], "type": { - "type": "intrinsic", - "name": "bool" + "name": "None", + "type": "literal", + "value": {} }, - "parameters": [] + "parentId": 1894 } - ] + ], + "permalink": "/python/api/class/SqlStorageClient#__init__", + "parentId": 1893 }, { - "id": 113, - "name": "get_env", - "module": "actor", "kind": 2048, "kindString": "Method", - "flags": {}, + "children": [], "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Return a dictionary with information parsed from all the `APIFY_XXX` environment variables.\\n\\nFor a list of all the environment variables,\\nsee the [Actor documentation](https://docs.apify.com/actors/development/environment-variables).\\nIf some variables are not defined or are invalid, the corresponding value in the resulting dictionary will be None.\"]}" + "text": "Async context manager entry." } ] }, - "children": [], + "decorations": [], + "flags": {}, "groups": [], + "id": 72308, + "module": "storage_clients._sql._storage_client", + "name": "__aenter__", + "parsedDocstring": { + "text": "Async context manager entry." + }, "sources": [ { - "filename": "/tmp-old-source-v17/src/apify/actor.py", - "line": 773, "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/actor.py#L773" + "fileName": "/src/crawlee/storage_clients/_sql/_storage_client.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 88, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/_sql/_storage_client.py#L88" } ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, "signatures": [ { - "id": 114, - "name": "get_env", - "modifiers": [], - "kind": 4096, - "kindString": "Call signature", - "flags": {}, "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Return a dictionary with information parsed from all the `APIFY_XXX` environment variables.\\n\\nFor a list of all the environment variables,\\nsee the [Actor documentation](https://docs.apify.com/actors/development/environment-variables).\\nIf some variables are not defined or are invalid, the corresponding value in the resulting dictionary will be None.\"]}" + "text": "Async context manager entry." } ] }, + "flags": {}, + "id": 72309, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "__aenter__", + "parameters": [], "type": { - "type": "intrinsic", - "name": "dict" + "name": "SqlStorageClient", + "type": "reference", + "target": "1893", + "ref": { + "id": 72303, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_sql/_storage_client.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 35 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/SqlStorageClient" + } }, - "parameters": [] + "parentId": 1898 } - ] + ], + "permalink": "/python/api/class/SqlStorageClient#__aenter__", + "parentId": 1893 }, { - "id": 115, - "name": "start", - "module": "actor", "kind": 2048, "kindString": "Method", - "flags": {}, + "children": [], "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Run an actor on the Apify platform.\\n\\nUnlike `Actor.call`, this method just starts the run without waiting for finish.\\n\", {\"Arguments\": [{\"param\": \"actor_id\", \"type\": \"str\", \"desc\": \"The ID of the actor to be run.\"}, {\"param\": \"run_input\", \"type\": \"Any, optional\", \"desc\": \"The input to pass to the actor run.\"}, {\"param\": \"token\", \"type\": \"str, optional\", \"desc\": \"The Apify API token to use for this request (defaults to the `APIFY_TOKEN` environment variable).\"}, {\"param\": \"content_type\", \"type\": \"str, optional\", \"desc\": \"The content type of the input.\"}, {\"param\": \"build\", \"type\": \"str, optional\", \"desc\": \"Specifies the actor build to run. It can be either a build tag or build number.\\nBy default, the run uses the build specified in the default run configuration for the actor (typically latest).\"}, {\"param\": \"memory_mbytes\", \"type\": \"int, optional\", \"desc\": \"Memory limit for the run, in megabytes.\\nBy default, the run uses a memory limit specified in the default run configuration for the actor.\"}, {\"param\": \"timeout_secs\", \"type\": \"int, optional\", \"desc\": \"Optional timeout for the run, in seconds.\\nBy default, the run uses timeout specified in the default run configuration for the actor.\"}, {\"param\": \"wait_for_finish\", \"type\": \"int, optional\", \"desc\": \"The maximum number of seconds the server waits for the run to finish.\\nBy default, it is 0, the maximum value is 300.\"}, {\"param\": \"webhooks\", \"type\": \"list of dict, optional\", \"desc\": \"Optional ad-hoc webhooks (https://docs.apify.com/webhooks/ad-hoc-webhooks)\\nassociated with the actor run which can be used to receive a notification,\\ne.g. when the actor finished or failed.\\nIf you already have a webhook set up for the actor or task, you do not have to add it again here.\\nEach webhook is represented by a dictionary containing these items:\\n* ``event_types``: list of ``WebhookEventType`` values which trigger the webhook\\n* ``request_url``: URL to which to send the webhook HTTP request\\n* ``payload_template`` (optional): Optional template for the request payload\\n\"}]}, {\"Returns\": [{\"param\": \"dict\", \"desc\": \"Info about the started actor run\"}]}]}" + "text": "Async context manager exit." } ] }, - "children": [], + "decorations": [], + "flags": {}, "groups": [], + "id": 72310, + "module": "storage_clients._sql._storage_client", + "name": "__aexit__", + "parsedDocstring": { + "text": "Async context manager exit." + }, "sources": [ { - "filename": "/tmp-old-source-v17/src/apify/actor.py", - "line": 788, "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/actor.py#L788" + "fileName": "/src/crawlee/storage_clients/_sql/_storage_client.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 92, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/_sql/_storage_client.py#L92" } ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, "signatures": [ { - "id": 116, - "name": "start", - "modifiers": [ - "async" - ], - "kind": 4096, - "kindString": "Call signature", - "flags": {}, "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Run an actor on the Apify platform.\\n\\nUnlike `Actor.call`, this method just starts the run without waiting for finish.\\n\", {\"" + "text": "Async context manager exit." } ] }, - "type": { - "type": "intrinsic", - "name": "dict" - }, + "flags": {}, + "id": 72311, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "__aexit__", "parameters": [ { - "id": 117, - "name": "actor_id", - "kind": 32768, - "kindString": "Parameter", - "flags": {}, - "type": { - "type": "intrinsic", - "name": "str" - } - }, - { - "id": 118, - "name": "run_input", - "kind": 32768, - "kindString": "Parameter", - "flags": {}, - "type": { - "type": "reference", - "name": "Any" - }, - "defaultValue": "None" - }, - { - "id": 119, - "name": "token", - "kind": 32768, - "kindString": "Parameter", - "flags": { - "keyword-only": "true" - }, - "type": { - "type": "reference", - "name": "str | None" - }, - "defaultValue": "None" - }, - { - "id": 120, - "name": "content_type", - "kind": 32768, - "kindString": "Parameter", "flags": { - "keyword-only": "true" + "isOptional": false, + "keyword-only": false }, - "type": { - "type": "reference", - "name": "str | None" - }, - "defaultValue": "None" - }, - { - "id": 121, - "name": "build", + "id": 72312, "kind": 32768, "kindString": "Parameter", - "flags": { - "keyword-only": "true" - }, + "name": "exc_type", "type": { - "type": "reference", - "name": "str | None" + "name": "type[BaseException] | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "type", + "typeArguments": [ + { + "type": "reference", + "name": "BaseException", + "target": 0, + "ref": {} + } + ], + "target": "3422", + "ref": { + "id": 73832, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_utils/sitemap.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 62 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api/class/SitemapSource#type" + } + }, + { + "type": "literal", + "value": {} + } + ] }, - "defaultValue": "None" + "parentId": 1901 }, { - "id": 122, - "name": "memory_mbytes", - "kind": 32768, - "kindString": "Parameter", "flags": { - "keyword-only": "true" + "isOptional": false, + "keyword-only": false }, - "type": { - "type": "reference", - "name": "int | None" - }, - "defaultValue": "None" - }, - { - "id": 123, - "name": "timeout_secs", + "id": 72313, "kind": 32768, "kindString": "Parameter", - "flags": { - "keyword-only": "true" - }, + "name": "exc_value", "type": { - "type": "reference", - "name": "int | None" + "name": "BaseException | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "BaseException", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] }, - "defaultValue": "None" + "parentId": 1901 }, { - "id": 124, - "name": "wait_for_finish", - "kind": 32768, - "kindString": "Parameter", "flags": { - "keyword-only": "true" - }, - "type": { - "type": "reference", - "name": "int | None" + "isOptional": false, + "keyword-only": false }, - "defaultValue": "None" - }, - { - "id": 125, - "name": "webhooks", + "id": 72314, "kind": 32768, "kindString": "Parameter", - "flags": { - "keyword-only": "true" - }, + "name": "exc_traceback", "type": { - "type": "reference", - "name": "list[dict] | None" + "name": "TracebackType | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "TracebackType", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] }, - "defaultValue": "None" + "parentId": 1901 } - ] + ], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "parentId": 1900 } - ] + ], + "permalink": "/python/api/class/SqlStorageClient#__aexit__", + "parentId": 1893 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Get the SQLAlchemy AsyncEngine instance." + } + ] + }, + "decorations": [ + { + "name": "property" + } + ], + "flags": {}, + "groups": [], + "id": 72315, + "module": "storage_clients._sql._storage_client", + "name": "engine", + "parsedDocstring": { + "text": "Get the SQLAlchemy AsyncEngine instance." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_sql/_storage_client.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 102, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/_sql/_storage_client.py#L102" + } + ], + "type": { + "name": "AsyncEngine", + "type": "reference", + "target": 0, + "ref": {} + }, + "permalink": "/python/api/class/SqlStorageClient#engine", + "parentId": 1893 }, { - "id": 126, - "name": "abort", - "module": "actor", "kind": 2048, "kindString": "Method", - "flags": {}, + "children": [], "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Abort given actor run on the Apify platform using the current user account (determined by the `APIFY_TOKEN` environment variable).\\n\", {\"Arguments\": [{\"param\": \"run_id\", \"type\": \"str\", \"desc\": \"The ID of the actor run to be aborted.\"}, {\"param\": \"token\", \"type\": \"str, optional\", \"desc\": \"The Apify API token to use for this request (defaults to the `APIFY_TOKEN` environment variable).\"}, {\"param\": \"gracefully\", \"type\": \"bool, optional\", \"desc\": \"If True, the actor run will abort gracefully.\\nIt will send ``aborting`` and ``persistStates`` events into the run and force-stop the run after 30 seconds.\\nIt is helpful in cases where you plan to resurrect the run later.\\n\"}]}, {\"Returns\": [{\"param\": \"dict\", \"desc\": \"Info about the aborted actor run\"}]}]}" + "text": "Get the database dialect name." } ] }, - "children": [], + "decorations": [], + "flags": {}, "groups": [], + "id": 72316, + "module": "storage_clients._sql._storage_client", + "name": "get_dialect_name", + "parsedDocstring": { + "text": "Get the database dialect name." + }, "sources": [ { - "filename": "/tmp-old-source-v17/src/apify/actor.py", - "line": 870, "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/actor.py#L870" + "fileName": "/src/crawlee/storage_clients/_sql/_storage_client.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 108, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/_sql/_storage_client.py#L108" } ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, "signatures": [ { - "id": 127, - "name": "abort", - "modifiers": [ - "async" - ], - "kind": 4096, - "kindString": "Call signature", - "flags": {}, "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Abort given actor run on the Apify platform using the current user account (determined by the `APIFY_TOKEN` environment variable).\\n\", {\"" + "text": "Get the database dialect name." } ] }, + "flags": {}, + "id": 72317, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "get_dialect_name", + "parameters": [], "type": { - "type": "intrinsic", - "name": "dict" - }, - "parameters": [ - { - "id": 128, - "name": "run_id", - "kind": 32768, - "kindString": "Parameter", - "flags": {}, - "type": { - "type": "intrinsic", - "name": "str" - } - }, - { - "id": 129, - "name": "token", - "kind": 32768, - "kindString": "Parameter", - "flags": { - "keyword-only": "true" - }, - "type": { - "type": "reference", - "name": "str | None" - }, - "defaultValue": "None" - }, - { - "id": 130, - "name": "gracefully", - "kind": 32768, - "kindString": "Parameter", - "flags": { - "keyword-only": "true" - }, - "type": { + "name": "str | None", + "type": "union", + "types": [ + { "type": "reference", - "name": "bool | None" + "name": "str", + "target": 0, + "ref": {} }, - "defaultValue": "None" - } - ] + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 1906 } - ] + ], + "permalink": "/python/api/class/SqlStorageClient#get_dialect_name", + "parentId": 1893 }, { - "id": 131, - "name": "call", - "module": "actor", "kind": 2048, "kindString": "Method", - "flags": {}, + "children": [], "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Start an actor on the Apify Platform and wait for it to finish before returning.\\n\\nIt waits indefinitely, unless the wait_secs argument is provided.\\n\", {\"Arguments\": [{\"param\": \"actor_id\", \"type\": \"str\", \"desc\": \"The ID of the actor to be run.\"}, {\"param\": \"run_input\", \"type\": \"Any, optional\", \"desc\": \"The input to pass to the actor run.\"}, {\"param\": \"token\", \"type\": \"str, optional\", \"desc\": \"The Apify API token to use for this request (defaults to the `APIFY_TOKEN` environment variable).\"}, {\"param\": \"content_type\", \"type\": \"str, optional\", \"desc\": \"The content type of the input.\"}, {\"param\": \"build\", \"type\": \"str, optional\", \"desc\": \"Specifies the actor build to run. It can be either a build tag or build number.\\nBy default, the run uses the build specified in the default run configuration for the actor (typically latest).\"}, {\"param\": \"memory_mbytes\", \"type\": \"int, optional\", \"desc\": \"Memory limit for the run, in megabytes.\\nBy default, the run uses a memory limit specified in the default run configuration for the actor.\"}, {\"param\": \"timeout_secs\", \"type\": \"int, optional\", \"desc\": \"Optional timeout for the run, in seconds.\\nBy default, the run uses timeout specified in the default run configuration for the actor.\"}, {\"param\": \"webhooks\", \"type\": \"list, optional\", \"desc\": \"Optional webhooks (https://docs.apify.com/webhooks) associated with the actor run,\\nwhich can be used to receive a notification, e.g. when the actor finished or failed.\\nIf you already have a webhook set up for the actor, you do not have to add it again here.\"}, {\"param\": \"wait_secs\", \"type\": \"int, optional\", \"desc\": \"The maximum number of seconds the server waits for the run to finish. If not provided, waits indefinitely.\\n\"}]}, {\"Returns\": [{\"param\": \"dict\", \"desc\": \"Info about the started actor run\"}]}]}" + "text": "Initialize the database schema.\n\nThis method creates all necessary tables if they don't exist.\nShould be called before using the storage client." } ] }, - "children": [], + "decorations": [], + "flags": {}, "groups": [], + "id": 72318, + "module": "storage_clients._sql._storage_client", + "name": "initialize", + "parsedDocstring": { + "text": "Initialize the database schema.\n\nThis method creates all necessary tables if they don't exist.\nShould be called before using the storage client." + }, "sources": [ { - "filename": "/tmp-old-source-v17/src/apify/actor.py", - "line": 913, "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/actor.py#L913" + "fileName": "/src/crawlee/storage_clients/_sql/_storage_client.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 112, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/_sql/_storage_client.py#L112" } ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, "signatures": [ { - "id": 132, - "name": "call", - "modifiers": [ - "async" - ], - "kind": 4096, - "kindString": "Call signature", - "flags": {}, "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Start an actor on the Apify Platform and wait for it to finish before returning.\\n\\nIt waits indefinitely, unless the wait_secs argument is provided.\\n\", {\"" + "text": "Initialize the database schema.\n\nThis method creates all necessary tables if they don't exist.\nShould be called before using the storage client." } ] }, - "type": { - "type": "reference", - "name": "dict | None" - }, + "flags": {}, + "id": 72319, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "initialize", "parameters": [ { - "id": 133, - "name": "actor_id", - "kind": 32768, - "kindString": "Parameter", - "flags": {}, - "type": { - "type": "intrinsic", - "name": "str" - } - }, - { - "id": 134, - "name": "run_input", - "kind": 32768, - "kindString": "Parameter", - "flags": {}, - "type": { - "type": "reference", - "name": "Any" - }, - "defaultValue": "None" - }, - { - "id": 135, - "name": "token", - "kind": 32768, - "kindString": "Parameter", - "flags": { - "keyword-only": "true" - }, - "type": { - "type": "reference", - "name": "str | None" - }, - "defaultValue": "None" - }, - { - "id": 136, - "name": "content_type", - "kind": 32768, - "kindString": "Parameter", - "flags": { - "keyword-only": "true" - }, - "type": { - "type": "reference", - "name": "str | None" - }, - "defaultValue": "None" - }, - { - "id": 137, - "name": "build", - "kind": 32768, - "kindString": "Parameter", - "flags": { - "keyword-only": "true" - }, - "type": { - "type": "reference", - "name": "str | None" - }, - "defaultValue": "None" - }, - { - "id": 138, - "name": "memory_mbytes", - "kind": 32768, - "kindString": "Parameter", - "flags": { - "keyword-only": "true" - }, - "type": { - "type": "reference", - "name": "int | None" - }, - "defaultValue": "None" - }, - { - "id": 139, - "name": "timeout_secs", - "kind": 32768, - "kindString": "Parameter", - "flags": { - "keyword-only": "true" - }, - "type": { - "type": "reference", - "name": "int | None" - }, - "defaultValue": "None" - }, - { - "id": 140, - "name": "webhooks", - "kind": 32768, - "kindString": "Parameter", "flags": { - "keyword-only": "true" + "isOptional": false, + "keyword-only": false }, - "type": { - "type": "reference", - "name": "list[dict] | None" - }, - "defaultValue": "None" - }, - { - "id": 141, - "name": "wait_secs", + "id": 72320, "kind": 32768, "kindString": "Parameter", - "flags": { - "keyword-only": "true" - }, + "name": "configuration", "type": { + "name": "Configuration", "type": "reference", - "name": "int | None" - }, - "defaultValue": "None" + "target": "2", + "ref": { + "id": 70412, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/configuration.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 20 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/Configuration" + } + }, + "parentId": 1909 } - ] + ], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "parentId": 1908 } - ] + ], + "permalink": "/python/api/class/SqlStorageClient#initialize", + "parentId": 1893 }, { - "id": 142, - "name": "call_task", - "module": "actor", "kind": 2048, "kindString": "Method", - "flags": {}, + "children": [], "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Start an actor task on the Apify Platform and wait for it to finish before returning.\\n\\nIt waits indefinitely, unless the wait_secs argument is provided.\\n\\nNote that an actor task is a saved input configuration and options for an actor.\\nIf you want to run an actor directly rather than an actor task, please use the `Actor.call`\\n\", {\"Arguments\": [{\"param\": \"task_id\", \"type\": \"str\", \"desc\": \"The ID of the actor to be run.\"}, {\"param\": \"task_input\", \"type\": \"Any, optional\", \"desc\": \"Overrides the input to pass to the actor run.\"}, {\"param\": \"token\", \"type\": \"str, optional\", \"desc\": \"The Apify API token to use for this request (defaults to the `APIFY_TOKEN` environment variable).\"}, {\"param\": \"content_type\", \"type\": \"str, optional\", \"desc\": \"The content type of the input.\"}, {\"param\": \"build\", \"type\": \"str, optional\", \"desc\": \"Specifies the actor build to run. It can be either a build tag or build number.\\nBy default, the run uses the build specified in the default run configuration for the actor (typically latest).\"}, {\"param\": \"memory_mbytes\", \"type\": \"int, optional\", \"desc\": \"Memory limit for the run, in megabytes.\\nBy default, the run uses a memory limit specified in the default run configuration for the actor.\"}, {\"param\": \"timeout_secs\", \"type\": \"int, optional\", \"desc\": \"Optional timeout for the run, in seconds.\\nBy default, the run uses timeout specified in the default run configuration for the actor.\"}, {\"param\": \"webhooks\", \"type\": \"list, optional\", \"desc\": \"Optional webhooks (https://docs.apify.com/webhooks) associated with the actor run,\\nwhich can be used to receive a notification, e.g. when the actor finished or failed.\\nIf you already have a webhook set up for the actor, you do not have to add it again here.\"}, {\"param\": \"wait_secs\", \"type\": \"int, optional\", \"desc\": \"The maximum number of seconds the server waits for the run to finish. If not provided, waits indefinitely.\\n\"}]}, {\"Returns\": [{\"param\": \"dict\", \"desc\": \"Info about the started actor run\"}]}]}" + "text": "Close the database connection pool." } ] }, - "children": [], + "decorations": [], + "flags": {}, "groups": [], + "id": 72321, + "module": "storage_clients._sql._storage_client", + "name": "close", + "parsedDocstring": { + "text": "Close the database connection pool." + }, "sources": [ { - "filename": "/tmp-old-source-v17/src/apify/actor.py", - "line": 989, "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/actor.py#L989" + "fileName": "/src/crawlee/storage_clients/_sql/_storage_client.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 157, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/_sql/_storage_client.py#L157" } ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, "signatures": [ { - "id": 143, - "name": "call_task", - "modifiers": [ - "async" - ], - "kind": 4096, - "kindString": "Call signature", - "flags": {}, "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Start an actor task on the Apify Platform and wait for it to finish before returning.\\n\\nIt waits indefinitely, unless the wait_secs argument is provided.\\n\\nNote that an actor task is a saved input configuration and options for an actor.\\nIf you want to run an actor directly rather than an actor task, please use the `Actor.call`\\n\", {\"" + "text": "Close the database connection pool." } ] }, + "flags": {}, + "id": 72322, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "close", + "parameters": [], "type": { - "type": "reference", - "name": "dict | None" + "name": "None", + "type": "literal", + "value": {} }, - "parameters": [ - { - "id": 144, - "name": "task_id", - "kind": 32768, - "kindString": "Parameter", - "flags": {}, - "type": { - "type": "intrinsic", - "name": "str" + "parentId": 1911 + } + ], + "permalink": "/python/api/class/SqlStorageClient#close", + "parentId": 1893 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Create a new database session.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 72323, + "module": "storage_clients._sql._storage_client", + "name": "create_session", + "parsedDocstring": { + "text": "Create a new database session.\n", + "returns": "A new AsyncSession instance." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_sql/_storage_client.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 167, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/_sql/_storage_client.py#L167" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "A new AsyncSession instance." + } + ], + "tag": "@returns" } - }, - { - "id": 145, - "name": "task_input", - "kind": 32768, - "kindString": "Parameter", - "flags": {}, - "type": { - "type": "reference", - "name": "dict | None" - }, - "defaultValue": "None" - }, - { - "id": 146, - "name": "build", - "kind": 32768, - "kindString": "Parameter", - "flags": { - "keyword-only": "true" - }, - "type": { - "type": "reference", - "name": "str | None" - }, - "defaultValue": "None" - }, - { - "id": 147, - "name": "memory_mbytes", - "kind": 32768, - "kindString": "Parameter", - "flags": { - "keyword-only": "true" - }, - "type": { - "type": "reference", - "name": "int | None" - }, - "defaultValue": "None" - }, - { - "id": 148, - "name": "timeout_secs", - "kind": 32768, - "kindString": "Parameter", - "flags": { - "keyword-only": "true" - }, - "type": { - "type": "reference", - "name": "int | None" - }, - "defaultValue": "None" - }, - { - "id": 149, - "name": "webhooks", - "kind": 32768, - "kindString": "Parameter", - "flags": { - "keyword-only": "true" - }, - "type": { - "type": "reference", - "name": "list[dict] | None" - }, - "defaultValue": "None" - }, - { - "id": 150, - "name": "wait_secs", - "kind": 32768, - "kindString": "Parameter", - "flags": { - "keyword-only": "true" - }, - "type": { - "type": "reference", - "name": "int | None" - }, - "defaultValue": "None" - }, - { - "id": 151, - "name": "token", - "kind": 32768, - "kindString": "Parameter", - "flags": { - "keyword-only": "true" - }, - "type": { - "type": "reference", - "name": "str | None" - }, - "defaultValue": "None" - } - ] + ], + "summary": [ + { + "kind": "text", + "text": "Create a new database session.\n" + } + ] + }, + "flags": {}, + "id": 72324, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "create_session", + "parameters": [], + "type": { + "name": "AsyncSession", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 1913 } - ] + ], + "permalink": "/python/api/class/SqlStorageClient#create_session", + "parentId": 1893 }, { - "id": 152, - "name": "metamorph", - "module": "actor", "kind": 2048, "kindString": "Method", - "flags": {}, + "children": [], "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Transform this actor run to an actor run of a different actor.\\n\\nThe platform stops the current actor container and starts a new container with the new actor instead.\\nAll the default storages are preserved,\\nand the new input is stored under the `INPUT-METAMORPH-1` key in the same default key-value store.\\n\", {\"Arguments\": [{\"param\": \"target_actor_id\", \"type\": \"str\", \"desc\": \"ID of the target actor that the run should be transformed into\"}, {\"param\": \"run_input\", \"type\": \"Any, optional\", \"desc\": \"The input to pass to the new run.\"}, {\"param\": \"target_actor_build\", \"type\": \"str, optional\", \"desc\": \"The build of the target actor. It can be either a build tag or build number.\\nBy default, the run uses the build specified in the default run configuration for the target actor (typically the latest build).\"}, {\"param\": \"content_type\", \"type\": \"str, optional\", \"desc\": \"The content type of the input.\"}, {\"param\": \"custom_after_sleep_millis\", \"type\": \"int, optional\", \"desc\": \"How long to sleep for after the metamorph, to wait for the container to be stopped.\\n\"}]}, {\"Returns\": [{\"param\": \"dict\", \"desc\": \"The actor run data.\"}]}]}" + "text": "Create a dataset client." } ] }, - "children": [], + "decorations": [ + { + "name": "abstractmethod" + } + ], + "flags": {}, "groups": [], + "id": 72325, + "module": "storage_clients._base._storage_client", + "name": "create_dataset_client", + "parsedDocstring": { + "text": "Create a dataset client." + }, "sources": [ { - "filename": "/tmp-old-source-v17/src/apify/actor.py", - "line": 1064, "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/actor.py#L1064" + "fileName": "/src/crawlee/storage_clients/_sql/_storage_client.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 178, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/_sql/_storage_client.py#L178" } ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, "signatures": [ { - "id": 153, - "name": "metamorph", - "modifiers": [ - "async" - ], - "kind": 4096, - "kindString": "Call signature", - "flags": {}, "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Transform this actor run to an actor run of a different actor.\\n\\nThe platform stops the current actor container and starts a new container with the new actor instead.\\nAll the default storages are preserved,\\nand the new input is stored under the `INPUT-METAMORPH-1` key in the same default key-value store.\\n\", {\"" + "text": "Create a dataset client." } ] }, - "type": { - "type": "reference", - "name": "None" - }, + "flags": {}, + "id": 71697, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "create_dataset_client", "parameters": [ { - "id": 154, - "name": "target_actor_id", + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 71698, "kind": 32768, "kindString": "Parameter", - "flags": {}, + "name": "id", "type": { - "type": "intrinsic", - "name": "str" - } + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 1287 }, { - "id": 155, - "name": "run_input", + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 71699, "kind": 32768, "kindString": "Parameter", - "flags": {}, + "name": "name", "type": { - "type": "reference", - "name": "Any" + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] }, - "defaultValue": "None" + "parentId": 1287 }, { - "id": 156, - "name": "target_actor_build", - "kind": 32768, - "kindString": "Parameter", + "defaultValue": "None", "flags": { - "keyword-only": "true" - }, - "type": { - "type": "reference", - "name": "str | None" + "isOptional": true, + "keyword-only": true }, - "defaultValue": "None" - }, - { - "id": 157, - "name": "content_type", + "id": 71700, "kind": 32768, "kindString": "Parameter", - "flags": { - "keyword-only": "true" - }, + "name": "alias", "type": { - "type": "reference", - "name": "str | None" + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] }, - "defaultValue": "None" + "parentId": 1287 }, { - "id": 158, - "name": "custom_after_sleep_millis", - "kind": 32768, - "kindString": "Parameter", + "defaultValue": "None", "flags": { - "keyword-only": "true" + "isOptional": true, + "keyword-only": true }, + "id": 71701, + "kind": 32768, + "kindString": "Parameter", + "name": "configuration", "type": { - "type": "reference", - "name": "int | None" + "name": "Configuration | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "Configuration", + "target": "2", + "ref": { + "id": 70412, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/configuration.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 20 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/Configuration" + } + }, + { + "type": "literal", + "value": {} + } + ] }, - "defaultValue": "None" + "parentId": 1287 + } + ], + "type": { + "name": "DatasetClient", + "type": "reference", + "target": "1220", + "ref": { + "id": 71630, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_dataset_client.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 13 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/DatasetClient" + } + }, + "overwrites": { + "name": "StorageClient.create_dataset_client", + "target": 1286, + "type": "reference", + "ref": { + "id": 71696, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 42 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/StorageClient#create_dataset_client" } - ] + }, + "parentId": 1915 } - ] + ], + "overwrites": { + "name": "StorageClient.create_dataset_client", + "target": 1286, + "type": "reference", + "ref": { + "id": 71696, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 42 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/StorageClient#create_dataset_client" + } + }, + "permalink": "/python/api/class/SqlStorageClient#create_dataset_client", + "parentId": 1893 }, { - "id": 159, - "name": "reboot", - "module": "actor", "kind": 2048, "kindString": "Method", - "flags": {}, + "children": [], "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Internally reboot this actor.\\n\\nThe system stops the current container and starts a new one, with the same run ID and default storages.\\n\", {\"Arguments\": [{\"param\": \"event_listeners_timeout_secs\", \"type\": \"int, optional\", \"desc\": \"How long should the actor wait for actor event listeners to finish before exiting\"}, {\"param\": \"custom_after_sleep_millis\", \"type\": \"int, optional\", \"desc\": \"How long to sleep for after the reboot, to wait for the container to be stopped.\"}]}]}" + "text": "Create a key-value store client." } ] }, - "children": [], + "decorations": [ + { + "name": "abstractmethod" + } + ], + "flags": {}, "groups": [], + "id": 72331, + "module": "storage_clients._base._storage_client", + "name": "create_kvs_client", + "parsedDocstring": { + "text": "Create a key-value store client." + }, "sources": [ { - "filename": "/tmp-old-source-v17/src/apify/actor.py", - "line": 1130, "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/actor.py#L1130" + "fileName": "/src/crawlee/storage_clients/_sql/_storage_client.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 200, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/_sql/_storage_client.py#L200" } ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, "signatures": [ { - "id": 160, - "name": "reboot", - "modifiers": [ - "async" - ], - "kind": 4096, - "kindString": "Call signature", - "flags": {}, "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Internally reboot this actor.\\n\\nThe system stops the current container and starts a new one, with the same run ID and default storages.\\n\", {\"" + "text": "Create a key-value store client." } ] }, - "type": { - "type": "reference", - "name": "None" - }, + "flags": {}, + "id": 71703, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "create_kvs_client", "parameters": [ { - "id": 161, - "name": "event_listeners_timeout_secs", + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 71704, "kind": 32768, "kindString": "Parameter", + "name": "id", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 1293 + }, + { + "defaultValue": "None", "flags": { - "keyword-only": "true" + "isOptional": true, + "keyword-only": true }, + "id": 71705, + "kind": 32768, + "kindString": "Parameter", + "name": "name", "type": { - "type": "reference", - "name": "int | None" + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] }, - "defaultValue": "EVENT_LISTENERS_TIMEOUT_SECS" + "parentId": 1293 }, { - "id": 162, - "name": "custom_after_sleep_millis", + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 71706, "kind": 32768, "kindString": "Parameter", + "name": "alias", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 1293 + }, + { + "defaultValue": "None", "flags": { - "keyword-only": "true" + "isOptional": true, + "keyword-only": true }, + "id": 71707, + "kind": 32768, + "kindString": "Parameter", + "name": "configuration", "type": { - "type": "reference", - "name": "int | None" + "name": "Configuration | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "Configuration", + "target": "2", + "ref": { + "id": 70412, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/configuration.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 20 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/Configuration" + } + }, + { + "type": "literal", + "value": {} + } + ] }, - "defaultValue": "None" + "parentId": 1293 + } + ], + "type": { + "name": "KeyValueStoreClient", + "type": "reference", + "target": "1254", + "ref": { + "id": 71664, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_key_value_store_client.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 14 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/KeyValueStoreClient" + } + }, + "overwrites": { + "name": "StorageClient.create_kvs_client", + "target": 1292, + "type": "reference", + "ref": { + "id": 71702, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 53 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/StorageClient#create_kvs_client" } - ] + }, + "parentId": 1921 } - ] + ], + "overwrites": { + "name": "StorageClient.create_kvs_client", + "target": 1292, + "type": "reference", + "ref": { + "id": 71702, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 53 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/StorageClient#create_kvs_client" + } + }, + "permalink": "/python/api/class/SqlStorageClient#create_kvs_client", + "parentId": 1893 }, { - "id": 163, - "name": "add_webhook", - "module": "actor", "kind": 2048, "kindString": "Method", - "flags": {}, + "children": [], "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Create an ad-hoc webhook for the current actor run.\\n\\nThis webhook lets you receive a notification when the actor run finished or failed.\\n\\nNote that webhooks are only supported for actors running on the Apify platform.\\nWhen running the actor locally, the function will print a warning and have no effect.\\n\\nFor more information about Apify actor webhooks, please see the [documentation](https://docs.apify.com/webhooks).\\n\", {\"Arguments\": [{\"param\": \"event_types\", \"type\": \"list of WebhookEventType\", \"desc\": \"List of event types that should trigger the webhook. At least one is required.\"}, {\"param\": \"request_url\", \"type\": \"str\", \"desc\": \"URL that will be invoked once the webhook is triggered.\"}, {\"param\": \"payload_template\", \"type\": \"str, optional\", \"desc\": \"Specification of the payload that will be sent to request_url\"}, {\"param\": \"ignore_ssl_errors\", \"type\": \"bool, optional\", \"desc\": \"Whether the webhook should ignore SSL errors returned by request_url\"}, {\"param\": \"do_not_retry\", \"type\": \"bool, optional\", \"desc\": \"Whether the webhook should retry sending the payload to request_url upon\\nfailure.\"}, {\"param\": \"idempotency_key\", \"type\": \"str, optional\", \"desc\": \"A unique identifier of a webhook. You can use it to ensure that you won't\\ncreate the same webhook multiple times.\\n\"}]}, {\"Returns\": [{\"param\": \"dict\", \"desc\": \"The created webhook\"}]}]}" + "text": "Create a request queue client." } ] }, - "children": [], + "decorations": [ + { + "name": "abstractmethod" + } + ], + "flags": {}, "groups": [], + "id": 72337, + "module": "storage_clients._base._storage_client", + "name": "create_rq_client", + "parsedDocstring": { + "text": "Create a request queue client." + }, "sources": [ { - "filename": "/tmp-old-source-v17/src/apify/actor.py", - "line": 1178, "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/actor.py#L1178" + "fileName": "/src/crawlee/storage_clients/_sql/_storage_client.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 222, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/_sql/_storage_client.py#L222" } ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, "signatures": [ { - "id": 164, - "name": "add_webhook", - "modifiers": [ - "async" - ], - "kind": 4096, - "kindString": "Call signature", - "flags": {}, "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Create an ad-hoc webhook for the current actor run.\\n\\nThis webhook lets you receive a notification when the actor run finished or failed.\\n\\nNote that webhooks are only supported for actors running on the Apify platform.\\nWhen running the actor locally, the function will print a warning and have no effect.\\n\\nFor more information about Apify actor webhooks, please see the [documentation](https://docs.apify.com/webhooks).\\n\", {\"" + "text": "Create a request queue client." } ] }, - "type": { - "type": "intrinsic", - "name": "dict" - }, + "flags": {}, + "id": 71709, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "create_rq_client", "parameters": [ { - "id": 165, - "name": "event_types", - "kind": 32768, - "kindString": "Parameter", + "defaultValue": "None", "flags": { - "keyword-only": "true" + "isOptional": true, + "keyword-only": true }, - "type": { - "type": "reference", - "name": "list[WebhookEventType]" - } - }, - { - "id": 166, - "name": "request_url", + "id": 71710, "kind": 32768, "kindString": "Parameter", - "flags": { - "keyword-only": "true" - }, + "name": "id", "type": { - "type": "intrinsic", - "name": "str" - } + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 1299 }, { - "id": 167, - "name": "payload_template", - "kind": 32768, - "kindString": "Parameter", + "defaultValue": "None", "flags": { - "keyword-only": "true" - }, - "type": { - "type": "reference", - "name": "str | None" + "isOptional": true, + "keyword-only": true }, - "defaultValue": "None" - }, - { - "id": 168, - "name": "ignore_ssl_errors", + "id": 71711, "kind": 32768, "kindString": "Parameter", - "flags": { - "keyword-only": "true" - }, + "name": "name", "type": { - "type": "reference", - "name": "bool | None" + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] }, - "defaultValue": "None" + "parentId": 1299 }, { - "id": 169, - "name": "do_not_retry", - "kind": 32768, - "kindString": "Parameter", + "defaultValue": "None", "flags": { - "keyword-only": "true" + "isOptional": true, + "keyword-only": true }, + "id": 71712, + "kind": 32768, + "kindString": "Parameter", + "name": "alias", "type": { - "type": "reference", - "name": "bool | None" + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] }, - "defaultValue": "None" + "parentId": 1299 }, { - "id": 170, - "name": "idempotency_key", - "kind": 32768, - "kindString": "Parameter", + "defaultValue": "None", "flags": { - "keyword-only": "true" + "isOptional": true, + "keyword-only": true }, + "id": 71713, + "kind": 32768, + "kindString": "Parameter", + "name": "configuration", "type": { - "type": "reference", - "name": "str | None" + "name": "Configuration | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "Configuration", + "target": "2", + "ref": { + "id": 70412, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/configuration.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 20 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/Configuration" + } + }, + { + "type": "literal", + "value": {} + } + ] }, - "defaultValue": "None" + "parentId": 1299 + } + ], + "type": { + "name": "RequestQueueClient", + "type": "reference", + "target": "1195", + "ref": { + "id": 71605, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_request_queue_client.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 13 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/RequestQueueClient" + } + }, + "overwrites": { + "name": "StorageClient.create_rq_client", + "target": 1298, + "type": "reference", + "ref": { + "id": 71708, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 64 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/StorageClient#create_rq_client" } - ] + }, + "parentId": 1927 } - ] + ], + "overwrites": { + "name": "StorageClient.create_rq_client", + "target": 1298, + "type": "reference", + "ref": { + "id": 71708, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 64 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/StorageClient#create_rq_client" + } + }, + "permalink": "/python/api/class/SqlStorageClient#create_rq_client", + "parentId": 1893 }, { - "id": 171, - "name": "set_status_message", - "module": "actor", "kind": 2048, "kindString": "Method", - "flags": {}, + "children": [], "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Set the status message for the current actor run.\\n\", {\"Arguments\": [{\"param\": \"status_message\", \"type\": \"str\", \"desc\": \"The status message to set to the run.\"}, {\"param\": \"is_terminal\", \"type\": \"bool, optional\", \"desc\": \"Set this flag to True if this is the final status message of the Actor run.\\n\"}]}, {\"Returns\": [{\"param\": \"dict\", \"desc\": \"The updated actor run object\"}]}]}" + "text": "Return a cache key that can differentiate between different storages of this and other clients.\n\nCan be based on configuration or on the client itself. By default, returns a module and name of the client\nclass." } ] }, - "children": [], + "decorations": [], + "flags": {}, "groups": [], + "id": 74612, + "module": "storage_clients._base._storage_client", + "name": "get_storage_client_cache_key", + "parsedDocstring": { + "text": "Return a cache key that can differentiate between different storages of this and other clients.\n\nCan be based on configuration or on the client itself. By default, returns a module and name of the client\nclass." + }, "sources": [ { - "filename": "/tmp-old-source-v17/src/apify/actor.py", - "line": 1249, "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/actor.py#L1249" + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 33, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/_base/_storage_client.py#L33" } ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, "signatures": [ { - "id": 172, - "name": "set_status_message", - "modifiers": [ - "async" - ], - "kind": 4096, - "kindString": "Call signature", - "flags": {}, "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Set the status message for the current actor run.\\n\", {\"" + "text": "Return a cache key that can differentiate between different storages of this and other clients.\n\nCan be based on configuration or on the client itself. By default, returns a module and name of the client\nclass." } ] }, - "type": { - "type": "reference", - "name": "dict | None" - }, + "flags": {}, + "id": 71694, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "get_storage_client_cache_key", "parameters": [ { - "id": 173, - "name": "status_message", - "kind": 32768, - "kindString": "Parameter", - "flags": {}, - "type": { - "type": "intrinsic", - "name": "str" - } - }, - { - "id": 174, - "name": "is_terminal", - "kind": 32768, - "kindString": "Parameter", "flags": { - "keyword-only": "true" + "isOptional": false, + "keyword-only": false }, + "id": 71695, + "kind": 32768, + "kindString": "Parameter", + "name": "configuration", "type": { + "name": "Configuration", "type": "reference", - "name": "bool | None" - }, - "defaultValue": "None" + "target": "2", + "ref": { + "id": 70412, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/configuration.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 20 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/Configuration" + } + }, + "parentId": 1284 + } + ], + "type": { + "name": "Hashable", + "type": "reference", + "target": 0, + "ref": {} + }, + "inheritedFrom": { + "name": "StorageClient.get_storage_client_cache_key", + "target": 1283, + "type": "reference", + "ref": { + "id": 71693, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 33 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/StorageClient#get_storage_client_cache_key" } - ] + }, + "parentId": 4202 } - ] + ], + "inheritedFrom": { + "name": "StorageClient.get_storage_client_cache_key", + "target": 1283, + "type": "reference", + "ref": { + "id": 71693, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 33 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/StorageClient#get_storage_client_cache_key" + } + }, + "permalink": "/python/api/class/SqlStorageClient#get_storage_client_cache_key", + "parentId": 1893 }, { - "id": 175, - "name": "create_proxy_configuration", - "module": "actor", "kind": 2048, "kindString": "Method", - "flags": {}, + "children": [], "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Create a ProxyConfiguration object with the passed proxy configuration.\\n\\nConfigures connection to a proxy server with the provided options.\\nProxy servers are used to prevent target websites from blocking your crawlers based on IP address rate limits or blacklists.\\n\\nFor more details and code examples, see the `ProxyConfiguration` class.\\n\", {\"Arguments\": [{\"param\": \"actor_proxy_input\", \"type\": \"dict, optional\", \"desc\": \"Proxy configuration field from the actor input, if actor has such input field.\\nIf you pass this argument, all the other arguments will be inferred from it.\"}, {\"param\": \"password\", \"type\": \"str, optional\", \"desc\": \"Password for the Apify Proxy. If not provided, will use os.environ['APIFY_PROXY_PASSWORD'], if available.\"}, {\"param\": \"groups\", \"type\": \"list of str, optional\", \"desc\": \"Proxy groups which the Apify Proxy should use, if provided.\"}, {\"param\": \"country_code\", \"type\": \"str, optional\", \"desc\": \"Country which the Apify Proxy should use, if provided.\"}, {\"param\": \"proxy_urls\", \"type\": \"list of str, optional\", \"desc\": \"Custom proxy server URLs which should be rotated through.\"}, {\"param\": \"new_url_function\", \"type\": \"Callable, optional\", \"desc\": \"Function which returns a custom proxy URL to be used.\\n\"}]}, {\"Returns\": [\"ProxyConfiguration, optional: ProxyConfiguration object with the passed configuration,\", \"or None, if no proxy should be used based on the configuration.\"]}]}" + "text": "Return statistics about rate limit errors encountered by the HTTP client in storage client." } ] }, - "children": [], + "decorations": [], + "flags": {}, "groups": [], + "id": 74613, + "module": "storage_clients._base._storage_client", + "name": "get_rate_limit_errors", + "parsedDocstring": { + "text": "Return statistics about rate limit errors encountered by the HTTP client in storage client." + }, "sources": [ { - "filename": "/tmp-old-source-v17/src/apify/actor.py", - "line": 1285, "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/actor.py#L1285" + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 74, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/_base/_storage_client.py#L74" } ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, "signatures": [ { - "id": 176, - "name": "create_proxy_configuration", - "modifiers": [ - "async" - ], - "kind": 4096, - "kindString": "Call signature", - "flags": {}, "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Create a ProxyConfiguration object with the passed proxy configuration.\\n\\nConfigures connection to a proxy server with the provided options.\\nProxy servers are used to prevent target websites from blocking your crawlers based on IP address rate limits or blacklists.\\n\\nFor more details and code examples, see the `ProxyConfiguration` class.\\n\", {\"" + "text": "Return statistics about rate limit errors encountered by the HTTP client in storage client." } ] }, + "flags": {}, + "id": 71715, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "get_rate_limit_errors", + "parameters": [], "type": { + "name": "dict[int, int]", "type": "reference", - "name": "ProxyConfiguration | None" + "target": 0, + "ref": {} }, - "parameters": [ - { - "id": 177, - "name": "actor_proxy_input", - "kind": 32768, - "kindString": "Parameter", - "flags": { - "keyword-only": "true" - }, - "type": { - "type": "reference", - "name": "dict | None" - }, - "defaultValue": "None" - }, - { - "id": 178, - "name": "password", - "kind": 32768, - "kindString": "Parameter", - "flags": { - "keyword-only": "true" - }, - "type": { - "type": "reference", - "name": "str | None" - }, - "defaultValue": "None" - }, - { - "id": 179, - "name": "groups", - "kind": 32768, - "kindString": "Parameter", - "flags": { - "keyword-only": "true" - }, - "type": { - "type": "reference", - "name": "list[str] | None" - }, - "defaultValue": "None" - }, - { - "id": 180, - "name": "country_code", - "kind": 32768, - "kindString": "Parameter", - "flags": { - "keyword-only": "true" - }, - "type": { - "type": "reference", - "name": "str | None" - }, - "defaultValue": "None" - }, - { - "id": 181, - "name": "proxy_urls", - "kind": 32768, - "kindString": "Parameter", - "flags": { - "keyword-only": "true" - }, - "type": { - "type": "reference", - "name": "list[str] | None" - }, - "defaultValue": "None" - }, + "inheritedFrom": { + "name": "StorageClient.get_rate_limit_errors", + "target": 1304, + "type": "reference", + "ref": { + "id": 71714, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 74 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/StorageClient#get_rate_limit_errors" + } + }, + "parentId": 4203 + } + ], + "inheritedFrom": { + "name": "StorageClient.get_rate_limit_errors", + "target": 1304, + "type": "reference", + "ref": { + "id": 71714, + "sources": [ { - "id": 182, - "name": "new_url_function", - "kind": 32768, - "kindString": "Parameter", - "flags": { - "keyword-only": "true" - }, - "type": { - "type": "reference", - "name": "Callable[[str | None], str] | Callable[[str | None], Awaitable[str]] | None" - }, - "defaultValue": "None" + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 74 } - ] + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/StorageClient#get_rate_limit_errors" } - ] + }, + "permalink": "/python/api/class/SqlStorageClient#get_rate_limit_errors", + "parentId": 1893 } ], - "groups": [ + "comment": { + "summary": [ + { + "kind": "text", + "text": "SQL implementation of the storage client.\n\nThis storage client provides access to datasets, key-value stores, and request queues that persist data\nto a SQL database using SQLAlchemy 2+. Each storage type uses two tables: one for metadata and one for\nrecords.\n\nThe client accepts either a database connection string or a pre-configured AsyncEngine. If neither is\nprovided, it creates a default SQLite database 'crawlee.db' in the storage directory.\n\nDatabase schema is automatically created during initialization. SQLite databases receive performance\noptimizations including WAL mode and increased cache size.\n\n\n:::warning Warning\nThis is an experimental feature. The behavior and interface may change in future versions.\n:::" + } + ] + }, + "decorations": [ { - "title": "Constructors", - "children": [ - 42 - ] - }, + "args": "('Storage clients')", + "name": "docs_group" + } + ], + "flags": {}, + "groups": [ { - "title": "Properties", "children": [ - 45, - 46, - 47, - 48 - ] + 72308, + 72310, + 72304, + 72321, + 72325, + 72331, + 72337, + 72323, + 72316, + 74613, + 74612, + 72318 + ], + "title": "Methods" }, { - "title": "Methods", "children": [ - 126, - 163, - 131, - 142, - 175, - 53, - 59, - 113, - 92, - 51, - 94, - 49, - 111, - 64, - 152, - 67, - 107, - 103, - 74, - 79, - 84, - 89, - 159, - 171, - 98, - 115 - ] + 72315 + ], + "title": "Properties" } ], + "id": 72303, + "module": "storage_clients._sql._storage_client", + "name": "SqlStorageClient", + "parsedDocstring": { + "text": "SQL implementation of the storage client.\n\nThis storage client provides access to datasets, key-value stores, and request queues that persist data\nto a SQL database using SQLAlchemy 2+. Each storage type uses two tables: one for metadata and one for\nrecords.\n\nThe client accepts either a database connection string or a pre-configured AsyncEngine. If neither is\nprovided, it creates a default SQLite database 'crawlee.db' in the storage directory.\n\nDatabase schema is automatically created during initialization. SQLite databases receive performance\noptimizations including WAL mode and increased cache size.\n\n\n:::warning Warning\nThis is an experimental feature. The behavior and interface may change in future versions.\n:::" + }, "sources": [ { - "filename": "/tmp-old-source-v17/src/apify/actor.py", - "line": 68, "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/actor.py#L68" + "fileName": "/src/crawlee/storage_clients/_sql/_storage_client.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 35, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/_sql/_storage_client.py#L35" } - ] + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "extendedTypes": [ + { + "name": "StorageClient", + "target": "1282", + "type": "reference", + "ref": { + "id": 71692, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 19 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/StorageClient" + } + } + ], + "permalink": "/python/api/class/SqlStorageClient", + "categories": [] }, { - "id": 183, - "name": "Configuration", - "module": "config", "kind": 128, "kindString": "Class", - "flags": {}, - "comment": { - "summary": [ - { - "kind": "text", - "text": "{\"content\": [\"A class for specifying the configuration of an actor.\\n\\nCan be used either globally via `Configuration.get_global_configuration()`,\\nor it can be specific to each `Actor` instance on the `actor.config` property.\"]}" - } - ] - }, "children": [ { - "id": 184, - "name": "__init__", - "module": "config", - "kind": 512, - "kindString": "Constructor", - "flags": {}, + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Get the number of requests in the loader that have been handled." + } + ] + }, + "decorations": [ + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 77548, + "module": "request_loaders._request_loader", + "name": "get_handled_count", + "parsedDocstring": { + "text": "Get the number of requests in the loader that have been handled." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 29, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/request_loaders/_request_loader.py#L29" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Get the number of requests in the loader that have been handled." + } + ] + }, + "flags": {}, + "id": 77549, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "get_handled_count", + "parameters": [], + "type": { + "name": "int", + "type": "reference", + "target": 0, + "ref": {} + }, + "overwrites": { + "name": "RequestLoader.get_handled_count", + "target": 2935, + "type": "reference", + "ref": { + "id": 77548, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 29 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestLoader#get_handled_count" + } + }, + "parentId": 2935 + } + ], + "permalink": "/python/api/class/RequestLoader#get_handled_count", + "parentId": 2934 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Create a `Configuration` instance.\\n\\nAll the parameters are loaded by default from environment variables when running on the Apify platform.\\nYou can override them here in the Configuration constructor, which might be useful for local testing of your actors.\\n\", {\"Arguments\": [{\"param\": \"api_base_url\", \"type\": \"str, optional\", \"desc\": \"The URL of the Apify API.\\nThis is the URL actually used for connecting to the API, so it can contain an IP address when running in a container on the platform.\"}, {\"param\": \"api_public_base_url\", \"type\": \"str, optional\", \"desc\": \"The public URL of the Apify API.\\nThis will always contain the public URL of the API, even when running in a container on the platform.\\nUseful for generating shareable URLs to key-value store records or datasets.\"}, {\"param\": \"container_port\", \"type\": \"int, optional\", \"desc\": \"The port on which the container can listen for HTTP requests.\"}, {\"param\": \"container_url\", \"type\": \"str, optional\", \"desc\": \"The URL on which the container can listen for HTTP requests.\"}, {\"param\": \"default_dataset_id\", \"type\": \"str, optional\", \"desc\": \"The ID of the default dataset for the actor.\"}, {\"param\": \"default_key_value_store_id\", \"type\": \"str, optional\", \"desc\": \"The ID of the default key-value store for the actor.\"}, {\"param\": \"default_request_queue_id\", \"type\": \"str, optional\", \"desc\": \"The ID of the default request queue for the actor.\"}, {\"param\": \"input_key\", \"type\": \"str, optional\", \"desc\": \"The key of the input record in the actor's default key-value store\"}, {\"param\": \"max_used_cpu_ratio\", \"type\": \"float, optional\", \"desc\": \"The CPU usage above which the SYSTEM_INFO event will report the CPU is overloaded.\"}, {\"param\": \"metamorph_after_sleep_millis\", \"type\": \"int, optional\", \"desc\": \"How long should the actor sleep after calling metamorph.\"}, {\"param\": \"persist_state_interval_millis\", \"type\": \"int, optional\", \"desc\": \"How often should the actor emit the PERSIST_STATE event.\"}, {\"param\": \"persist_storage\", \"type\": \"bool, optional\", \"desc\": \"Whether the actor should persist its used storages to the filesystem when running locally.\"}, {\"param\": \"proxy_hostname\", \"type\": \"str, optional\", \"desc\": \"The hostname of Apify Proxy.\"}, {\"param\": \"proxy_password\", \"type\": \"str, optional\", \"desc\": \"The password for Apify Proxy.\"}, {\"param\": \"proxy_port\", \"type\": \"str, optional\", \"desc\": \"The port of Apify Proxy.\"}, {\"param\": \"proxy_status_url\", \"type\": \"str, optional\", \"desc\": \"The URL on which the Apify Proxy status page is available.\"}, {\"param\": \"purge_on_start\", \"type\": \"str, optional\", \"desc\": \"Whether the actor should purge its default storages on startup, when running locally.\"}, {\"param\": \"token\", \"type\": \"str, optional\", \"desc\": \"The API token for the Apify API this actor should use.\"}, {\"param\": \"system_info_interval_millis\", \"type\": \"str, optional\", \"desc\": \"How often should the actor emit the SYSTEM_INFO event when running locally.\"}, {\"param\": \"standby_port\", \"type\": \"int, optional\", \"desc\": \"The port on which the container can listen for Actor Standby HTTP requests.\"}]}]}" + "text": "Get an offline approximation of the total number of requests in the loader (i.e. pending + handled)." } ] }, - "children": [], + "decorations": [ + { + "name": "abstractmethod" + } + ], + "flags": {}, "groups": [], + "id": 77550, + "module": "request_loaders._request_loader", + "name": "get_total_count", + "parsedDocstring": { + "text": "Get an offline approximation of the total number of requests in the loader (i.e. pending + handled)." + }, "sources": [ { - "filename": "/tmp-old-source-v17/src/apify/config.py", - "line": 17, "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/config.py#L17" + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 33, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/request_loaders/_request_loader.py#L33" } ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, "signatures": [ { - "id": 185, - "name": "__init__", - "modifiers": [], - "kind": 4096, - "kindString": "Call signature", - "flags": {}, "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Create a `Configuration` instance.\\n\\nAll the parameters are loaded by default from environment variables when running on the Apify platform.\\nYou can override them here in the Configuration constructor, which might be useful for local testing of your actors.\\n\", {\"" + "text": "Get an offline approximation of the total number of requests in the loader (i.e. pending + handled)." } ] }, + "flags": {}, + "id": 77551, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "get_total_count", + "parameters": [], "type": { + "name": "int", "type": "reference", - "name": "None" + "target": 0, + "ref": {} }, - "parameters": [ - { - "id": 186, - "name": "api_base_url", - "kind": 32768, - "kindString": "Parameter", - "flags": { - "keyword-only": "true" - }, - "type": { - "type": "reference", - "name": "str | None" - }, - "defaultValue": "None" - }, - { - "id": 187, - "name": "api_public_base_url", - "kind": 32768, - "kindString": "Parameter", - "flags": { - "keyword-only": "true" - }, - "type": { - "type": "reference", - "name": "str | None" - }, - "defaultValue": "None" - }, - { - "id": 188, - "name": "container_port", - "kind": 32768, - "kindString": "Parameter", - "flags": { - "keyword-only": "true" - }, - "type": { - "type": "reference", - "name": "int | None" - }, - "defaultValue": "None" - }, - { - "id": 189, - "name": "container_url", - "kind": 32768, - "kindString": "Parameter", - "flags": { - "keyword-only": "true" - }, - "type": { - "type": "reference", - "name": "str | None" - }, - "defaultValue": "None" - }, - { - "id": 190, - "name": "default_dataset_id", - "kind": 32768, - "kindString": "Parameter", - "flags": { - "keyword-only": "true" - }, - "type": { - "type": "reference", - "name": "str | None" - }, - "defaultValue": "None" - }, - { - "id": 191, - "name": "default_key_value_store_id", - "kind": 32768, - "kindString": "Parameter", - "flags": { - "keyword-only": "true" - }, - "type": { - "type": "reference", - "name": "str | None" - }, - "defaultValue": "None" - }, - { - "id": 192, - "name": "default_request_queue_id", - "kind": 32768, - "kindString": "Parameter", - "flags": { - "keyword-only": "true" - }, - "type": { - "type": "reference", - "name": "str | None" - }, - "defaultValue": "None" - }, - { - "id": 193, - "name": "input_key", - "kind": 32768, - "kindString": "Parameter", - "flags": { - "keyword-only": "true" - }, - "type": { - "type": "reference", - "name": "str | None" - }, - "defaultValue": "None" - }, - { - "id": 194, - "name": "max_used_cpu_ratio", - "kind": 32768, - "kindString": "Parameter", - "flags": { - "keyword-only": "true" - }, - "type": { - "type": "reference", - "name": "float | None" - }, - "defaultValue": "None" - }, - { - "id": 195, - "name": "metamorph_after_sleep_millis", - "kind": 32768, - "kindString": "Parameter", - "flags": { - "keyword-only": "true" - }, - "type": { - "type": "reference", - "name": "int | None" - }, - "defaultValue": "None" - }, - { - "id": 196, - "name": "persist_state_interval_millis", - "kind": 32768, - "kindString": "Parameter", - "flags": { - "keyword-only": "true" - }, - "type": { - "type": "reference", - "name": "int | None" - }, - "defaultValue": "None" - }, - { - "id": 197, - "name": "persist_storage", - "kind": 32768, - "kindString": "Parameter", - "flags": { - "keyword-only": "true" - }, - "type": { - "type": "reference", - "name": "bool | None" - }, - "defaultValue": "None" - }, - { - "id": 198, - "name": "proxy_hostname", - "kind": 32768, - "kindString": "Parameter", - "flags": { - "keyword-only": "true" - }, - "type": { - "type": "reference", - "name": "str | None" - }, - "defaultValue": "None" - }, - { - "id": 199, - "name": "proxy_password", - "kind": 32768, - "kindString": "Parameter", - "flags": { - "keyword-only": "true" - }, - "type": { - "type": "reference", - "name": "str | None" - }, - "defaultValue": "None" - }, - { - "id": 200, - "name": "proxy_port", - "kind": 32768, - "kindString": "Parameter", - "flags": { - "keyword-only": "true" - }, - "type": { - "type": "reference", - "name": "int | None" - }, - "defaultValue": "None" - }, - { - "id": 201, - "name": "proxy_status_url", - "kind": 32768, - "kindString": "Parameter", - "flags": { - "keyword-only": "true" - }, - "type": { - "type": "reference", - "name": "str | None" - }, - "defaultValue": "None" - }, - { - "id": 202, - "name": "purge_on_start", - "kind": 32768, - "kindString": "Parameter", - "flags": { - "keyword-only": "true" - }, - "type": { - "type": "reference", - "name": "bool | None" - }, - "defaultValue": "None" - }, - { - "id": 203, - "name": "token", - "kind": 32768, - "kindString": "Parameter", - "flags": { - "keyword-only": "true" - }, - "type": { - "type": "reference", - "name": "str | None" - }, - "defaultValue": "None" - }, - { - "id": 204, - "name": "standby_port", - "kind": 32768, - "kindString": "Parameter", - "flags": { - "keyword-only": "true" - }, - "type": { - "type": "reference", - "name": "int | None" - }, - "defaultValue": "None" - }, - { - "id": 205, - "name": "system_info_interval_millis", - "kind": 32768, - "kindString": "Parameter", - "flags": { - "keyword-only": "true" - }, - "type": { - "type": "reference", - "name": "int | None" - }, - "defaultValue": "None" + "overwrites": { + "name": "RequestLoader.get_total_count", + "target": 2937, + "type": "reference", + "ref": { + "id": 77550, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 33 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestLoader#get_total_count" } - ] + }, + "parentId": 2937 } - ] + ], + "permalink": "/python/api/class/RequestLoader#get_total_count", + "parentId": 2934 }, { - "id": 206, - "name": "get_global_configuration", - "module": "config", "kind": 2048, "kindString": "Method", - "flags": {}, + "children": [], "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Retrieve the global configuration.\\n\\nThe global configuration applies when you call actor methods via their static versions, e.g. `Actor.init()`.\\nAlso accessible via `Actor.config`.\"]}" + "text": "Return True if there are no more requests in the loader (there might still be unfinished requests)." } ] }, - "children": [], + "decorations": [ + { + "name": "abstractmethod" + } + ], + "flags": {}, "groups": [], + "id": 77552, + "module": "request_loaders._request_loader", + "name": "is_empty", + "parsedDocstring": { + "text": "Return True if there are no more requests in the loader (there might still be unfinished requests)." + }, "sources": [ { - "filename": "/tmp-old-source-v17/src/apify/config.py", - "line": 124, "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/config.py#L124" + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 37, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/request_loaders/_request_loader.py#L37" } ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, "signatures": [ { - "id": 207, - "name": "get_global_configuration", - "modifiers": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Return True if there are no more requests in the loader (there might still be unfinished requests)." + } + ] + }, + "flags": {}, + "id": 77553, "kind": 4096, "kindString": "Call signature", - "flags": {}, + "modifiers": [ + "async" + ], + "name": "is_empty", + "parameters": [], + "type": { + "name": "bool", + "type": "reference", + "target": 0, + "ref": {} + }, + "overwrites": { + "name": "RequestLoader.is_empty", + "target": 2939, + "type": "reference", + "ref": { + "id": 77552, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 37 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestLoader#is_empty" + } + }, + "parentId": 2939 + } + ], + "permalink": "/python/api/class/RequestLoader#is_empty", + "parentId": 2934 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Return True if all requests have been handled." + } + ] + }, + "decorations": [ + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 77554, + "module": "request_loaders._request_loader", + "name": "is_finished", + "parsedDocstring": { + "text": "Return True if all requests have been handled." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 41, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/request_loaders/_request_loader.py#L41" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Retrieve the global configuration.\\n\\nThe global configuration applies when you call actor methods via their static versions, e.g. `Actor.init()`.\\nAlso accessible via `Actor.config`.\"]}" + "text": "Return True if all requests have been handled." } ] }, + "flags": {}, + "id": 77555, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "is_finished", + "parameters": [], "type": { + "name": "bool", "type": "reference", - "name": "Configuration", - "id": 183 + "target": 0, + "ref": {} }, - "parameters": [] + "overwrites": { + "name": "RequestLoader.is_finished", + "target": 2941, + "type": "reference", + "ref": { + "id": 77554, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 41 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestLoader#is_finished" + } + }, + "parentId": 2941 } - ] - } - ], - "groups": [ - { - "title": "Constructors", - "children": [ - 184 - ] + ], + "permalink": "/python/api/class/RequestLoader#is_finished", + "parentId": 2934 }, { - "title": "Methods", - "children": [ - 206 - ] - } - ], - "sources": [ - { - "filename": "/tmp-old-source-v17/src/apify/config.py", - "line": 8, - "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/config.py#L8" - } - ] - }, - { - "id": 208, - "name": "StorageTypes", - "module": "consts", - "kind": 8, - "kindString": "Enumeration", - "flags": {}, - "comment": { - "summary": [ - { - "kind": "text", - "text": "{\"content\": [\"Possible Apify storage types.\"]}" - } - ] - }, - "children": [], - "groups": [], - "sources": [ + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Return the next request to be processed, or `None` if there are no more pending requests.\n\nThe method should return `None` if and only if `is_finished` would return `True`. In other cases, the method\nshould wait until a request appears." + } + ] + }, + "decorations": [ + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 77556, + "module": "request_loaders._request_loader", + "name": "fetch_next_request", + "parsedDocstring": { + "text": "Return the next request to be processed, or `None` if there are no more pending requests.\n\nThe method should return `None` if and only if `is_finished` would return `True`. In other cases, the method\nshould wait until a request appears." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 45, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/request_loaders/_request_loader.py#L45" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Return the next request to be processed, or `None` if there are no more pending requests.\n\nThe method should return `None` if and only if `is_finished` would return `True`. In other cases, the method\nshould wait until a request appears." + } + ] + }, + "flags": {}, + "id": 77557, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "fetch_next_request", + "parameters": [], + "type": { + "name": "Request | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "Request", + "target": "409", + "ref": { + "id": 75022, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_request.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 145 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/Request" + } + }, + { + "type": "literal", + "value": {} + } + ] + }, + "overwrites": { + "name": "RequestLoader.fetch_next_request", + "target": 2943, + "type": "reference", + "ref": { + "id": 77556, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 45 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestLoader#fetch_next_request" + } + }, + "parentId": 2943 + } + ], + "permalink": "/python/api/class/RequestLoader#fetch_next_request", + "parentId": 2934 + }, { - "filename": "/tmp-old-source-v17/src/apify/consts.py", - "line": 46, - "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/consts.py#L46" - } - ] - }, - { - "id": 209, - "name": "ActorLogFormatter", - "module": "log", - "kind": 128, - "kindString": "Class", - "flags": {}, - "comment": { - "summary": [ - { - "kind": "text", - "text": "{\"content\": [\"Log formatter that prints out the log message nicely formatted, with colored level and stringified extra fields.\\n\\nIt formats the log records so that they:\\n- start with the level (colorized, and padded to 5 chars so that it is nicely aligned)\\n- then have the actual log message, if it's multiline then it's nicely indented\\n- then have the stringified extra log fields\\n- then, if an exception is a part of the log record, prints the formatted exception.\"]}" - } - ] - }, - "children": [ + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Mark a request as handled after a successful processing (or after giving up retrying)." + } + ] + }, + "decorations": [ + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 77558, + "module": "request_loaders._request_loader", + "name": "mark_request_as_handled", + "parsedDocstring": { + "text": "Mark a request as handled after a successful processing (or after giving up retrying)." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 53, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/request_loaders/_request_loader.py#L53" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Mark a request as handled after a successful processing (or after giving up retrying)." + } + ] + }, + "flags": {}, + "id": 77559, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "mark_request_as_handled", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 77560, + "kind": 32768, + "kindString": "Parameter", + "name": "request", + "type": { + "name": "Request", + "type": "reference", + "target": "409", + "ref": { + "id": 75022, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_request.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 145 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/Request" + } + }, + "parentId": 2946 + } + ], + "type": { + "name": "ProcessedRequest | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "ProcessedRequest", + "target": "1340", + "ref": { + "id": 75953, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 140 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/ProcessedRequest" + } + }, + { + "type": "literal", + "value": {} + } + ] + }, + "overwrites": { + "name": "RequestLoader.mark_request_as_handled", + "target": 2945, + "type": "reference", + "ref": { + "id": 77558, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 53 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestLoader#mark_request_as_handled" + } + }, + "parentId": 2945 + } + ], + "permalink": "/python/api/class/RequestLoader#mark_request_as_handled", + "parentId": 2934 + }, { - "id": 210, - "name": "__init__", - "module": "log", - "kind": 512, - "kindString": "Constructor", - "flags": {}, + "kind": 2048, + "kindString": "Method", + "children": [], "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Create an instance of the ActorLogFormatter.\\n\", {\"Arguments\": [{\"param\": \"include_logger_name\", \"desc\": \"Include logger name at the beginning of the log line. Defaults to False.\"}, {\"param\": \"args\", \"desc\": \"Arguments passed to the parent class.\"}, {\"param\": \"kwargs\", \"desc\": \"Keyword arguments passed to the parent class.\"}]}]}" + "text": "Combine the loader with a request manager to support adding and reclaiming requests.\n" } ] }, - "children": [], + "decorations": [], + "flags": {}, "groups": [], + "id": 77561, + "module": "request_loaders._request_loader", + "name": "to_tandem", + "parsedDocstring": { + "text": "Combine the loader with a request manager to support adding and reclaiming requests.\n", + "args": { + "request_manager": "Request manager to combine the loader with.\nIf None is given, the default request queue is used." + } + }, "sources": [ { - "filename": "/tmp-old-source-v17/src/apify/log.py", - "line": 59, "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/log.py#L59" + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 56, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/request_loaders/_request_loader.py#L56" } ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, "signatures": [ { - "id": 211, - "name": "__init__", - "modifiers": [], - "kind": 4096, - "kindString": "Call signature", - "flags": {}, "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Create an instance of the ActorLogFormatter.\\n\", {\"" + "text": "Combine the loader with a request manager to support adding and reclaiming requests.\n" } ] }, - "type": { - "type": "reference", - "name": "None" - }, + "flags": {}, + "id": 77562, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "to_tandem", "parameters": [ { - "id": 212, - "name": "include_logger_name", - "kind": 32768, - "kindString": "Parameter", - "flags": {}, - "type": { - "type": "intrinsic", - "name": "bool" + "comment": { + "summary": [ + { + "kind": "text", + "text": "Request manager to combine the loader with.\nIf None is given, the default request queue is used." + } + ] }, - "defaultValue": "False" - }, - { - "id": 213, - "name": "args", - "kind": 32768, - "kindString": "Parameter", - "flags": {}, - "type": { - "type": "reference", - "name": "Any" - } - }, - { - "id": 214, - "name": "kwargs", + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": false + }, + "id": 77563, "kind": 32768, "kindString": "Parameter", - "flags": {}, + "name": "request_manager", "type": { - "type": "reference", - "name": "Any" - } + "name": "RequestManager | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "RequestManager", + "target": "3001", + "ref": { + "id": 77614, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_manager.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 18 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/RequestManager" + } + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2949 + } + ], + "type": { + "name": "RequestManagerTandem", + "type": "reference", + "target": "2803", + "ref": { + "id": 77416, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_manager_tandem.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 24 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/RequestManagerTandem" } - ] + }, + "parentId": 2948 } - ] - } - ], - "groups": [ - { - "title": "Constructors", - "children": [ - 210 - ] + ], + "permalink": "/python/api/class/RequestLoader#to_tandem", + "parentId": 2934 } ], - "sources": [ - { - "filename": "/tmp-old-source-v17/src/apify/log.py", - "line": 42, - "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/log.py#L42" - } - ] - }, - { - "id": 215, - "name": "is_url", - "module": "proxy_configuration", - "kind": 2048, - "kindString": "Method", - "flags": {}, "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Check if the given string is a valid URL.\"]}" + "text": "An abstract class defining the interface for classes that provide access to a read-only stream of requests.\n\nRequest loaders are used to manage and provide access to a storage of crawling requests.\n\nKey responsibilities:\n- Fetching the next request to be processed.\n- Marking requests as successfully handled after processing.\n- Managing state information such as the total and handled request counts." } ] }, - "children": [], - "groups": [], + "decorations": [ + { + "args": "('Request loaders')", + "name": "docs_group" + } + ], + "flags": {}, + "groups": [ + { + "children": [ + 77556, + 77548, + 77550, + 77552, + 77554, + 77558, + 77561 + ], + "title": "Methods" + } + ], + "id": 77547, + "module": "request_loaders._request_loader", + "name": "RequestLoader", + "parsedDocstring": { + "text": "An abstract class defining the interface for classes that provide access to a read-only stream of requests.\n\nRequest loaders are used to manage and provide access to a storage of crawling requests.\n\nKey responsibilities:\n- Fetching the next request to be processed.\n- Marking requests as successfully handled after processing.\n- Managing state information such as the total and handled request counts." + }, "sources": [ { - "filename": "/tmp-old-source-v17/src/apify/proxy_configuration.py", - "line": 25, "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/proxy_configuration.py#L25" + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 17, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/request_loaders/_request_loader.py#L17" } ], - "signatures": [ + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "extendedBy": [ { - "id": 216, - "name": "is_url", - "modifiers": [], - "kind": 4096, - "kindString": "Call signature", - "flags": {}, - "comment": { - "summary": [ + "name": "RequestList", + "target": "2913", + "type": "reference", + "ref": { + "id": 77526, + "sources": [ { - "kind": "text", - "text": "{\"content\": [\"Check if the given string is a valid URL.\"]}" + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_list.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 32 } - ] - }, - "type": { - "type": "intrinsic", - "name": "bool" - }, - "parameters": [ - { - "id": 217, - "name": "url", - "kind": 32768, - "kindString": "Parameter", - "flags": {}, - "type": { - "type": "intrinsic", - "name": "str" + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/RequestList" + } + }, + { + "name": "SitemapRequestLoader", + "target": "2963", + "type": "reference", + "ref": { + "id": 77576, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_sitemap_request_loader.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 94 } - } - ] + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/SitemapRequestLoader" + } + }, + { + "name": "RequestManager", + "target": "3001", + "type": "reference", + "ref": { + "id": 77614, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_manager.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 18 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/RequestManager" + } } - ] + ], + "permalink": "/python/api/class/RequestLoader", + "categories": [] }, { - "id": 218, - "name": "ProxyInfo", - "module": "proxy_configuration", "kind": 128, "kindString": "Class", - "flags": {}, - "comment": { - "summary": [ - { - "kind": "text", - "text": "{\"content\": [\"Provides information about a proxy connection that is used for requests.\"]}" - } - ] - }, "children": [ { - "id": 219, - "name": "url", - "module": "proxy_configuration", - "kind": 1024, - "kindString": "Property", - "flags": {}, + "kind": 2048, + "kindString": "Method", + "children": [], "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"The URL of the proxy.\"]}" + "text": "Remove persistent state either from the Apify Cloud storage or from the local database." } ] }, - "type": { - "type": "intrinsic", - "name": "str" - }, - "children": [], + "decorations": [ + { + "name": "abstractmethod" + } + ], + "flags": {}, "groups": [], + "id": 80616, + "module": "request_loaders._request_manager", + "name": "drop", + "parsedDocstring": { + "text": "Remove persistent state either from the Apify Cloud storage or from the local database." + }, "sources": [ { - "filename": "/tmp-old-source-v17/src/apify/proxy_configuration.py", - "line": 68, "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/proxy_configuration.py#L68" + "fileName": "/src/crawlee/request_loaders/_request_manager.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 22, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/request_loaders/_request_manager.py#L22" } - ] - }, - { - "id": 220, - "name": "hostname", - "module": "proxy_configuration", - "kind": 1024, - "kindString": "Property", - "flags": {}, - "comment": { - "summary": [ - { - "kind": "text", - "text": "{\"content\": [\"The hostname of the proxy.\"]}" - } - ] - }, + ], "type": { - "type": "intrinsic", - "name": "str" + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} }, - "children": [], - "groups": [], - "sources": [ + "signatures": [ { - "filename": "/tmp-old-source-v17/src/apify/proxy_configuration.py", - "line": 71, - "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/proxy_configuration.py#L71" + "comment": { + "summary": [ + { + "kind": "text", + "text": "Remove persistent state either from the Apify Cloud storage or from the local database." + } + ] + }, + "flags": {}, + "id": 80617, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "drop", + "parameters": [], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "overwrites": { + "name": "Storage.drop", + "target": 2342, + "type": "reference", + "ref": { + "id": 79956, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_base.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 57 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/Storage#drop" + } + }, + "parentId": 3002 } - ] + ], + "permalink": "/python/api/class/RequestManager#drop", + "parentId": 3001 }, { - "id": 221, - "name": "port", - "module": "proxy_configuration", - "kind": 1024, - "kindString": "Property", - "flags": {}, + "kind": 2048, + "kindString": "Method", + "children": [], "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"The proxy port.\"]}" + "text": "Remove all queued requests from this manager while keeping it usable.\n\nDefault implementation is a no-op. Subclasses with persistent state should override to clear it." } ] }, - "type": { - "type": "intrinsic", - "name": "int" - }, - "children": [], + "decorations": [], + "flags": {}, "groups": [], + "id": 80618, + "module": "request_loaders._request_manager", + "name": "purge", + "parsedDocstring": { + "text": "Remove all queued requests from this manager while keeping it usable.\n\nDefault implementation is a no-op. Subclasses with persistent state should override to clear it." + }, "sources": [ { - "filename": "/tmp-old-source-v17/src/apify/proxy_configuration.py", - "line": 74, "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/proxy_configuration.py#L74" + "fileName": "/src/crawlee/request_loaders/_request_manager.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 25, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/request_loaders/_request_manager.py#L25" } - ] - }, - { - "id": 222, - "name": "username", - "module": "proxy_configuration", - "kind": 1024, - "kindString": "Property", - "flags": {}, + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Remove all queued requests from this manager while keeping it usable.\n\nDefault implementation is a no-op. Subclasses with persistent state should override to clear it." + } + ] + }, + "flags": {}, + "id": 80619, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "purge", + "parameters": [], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "overwrites": { + "name": "Storage.purge", + "target": 2344, + "type": "reference", + "ref": { + "id": 79958, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_base.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 61 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/Storage#purge" + } + }, + "parentId": 3004 + } + ], + "permalink": "/python/api/class/RequestManager#purge", + "parentId": 3001 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"The username for the proxy.\"]}" + "text": "Add a single request to the manager and store it in underlying resource client.\n" } ] }, - "type": { - "type": "reference", - "name": "NotRequired[str]" - }, - "children": [], + "decorations": [ + { + "name": "abstractmethod" + } + ], + "flags": {}, "groups": [], + "id": 80620, + "module": "request_loaders._request_manager", + "name": "add_request", + "parsedDocstring": { + "text": "Add a single request to the manager and store it in underlying resource client.\n", + "args": { + "request": "The request object (or its string representation) to be added to the manager.", + "forefront": "Determines whether the request should be added to the beginning (if True) or the end (if False)\nof the manager.\n" + }, + "returns": "Information about the request addition to the manager or None if the request was not added." + }, "sources": [ { - "filename": "/tmp-old-source-v17/src/apify/proxy_configuration.py", - "line": 77, "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/proxy_configuration.py#L77" + "fileName": "/src/crawlee/request_loaders/_request_manager.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 32, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/request_loaders/_request_manager.py#L32" } - ] - }, - { - "id": 223, - "name": "password", - "module": "proxy_configuration", - "kind": 1024, - "kindString": "Property", - "flags": {}, - "comment": { - "summary": [ - { - "kind": "text", - "text": "{\"content\": [\"The password for the proxy.\"]}" - } - ] - }, + ], "type": { - "type": "intrinsic", - "name": "str" + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} }, - "children": [], - "groups": [], - "sources": [ + "signatures": [ { - "filename": "/tmp-old-source-v17/src/apify/proxy_configuration.py", - "line": 80, - "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/proxy_configuration.py#L80" + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "Information about the request addition to the manager or None if the request was not added." + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Add a single request to the manager and store it in underlying resource client.\n" + } + ] + }, + "flags": {}, + "id": 80621, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "add_request", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The request object (or its string representation) to be added to the manager." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 80622, + "kind": 32768, + "kindString": "Parameter", + "name": "request", + "type": { + "name": "str | Request", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "reference", + "name": "Request", + "target": "409", + "ref": { + "id": 78023, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_request.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 145 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/Request" + } + } + ] + }, + "parentId": 3007 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Determines whether the request should be added to the beginning (if True) or the end (if False)\nof the manager.\n" + } + ] + }, + "defaultValue": "False", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 80623, + "kind": 32768, + "kindString": "Parameter", + "name": "forefront", + "type": { + "name": "bool", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 3007 + } + ], + "type": { + "name": "ProcessedRequest | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "ProcessedRequest", + "target": "1340", + "ref": { + "id": 78954, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 140 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/ProcessedRequest" + } + }, + { + "type": "literal", + "value": {} + } + ] + }, + "overwrites": { + "name": "RequestManager.add_request", + "target": 3006, + "type": "reference", + "ref": { + "id": 80620, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_manager.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 32 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestManager#add_request" + } + }, + "parentId": 3006 } - ] + ], + "permalink": "/python/api/class/RequestManager#add_request", + "parentId": 3001 }, { - "id": 224, - "name": "groups", - "module": "proxy_configuration", - "kind": 1024, - "kindString": "Property", - "flags": {}, + "kind": 2048, + "kindString": "Method", + "children": [], "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"An array of proxy groups to be used by the [Apify Proxy](https://docs.apify.com/proxy).\\nIf not provided, the proxy will select the groups automatically.\"]}" + "text": "Add requests to the manager in batches.\n" } ] }, - "type": { - "type": "reference", - "name": "NotRequired[list[str]]" - }, - "children": [], + "decorations": [], + "flags": {}, "groups": [], + "id": 80624, + "module": "request_loaders._request_manager", + "name": "add_requests", + "parsedDocstring": { + "text": "Add requests to the manager in batches.\n", + "args": { + "requests": "Requests to enqueue.", + "forefront": "If True, add requests to the beginning of the queue.", + "batch_size": "The number of requests to add in one batch.", + "wait_time_between_batches": "Time to wait between adding batches.", + "wait_for_all_requests_to_be_added": "If True, wait for all requests to be added before returning.", + "wait_for_all_requests_to_be_added_timeout": "Timeout for waiting for all requests to be added." + } + }, "sources": [ { - "filename": "/tmp-old-source-v17/src/apify/proxy_configuration.py", - "line": 83, "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/proxy_configuration.py#L83" + "fileName": "/src/crawlee/request_loaders/_request_manager.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 49, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/request_loaders/_request_manager.py#L49" } - ] - }, - { - "id": 225, - "name": "country_code", - "module": "proxy_configuration", - "kind": 1024, - "kindString": "Property", - "flags": {}, - "comment": { - "summary": [ - { - "kind": "text", - "text": "{\"content\": [\"If set and relevant proxies are available in your Apify account, all proxied requests will\\nuse IP addresses that are geolocated to the specified country. For example `GB` for IPs\\nfrom Great Britain. Note that online services often have their own rules for handling\\ngeolocation and thus the country selection is a best attempt at geolocation, rather than\\na guaranteed hit. This parameter is optional, by default, each proxied request is assigned\\nan IP address from a random country. The country code needs to be a two letter ISO country code.\\nSee the [full list of available country codes](https://en.wikipedia.org/wiki/ISO_3166-1_alpha-2#Officially_assigned_code_elements).\\nThis parameter is optional, by default, the proxy uses all available proxy servers from all countries.\"]}" - } - ] - }, + ], "type": { + "name": "Undefined", "type": "reference", - "name": "NotRequired[str]" + "target": 0, + "ref": {} }, - "children": [], - "groups": [], - "sources": [ + "signatures": [ { - "filename": "/tmp-old-source-v17/src/apify/proxy_configuration.py", - "line": 88, - "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/proxy_configuration.py#L88" + "comment": { + "summary": [ + { + "kind": "text", + "text": "Add requests to the manager in batches.\n" + } + ] + }, + "flags": {}, + "id": 80625, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "add_requests", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Requests to enqueue." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 80626, + "kind": 32768, + "kindString": "Parameter", + "name": "requests", + "type": { + "name": "Sequence", + "type": "reference", + "typeArguments": [ + { + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "reference", + "name": "Request", + "target": "409", + "ref": { + "id": 78023, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_request.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 145 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/Request" + } + } + ] + } + ], + "target": 0, + "ref": {} + }, + "parentId": 3011 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "If True, add requests to the beginning of the queue." + } + ] + }, + "defaultValue": "False", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 80627, + "kind": 32768, + "kindString": "Parameter", + "name": "forefront", + "type": { + "name": "bool", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 3011 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The number of requests to add in one batch." + } + ] + }, + "defaultValue": "1000", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 80628, + "kind": 32768, + "kindString": "Parameter", + "name": "batch_size", + "type": { + "name": "int", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 3011 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Time to wait between adding batches." + } + ] + }, + "defaultValue": "timedelta(seconds=1)", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 80629, + "kind": 32768, + "kindString": "Parameter", + "name": "wait_time_between_batches", + "type": { + "name": "timedelta", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 3011 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "If True, wait for all requests to be added before returning." + } + ] + }, + "defaultValue": "False", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 80630, + "kind": 32768, + "kindString": "Parameter", + "name": "wait_for_all_requests_to_be_added", + "type": { + "name": "bool", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 3011 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Timeout for waiting for all requests to be added." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 80631, + "kind": 32768, + "kindString": "Parameter", + "name": "wait_for_all_requests_to_be_added_timeout", + "type": { + "name": "timedelta | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "timedelta", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 3011 + } + ], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "overwrites": { + "name": "RequestManager.add_requests", + "target": 3010, + "type": "reference", + "ref": { + "id": 80624, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_manager.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 49 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestManager#add_requests" + } + }, + "parentId": 3010 } - ] + ], + "permalink": "/python/api/class/RequestManager#add_requests", + "parentId": 3001 }, { - "id": 226, - "name": "session_id", - "module": "proxy_configuration", - "kind": 1024, - "kindString": "Property", - "flags": {}, + "kind": 2048, + "kindString": "Method", + "children": [], "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"The identifier of the used proxy session, if used. Using the same session ID guarantees getting the same proxy URL.\"]}" + "text": "Reclaims a failed request back to the source, so that it can be returned for processing later again.\n\nIt is possible to modify the request data by supplying an updated request as a parameter." } ] }, - "type": { - "type": "reference", - "name": "NotRequired[str]" - }, - "children": [], - "groups": [], - "sources": [ + "decorations": [ { - "filename": "/tmp-old-source-v17/src/apify/proxy_configuration.py", - "line": 99, - "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/proxy_configuration.py#L99" + "name": "abstractmethod" } - ] - } - ], - "groups": [ - { - "title": "Properties", - "children": [ - 225, - 224, - 220, - 223, - 221, - 226, - 219, - 222 - ] - } - ], - "sources": [ - { - "filename": "/tmp-old-source-v17/src/apify/proxy_configuration.py", - "line": 65, - "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/proxy_configuration.py#L65" - } - ] - }, - { - "id": 227, - "name": "ProxyConfiguration", - "module": "proxy_configuration", - "kind": 128, - "kindString": "Class", - "flags": {}, - "comment": { - "summary": [ - { - "kind": "text", - "text": "{\"content\": [\"Configures a connection to a proxy server with the provided options.\\n\\nProxy servers are used to prevent target websites from blocking your crawlers based on IP address rate limits or blacklists.\\nThe default servers used by this class are managed by [Apify Proxy](https://docs.apify.com/proxy).\\nTo be able to use Apify Proxy, you need an Apify account and access to the selected proxies. If you provide no configuration option,\\nthe proxies will be managed automatically using a smart algorithm.\\n\\nIf you want to use your own proxies, use the `proxy_urls` or `new_url_function` constructor options.\\nYour list of proxy URLs will be rotated by the configuration, if this option is provided.\"]}" - } - ] - }, - "children": [ - { - "id": 228, - "name": "initialize", - "module": "proxy_configuration", - "kind": 2048, - "kindString": "Method", + ], "flags": {}, - "comment": { - "summary": [ - { - "kind": "text", - "text": "{\"content\": [\"Load the Apify Proxy password if the API token is provided and check access to Apify Proxy and provided proxy groups.\\n\\nOnly called if Apify Proxy configuration is used.\\nAlso checks if country has access to Apify Proxy groups if the country code is provided.\\n\\nYou should use the Actor.create_proxy_configuration function\\nto create a pre-initialized `ProxyConfiguration` instance instead of calling this manually.\"]}" - } - ] - }, - "children": [], "groups": [], + "id": 80632, + "module": "request_loaders._request_manager", + "name": "reclaim_request", + "parsedDocstring": { + "text": "Reclaims a failed request back to the source, so that it can be returned for processing later again.\n\nIt is possible to modify the request data by supplying an updated request as a parameter." + }, "sources": [ { - "filename": "/tmp-old-source-v17/src/apify/proxy_configuration.py", - "line": 195, "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/proxy_configuration.py#L195" + "fileName": "/src/crawlee/request_loaders/_request_manager.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 77, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/request_loaders/_request_manager.py#L77" } ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, "signatures": [ { - "id": 229, - "name": "initialize", - "modifiers": [ - "async" - ], - "kind": 4096, - "kindString": "Call signature", - "flags": {}, "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Load the Apify Proxy password if the API token is provided and check access to Apify Proxy and provided proxy groups.\\n\\nOnly called if Apify Proxy configuration is used.\\nAlso checks if country has access to Apify Proxy groups if the country code is provided.\\n\\nYou should use the Actor.create_proxy_configuration function\\nto create a pre-initialized `ProxyConfiguration` instance instead of calling this manually.\"]}" + "text": "Reclaims a failed request back to the source, so that it can be returned for processing later again.\n\nIt is possible to modify the request data by supplying an updated request as a parameter." } ] }, + "flags": {}, + "id": 80633, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "reclaim_request", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 80634, + "kind": 32768, + "kindString": "Parameter", + "name": "request", + "type": { + "name": "Request", + "type": "reference", + "target": "409", + "ref": { + "id": 78023, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_request.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 145 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/Request" + } + }, + "parentId": 3019 + }, + { + "defaultValue": "False", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 80635, + "kind": 32768, + "kindString": "Parameter", + "name": "forefront", + "type": { + "name": "bool", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 3019 + } + ], "type": { + "name": "ProcessedRequest | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "ProcessedRequest", + "target": "1340", + "ref": { + "id": 78954, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 140 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/ProcessedRequest" + } + }, + { + "type": "literal", + "value": {} + } + ] + }, + "overwrites": { + "name": "RequestManager.reclaim_request", + "target": 3018, "type": "reference", - "name": "None" + "ref": { + "id": 80632, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_manager.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 77 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestManager#reclaim_request" + } }, - "parameters": [] + "parentId": 3018 } - ] + ], + "permalink": "/python/api/class/RequestManager#reclaim_request", + "parentId": 3001 }, { - "id": 230, - "name": "new_url", - "module": "proxy_configuration", "kind": 2048, "kindString": "Method", - "flags": {}, + "children": [], "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Return a new proxy URL based on provided configuration options and the `sessionId` parameter.\\n\", {\"Arguments\": [{\"param\": \"session_id\", \"type\": \"int or str, optional\", \"desc\": \"Represents the identifier of a proxy session (https://docs.apify.com/proxy#sessions).\\nAll the HTTP requests going through the proxy with the same session identifier\\nwill use the same target proxy server (i.e. the same IP address).\\nThe identifier must not be longer than 50 characters and include only the following: `0-9`, `a-z`, `A-Z`, `\\\".\\\"`, `\\\"_\\\"` and `\\\"~\\\"`.\\n\"}]}, {\"Returns\": [{\"param\": \"str\", \"desc\": \"A string with a proxy URL, including authentication credentials and port number.\\nFor example, `http://bob:password123@proxy.example.com:8000`\"}]}]}" + "text": "Get the number of requests in the loader that have been handled." } ] }, - "children": [], + "decorations": [ + { + "name": "abstractmethod" + } + ], + "flags": {}, "groups": [], + "id": 81757, + "module": "request_loaders._request_loader", + "name": "get_handled_count", + "parsedDocstring": { + "text": "Get the number of requests in the loader that have been handled." + }, "sources": [ { - "filename": "/tmp-old-source-v17/src/apify/proxy_configuration.py", - "line": 208, "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/proxy_configuration.py#L208" + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 29, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/request_loaders/_request_loader.py#L29" } ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, "signatures": [ { - "id": 231, - "name": "new_url", - "modifiers": [ - "async" - ], - "kind": 4096, - "kindString": "Call signature", - "flags": {}, "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Return a new proxy URL based on provided configuration options and the `sessionId` parameter.\\n\", {\"" + "text": "Get the number of requests in the loader that have been handled." } ] }, + "flags": {}, + "id": 80550, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "get_handled_count", + "parameters": [], "type": { - "type": "intrinsic", - "name": "str" + "name": "int", + "type": "reference", + "target": 0, + "ref": {} }, - "parameters": [ + "overwrites": { + "name": "RequestManager.get_handled_count", + "target": 4143, + "type": "reference", + "ref": { + "id": 81757, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 29 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestManager#get_handled_count" + } + }, + "parentId": 4143 + } + ], + "inheritedFrom": { + "name": "RequestLoader.get_handled_count", + "target": 2935, + "type": "reference", + "ref": { + "id": 80549, + "sources": [ { - "id": 232, - "name": "session_id", - "kind": 32768, - "kindString": "Parameter", - "flags": {}, - "type": { - "type": "reference", - "name": "int | str | None" - }, - "defaultValue": "None" + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 29 } - ] + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestLoader#get_handled_count" } - ] + }, + "permalink": "/python/api/class/RequestManager#get_handled_count", + "parentId": 3001 }, { - "id": 233, - "name": "new_proxy_info", - "module": "proxy_configuration", "kind": 2048, "kindString": "Method", - "flags": {}, + "children": [], "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Create a new ProxyInfo object.\\n\\nUse it if you want to work with a rich representation of a proxy URL.\\nIf you need the URL string only, use `ProxyConfiguration.new_url`.\\n\", {\"Arguments\": [{\"param\": \"session_id\", \"type\": \"int or str, optional\", \"desc\": \"Represents the identifier of a proxy session (https://docs.apify.com/proxy#sessions).\\nAll the HTTP requests going through the proxy with the same session identifier\\nwill use the same target proxy server (i.e. the same IP address).\\nThe identifier must not be longer than 50 characters and include only the following: `0-9`, `a-z`, `A-Z`, `\\\".\\\"`, `\\\"_\\\"` and `\\\"~\\\"`.\\n\"}]}, {\"Returns\": [{\"param\": \"ProxyInfo\", \"desc\": \"Dictionary that represents information about the proxy and its configuration.\"}]}]}" + "text": "Get an offline approximation of the total number of requests in the loader (i.e. pending + handled)." } ] }, - "children": [], + "decorations": [ + { + "name": "abstractmethod" + } + ], + "flags": {}, "groups": [], + "id": 81758, + "module": "request_loaders._request_loader", + "name": "get_total_count", + "parsedDocstring": { + "text": "Get an offline approximation of the total number of requests in the loader (i.e. pending + handled)." + }, "sources": [ { - "filename": "/tmp-old-source-v17/src/apify/proxy_configuration.py", - "line": 251, "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/proxy_configuration.py#L251" + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 33, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/request_loaders/_request_loader.py#L33" } ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, "signatures": [ - { - "id": 234, - "name": "new_proxy_info", - "modifiers": [ - "async" - ], - "kind": 4096, - "kindString": "Call signature", - "flags": {}, - "comment": { - "summary": [ - { - "kind": "text", - "text": "{\"content\": [\"Create a new ProxyInfo object.\\n\\nUse it if you want to work with a rich representation of a proxy URL.\\nIf you need the URL string only, use `ProxyConfiguration.new_url`.\\n\", {\"" - } - ] - }, - "type": { - "type": "reference", - "name": "ProxyInfo", - "id": 218 - }, - "parameters": [ - { - "id": 235, - "name": "session_id", - "kind": 32768, - "kindString": "Parameter", - "flags": {}, - "type": { - "type": "reference", - "name": "int | str | None" - }, - "defaultValue": "None" - } - ] - } - ] - } - ], - "groups": [ - { - "title": "Methods", - "children": [ - 228, - 233, - 230 - ] - } - ], - "sources": [ - { - "filename": "/tmp-old-source-v17/src/apify/proxy_configuration.py", - "line": 103, - "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/proxy_configuration.py#L103" - } - ] - }, - { - "id": 236, - "name": "ResourceClientType", - "module": "_memory_storage.resource_clients.base_resource_collection_client", - "kind": 1024, - "kindString": "Property", - "flags": {}, - "comment": { - "summary": [ - { - "kind": "text", - "text": "{\"content\": [\"noqa: PLC0105\"]}" - } - ] - }, - "children": [], - "groups": [], - "sources": [ - { - "filename": "/tmp-old-source-v17/src/apify/_memory_storage/resource_clients/base_resource_collection_client.py", - "line": 17, - "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/_memory_storage/resource_clients/base_resource_collection_client.py#L17" - } - ] - }, - { - "id": 237, - "name": "ApifyHttpProxyMiddleware", - "module": "scrapy.middlewares.apify_proxy", - "kind": 128, - "kindString": "Class", - "flags": {}, - "comment": { - "summary": [ - { - "kind": "text", - "text": "{\"content\": [\"Apify HTTP proxy middleware for Scrapy.\\n\\nThis middleware enhances request processing by adding a 'proxy' field to the request's meta and an authentication\\nheader. It draws inspiration from the `HttpProxyMiddleware` included by default in Scrapy projects. The proxy URL\\nis sourced from the settings under the `APIFY_PROXY_SETTINGS` key. The value of this key, a dictionary, should be\\nprovided by the Actor input. An example of the proxy settings:\\n\\nproxy_settings = {'useApifyProxy': true, 'apifyProxyGroups': []}\"]}" - } - ] - }, - "children": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Get an offline approximation of the total number of requests in the loader (i.e. pending + handled)." + } + ] + }, + "flags": {}, + "id": 80552, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "get_total_count", + "parameters": [], + "type": { + "name": "int", + "type": "reference", + "target": 0, + "ref": {} + }, + "overwrites": { + "name": "RequestManager.get_total_count", + "target": 4144, + "type": "reference", + "ref": { + "id": 81758, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 33 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestManager#get_total_count" + } + }, + "parentId": 4144 + } + ], + "inheritedFrom": { + "name": "RequestLoader.get_total_count", + "target": 2937, + "type": "reference", + "ref": { + "id": 80551, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 33 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestLoader#get_total_count" + } + }, + "permalink": "/python/api/class/RequestManager#get_total_count", + "parentId": 3001 + }, { - "id": 238, - "name": "__init__", - "module": "scrapy.middlewares.apify_proxy", - "kind": 512, - "kindString": "Constructor", - "flags": {}, + "kind": 2048, + "kindString": "Method", + "children": [], "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Create a new instance.\\n\", {\"Arguments\": [{\"param\": \"proxy_settings\", \"desc\": \"Dictionary containing proxy settings, provided by the Actor input.\"}, {\"param\": \"auth_encoding\", \"desc\": \"Encoding for basic authentication (default is 'latin-1').\"}]}]}" + "text": "Return True if there are no more requests in the loader (there might still be unfinished requests)." } ] }, - "children": [], + "decorations": [ + { + "name": "abstractmethod" + } + ], + "flags": {}, "groups": [], + "id": 81759, + "module": "request_loaders._request_loader", + "name": "is_empty", + "parsedDocstring": { + "text": "Return True if there are no more requests in the loader (there might still be unfinished requests)." + }, "sources": [ { - "filename": "/tmp-old-source-v17/src/apify/scrapy/middlewares/apify_proxy.py", - "line": 31, "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/scrapy/middlewares/apify_proxy.py#L31" + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 37, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/request_loaders/_request_loader.py#L37" } ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, "signatures": [ { - "id": 239, - "name": "__init__", - "modifiers": [], - "kind": 4096, - "kindString": "Call signature", - "flags": {}, "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Create a new instance.\\n\", {\"" + "text": "Return True if there are no more requests in the loader (there might still be unfinished requests)." } ] }, + "flags": {}, + "id": 80554, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "is_empty", + "parameters": [], "type": { + "name": "bool", "type": "reference", - "name": "None" + "target": 0, + "ref": {} }, - "parameters": [ + "overwrites": { + "name": "RequestManager.is_empty", + "target": 4145, + "type": "reference", + "ref": { + "id": 81759, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 37 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestManager#is_empty" + } + }, + "parentId": 4145 + } + ], + "inheritedFrom": { + "name": "RequestLoader.is_empty", + "target": 2939, + "type": "reference", + "ref": { + "id": 80553, + "sources": [ { - "id": 240, - "name": "proxy_settings", - "kind": 32768, - "kindString": "Parameter", - "flags": {}, - "type": { - "type": "intrinsic", - "name": "dict" - } + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 37 } - ] + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestLoader#is_empty" } - ] + }, + "permalink": "/python/api/class/RequestManager#is_empty", + "parentId": 3001 }, { - "id": 241, - "name": "from_crawler", - "module": "scrapy.middlewares.apify_proxy", "kind": 2048, "kindString": "Method", - "flags": {}, + "children": [], "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Create an instance of ApifyHttpProxyMiddleware from a Scrapy Crawler.\\n\", {\"Arguments\": [{\"param\": \"cls\", \"desc\": \"Class type.\"}, {\"param\": \"crawler\", \"desc\": \"Scrapy Crawler object.\\n\"}]}, {\"Returns\": [{\"param\": \"ApifyHttpProxyMiddleware\", \"desc\": \"Instance of the class.\"}]}]}" + "text": "Return True if all requests have been handled." } ] }, - "children": [], + "decorations": [ + { + "name": "abstractmethod" + } + ], + "flags": {}, "groups": [], + "id": 81760, + "module": "request_loaders._request_loader", + "name": "is_finished", + "parsedDocstring": { + "text": "Return True if all requests have been handled." + }, "sources": [ { - "filename": "/tmp-old-source-v17/src/apify/scrapy/middlewares/apify_proxy.py", - "line": 42, "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/scrapy/middlewares/apify_proxy.py#L42" + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 41, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/request_loaders/_request_loader.py#L41" } ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, "signatures": [ { - "id": 242, - "name": "from_crawler", - "modifiers": [], - "kind": 4096, - "kindString": "Call signature", - "flags": {}, "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Create an instance of ApifyHttpProxyMiddleware from a Scrapy Crawler.\\n\", {\"" + "text": "Return True if all requests have been handled." } ] }, + "flags": {}, + "id": 80556, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "is_finished", + "parameters": [], "type": { + "name": "bool", "type": "reference", - "name": "ApifyHttpProxyMiddleware", - "id": 237 + "target": 0, + "ref": {} }, - "parameters": [ + "overwrites": { + "name": "RequestManager.is_finished", + "target": 4146, + "type": "reference", + "ref": { + "id": 81760, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 41 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestManager#is_finished" + } + }, + "parentId": 4146 + } + ], + "inheritedFrom": { + "name": "RequestLoader.is_finished", + "target": 2941, + "type": "reference", + "ref": { + "id": 80555, + "sources": [ { - "id": 243, - "name": "crawler", - "kind": 32768, - "kindString": "Parameter", - "flags": {}, - "type": { - "type": "reference", - "name": "Crawler" - } + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 41 } - ] + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestLoader#is_finished" } - ] + }, + "permalink": "/python/api/class/RequestManager#is_finished", + "parentId": 3001 }, { - "id": 244, - "name": "process_request", - "module": "scrapy.middlewares.apify_proxy", "kind": 2048, "kindString": "Method", - "flags": {}, + "children": [], "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Process a Scrapy request by assigning a new proxy.\\n\", {\"Arguments\": [{\"param\": \"request\", \"desc\": \"Scrapy Request object.\"}, {\"param\": \"spider\", \"desc\": \"Scrapy Spider object.\\n\"}]}, {\"Raises\": [{\"param\": \"ValueError\", \"desc\": \"If username and password are not provided in the proxy URL.\\n\"}]}, {\"Returns\": [{\"param\": \"None\", \"desc\": \"The request is processed and middleware pipeline can continue.\"}]}]}" + "text": "Return the next request to be processed, or `None` if there are no more pending requests.\n\nThe method should return `None` if and only if `is_finished` would return `True`. In other cases, the method\nshould wait until a request appears." } ] }, - "children": [], + "decorations": [ + { + "name": "abstractmethod" + } + ], + "flags": {}, "groups": [], + "id": 81761, + "module": "request_loaders._request_loader", + "name": "fetch_next_request", + "parsedDocstring": { + "text": "Return the next request to be processed, or `None` if there are no more pending requests.\n\nThe method should return `None` if and only if `is_finished` would return `True`. In other cases, the method\nshould wait until a request appears." + }, "sources": [ { - "filename": "/tmp-old-source-v17/src/apify/scrapy/middlewares/apify_proxy.py", - "line": 68, "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/scrapy/middlewares/apify_proxy.py#L68" + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 45, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/request_loaders/_request_loader.py#L45" } ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, "signatures": [ { - "id": 245, - "name": "process_request", - "modifiers": [ - "async" - ], - "kind": 4096, - "kindString": "Call signature", - "flags": {}, "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Process a Scrapy request by assigning a new proxy.\\n\", {\"" + "text": "Return the next request to be processed, or `None` if there are no more pending requests.\n\nThe method should return `None` if and only if `is_finished` would return `True`. In other cases, the method\nshould wait until a request appears." } ] }, + "flags": {}, + "id": 80558, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "fetch_next_request", + "parameters": [], "type": { + "name": "Request | None", "type": "reference", - "name": "None" + "target": 0, + "ref": {} }, - "parameters": [ - { - "id": 246, - "name": "request", - "kind": 32768, - "kindString": "Parameter", - "flags": {}, - "type": { - "type": "reference", - "name": "Request" - } - }, + "overwrites": { + "name": "RequestManager.fetch_next_request", + "target": 4147, + "type": "reference", + "ref": { + "id": 81761, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 45 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestManager#fetch_next_request" + } + }, + "parentId": 4147 + } + ], + "inheritedFrom": { + "name": "RequestLoader.fetch_next_request", + "target": 2943, + "type": "reference", + "ref": { + "id": 80557, + "sources": [ { - "id": 247, - "name": "spider", - "kind": 32768, - "kindString": "Parameter", - "flags": {}, - "type": { - "type": "reference", - "name": "Spider" - } + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 45 } - ] + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestLoader#fetch_next_request" } - ] + }, + "permalink": "/python/api/class/RequestManager#fetch_next_request", + "parentId": 3001 }, { - "id": 248, - "name": "process_exception", - "module": "scrapy.middlewares.apify_proxy", "kind": 2048, "kindString": "Method", - "flags": {}, + "children": [], "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Process an exception that occurs during request processing.\\n\", {\"Arguments\": [{\"param\": \"request\", \"desc\": \"Scrapy Request object.\"}, {\"param\": \"exception\", \"desc\": \"Exception object.\"}, {\"param\": \"spider\", \"desc\": \"Scrapy Spider object.\\n\"}]}, {\"Returns\": [\"If a TunnelError occurs, return the request object to halt its processing in the middleware pipeline.\", \"Return None otherwise to allow the continuation of request processing.\"]}]}" + "text": "Mark a request as handled after a successful processing (or after giving up retrying)." } ] }, - "children": [], + "decorations": [ + { + "name": "abstractmethod" + } + ], + "flags": {}, "groups": [], + "id": 81762, + "module": "request_loaders._request_loader", + "name": "mark_request_as_handled", + "parsedDocstring": { + "text": "Mark a request as handled after a successful processing (or after giving up retrying)." + }, "sources": [ { - "filename": "/tmp-old-source-v17/src/apify/scrapy/middlewares/apify_proxy.py", - "line": 93, "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/scrapy/middlewares/apify_proxy.py#L93" + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 53, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/request_loaders/_request_loader.py#L53" } ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, "signatures": [ { - "id": 249, - "name": "process_exception", - "modifiers": [], - "kind": 4096, - "kindString": "Call signature", - "flags": {}, "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Process an exception that occurs during request processing.\\n\", {\"" + "text": "Mark a request as handled after a successful processing (or after giving up retrying)." } ] }, - "type": { - "type": "reference", - "name": "None | Request" - }, + "flags": {}, + "id": 80560, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "mark_request_as_handled", "parameters": [ { - "id": 250, - "name": "request", + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 80561, "kind": 32768, "kindString": "Parameter", - "flags": {}, + "name": "request", "type": { + "name": "Request", "type": "reference", - "name": "Request" - } - }, + "target": "409", + "ref": { + "id": 78023, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_request.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 145 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/Request" + } + }, + "parentId": 2946 + } + ], + "type": { + "name": "ProcessedRequest | None", + "type": "reference", + "target": 0, + "ref": {} + }, + "overwrites": { + "name": "RequestManager.mark_request_as_handled", + "target": 4148, + "type": "reference", + "ref": { + "id": 81762, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 53 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestManager#mark_request_as_handled" + } + }, + "parentId": 4148 + } + ], + "inheritedFrom": { + "name": "RequestLoader.mark_request_as_handled", + "target": 2945, + "type": "reference", + "ref": { + "id": 80559, + "sources": [ { - "id": 251, - "name": "exception", - "kind": 32768, - "kindString": "Parameter", - "flags": {}, - "type": { - "type": "reference", - "name": "Exception" + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 53 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestLoader#mark_request_as_handled" + } + }, + "permalink": "/python/api/class/RequestManager#mark_request_as_handled", + "parentId": 3001 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Combine the loader with a request manager to support adding and reclaiming requests.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 81763, + "module": "request_loaders._request_loader", + "name": "to_tandem", + "parsedDocstring": { + "text": "Combine the loader with a request manager to support adding and reclaiming requests.\n", + "args": { + "request_manager": "Request manager to combine the loader with.\nIf None is given, the default request queue is used." + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 56, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/request_loaders/_request_loader.py#L56" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Combine the loader with a request manager to support adding and reclaiming requests.\n" } - }, + ] + }, + "flags": {}, + "id": 80563, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "to_tandem", + "parameters": [ { - "id": 252, - "name": "spider", + "comment": { + "summary": [ + { + "kind": "text", + "text": "Request manager to combine the loader with.\nIf None is given, the default request queue is used." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": false + }, + "id": 80564, "kind": 32768, "kindString": "Parameter", - "flags": {}, + "name": "request_manager", "type": { + "name": "RequestManager | None", "type": "reference", - "name": "Spider" - } + "target": 0, + "ref": {} + }, + "parentId": 2949 + } + ], + "type": { + "name": "RequestManagerTandem", + "type": "reference", + "target": "2803", + "ref": { + "id": 80417, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_manager_tandem.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 24 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/RequestManagerTandem" + } + }, + "inheritedFrom": { + "name": "RequestLoader.to_tandem", + "target": 2948, + "type": "reference", + "ref": { + "id": 80562, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 56 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestLoader#to_tandem" } - ] + }, + "parentId": 4149 } - ] + ], + "inheritedFrom": { + "name": "RequestLoader.to_tandem", + "target": 2948, + "type": "reference", + "ref": { + "id": 80562, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 56 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestLoader#to_tandem" + } + }, + "permalink": "/python/api/class/RequestManager#to_tandem", + "parentId": 3001 } ], - "groups": [ + "comment": { + "summary": [ + { + "kind": "text", + "text": "Base class that extends `RequestLoader` with the capability to enqueue new requests and reclaim failed ones." + } + ] + }, + "decorations": [ { - "title": "Constructors", - "children": [ - 238 - ] - }, + "args": "('Request loaders')", + "name": "docs_group" + } + ], + "flags": {}, + "groups": [ { - "title": "Methods", "children": [ - 241, - 248, - 244 - ] + 80620, + 80624, + 80616, + 81761, + 81757, + 81758, + 81759, + 81760, + 81762, + 80618, + 80632, + 81763 + ], + "title": "Methods" } ], + "id": 80615, + "module": "request_loaders._request_manager", + "name": "RequestManager", + "parsedDocstring": { + "text": "Base class that extends `RequestLoader` with the capability to enqueue new requests and reclaim failed ones." + }, "sources": [ { - "filename": "/tmp-old-source-v17/src/apify/scrapy/middlewares/apify_proxy.py", - "line": 20, "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/scrapy/middlewares/apify_proxy.py#L20" + "fileName": "/src/crawlee/request_loaders/_request_manager.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 18, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/request_loaders/_request_manager.py#L18" } - ] + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "extendedTypes": [ + { + "name": "RequestLoader", + "target": "2934", + "type": "reference", + "ref": { + "id": 80548, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 17 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/RequestLoader" + } + } + ], + "extendedBy": [ + { + "name": "RequestQueue", + "target": "2348", + "type": "reference", + "ref": { + "id": 79962, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_request_queue.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 33 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/RequestQueue" + } + }, + { + "name": "RequestManagerTandem", + "target": "2803", + "type": "reference", + "ref": { + "id": 80417, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_manager_tandem.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 24 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/RequestManagerTandem" + } + }, + { + "name": "ThrottlingRequestManager", + "target": "2841", + "type": "reference", + "ref": { + "id": 80455, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_throttling_request_manager.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 35 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/ThrottlingRequestManager" + } + } + ], + "permalink": "/python/api/class/RequestManager", + "categories": [] }, { - "id": 253, - "name": "ActorDatasetPushPipeline", - "module": "scrapy.pipelines.actor_dataset_push", "kind": 128, "kindString": "Class", - "flags": {}, - "comment": { - "summary": [ - { - "kind": "text", - "text": "{\"content\": [\"A Scrapy pipeline for pushing items to an Actor's default dataset.\\n\\nThis pipeline is designed to be enabled only when the Scrapy project is run as an Actor.\"]}" - } - ] - }, "children": [ { - "id": 254, - "name": "process_item", - "module": "scrapy.pipelines.actor_dataset_push", "kind": 2048, "kindString": "Method", - "flags": {}, + "children": [], "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Pushes the provided Scrapy item to the Actor's default dataset.\"]}" + "text": "" } ] }, - "children": [], + "decorations": [], + "flags": {}, "groups": [], + "id": 84567, + "module": "request_loaders._request_manager_tandem", + "name": "__init__", + "parsedDocstring": { + "text": "" + }, "sources": [ { - "filename": "/tmp-old-source-v17/src/apify/scrapy/pipelines/actor_dataset_push.py", - "line": 21, "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/scrapy/pipelines/actor_dataset_push.py#L21" + "fileName": "/src/crawlee/request_loaders/_request_manager_tandem.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 31, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/request_loaders/_request_manager_tandem.py#L31" } ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, "signatures": [ { - "id": 255, - "name": "process_item", - "modifiers": [ - "async" - ], + "flags": {}, + "id": 84568, "kind": 4096, "kindString": "Call signature", - "flags": {}, - "comment": { - "summary": [ - { - "kind": "text", - "text": "{\"content\": [\"Pushes the provided Scrapy item to the Actor's default dataset.\"]}" - } - ] - }, - "type": { - "type": "reference", - "name": "Item" - }, + "modifiers": [], + "name": "__init__", "parameters": [ { - "id": 256, - "name": "item", + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 84569, "kind": 32768, "kindString": "Parameter", - "flags": {}, + "name": "request_loader", "type": { + "name": "RequestLoader", "type": "reference", - "name": "Item" - } + "target": "2934", + "ref": { + "id": 84697, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 17 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/RequestLoader" + } + }, + "parentId": 2805 }, { - "id": 257, - "name": "spider", + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 84570, "kind": 32768, "kindString": "Parameter", - "flags": {}, + "name": "request_manager", "type": { + "name": "RequestManager", "type": "reference", - "name": "Spider" - } + "target": "3001", + "ref": { + "id": 84764, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_manager.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 18 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/RequestManager" + } + }, + "parentId": 2805 } - ] + ], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "parentId": 2804 } - ] - } - ], - "groups": [ - { - "title": "Methods", - "children": [ - 254 - ] - } - ], - "sources": [ - { - "filename": "/tmp-old-source-v17/src/apify/scrapy/pipelines/actor_dataset_push.py", - "line": 15, - "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/scrapy/pipelines/actor_dataset_push.py#L15" - } - ] - }, - { - "id": 258, - "name": "to_apify_request", - "module": "scrapy.requests", - "kind": 2048, - "kindString": "Method", - "flags": {}, - "comment": { - "summary": [ - { - "kind": "text", - "text": "{\"content\": [\"Convert a Scrapy request to an Apify request.\\n\", {\"Arguments\": [{\"param\": \"scrapy_request\", \"desc\": \"The Scrapy request to be converted.\"}, {\"param\": \"spider\", \"desc\": \"The Scrapy spider that the request is associated with.\\n\"}]}, {\"Returns\": [\"The converted Apify request if the conversion was successful, otherwise None.\"]}]}" - } - ] - }, - "children": [], - "groups": [], - "sources": [ - { - "filename": "/tmp-old-source-v17/src/apify/scrapy/requests.py", - "line": 28, - "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/scrapy/requests.py#L28" - } - ], - "signatures": [ + ], + "permalink": "/python/api/class/RequestManagerTandem#__init__", + "parentId": 2803 + }, { - "id": 259, - "name": "to_apify_request", - "modifiers": [], - "kind": 4096, - "kindString": "Call signature", - "flags": {}, + "kind": 2048, + "kindString": "Method", + "children": [], "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Convert a Scrapy request to an Apify request.\\n\", {\"" + "text": "Get the number of requests in the loader that have been handled." } ] }, + "decorations": [ + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 84571, + "module": "request_loaders._request_loader", + "name": "get_handled_count", + "parsedDocstring": { + "text": "Get the number of requests in the loader that have been handled." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_manager_tandem.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 36, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/request_loaders/_request_manager_tandem.py#L36" + } + ], "type": { + "name": "Undefined", "type": "reference", - "name": "dict | None" + "target": 0, + "ref": {} }, - "parameters": [ + "signatures": [ { - "id": 260, - "name": "scrapy_request", - "kind": 32768, - "kindString": "Parameter", + "comment": { + "summary": [ + { + "kind": "text", + "text": "Get the number of requests in the loader that have been handled." + } + ] + }, "flags": {}, + "id": 84699, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "get_handled_count", + "parameters": [], "type": { + "name": "int", "type": "reference", - "name": "Request" - } - }, - { - "id": 261, - "name": "spider", - "kind": 32768, - "kindString": "Parameter", - "flags": {}, - "type": { + "target": 0, + "ref": {} + }, + "overwrites": { + "name": "RequestManager.get_handled_count", + "target": 4143, "type": "reference", - "name": "Spider" - } + "ref": { + "id": 85906, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 29 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestManager#get_handled_count" + } + }, + "parentId": 2808 } - ] - } - ] - }, - { - "id": 262, - "name": "to_scrapy_request", - "module": "scrapy.requests", - "kind": 2048, - "kindString": "Method", - "flags": {}, - "comment": { - "summary": [ - { - "kind": "text", - "text": "{\"content\": [\"Convert an Apify request to a Scrapy request.\\n\", {\"Arguments\": [{\"param\": \"apify_request\", \"desc\": \"The Apify request to be converted.\"}, {\"param\": \"spider\", \"desc\": \"The Scrapy spider that the request is associated with.\\n\"}]}, {\"Raises\": [{\"param\": \"TypeError\", \"desc\": \"If the apify_request is not a dictionary.\"}, {\"param\": \"ValueError\", \"desc\": \"If the apify_request does not contain the required keys.\\n\"}]}, {\"Returns\": [\"The converted Scrapy request.\"]}]}" - } - ] - }, - "children": [], - "groups": [], - "sources": [ - { - "filename": "/tmp-old-source-v17/src/apify/scrapy/requests.py", - "line": 96, - "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/scrapy/requests.py#L96" - } - ], - "signatures": [ + ], + "overwrites": { + "name": "RequestManager.get_handled_count", + "target": 4143, + "type": "reference", + "ref": { + "id": 85906, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 29 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestManager#get_handled_count" + } + }, + "permalink": "/python/api/class/RequestManagerTandem#get_handled_count", + "parentId": 2803 + }, { - "id": 263, - "name": "to_scrapy_request", - "modifiers": [], - "kind": 4096, - "kindString": "Call signature", - "flags": {}, + "kind": 2048, + "kindString": "Method", + "children": [], "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Convert an Apify request to a Scrapy request.\\n\", {\"" + "text": "Get an offline approximation of the total number of requests in the loader (i.e. pending + handled)." } ] }, + "decorations": [ + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 84573, + "module": "request_loaders._request_loader", + "name": "get_total_count", + "parsedDocstring": { + "text": "Get an offline approximation of the total number of requests in the loader (i.e. pending + handled)." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_manager_tandem.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 40, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/request_loaders/_request_manager_tandem.py#L40" + } + ], "type": { + "name": "Undefined", "type": "reference", - "name": "Request" + "target": 0, + "ref": {} }, - "parameters": [ + "signatures": [ { - "id": 264, - "name": "apify_request", - "kind": 32768, - "kindString": "Parameter", + "comment": { + "summary": [ + { + "kind": "text", + "text": "Get an offline approximation of the total number of requests in the loader (i.e. pending + handled)." + } + ] + }, "flags": {}, + "id": 84701, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "get_total_count", + "parameters": [], "type": { - "type": "intrinsic", - "name": "dict" + "name": "int", + "type": "reference", + "target": 0, + "ref": {} + }, + "overwrites": { + "name": "RequestManager.get_total_count", + "target": 4144, + "type": "reference", + "ref": { + "id": 85907, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 33 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestManager#get_total_count" + } + }, + "parentId": 2810 + } + ], + "overwrites": { + "name": "RequestManager.get_total_count", + "target": 4144, + "type": "reference", + "ref": { + "id": 85907, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 33 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestManager#get_total_count" + } + }, + "permalink": "/python/api/class/RequestManagerTandem#get_total_count", + "parentId": 2803 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Return True if there are no more requests in the loader (there might still be unfinished requests)." } - }, + ] + }, + "decorations": [ { - "id": 265, - "name": "spider", - "kind": 32768, - "kindString": "Parameter", + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 84575, + "module": "request_loaders._request_loader", + "name": "is_empty", + "parsedDocstring": { + "text": "Return True if there are no more requests in the loader (there might still be unfinished requests)." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_manager_tandem.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 44, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/request_loaders/_request_manager_tandem.py#L44" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Return True if there are no more requests in the loader (there might still be unfinished requests)." + } + ] + }, "flags": {}, + "id": 84703, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "is_empty", + "parameters": [], "type": { + "name": "bool", "type": "reference", - "name": "Spider" - } + "target": 0, + "ref": {} + }, + "overwrites": { + "name": "RequestManager.is_empty", + "target": 4145, + "type": "reference", + "ref": { + "id": 85908, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 37 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestManager#is_empty" + } + }, + "parentId": 2812 } - ] - } - ] - }, - { - "id": 266, - "name": "ApifyScheduler", - "module": "scrapy.scheduler", - "kind": 128, - "kindString": "Class", - "flags": {}, - "comment": { - "summary": [ - { - "kind": "text", - "text": "{\"content\": [\"A Scrapy scheduler that uses the Apify Request Queue to manage requests.\\n\\nThis scheduler requires the asyncio Twisted reactor to be installed.\"]}" - } - ] - }, - "children": [ + ], + "overwrites": { + "name": "RequestManager.is_empty", + "target": 4145, + "type": "reference", + "ref": { + "id": 85908, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 37 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestManager#is_empty" + } + }, + "permalink": "/python/api/class/RequestManagerTandem#is_empty", + "parentId": 2803 + }, { - "id": 267, - "name": "__init__", - "module": "scrapy.scheduler", - "kind": 512, - "kindString": "Constructor", - "flags": {}, + "kind": 2048, + "kindString": "Method", + "children": [], "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Create a new instance.\"]}" + "text": "Return True if all requests have been handled." } ] }, - "children": [], + "decorations": [ + { + "name": "abstractmethod" + } + ], + "flags": {}, "groups": [], + "id": 84577, + "module": "request_loaders._request_loader", + "name": "is_finished", + "parsedDocstring": { + "text": "Return True if all requests have been handled." + }, "sources": [ { - "filename": "/tmp-old-source-v17/src/apify/scrapy/scheduler.py", - "line": 28, "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/scrapy/scheduler.py#L28" + "fileName": "/src/crawlee/request_loaders/_request_manager_tandem.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 48, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/request_loaders/_request_manager_tandem.py#L48" } ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, "signatures": [ { - "id": 268, - "name": "__init__", - "modifiers": [], - "kind": 4096, - "kindString": "Call signature", - "flags": {}, "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Create a new instance.\"]}" + "text": "Return True if all requests have been handled." } ] }, + "flags": {}, + "id": 84705, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "is_finished", + "parameters": [], "type": { + "name": "bool", + "type": "reference", + "target": 0, + "ref": {} + }, + "overwrites": { + "name": "RequestManager.is_finished", + "target": 4146, "type": "reference", - "name": "None" + "ref": { + "id": 85909, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 41 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestManager#is_finished" + } }, - "parameters": [] + "parentId": 2814 } - ] + ], + "overwrites": { + "name": "RequestManager.is_finished", + "target": 4146, + "type": "reference", + "ref": { + "id": 85909, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 41 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestManager#is_finished" + } + }, + "permalink": "/python/api/class/RequestManagerTandem#is_finished", + "parentId": 2803 }, { - "id": 269, - "name": "open", - "module": "scrapy.scheduler", "kind": 2048, "kindString": "Method", - "flags": {}, + "children": [], "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Open the scheduler.\\n\", {\"Arguments\": [{\"param\": \"spider\", \"desc\": \"The spider that the scheduler is associated with.\"}]}]}" + "text": "Add a single request to the manager and store it in underlying resource client.\n" } ] }, - "children": [], + "decorations": [ + { + "name": "abstractmethod" + } + ], + "flags": {}, "groups": [], + "id": 84579, + "module": "request_loaders._request_manager", + "name": "add_request", + "parsedDocstring": { + "text": "Add a single request to the manager and store it in underlying resource client.\n", + "args": { + "request": "The request object (or its string representation) to be added to the manager.", + "forefront": "Determines whether the request should be added to the beginning (if True) or the end (if False)\nof the manager.\n" + }, + "returns": "Information about the request addition to the manager or None if the request was not added." + }, "sources": [ { - "filename": "/tmp-old-source-v17/src/apify/scrapy/scheduler.py", - "line": 39, "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/scrapy/scheduler.py#L39" + "fileName": "/src/crawlee/request_loaders/_request_manager_tandem.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 52, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/request_loaders/_request_manager_tandem.py#L52" } ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, "signatures": [ { - "id": 270, - "name": "open", - "modifiers": [], - "kind": 4096, - "kindString": "Call signature", - "flags": {}, "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "Information about the request addition to the manager or None if the request was not added." + } + ], + "tag": "@returns" + } + ], "summary": [ { "kind": "text", - "text": "{\"content\": [\"Open the scheduler.\\n\", {\"" + "text": "Add a single request to the manager and store it in underlying resource client.\n" } ] }, - "type": { - "type": "reference", - "name": "None" - }, + "flags": {}, + "id": 84770, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "add_request", "parameters": [ { - "id": 271, - "name": "spider", + "comment": { + "summary": [ + { + "kind": "text", + "text": "The request object (or its string representation) to be added to the manager." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 84771, + "kind": 32768, + "kindString": "Parameter", + "name": "request", + "type": { + "name": "str | Request", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "reference", + "name": "Request", + "target": "409", + "ref": { + "id": 82172, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_request.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 145 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/Request" + } + } + ] + }, + "parentId": 3007 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Determines whether the request should be added to the beginning (if True) or the end (if False)\nof the manager.\n" + } + ] + }, + "defaultValue": "False", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 84772, "kind": 32768, "kindString": "Parameter", - "flags": {}, + "name": "forefront", "type": { + "name": "bool", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 3007 + } + ], + "type": { + "name": "ProcessedRequest | None", + "type": "union", + "types": [ + { "type": "reference", - "name": "Spider" + "name": "ProcessedRequest", + "target": "1340", + "ref": { + "id": 83103, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 140 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/ProcessedRequest" + } + }, + { + "type": "literal", + "value": {} } + ] + }, + "overwrites": { + "name": "RequestManager.add_request", + "target": 3006, + "type": "reference", + "ref": { + "id": 84769, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_manager.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 32 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestManager#add_request" } - ] + }, + "parentId": 2816 } - ] + ], + "overwrites": { + "name": "RequestManager.add_request", + "target": 3006, + "type": "reference", + "ref": { + "id": 84769, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_manager.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 32 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestManager#add_request" + } + }, + "permalink": "/python/api/class/RequestManagerTandem#add_request", + "parentId": 2803 }, - { - "id": 272, - "name": "has_pending_requests", - "module": "scrapy.scheduler", + { "kind": 2048, "kindString": "Method", - "flags": {}, + "children": [], "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Check if the scheduler has any pending requests.\\n\", {\"Returns\": [\"True if the scheduler has any pending requests, False otherwise.\"]}]}" + "text": "Add requests to the manager in batches.\n" } ] }, - "children": [], + "decorations": [], + "flags": {}, "groups": [], + "id": 84583, + "module": "request_loaders._request_manager", + "name": "add_requests", + "parsedDocstring": { + "text": "Add requests to the manager in batches.\n", + "args": { + "requests": "Requests to enqueue.", + "forefront": "If True, add requests to the beginning of the queue.", + "batch_size": "The number of requests to add in one batch.", + "wait_time_between_batches": "Time to wait between adding batches.", + "wait_for_all_requests_to_be_added": "If True, wait for all requests to be added before returning.", + "wait_for_all_requests_to_be_added_timeout": "Timeout for waiting for all requests to be added." + } + }, "sources": [ { - "filename": "/tmp-old-source-v17/src/apify/scrapy/scheduler.py", - "line": 53, "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/scrapy/scheduler.py#L53" + "fileName": "/src/crawlee/request_loaders/_request_manager_tandem.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 56, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/request_loaders/_request_manager_tandem.py#L56" } ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, "signatures": [ { - "id": 273, - "name": "has_pending_requests", - "modifiers": [], - "kind": 4096, - "kindString": "Call signature", - "flags": {}, "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Check if the scheduler has any pending requests.\\n\", {\"" + "text": "Add requests to the manager in batches.\n" } ] }, + "flags": {}, + "id": 84774, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "add_requests", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Requests to enqueue." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 84775, + "kind": 32768, + "kindString": "Parameter", + "name": "requests", + "type": { + "name": "Sequence", + "type": "reference", + "typeArguments": [ + { + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "reference", + "name": "Request", + "target": "409", + "ref": { + "id": 82172, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_request.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 145 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/Request" + } + } + ] + } + ], + "target": 0, + "ref": {} + }, + "parentId": 3011 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "If True, add requests to the beginning of the queue." + } + ] + }, + "defaultValue": "False", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 84776, + "kind": 32768, + "kindString": "Parameter", + "name": "forefront", + "type": { + "name": "bool", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 3011 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The number of requests to add in one batch." + } + ] + }, + "defaultValue": "1000", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 84777, + "kind": 32768, + "kindString": "Parameter", + "name": "batch_size", + "type": { + "name": "int", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 3011 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Time to wait between adding batches." + } + ] + }, + "defaultValue": "timedelta(seconds=1)", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 84778, + "kind": 32768, + "kindString": "Parameter", + "name": "wait_time_between_batches", + "type": { + "name": "timedelta", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 3011 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "If True, wait for all requests to be added before returning." + } + ] + }, + "defaultValue": "False", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 84779, + "kind": 32768, + "kindString": "Parameter", + "name": "wait_for_all_requests_to_be_added", + "type": { + "name": "bool", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 3011 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Timeout for waiting for all requests to be added." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 84780, + "kind": 32768, + "kindString": "Parameter", + "name": "wait_for_all_requests_to_be_added_timeout", + "type": { + "name": "timedelta | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "timedelta", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 3011 + } + ], "type": { - "type": "intrinsic", - "name": "bool" + "name": "None", + "type": "literal", + "value": {} + }, + "overwrites": { + "name": "RequestManager.add_requests", + "target": 3010, + "type": "reference", + "ref": { + "id": 84773, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_manager.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 49 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestManager#add_requests" + } }, - "parameters": [] + "parentId": 2820 } - ] + ], + "overwrites": { + "name": "RequestManager.add_requests", + "target": 3010, + "type": "reference", + "ref": { + "id": 84773, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_manager.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 49 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestManager#add_requests" + } + }, + "permalink": "/python/api/class/RequestManagerTandem#add_requests", + "parentId": 2803 }, { - "id": 274, - "name": "enqueue_request", - "module": "scrapy.scheduler", "kind": 2048, "kindString": "Method", - "flags": {}, + "children": [], "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Add a request to the scheduler.\\n\\nThis could be called from either from a spider or a downloader middleware (e.g. redirect, retry, ...).\\n\", {\"Arguments\": [{\"param\": \"request\", \"desc\": \"The request to add to the scheduler.\\n\"}]}, {\"Returns\": [\"True if the request was successfully enqueued, False otherwise.\"]}]}" + "text": "Return the next request to be processed, or `None` if there are no more pending requests.\n\nThe method should return `None` if and only if `is_finished` would return `True`. In other cases, the method\nshould wait until a request appears." } ] }, - "children": [], + "decorations": [ + { + "name": "abstractmethod" + } + ], + "flags": {}, "groups": [], + "id": 84591, + "module": "request_loaders._request_loader", + "name": "fetch_next_request", + "parsedDocstring": { + "text": "Return the next request to be processed, or `None` if there are no more pending requests.\n\nThe method should return `None` if and only if `is_finished` would return `True`. In other cases, the method\nshould wait until a request appears." + }, "sources": [ { - "filename": "/tmp-old-source-v17/src/apify/scrapy/scheduler.py", - "line": 70, "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/scrapy/scheduler.py#L70" + "fileName": "/src/crawlee/request_loaders/_request_manager_tandem.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 76, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/request_loaders/_request_manager_tandem.py#L76" } ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, "signatures": [ { - "id": 275, - "name": "enqueue_request", - "modifiers": [], - "kind": 4096, - "kindString": "Call signature", - "flags": {}, "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Add a request to the scheduler.\\n\\nThis could be called from either from a spider or a downloader middleware (e.g. redirect, retry, ...).\\n\", {\"" + "text": "Return the next request to be processed, or `None` if there are no more pending requests.\n\nThe method should return `None` if and only if `is_finished` would return `True`. In other cases, the method\nshould wait until a request appears." } ] }, + "flags": {}, + "id": 84707, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "fetch_next_request", + "parameters": [], "type": { - "type": "intrinsic", - "name": "bool" + "name": "Request | None", + "type": "reference", + "target": 0, + "ref": {} }, - "parameters": [ + "overwrites": { + "name": "RequestManager.fetch_next_request", + "target": 4147, + "type": "reference", + "ref": { + "id": 85910, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 45 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestManager#fetch_next_request" + } + }, + "parentId": 2828 + } + ], + "overwrites": { + "name": "RequestManager.fetch_next_request", + "target": 4147, + "type": "reference", + "ref": { + "id": 85910, + "sources": [ { - "id": 276, - "name": "request", - "kind": 32768, - "kindString": "Parameter", - "flags": {}, - "type": { - "type": "reference", - "name": "Request" - } + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 45 } - ] + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestManager#fetch_next_request" } - ] + }, + "permalink": "/python/api/class/RequestManagerTandem#fetch_next_request", + "parentId": 2803 }, { - "id": 277, - "name": "next_request", - "module": "scrapy.scheduler", "kind": 2048, "kindString": "Method", - "flags": {}, + "children": [], "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Fetch the next request from the scheduler.\\n\", {\"Returns\": [\"The next request, or None if there are no more requests.\"]}]}" + "text": "Reclaims a failed request back to the source, so that it can be returned for processing later again.\n\nIt is possible to modify the request data by supplying an updated request as a parameter." } ] }, - "children": [], + "decorations": [ + { + "name": "abstractmethod" + } + ], + "flags": {}, "groups": [], + "id": 84593, + "module": "request_loaders._request_manager", + "name": "reclaim_request", + "parsedDocstring": { + "text": "Reclaims a failed request back to the source, so that it can be returned for processing later again.\n\nIt is possible to modify the request data by supplying an updated request as a parameter." + }, "sources": [ { - "filename": "/tmp-old-source-v17/src/apify/scrapy/scheduler.py", - "line": 111, "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/scrapy/scheduler.py#L111" + "fileName": "/src/crawlee/request_loaders/_request_manager_tandem.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 102, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/request_loaders/_request_manager_tandem.py#L102" } ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, "signatures": [ { - "id": 278, - "name": "next_request", - "modifiers": [], - "kind": 4096, - "kindString": "Call signature", - "flags": {}, "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Fetch the next request from the scheduler.\\n\", {\"" + "text": "Reclaims a failed request back to the source, so that it can be returned for processing later again.\n\nIt is possible to modify the request data by supplying an updated request as a parameter." } ] }, - "type": { - "type": "reference", - "name": "Request | None" - }, - "parameters": [] - } - ] - } - ], - "groups": [ - { - "title": "Constructors", - "children": [ - 267 - ] - }, - { - "title": "Methods", - "children": [ - 274, - 272, - 277, - 269 - ] - } - ], - "sources": [ - { - "filename": "/tmp-old-source-v17/src/apify/scrapy/scheduler.py", - "line": 22, - "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/scrapy/scheduler.py#L22" - } - ] - }, - { - "id": 279, - "name": "get_basic_auth_header", - "module": "scrapy.utils", - "kind": 2048, - "kindString": "Method", - "flags": {}, - "comment": { - "summary": [ - { - "kind": "text", - "text": "{\"content\": [\"Generate a basic authentication header for the given username and password.\"]}" - } - ] - }, - "children": [], - "groups": [], - "sources": [ - { - "filename": "/tmp-old-source-v17/src/apify/scrapy/utils.py", - "line": 22, - "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/scrapy/utils.py#L22" - } - ], - "signatures": [ + "flags": {}, + "id": 84782, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "reclaim_request", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 84783, + "kind": 32768, + "kindString": "Parameter", + "name": "request", + "type": { + "name": "Request", + "type": "reference", + "target": "409", + "ref": { + "id": 82172, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_request.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 145 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/Request" + } + }, + "parentId": 3019 + }, + { + "defaultValue": "False", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 84784, + "kind": 32768, + "kindString": "Parameter", + "name": "forefront", + "type": { + "name": "bool", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 3019 + } + ], + "type": { + "name": "ProcessedRequest | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "ProcessedRequest", + "target": "1340", + "ref": { + "id": 83103, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 140 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/ProcessedRequest" + } + }, + { + "type": "literal", + "value": {} + } + ] + }, + "overwrites": { + "name": "RequestManager.reclaim_request", + "target": 3018, + "type": "reference", + "ref": { + "id": 84781, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_manager.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 77 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestManager#reclaim_request" + } + }, + "parentId": 2830 + } + ], + "overwrites": { + "name": "RequestManager.reclaim_request", + "target": 3018, + "type": "reference", + "ref": { + "id": 84781, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_manager.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 77 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestManager#reclaim_request" + } + }, + "permalink": "/python/api/class/RequestManagerTandem#reclaim_request", + "parentId": 2803 + }, { - "id": 280, - "name": "get_basic_auth_header", - "modifiers": [], - "kind": 4096, - "kindString": "Call signature", - "flags": {}, + "kind": 2048, + "kindString": "Method", + "children": [], "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Generate a basic authentication header for the given username and password.\"]}" + "text": "Mark a request as handled after a successful processing (or after giving up retrying)." } ] }, + "decorations": [ + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 84597, + "module": "request_loaders._request_loader", + "name": "mark_request_as_handled", + "parsedDocstring": { + "text": "Mark a request as handled after a successful processing (or after giving up retrying)." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_manager_tandem.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 106, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/request_loaders/_request_manager_tandem.py#L106" + } + ], "type": { + "name": "Undefined", "type": "reference", - "name": "bytes" + "target": 0, + "ref": {} }, - "parameters": [ - { - "id": 281, - "name": "username", - "kind": 32768, - "kindString": "Parameter", - "flags": {}, - "type": { - "type": "intrinsic", - "name": "str" - } - }, - { - "id": 282, - "name": "password", - "kind": 32768, - "kindString": "Parameter", - "flags": {}, - "type": { - "type": "intrinsic", - "name": "str" - } - }, + "signatures": [ { - "id": 283, - "name": "auth_encoding", - "kind": 32768, - "kindString": "Parameter", + "comment": { + "summary": [ + { + "kind": "text", + "text": "Mark a request as handled after a successful processing (or after giving up retrying)." + } + ] + }, "flags": {}, + "id": 84709, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "mark_request_as_handled", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 84710, + "kind": 32768, + "kindString": "Parameter", + "name": "request", + "type": { + "name": "Request", + "type": "reference", + "target": "409", + "ref": { + "id": 82172, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_request.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 145 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/Request" + } + }, + "parentId": 2946 + } + ], "type": { - "type": "intrinsic", - "name": "str" + "name": "ProcessedRequest | None", + "type": "reference", + "target": 0, + "ref": {} }, - "defaultValue": "'latin-1'" + "overwrites": { + "name": "RequestManager.mark_request_as_handled", + "target": 4148, + "type": "reference", + "ref": { + "id": 85911, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 53 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestManager#mark_request_as_handled" + } + }, + "parentId": 2834 } - ] - } - ] - }, - { - "id": 284, - "name": "get_running_event_loop_id", - "module": "scrapy.utils", - "kind": 2048, - "kindString": "Method", - "flags": {}, - "comment": { - "summary": [ - { - "kind": "text", - "text": "{\"content\": [\"Get the ID of the currently running event loop.\\n\\nIt could be useful mainly for debugging purposes.\\n\", {\"Returns\": [\"The ID of the event loop.\"]}]}" - } - ] - }, - "children": [], - "groups": [], - "sources": [ - { - "filename": "/tmp-old-source-v17/src/apify/scrapy/utils.py", - "line": 29, - "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/scrapy/utils.py#L29" - } - ], - "signatures": [ + ], + "overwrites": { + "name": "RequestManager.mark_request_as_handled", + "target": 4148, + "type": "reference", + "ref": { + "id": 85911, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 53 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestManager#mark_request_as_handled" + } + }, + "permalink": "/python/api/class/RequestManagerTandem#mark_request_as_handled", + "parentId": 2803 + }, { - "id": 285, - "name": "get_running_event_loop_id", - "modifiers": [], - "kind": 4096, - "kindString": "Call signature", - "flags": {}, + "kind": 2048, + "kindString": "Method", + "children": [], "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Get the ID of the currently running event loop.\\n\\nIt could be useful mainly for debugging purposes.\\n\", {\"" + "text": "Remove persistent state either from the Apify Cloud storage or from the local database." } ] }, - "type": { - "type": "intrinsic", - "name": "int" - }, - "parameters": [] - } - ] - }, - { - "id": 286, - "name": "apply_apify_settings", - "module": "scrapy.utils", - "kind": 2048, - "kindString": "Method", - "flags": {}, - "comment": { - "summary": [ - { - "kind": "text", - "text": "{\"content\": [\"Integrates Apify configuration into a Scrapy project settings.\\n\\nNote: The function directly modifies the passed `settings` object and also returns it.\\n\", {\"Arguments\": [{\"param\": \"settings\", \"desc\": \"Scrapy project settings to be modified.\"}, {\"param\": \"proxy_config\", \"desc\": \"Proxy configuration to be stored in the settings.\\n\"}]}, {\"Returns\": [\"Scrapy project settings with custom configurations.\"]}]}" - } - ] - }, - "children": [], - "groups": [], - "sources": [ - { - "filename": "/tmp-old-source-v17/src/apify/scrapy/utils.py", - "line": 40, - "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/scrapy/utils.py#L40" - } - ], - "signatures": [ - { - "id": 287, - "name": "apply_apify_settings", - "modifiers": [], - "kind": 4096, - "kindString": "Call signature", + "decorations": [ + { + "name": "abstractmethod" + } + ], "flags": {}, - "comment": { - "summary": [ - { - "kind": "text", - "text": "{\"content\": [\"Integrates Apify configuration into a Scrapy project settings.\\n\\nNote: The function directly modifies the passed `settings` object and also returns it.\\n\", {\"" - } - ] + "groups": [], + "id": 84600, + "module": "request_loaders._request_manager", + "name": "drop", + "parsedDocstring": { + "text": "Remove persistent state either from the Apify Cloud storage or from the local database." }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_manager_tandem.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 110, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/request_loaders/_request_manager_tandem.py#L110" + } + ], "type": { + "name": "Undefined", "type": "reference", - "name": "Settings" + "target": 0, + "ref": {} }, - "parameters": [ - { - "id": 288, - "name": "settings", - "kind": 32768, - "kindString": "Parameter", - "flags": { - "keyword-only": "true" - }, - "type": { - "type": "reference", - "name": "Settings | None" - }, - "defaultValue": "None" - }, + "signatures": [ { - "id": 289, - "name": "proxy_config", - "kind": 32768, - "kindString": "Parameter", - "flags": { - "keyword-only": "true" - }, - "type": { - "type": "reference", - "name": "dict | None" + "comment": { + "summary": [ + { + "kind": "text", + "text": "Remove persistent state either from the Apify Cloud storage or from the local database." + } + ] }, - "defaultValue": "None" - } - ] - } - ] - }, - { - "id": 290, - "name": "open_queue_with_custom_client", - "module": "scrapy.utils", - "kind": 2048, - "kindString": "Method", - "flags": {}, - "comment": { - "summary": [ - { - "kind": "text", - "text": "{\"content\": [\"Open a Request Queue with custom Apify Client.\\n\\nTODO: add support for custom client to Actor.open_request_queue(), so that\\nwe don't have to do this hacky workaround\"]}" - } - ] - }, - "children": [], - "groups": [], - "sources": [ - { - "filename": "/tmp-old-source-v17/src/apify/scrapy/utils.py", - "line": 76, - "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/scrapy/utils.py#L76" - } - ], - "signatures": [ - { - "id": 291, - "name": "open_queue_with_custom_client", - "modifiers": [ - "async" + "flags": {}, + "id": 84766, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "drop", + "parameters": [], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "overwrites": { + "name": "Storage.drop", + "target": 2342, + "type": "reference", + "ref": { + "id": 84105, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_base.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 57 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/Storage#drop" + } + }, + "parentId": 2837 + } ], - "kind": 4096, - "kindString": "Call signature", - "flags": {}, - "comment": { - "summary": [ - { - "kind": "text", - "text": "{\"content\": [\"Open a Request Queue with custom Apify Client.\\n\\nTODO: add support for custom client to Actor.open_request_queue(), so that\\nwe don't have to do this hacky workaround\"]}" - } - ] - }, - "type": { + "overwrites": { + "name": "RequestManager.drop", + "target": 3002, "type": "reference", - "name": "RequestQueue", - "id": 376 + "ref": { + "id": 84765, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_manager.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 22 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestManager#drop" + } }, - "parameters": [] - } - ] - }, - { - "id": 292, - "name": "Dataset", - "module": "storages.dataset", - "kind": 128, - "kindString": "Class", - "flags": {}, - "comment": { - "summary": [ - { - "kind": "text", - "text": "{\"content\": [\"The `Dataset` class represents a store for structured data where each object stored has the same attributes.\\n\\nYou can imagine it as a table, where each object is a row and its attributes are columns.\\nDataset is an append-only storage - you can only add new records to it but you cannot modify or remove existing records.\\nTypically it is used to store crawling results.\\n\\nDo not instantiate this class directly, use the `Actor.open_dataset()` function instead.\\n\\n`Dataset` stores its data either on local disk or in the Apify cloud,\\ndepending on whether the `APIFY_LOCAL_STORAGE_DIR` or `APIFY_TOKEN` environment variables are set.\\n\\nIf the `APIFY_LOCAL_STORAGE_DIR` environment variable is set, the data is stored in\\nthe local directory in the following files:\\n```\\n{APIFY_LOCAL_STORAGE_DIR}/datasets/{DATASET_ID}/{INDEX}.json\\n```\", \"Note that `{DATASET_ID}` is the name or ID of the dataset. The default dataset has ID: `default`,\\nunless you override it by setting the `APIFY_DEFAULT_DATASET_ID` environment variable.\\nEach dataset item is stored as a separate JSON file, where `{INDEX}` is a zero-based index of the item in the dataset.\\n\\nIf the `APIFY_TOKEN` environment variable is set but `APIFY_LOCAL_STORAGE_DIR` is not, the data is stored in the\\n[Apify Dataset](https://docs.apify.com/storage/dataset) cloud storage.\"]}" - } - ] - }, - "children": [ + "permalink": "/python/api/class/RequestManagerTandem#drop", + "parentId": 2803 + }, { - "id": 293, - "name": "push_data", - "module": "storages.dataset", "kind": 2048, "kindString": "Method", - "flags": {}, + "children": [], "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Store an object or an array of objects to the dataset.\\n\\nThe size of the data is limited by the receiving API and therefore `push_data()` will only\\nallow objects whose JSON representation is smaller than 9MB. When an array is passed,\\nnone of the included objects may be larger than 9MB, but the array itself may be of any size.\\n\", {\"Arguments\": [{\"param\": \"data\", \"type\": \"JSONSerializable\", \"desc\": \"dict or array of dicts containing data to be stored in the default dataset.\\nThe JSON representation of each item must be smaller than 9MB.\"}]}]}" + "text": "Remove all queued requests from this manager while keeping it usable.\n\nDefault implementation is a no-op. Subclasses with persistent state should override to clear it." } ] }, - "children": [], + "decorations": [], + "flags": {}, "groups": [], + "id": 85914, + "module": "request_loaders._request_manager", + "name": "purge", + "parsedDocstring": { + "text": "Remove all queued requests from this manager while keeping it usable.\n\nDefault implementation is a no-op. Subclasses with persistent state should override to clear it." + }, "sources": [ { - "filename": "/tmp-old-source-v17/src/apify/storages/dataset.py", - "line": 153, "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/storages/dataset.py#L153" + "fileName": "/src/crawlee/request_loaders/_request_manager.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 25, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/request_loaders/_request_manager.py#L25" } ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, "signatures": [ { - "id": 294, - "name": "push_data", - "modifiers": [ - "async" - ], - "kind": 4096, - "kindString": "Call signature", - "flags": {}, "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Store an object or an array of objects to the dataset.\\n\\nThe size of the data is limited by the receiving API and therefore `push_data()` will only\\nallow objects whose JSON representation is smaller than 9MB. When an array is passed,\\nnone of the included objects may be larger than 9MB, but the array itself may be of any size.\\n\", {\"" + "text": "Remove all queued requests from this manager while keeping it usable.\n\nDefault implementation is a no-op. Subclasses with persistent state should override to clear it." } ] }, + "flags": {}, + "id": 84768, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "purge", + "parameters": [], "type": { + "name": "None", "type": "reference", - "name": "None" + "target": 0, + "ref": {} }, - "parameters": [ + "inheritedFrom": { + "name": "RequestManager.purge", + "target": 3004, + "type": "reference", + "ref": { + "id": 84767, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_manager.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 25 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestManager#purge" + } + }, + "parentId": 4151 + } + ], + "inheritedFrom": { + "name": "RequestManager.purge", + "target": 3004, + "type": "reference", + "ref": { + "id": 84767, + "sources": [ { - "id": 295, - "name": "data", - "kind": 32768, - "kindString": "Parameter", - "flags": {}, - "type": { - "type": "reference", - "name": "JSONSerializable" - } + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_manager.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 25 } - ] + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestManager#purge" } - ] + }, + "permalink": "/python/api/class/RequestManagerTandem#purge", + "parentId": 2803 }, { - "id": 296, - "name": "get_data", - "module": "storages.dataset", "kind": 2048, "kindString": "Method", - "flags": {}, + "children": [], "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Get items from the dataset.\\n\", {\"Arguments\": [{\"param\": \"offset\", \"type\": \"int, optional\", \"desc\": \"Number of items that should be skipped at the start. The default value is 0\"}, {\"param\": \"limit\", \"type\": \"int, optional\", \"desc\": \"Maximum number of items to return. By default there is no limit.\"}, {\"param\": \"desc\", \"type\": \"bool, optional\", \"desc\": \"By default, results are returned in the same order as they were stored.\\nTo reverse the order, set this parameter to True.\"}, {\"param\": \"clean\", \"type\": \"bool, optional\", \"desc\": \"If True, returns only non-empty items and skips hidden fields (i.e. fields starting with the # character).\\nThe clean parameter is just a shortcut for skip_hidden=True and skip_empty=True parameters.\\nNote that since some objects might be skipped from the output, that the result might contain less items than the limit value.\"}, {\"param\": \"fields\", \"type\": \"list of str, optional\", \"desc\": \"A list of fields which should be picked from the items,\\nonly these fields will remain in the resulting record objects.\\nNote that the fields in the outputted items are sorted the same way as they are specified in the fields parameter.\\nYou can use this feature to effectively fix the output format.\"}, {\"param\": \"omit\", \"type\": \"list of str, optional\", \"desc\": \"A list of fields which should be omitted from the items.\"}, {\"param\": \"unwind\", \"type\": \"str, optional\", \"desc\": \"Name of a field which should be unwound.\\nIf the field is an array then every element of the array will become a separate record and merged with parent object.\\nIf the unwound field is an object then it is merged with the parent object.\\nIf the unwound field is missing or its value is neither an array nor an object and therefore cannot be merged with a parent object,\\nthen the item gets preserved as it is. Note that the unwound items ignore the desc parameter.\"}, {\"param\": \"skip_empty\", \"type\": \"bool, optional\", \"desc\": \"If True, then empty items are skipped from the output.\\nNote that if used, the results might contain less items than the limit value.\"}, {\"param\": \"skip_hidden\", \"type\": \"bool, optional\", \"desc\": \"If True, then hidden fields are skipped from the output, i.e. fields starting with the # character.\"}, {\"param\": \"flatten\", \"type\": \"list of str, optional\", \"desc\": \"A list of fields that should be flattened\"}, {\"param\": \"view\", \"type\": \"str, optional\", \"desc\": \"Name of the dataset view to be used\\n\"}]}, {\"Returns\": [{\"param\": \"ListPage\", \"desc\": \"A page of the list of dataset items according to the specified filters.\"}]}]}" + "text": "Combine the loader with a request manager to support adding and reclaiming requests.\n" } ] }, - "children": [], + "decorations": [], + "flags": {}, "groups": [], + "id": 85915, + "module": "request_loaders._request_loader", + "name": "to_tandem", + "parsedDocstring": { + "text": "Combine the loader with a request manager to support adding and reclaiming requests.\n", + "args": { + "request_manager": "Request manager to combine the loader with.\nIf None is given, the default request queue is used." + } + }, "sources": [ { - "filename": "/tmp-old-source-v17/src/apify/storages/dataset.py", - "line": 182, "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/storages/dataset.py#L182" + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 56, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/request_loaders/_request_loader.py#L56" } ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, "signatures": [ { - "id": 297, - "name": "get_data", - "modifiers": [ - "async" - ], - "kind": 4096, - "kindString": "Call signature", - "flags": {}, "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Get items from the dataset.\\n\", {\"" + "text": "Combine the loader with a request manager to support adding and reclaiming requests.\n" } ] }, - "type": { - "type": "reference", - "name": "ListPage" - }, + "flags": {}, + "id": 84712, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "to_tandem", "parameters": [ { - "id": 298, - "name": "offset", - "kind": 32768, - "kindString": "Parameter", - "flags": { - "keyword-only": "true" - }, - "type": { - "type": "reference", - "name": "int | None" - }, - "defaultValue": "None" - }, - { - "id": 299, - "name": "limit", - "kind": 32768, - "kindString": "Parameter", - "flags": { - "keyword-only": "true" - }, - "type": { - "type": "reference", - "name": "int | None" - }, - "defaultValue": "None" - }, - { - "id": 300, - "name": "clean", - "kind": 32768, - "kindString": "Parameter", - "flags": { - "keyword-only": "true" - }, - "type": { - "type": "reference", - "name": "bool | None" - }, - "defaultValue": "None" - }, - { - "id": 301, - "name": "desc", - "kind": 32768, - "kindString": "Parameter", - "flags": { - "keyword-only": "true" - }, - "type": { - "type": "reference", - "name": "bool | None" - }, - "defaultValue": "None" - }, - { - "id": 302, - "name": "fields", - "kind": 32768, - "kindString": "Parameter", - "flags": { - "keyword-only": "true" - }, - "type": { - "type": "reference", - "name": "list[str] | None" - }, - "defaultValue": "None" - }, - { - "id": 303, - "name": "omit", - "kind": 32768, - "kindString": "Parameter", - "flags": { - "keyword-only": "true" - }, - "type": { - "type": "reference", - "name": "list[str] | None" - }, - "defaultValue": "None" - }, - { - "id": 304, - "name": "unwind", - "kind": 32768, - "kindString": "Parameter", - "flags": { - "keyword-only": "true" - }, - "type": { - "type": "reference", - "name": "str | None" - }, - "defaultValue": "None" - }, - { - "id": 305, - "name": "skip_empty", - "kind": 32768, - "kindString": "Parameter", - "flags": { - "keyword-only": "true" - }, - "type": { - "type": "reference", - "name": "bool | None" + "comment": { + "summary": [ + { + "kind": "text", + "text": "Request manager to combine the loader with.\nIf None is given, the default request queue is used." + } + ] }, - "defaultValue": "None" - }, - { - "id": 306, - "name": "skip_hidden", - "kind": 32768, - "kindString": "Parameter", + "defaultValue": "None", "flags": { - "keyword-only": "true" - }, - "type": { - "type": "reference", - "name": "bool | None" + "isOptional": true, + "keyword-only": false }, - "defaultValue": "None" - }, - { - "id": 307, - "name": "flatten", + "id": 84713, "kind": 32768, "kindString": "Parameter", - "flags": { - "keyword-only": "true" - }, + "name": "request_manager", "type": { + "name": "RequestManager | None", "type": "reference", - "name": "list[str] | None" + "target": 0, + "ref": {} }, - "defaultValue": "None" - }, + "parentId": 2949 + } + ], + "type": { + "name": "RequestManagerTandem", + "type": "reference", + "target": "2803", + "ref": { + "id": 84566, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_manager_tandem.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 24 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/RequestManagerTandem" + } + }, + "inheritedFrom": { + "name": "RequestLoader.to_tandem", + "target": 2948, + "type": "reference", + "ref": { + "id": 84711, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 56 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestLoader#to_tandem" + } + }, + "parentId": 4152 + } + ], + "inheritedFrom": { + "name": "RequestLoader.to_tandem", + "target": 2948, + "type": "reference", + "ref": { + "id": 84711, + "sources": [ { - "id": 308, - "name": "view", - "kind": 32768, - "kindString": "Parameter", - "flags": { - "keyword-only": "true" - }, - "type": { - "type": "reference", - "name": "str | None" - }, - "defaultValue": "None" + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 56 } - ] + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestLoader#to_tandem" } - ] - }, + }, + "permalink": "/python/api/class/RequestManagerTandem#to_tandem", + "parentId": 2803 + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Implements a tandem behaviour for a pair of `RequestLoader` and `RequestManager`.\n\nIn this scenario, the contents of the \"loader\" get transferred into the \"manager\", allowing processing the requests\nfrom both sources and also enqueueing new requests (not possible with plain `RequestManager`)." + } + ] + }, + "decorations": [ + { + "args": "('Request loaders')", + "name": "docs_group" + } + ], + "flags": {}, + "groups": [ + { + "children": [ + 84567, + 84579, + 84583, + 84600, + 84591, + 84571, + 84573, + 84575, + 84577, + 84597, + 85914, + 84593, + 85915 + ], + "title": "Methods" + } + ], + "id": 84566, + "module": "request_loaders._request_manager_tandem", + "name": "RequestManagerTandem", + "parsedDocstring": { + "text": "Implements a tandem behaviour for a pair of `RequestLoader` and `RequestManager`.\n\nIn this scenario, the contents of the \"loader\" get transferred into the \"manager\", allowing processing the requests\nfrom both sources and also enqueueing new requests (not possible with plain `RequestManager`)." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_manager_tandem.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 24, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/request_loaders/_request_manager_tandem.py#L24" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "extendedTypes": [ + { + "name": "RequestManager", + "target": "3001", + "type": "reference", + "ref": { + "id": 84764, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_manager.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 18 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/RequestManager" + } + } + ], + "permalink": "/python/api/class/RequestManagerTandem", + "categories": [] + }, + { + "kind": 128, + "kindString": "Class", + "children": [ { - "id": 309, - "name": "export_to", - "module": "storages.dataset", "kind": 2048, "kindString": "Method", - "flags": {}, + "children": [], "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Save the entirety of the dataset's contents into one file within a key-value store.\\n\", {\"Arguments\": [{\"param\": \"key\", \"type\": \"str\", \"desc\": \"The key to save the data under.\"}, {\"param\": \"to_key_value_store_id\", \"type\": \"str, optional\", \"desc\": \"The id of the key-value store in which the result will be saved.\"}, {\"param\": \"to_key_value_store_name\", \"type\": \"str, optional\", \"desc\": \"The name of the key-value store in which the result will be saved.\\nYou must specify only one of `to_key_value_store_id` and `to_key_value_store_name` arguments.\\nIf you omit both, it uses the default key-value store.\"}, {\"param\": \"content_type\", \"type\": \"str, optional\", \"desc\": \"Either 'text/csv' or 'application/json'. Defaults to JSON.\"}]}]}" + "text": "Initialize the sitemap request loader.\n" } ] }, - "children": [], + "decorations": [], + "flags": {}, "groups": [], + "id": 88879, + "module": "request_loaders._sitemap_request_loader", + "name": "__init__", + "parsedDocstring": { + "text": "Initialize the sitemap request loader.\n", + "args": { + "sitemap_urls": "Configuration options for the loader.", + "proxy_info": "Optional proxy to use for fetching sitemaps.", + "include": "List of glob or regex patterns to include URLs.", + "exclude": "List of glob or regex patterns to exclude URLs.", + "enqueue_strategy": "Strategy used to decide which sitemap-derived URLs (both nested-sitemap entries and\nURL entries) are kept relative to the parent sitemap URL. Defaults to `'same-hostname'`, matching\nthe sitemap protocol's same-host expectation and the `enqueue_links` default; pass `'all'` to\ndisable filtering. Note: regardless of `enqueue_strategy`, entries with non-`http(s)` schemes are\nalways filtered out.", + "max_buffer_size": "Maximum number of URLs to buffer in memory.", + "http_client": "the instance of `HttpClient` to use for fetching sitemaps.", + "persist_state_key": "A key for persisting the loader's state in the KeyValueStore.\nWhen provided, allows resuming from where it left off after interruption.\nIf None, no state persistence occurs.", + "transform_request_function": "An optional function to transform requests\ngenerated by the loader. It receives `RequestOptions` with `url` and should return either\nmodified `RequestOptions` or a `RequestTransformAction`." + } + }, "sources": [ { - "filename": "/tmp-old-source-v17/src/apify/storages/dataset.py", - "line": 272, "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/storages/dataset.py#L272" + "fileName": "/src/crawlee/request_loaders/_sitemap_request_loader.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 109, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/request_loaders/_sitemap_request_loader.py#L109" } ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, "signatures": [ { - "id": 310, - "name": "export_to", - "modifiers": [ - "async" - ], - "kind": 4096, - "kindString": "Call signature", - "flags": {}, "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Save the entirety of the dataset's contents into one file within a key-value store.\\n\", {\"" + "text": "Initialize the sitemap request loader.\n" } ] }, - "type": { - "type": "reference", - "name": "None" - }, + "flags": {}, + "id": 88880, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "__init__", "parameters": [ { - "id": 311, - "name": "key", + "comment": { + "summary": [ + { + "kind": "text", + "text": "Configuration options for the loader." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 88881, "kind": 32768, "kindString": "Parameter", - "flags": {}, + "name": "sitemap_urls", "type": { - "type": "intrinsic", - "name": "str" - } + "name": "list", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + }, + "parentId": 2965 }, { - "id": 312, - "name": "to_key_value_store_id", + "comment": { + "summary": [ + { + "kind": "text", + "text": "the instance of `HttpClient` to use for fetching sitemaps." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 88882, "kind": 32768, "kindString": "Parameter", + "name": "http_client", + "type": { + "name": "HttpClient", + "type": "reference", + "target": "3137", + "ref": { + "id": 89052, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/http_clients/_base.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 75 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/HttpClient" + } + }, + "parentId": 2965 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Optional proxy to use for fetching sitemaps." + } + ] + }, + "defaultValue": "None", "flags": { - "keyword-only": "true" + "isOptional": true, + "keyword-only": true }, + "id": 88883, + "kind": 32768, + "kindString": "Parameter", + "name": "proxy_info", "type": { - "type": "reference", - "name": "str | None" + "name": "ProxyInfo | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "ProxyInfo", + "target": "21", + "ref": { + "id": 85936, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/proxy_configuration.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 26 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/ProxyInfo" + } + }, + { + "type": "literal", + "value": {} + } + ] }, - "defaultValue": "None" + "parentId": 2965 }, { - "id": 313, - "name": "to_key_value_store_name", + "comment": { + "summary": [ + { + "kind": "text", + "text": "List of glob or regex patterns to include URLs." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 88884, "kind": 32768, "kindString": "Parameter", + "name": "include", + "type": { + "name": "list[re.Pattern[Any] | Glob] | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "list", + "typeArguments": [ + { + "type": "union", + "types": [ + { + "type": "reference", + "name": "re.Pattern", + "typeArguments": [ + { + "type": "reference", + "name": "Any", + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + }, + { + "type": "reference", + "name": "Glob", + "target": "3253", + "ref": { + "id": 89168, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_utils/globs.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 11 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/Glob" + } + } + ] + } + ], + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2965 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "List of glob or regex patterns to exclude URLs." + } + ] + }, + "defaultValue": "None", "flags": { - "keyword-only": "true" + "isOptional": true, + "keyword-only": true }, + "id": 88885, + "kind": 32768, + "kindString": "Parameter", + "name": "exclude", "type": { - "type": "reference", - "name": "str | None" + "name": "list[re.Pattern[Any] | Glob] | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "list", + "typeArguments": [ + { + "type": "union", + "types": [ + { + "type": "reference", + "name": "re.Pattern", + "typeArguments": [ + { + "type": "reference", + "name": "Any", + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + }, + { + "type": "reference", + "name": "Glob", + "target": "3253", + "ref": { + "id": 89168, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_utils/globs.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 11 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/Glob" + } + } + ] + } + ], + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] }, - "defaultValue": "None" + "parentId": 2965 }, { - "id": 314, - "name": "content_type", + "comment": { + "summary": [ + { + "kind": "text", + "text": "Strategy used to decide which sitemap-derived URLs (both nested-sitemap entries and\nURL entries) are kept relative to the parent sitemap URL. Defaults to `'same-hostname'`, matching\nthe sitemap protocol's same-host expectation and the `enqueue_links` default; pass `'all'` to\ndisable filtering. Note: regardless of `enqueue_strategy`, entries with non-`http(s)` schemes are\nalways filtered out." + } + ] + }, + "defaultValue": "'same-hostname'", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 88886, "kind": 32768, "kindString": "Parameter", + "name": "enqueue_strategy", + "type": { + "name": "EnqueueStrategy", + "type": "reference", + "target": "88", + "ref": { + "id": 86003, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_types.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 47 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api#EnqueueStrategy" + } + }, + "parentId": 2965 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Maximum number of URLs to buffer in memory." + } + ] + }, + "defaultValue": "200", "flags": { - "keyword-only": "true" + "isOptional": true, + "keyword-only": true }, + "id": 88887, + "kind": 32768, + "kindString": "Parameter", + "name": "max_buffer_size", "type": { + "name": "int", "type": "reference", - "name": "str | None" + "target": 0, + "ref": {} + }, + "parentId": 2965 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "A key for persisting the loader's state in the KeyValueStore.\nWhen provided, allows resuming from where it left off after interruption.\nIf None, no state persistence occurs." + } + ] }, - "defaultValue": "None" + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 88888, + "kind": 32768, + "kindString": "Parameter", + "name": "persist_state_key", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2965 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "An optional function to transform requests\ngenerated by the loader. It receives `RequestOptions` with `url` and should return either\nmodified `RequestOptions` or a `RequestTransformAction`." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 88889, + "kind": 32768, + "kindString": "Parameter", + "name": "transform_request_function", + "type": { + "name": "Callable[[RequestOptions], RequestOptions | RequestTransformAction] | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "Callable", + "typeArguments": [ + { + "type": "reference", + "name": "[RequestOptions]", + "target": 0, + "ref": {} + }, + { + "type": "union", + "types": [ + { + "type": "reference", + "name": "RequestOptions", + "target": "393", + "ref": { + "id": 86308, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_request.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 121 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/RequestOptions" + } + }, + { + "type": "reference", + "name": "RequestTransformAction", + "target": "87", + "ref": { + "id": 86002, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_types.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 45 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api#RequestTransformAction" + } + } + ] + } + ], + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2965 } - ] + ], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "parentId": 2964 } - ] + ], + "permalink": "/python/api/class/SitemapRequestLoader#__init__", + "parentId": 2963 }, { - "id": 315, - "name": "export_to_json", - "module": "storages.dataset", "kind": 2048, "kindString": "Method", - "flags": {}, + "children": [], "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Save the entirety of the dataset's contents into one JSON file within a key-value store.\\n\", {\"Arguments\": [{\"param\": \"key\", \"type\": \"str\", \"desc\": \"The key to save the data under.\"}, {\"param\": \"from_dataset_id\", \"type\": \"str, optional\", \"desc\": \"The ID of the dataset in case of calling the class method. Uses default dataset if omitted.\"}, {\"param\": \"from_dataset_name\", \"type\": \"str, optional\", \"desc\": \"The name of the dataset in case of calling the class method. Uses default dataset if omitted.\\nYou must specify only one of `from_dataset_id` and `from_dataset_name` arguments.\\nIf you omit both, it uses the default dataset.\"}, {\"param\": \"to_key_value_store_id\", \"type\": \"str, optional\", \"desc\": \"The id of the key-value store in which the result will be saved.\"}, {\"param\": \"to_key_value_store_name\", \"type\": \"str, optional\", \"desc\": \"The name of the key-value store in which the result will be saved.\\nYou must specify only one of `to_key_value_store_id` and `to_key_value_store_name` arguments.\\nIf you omit both, it uses the default key-value store.\"}]}]}" + "text": "Enter the context manager." } ] }, - "children": [], + "decorations": [], + "flags": {}, "groups": [], + "id": 88890, + "module": "request_loaders._sitemap_request_loader", + "name": "__aenter__", + "parsedDocstring": { + "text": "Enter the context manager." + }, "sources": [ { - "filename": "/tmp-old-source-v17/src/apify/storages/dataset.py", - "line": 317, "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/storages/dataset.py#L317" + "fileName": "/src/crawlee/request_loaders/_sitemap_request_loader.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 171, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/request_loaders/_sitemap_request_loader.py#L171" } ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, "signatures": [ { - "id": 316, - "name": "export_to_json", - "modifiers": [ - "async" - ], - "kind": 4096, - "kindString": "Call signature", - "flags": {}, "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Save the entirety of the dataset's contents into one JSON file within a key-value store.\\n\", {\"" + "text": "Enter the context manager." } ] }, + "flags": {}, + "id": 88891, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "__aenter__", + "parameters": [], "type": { + "name": "SitemapRequestLoader", "type": "reference", - "name": "None" - }, - "parameters": [ - { - "id": 317, - "name": "key", - "kind": 32768, - "kindString": "Parameter", - "flags": {}, - "type": { - "type": "intrinsic", - "name": "str" - } - }, - { - "id": 318, - "name": "from_dataset_id", - "kind": 32768, - "kindString": "Parameter", - "flags": { - "keyword-only": "true" - }, - "type": { - "type": "reference", - "name": "str | None" - }, - "defaultValue": "None" - }, - { - "id": 319, - "name": "from_dataset_name", - "kind": 32768, - "kindString": "Parameter", - "flags": { - "keyword-only": "true" - }, - "type": { - "type": "reference", - "name": "str | None" - }, - "defaultValue": "None" - }, - { - "id": 320, - "name": "to_key_value_store_id", - "kind": 32768, - "kindString": "Parameter", - "flags": { - "keyword-only": "true" - }, - "type": { - "type": "reference", - "name": "str | None" - }, - "defaultValue": "None" - }, - { - "id": 321, - "name": "to_key_value_store_name", - "kind": 32768, - "kindString": "Parameter", - "flags": { - "keyword-only": "true" - }, - "type": { - "type": "reference", - "name": "str | None" - }, - "defaultValue": "None" + "target": "2963", + "ref": { + "id": 88878, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_sitemap_request_loader.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 94 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/SitemapRequestLoader" } - ] + }, + "parentId": 2975 } - ] + ], + "permalink": "/python/api/class/SitemapRequestLoader#__aenter__", + "parentId": 2963 }, { - "id": 322, - "name": "export_to_csv", - "module": "storages.dataset", "kind": 2048, "kindString": "Method", - "flags": {}, + "children": [], "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Save the entirety of the dataset's contents into one CSV file within a key-value store.\\n\", {\"Arguments\": [{\"param\": \"key\", \"type\": \"str\", \"desc\": \"The key to save the data under.\"}, {\"param\": \"from_dataset_id\", \"type\": \"str, optional\", \"desc\": \"The ID of the dataset in case of calling the class method. Uses default dataset if omitted.\"}, {\"param\": \"from_dataset_name\", \"type\": \"str, optional\", \"desc\": \"The name of the dataset in case of calling the class method. Uses default dataset if omitted.\\nYou must specify only one of `from_dataset_id` and `from_dataset_name` arguments.\\nIf you omit both, it uses the default dataset.\"}, {\"param\": \"to_key_value_store_id\", \"type\": \"str, optional\", \"desc\": \"The id of the key-value store in which the result will be saved.\"}, {\"param\": \"to_key_value_store_name\", \"type\": \"str, optional\", \"desc\": \"The name of the key-value store in which the result will be saved.\\nYou must specify only one of `to_key_value_store_id` and `to_key_value_store_name` arguments.\\nIf you omit both, it uses the default key-value store.\"}]}]}" + "text": "Exit the context manager." } ] }, - "children": [], + "decorations": [], + "flags": {}, "groups": [], + "id": 88892, + "module": "request_loaders._sitemap_request_loader", + "name": "__aexit__", + "parsedDocstring": { + "text": "Exit the context manager." + }, "sources": [ { - "filename": "/tmp-old-source-v17/src/apify/storages/dataset.py", - "line": 359, "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/storages/dataset.py#L359" + "fileName": "/src/crawlee/request_loaders/_sitemap_request_loader.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 176, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/request_loaders/_sitemap_request_loader.py#L176" } ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, "signatures": [ { - "id": 323, - "name": "export_to_csv", - "modifiers": [ - "async" - ], - "kind": 4096, - "kindString": "Call signature", - "flags": {}, "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Save the entirety of the dataset's contents into one CSV file within a key-value store.\\n\", {\"" + "text": "Exit the context manager." } ] }, - "type": { - "type": "reference", - "name": "None" - }, + "flags": {}, + "id": 88893, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "__aexit__", "parameters": [ { - "id": 324, - "name": "key", + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 88894, "kind": 32768, "kindString": "Parameter", - "flags": {}, + "name": "exc_type", "type": { - "type": "intrinsic", - "name": "str" - } + "name": "type[BaseException] | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "type", + "typeArguments": [ + { + "type": "reference", + "name": "BaseException", + "target": 0, + "ref": {} + } + ], + "target": "3422", + "ref": { + "id": 89337, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_utils/sitemap.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 62 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api/class/SitemapSource#type" + } + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2978 }, { - "id": 325, - "name": "from_dataset_id", - "kind": 32768, - "kindString": "Parameter", "flags": { - "keyword-only": "true" - }, - "type": { - "type": "reference", - "name": "str | None" + "isOptional": false, + "keyword-only": false }, - "defaultValue": "None" - }, - { - "id": 326, - "name": "from_dataset_name", + "id": 88895, "kind": 32768, "kindString": "Parameter", - "flags": { - "keyword-only": "true" - }, + "name": "exc_value", "type": { - "type": "reference", - "name": "str | None" + "name": "BaseException | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "BaseException", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] }, - "defaultValue": "None" + "parentId": 2978 }, { - "id": 327, - "name": "to_key_value_store_id", - "kind": 32768, - "kindString": "Parameter", "flags": { - "keyword-only": "true" - }, - "type": { - "type": "reference", - "name": "str | None" + "isOptional": false, + "keyword-only": false }, - "defaultValue": "None" - }, - { - "id": 328, - "name": "to_key_value_store_name", + "id": 88896, "kind": 32768, "kindString": "Parameter", - "flags": { - "keyword-only": "true" - }, + "name": "exc_traceback", "type": { - "type": "reference", - "name": "str | None" + "name": "TracebackType | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "TracebackType", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] }, - "defaultValue": "None" + "parentId": 2978 } - ] + ], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "parentId": 2977 } - ] + ], + "permalink": "/python/api/class/SitemapRequestLoader#__aexit__", + "parentId": 2963 }, { - "id": 329, - "name": "get_info", - "module": "storages.dataset", "kind": 2048, "kindString": "Method", - "flags": {}, + "children": [], "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Get an object containing general information about the dataset.\\n\", {\"Returns\": [{\"param\": \"dict\", \"desc\": \"Object returned by calling the GET dataset API endpoint.\"}]}]}" + "text": "Return the total number of URLs found so far." } ] }, - "children": [], + "decorations": [ + { + "name": "override" + } + ], + "flags": {}, "groups": [], + "id": 88897, + "module": "request_loaders._sitemap_request_loader", + "name": "get_total_count", + "parsedDocstring": { + "text": "Return the total number of URLs found so far." + }, "sources": [ { - "filename": "/tmp-old-source-v17/src/apify/storages/dataset.py", - "line": 400, "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/storages/dataset.py#L400" + "fileName": "/src/crawlee/request_loaders/_sitemap_request_loader.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 186, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/request_loaders/_sitemap_request_loader.py#L186" } ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, "signatures": [ { - "id": 330, - "name": "get_info", - "modifiers": [ - "async" - ], - "kind": 4096, - "kindString": "Call signature", - "flags": {}, "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Get an object containing general information about the dataset.\\n\", {\"" + "text": "Return the total number of URLs found so far." } ] }, + "flags": {}, + "id": 88898, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "get_total_count", + "parameters": [], "type": { + "name": "int", + "type": "reference", + "target": 0, + "ref": {} + }, + "overwrites": { + "name": "RequestLoader.get_total_count", + "target": 2937, "type": "reference", - "name": "dict | None" + "ref": { + "id": 88852, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 33 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestLoader#get_total_count" + } }, - "parameters": [] + "parentId": 2982 } - ] + ], + "overwrites": { + "name": "RequestLoader.get_total_count", + "target": 2937, + "type": "reference", + "ref": { + "id": 88852, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 33 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestLoader#get_total_count" + } + }, + "permalink": "/python/api/class/SitemapRequestLoader#get_total_count", + "parentId": 2963 }, { - "id": 331, - "name": "iterate_items", - "module": "storages.dataset", "kind": 2048, "kindString": "Method", - "flags": {}, + "children": [], "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Iterate over the items in the dataset.\\n\", {\"Arguments\": [{\"param\": \"offset\", \"type\": \"int, optional\", \"desc\": \"Number of items that should be skipped at the start. The default value is 0\"}, {\"param\": \"limit\", \"type\": \"int, optional\", \"desc\": \"Maximum number of items to return. By default there is no limit.\"}, {\"param\": \"desc\", \"type\": \"bool, optional\", \"desc\": \"By default, results are returned in the same order as they were stored.\\nTo reverse the order, set this parameter to True.\"}, {\"param\": \"clean\", \"type\": \"bool, optional\", \"desc\": \"If True, returns only non-empty items and skips hidden fields (i.e. fields starting with the # character).\\nThe clean parameter is just a shortcut for skip_hidden=True and skip_empty=True parameters.\\nNote that since some objects might be skipped from the output, that the result might contain less items than the limit value.\"}, {\"param\": \"fields\", \"type\": \"list of str, optional\", \"desc\": \"A list of fields which should be picked from the items,\\nonly these fields will remain in the resulting record objects.\\nNote that the fields in the outputted items are sorted the same way as they are specified in the fields parameter.\\nYou can use this feature to effectively fix the output format.\"}, {\"param\": \"omit\", \"type\": \"list of str, optional\", \"desc\": \"A list of fields which should be omitted from the items.\"}, {\"param\": \"unwind\", \"type\": \"str, optional\", \"desc\": \"Name of a field which should be unwound.\\nIf the field is an array then every element of the array will become a separate record and merged with parent object.\\nIf the unwound field is an object then it is merged with the parent object.\\nIf the unwound field is missing or its value is neither an array nor an object and therefore cannot be merged with a parent object,\\nthen the item gets preserved as it is. Note that the unwound items ignore the desc parameter.\"}, {\"param\": \"skip_empty\", \"type\": \"bool, optional\", \"desc\": \"If True, then empty items are skipped from the output.\\nNote that if used, the results might contain less items than the limit value.\"}, {\"param\": \"skip_hidden\", \"type\": \"bool, optional\", \"desc\": \"If True, then hidden fields are skipped from the output, i.e. fields starting with the # character.\\n\"}]}, {\"Yields\": [{\"param\": \"dict\", \"desc\": \"An item from the dataset\"}]}]}" + "text": "Return the number of URLs that have been handled." } ] }, - "children": [], + "decorations": [ + { + "name": "override" + } + ], + "flags": {}, "groups": [], + "id": 88899, + "module": "request_loaders._sitemap_request_loader", + "name": "get_handled_count", + "parsedDocstring": { + "text": "Return the number of URLs that have been handled." + }, "sources": [ { - "filename": "/tmp-old-source-v17/src/apify/storages/dataset.py", - "line": 408, "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/storages/dataset.py#L408" + "fileName": "/src/crawlee/request_loaders/_sitemap_request_loader.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 192, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/request_loaders/_sitemap_request_loader.py#L192" } ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, "signatures": [ { - "id": 332, - "name": "iterate_items", - "modifiers": [], - "kind": 4096, - "kindString": "Call signature", - "flags": {}, "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Iterate over the items in the dataset.\\n\", {\"" + "text": "Return the number of URLs that have been handled." } ] }, + "flags": {}, + "id": 88900, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "get_handled_count", + "parameters": [], "type": { + "name": "int", "type": "reference", - "name": "AsyncIterator[dict]" + "target": 0, + "ref": {} }, - "parameters": [ - { - "id": 333, - "name": "offset", - "kind": 32768, - "kindString": "Parameter", - "flags": { - "keyword-only": "true" - }, - "type": { - "type": "intrinsic", - "name": "int" - }, - "defaultValue": "0" - }, - { - "id": 334, - "name": "limit", - "kind": 32768, - "kindString": "Parameter", - "flags": { - "keyword-only": "true" - }, - "type": { - "type": "reference", - "name": "int | None" - }, - "defaultValue": "None" - }, - { - "id": 335, - "name": "clean", - "kind": 32768, - "kindString": "Parameter", - "flags": { - "keyword-only": "true" - }, - "type": { - "type": "reference", - "name": "bool | None" - }, - "defaultValue": "None" - }, - { - "id": 336, - "name": "desc", - "kind": 32768, - "kindString": "Parameter", - "flags": { - "keyword-only": "true" - }, - "type": { - "type": "reference", - "name": "bool | None" - }, - "defaultValue": "None" - }, - { - "id": 337, - "name": "fields", - "kind": 32768, - "kindString": "Parameter", - "flags": { - "keyword-only": "true" - }, - "type": { - "type": "reference", - "name": "list[str] | None" - }, - "defaultValue": "None" - }, - { - "id": 338, - "name": "omit", - "kind": 32768, - "kindString": "Parameter", - "flags": { - "keyword-only": "true" - }, - "type": { - "type": "reference", - "name": "list[str] | None" - }, - "defaultValue": "None" - }, - { - "id": 339, - "name": "unwind", - "kind": 32768, - "kindString": "Parameter", - "flags": { - "keyword-only": "true" - }, - "type": { - "type": "reference", - "name": "str | None" - }, - "defaultValue": "None" - }, - { - "id": 340, - "name": "skip_empty", - "kind": 32768, - "kindString": "Parameter", - "flags": { - "keyword-only": "true" - }, - "type": { - "type": "reference", - "name": "bool | None" - }, - "defaultValue": "None" - }, + "overwrites": { + "name": "RequestLoader.get_handled_count", + "target": 2935, + "type": "reference", + "ref": { + "id": 88850, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 29 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestLoader#get_handled_count" + } + }, + "parentId": 2984 + } + ], + "overwrites": { + "name": "RequestLoader.get_handled_count", + "target": 2935, + "type": "reference", + "ref": { + "id": 88850, + "sources": [ { - "id": 341, - "name": "skip_hidden", - "kind": 32768, - "kindString": "Parameter", - "flags": { - "keyword-only": "true" - }, - "type": { - "type": "reference", - "name": "bool | None" - }, - "defaultValue": "None" + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 29 } - ] + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestLoader#get_handled_count" } - ] + }, + "permalink": "/python/api/class/SitemapRequestLoader#get_handled_count", + "parentId": 2963 }, { - "id": 342, - "name": "drop", - "module": "storages.dataset", "kind": 2048, "kindString": "Method", - "flags": {}, + "children": [], "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Remove the dataset either from the Apify cloud storage or from the local directory.\"]}" + "text": "Check if there are no more URLs to process." } ] }, - "children": [], + "decorations": [ + { + "name": "override" + } + ], + "flags": {}, "groups": [], + "id": 88901, + "module": "request_loaders._sitemap_request_loader", + "name": "is_empty", + "parsedDocstring": { + "text": "Check if there are no more URLs to process." + }, "sources": [ { - "filename": "/tmp-old-source-v17/src/apify/storages/dataset.py", - "line": 460, "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/storages/dataset.py#L460" + "fileName": "/src/crawlee/request_loaders/_sitemap_request_loader.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 198, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/request_loaders/_sitemap_request_loader.py#L198" } ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, "signatures": [ { - "id": 343, - "name": "drop", - "modifiers": [ - "async" - ], - "kind": 4096, - "kindString": "Call signature", - "flags": {}, "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Remove the dataset either from the Apify cloud storage or from the local directory.\"]}" + "text": "Check if there are no more URLs to process." } ] }, + "flags": {}, + "id": 88902, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "is_empty", + "parameters": [], "type": { + "name": "bool", + "type": "reference", + "target": 0, + "ref": {} + }, + "overwrites": { + "name": "RequestLoader.is_empty", + "target": 2939, "type": "reference", - "name": "None" + "ref": { + "id": 88854, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 37 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestLoader#is_empty" + } }, - "parameters": [] + "parentId": 2986 } - ] + ], + "overwrites": { + "name": "RequestLoader.is_empty", + "target": 2939, + "type": "reference", + "ref": { + "id": 88854, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 37 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestLoader#is_empty" + } + }, + "permalink": "/python/api/class/SitemapRequestLoader#is_empty", + "parentId": 2963 }, { - "id": 344, - "name": "open", - "module": "storages.dataset", "kind": 2048, "kindString": "Method", - "flags": {}, + "children": [], "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Open a dataset.\\n\\nDatasets are used to store structured data where each object stored has the same attributes,\\nsuch as online store products or real estate offers.\\nThe actual data is stored either on the local filesystem or in the Apify cloud.\\n\", {\"Arguments\": [{\"param\": \"id\", \"type\": \"str, optional\", \"desc\": \"ID of the dataset to be opened.\\nIf neither `id` nor `name` are provided, the method returns the default dataset associated with the actor run.\\nIf the dataset with the given ID does not exist, it raises an error.\"}, {\"param\": \"name\", \"type\": \"str, optional\", \"desc\": \"Name of the dataset to be opened.\\nIf neither `id` nor `name` are provided, the method returns the default dataset associated with the actor run.\\nIf the dataset with the given name does not exist, it is created.\"}, {\"param\": \"force_cloud\", \"type\": \"bool, optional\", \"desc\": \"If set to True, it will open a dataset on the Apify Platform even when running the actor locally.\\nDefaults to False.\"}, {\"param\": \"config\", \"type\": \"Configuration, optional\", \"desc\": \"A `Configuration` instance, uses global configuration if omitted.\\n\"}]}, {\"Returns\": [{\"param\": \"Dataset\", \"desc\": \"An instance of the `Dataset` class for the given ID or name.\"}]}]}" + "text": "Check if all URLs have been processed." } ] }, - "children": [], + "decorations": [ + { + "name": "override" + } + ], + "flags": {}, "groups": [], + "id": 88903, + "module": "request_loaders._sitemap_request_loader", + "name": "is_finished", + "parsedDocstring": { + "text": "Check if all URLs have been processed." + }, "sources": [ { - "filename": "/tmp-old-source-v17/src/apify/storages/dataset.py", - "line": 466, "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/storages/dataset.py#L466" + "fileName": "/src/crawlee/request_loaders/_sitemap_request_loader.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 204, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/request_loaders/_sitemap_request_loader.py#L204" } ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, "signatures": [ { - "id": 345, - "name": "open", - "modifiers": [ - "async" - ], - "kind": 4096, - "kindString": "Call signature", - "flags": {}, "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Open a dataset.\\n\\nDatasets are used to store structured data where each object stored has the same attributes,\\nsuch as online store products or real estate offers.\\nThe actual data is stored either on the local filesystem or in the Apify cloud.\\n\", {\"" + "text": "Check if all URLs have been processed." } ] }, + "flags": {}, + "id": 88904, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "is_finished", + "parameters": [], "type": { + "name": "bool", "type": "reference", - "name": "Dataset", - "id": 292 + "target": 0, + "ref": {} }, - "parameters": [ - { - "id": 346, - "name": "id", - "kind": 32768, - "kindString": "Parameter", - "flags": { - "keyword-only": "true" - }, - "type": { - "type": "reference", - "name": "str | None" - }, - "defaultValue": "None" - }, - { - "id": 347, - "name": "name", - "kind": 32768, - "kindString": "Parameter", - "flags": { - "keyword-only": "true" - }, - "type": { - "type": "reference", - "name": "str | None" - }, - "defaultValue": "None" - }, - { - "id": 348, - "name": "force_cloud", - "kind": 32768, - "kindString": "Parameter", - "flags": { - "keyword-only": "true" - }, - "type": { - "type": "intrinsic", - "name": "bool" - }, - "defaultValue": "False" - }, + "overwrites": { + "name": "RequestLoader.is_finished", + "target": 2941, + "type": "reference", + "ref": { + "id": 88856, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 41 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestLoader#is_finished" + } + }, + "parentId": 2988 + } + ], + "overwrites": { + "name": "RequestLoader.is_finished", + "target": 2941, + "type": "reference", + "ref": { + "id": 88856, + "sources": [ { - "id": 349, - "name": "config", - "kind": 32768, - "kindString": "Parameter", - "flags": { - "keyword-only": "true" - }, - "type": { - "type": "reference", - "name": "Configuration | None" - }, - "defaultValue": "None" + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 41 } - ] + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestLoader#is_finished" } - ] - } - ], - "groups": [ - { - "title": "Methods", - "children": [ - 342, - 309, - 322, - 315, - 296, - 329, - 331, - 344, - 293 - ] - } - ], - "sources": [ - { - "filename": "/tmp-old-source-v17/src/apify/storages/dataset.py", - "line": 73, - "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/storages/dataset.py#L73" - } - ] - }, - { - "id": 350, - "name": "IterateKeysInfo", - "module": "storages.key_value_store", - "kind": 128, - "kindString": "Class", - "flags": {}, - "comment": { - "summary": [ - { - "kind": "text", - "text": "{\"content\": [\"Contains information about a key-value store record.\"]}" - } - ] - }, - "children": [], - "groups": [], - "sources": [ - { - "filename": "/tmp-old-source-v17/src/apify/storages/key_value_store.py", - "line": 22, - "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/storages/key_value_store.py#L22" - } - ] - }, - { - "id": 351, - "name": "IterateKeysTuple", - "module": "storages.key_value_store", - "kind": 128, - "kindString": "Class", - "flags": {}, - "comment": { - "summary": [ - { - "kind": "text", - "text": "{\"content\": [\"A tuple representing a key-value store record.\"]}" - } - ] - }, - "children": [], - "groups": [], - "sources": [ - { - "filename": "/tmp-old-source-v17/src/apify/storages/key_value_store.py", - "line": 28, - "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/storages/key_value_store.py#L28" - } - ] - }, - { - "id": 352, - "name": "KeyValueStore", - "module": "storages.key_value_store", - "kind": 128, - "kindString": "Class", - "flags": {}, - "comment": { - "summary": [ - { - "kind": "text", - "text": "{\"content\": [\"The `KeyValueStore` class represents a key-value store.\\n\\nYou can imagine it as a simple data storage that is used\\nfor saving and reading data records or files. Each data record is\\nrepresented by a unique key and associated with a MIME content type.\\n\\nDo not instantiate this class directly, use the `Actor.open_key_value_store()` function instead.\\n\\nEach crawler run is associated with a default key-value store, which is created exclusively\\nfor the run. By convention, the crawler input and output are stored into the\\ndefault key-value store under the `INPUT` and `OUTPUT` key, respectively.\\nTypically, input and output are JSON files, although it can be any other format.\\nTo access the default key-value store directly, you can use the\\n`KeyValueStore.get_value` and `KeyValueStore.set_value` convenience functions.\\n\\n`KeyValueStore` stores its data either on local disk or in the Apify cloud,\\ndepending on whether the `APIFY_LOCAL_STORAGE_DIR` or `APIFY_TOKEN` environment variables are set.\\n\\nIf the `APIFY_LOCAL_STORAGE_DIR` environment variable is set, the data is stored in\\nthe local directory in the following files:\\n```\\n{APIFY_LOCAL_STORAGE_DIR}/key_value_stores/{STORE_ID}/{INDEX}.{EXT}\\n```\", \"Note that `{STORE_ID}` is the name or ID of the key-value store. The default key-value store has ID: `default`,\\nunless you override it by setting the `APIFY_DEFAULT_KEY_VALUE_STORE_ID` environment variable.\\nThe `{KEY}` is the key of the record and `{EXT}` corresponds to the MIME content type of the data value.\\n\\nIf the `APIFY_TOKEN` environment variable is set but `APIFY_LOCAL_STORAGE_DIR` is not, the data is stored in the\\n[Apify Key-value store](https://docs.apify.com/storage/key-value-store) cloud storage.\"]}" - } - ] - }, - "children": [ + }, + "permalink": "/python/api/class/SitemapRequestLoader#is_finished", + "parentId": 2963 + }, { - "id": 353, - "name": "get_value", - "module": "storages.key_value_store", "kind": 2048, "kindString": "Method", - "flags": {}, + "children": [], "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Get a value from the key-value store.\\n\", {\"Arguments\": [{\"param\": \"key\", \"type\": \"str\", \"desc\": \"Key of the record to retrieve.\"}, {\"param\": \"default_value\", \"type\": \"Any, optional\", \"desc\": \"Default value returned in case the record does not exist.\\n\"}]}, {\"Returns\": [{\"param\": \"Any\", \"desc\": \"The value associated with the given key. `default_value` is used in case the record does not exist.\"}]}]}" + "text": "Fetch the next request to process." } ] }, - "children": [], + "decorations": [ + { + "name": "override" + } + ], + "flags": {}, "groups": [], + "id": 88905, + "module": "request_loaders._sitemap_request_loader", + "name": "fetch_next_request", + "parsedDocstring": { + "text": "Fetch the next request to process." + }, "sources": [ { - "filename": "/tmp-old-source-v17/src/apify/storages/key_value_store.py", - "line": 137, "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/storages/key_value_store.py#L137" + "fileName": "/src/crawlee/request_loaders/_sitemap_request_loader.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 210, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/request_loaders/_sitemap_request_loader.py#L210" } ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, "signatures": [ { - "id": 354, - "name": "get_value", - "modifiers": [ - "async" - ], - "kind": 4096, - "kindString": "Call signature", - "flags": {}, "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Get a value from the key-value store.\\n\", {\"" + "text": "Fetch the next request to process." } ] }, + "flags": {}, + "id": 88906, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "fetch_next_request", + "parameters": [], "type": { + "name": "Request | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "Request", + "target": "409", + "ref": { + "id": 86324, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_request.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 145 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/Request" + } + }, + { + "type": "literal", + "value": {} + } + ] + }, + "overwrites": { + "name": "RequestLoader.fetch_next_request", + "target": 2943, "type": "reference", - "name": "T | None" + "ref": { + "id": 88858, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 45 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestLoader#fetch_next_request" + } }, - "parameters": [ - { - "id": 355, - "name": "key", - "kind": 32768, - "kindString": "Parameter", - "flags": {}, - "type": { - "type": "intrinsic", - "name": "str" - } - }, + "parentId": 2990 + } + ], + "overwrites": { + "name": "RequestLoader.fetch_next_request", + "target": 2943, + "type": "reference", + "ref": { + "id": 88858, + "sources": [ { - "id": 356, - "name": "default_value", - "kind": 32768, - "kindString": "Parameter", - "flags": {}, - "type": { - "type": "reference", - "name": "T | None" - }, - "defaultValue": "None" + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 45 } - ] + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestLoader#fetch_next_request" } - ] + }, + "permalink": "/python/api/class/SitemapRequestLoader#fetch_next_request", + "parentId": 2963 }, { - "id": 357, - "name": "iterate_keys", - "module": "storages.key_value_store", "kind": 2048, "kindString": "Method", - "flags": {}, + "children": [], "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Iterate over the keys in the key-value store.\\n\", {\"Arguments\": [{\"param\": \"exclusive_start_key\", \"type\": \"str, optional\", \"desc\": \"All keys up to this one (including) are skipped from the result.\\n\"}]}, {\"Yields\": [{\"param\": \"IterateKeysTuple\", \"desc\": \"A tuple `(key, info)`,\\nwhere `key` is the record key, and `info` is an object that contains a single property `size`\\nindicating size of the record in bytes.\"}]}]}" + "text": "Mark a request as successfully handled." } ] }, - "children": [], + "decorations": [ + { + "name": "override" + } + ], + "flags": {}, "groups": [], + "id": 88907, + "module": "request_loaders._sitemap_request_loader", + "name": "mark_request_as_handled", + "parsedDocstring": { + "text": "Mark a request as successfully handled." + }, "sources": [ { - "filename": "/tmp-old-source-v17/src/apify/storages/key_value_store.py", - "line": 154, "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/storages/key_value_store.py#L154" + "fileName": "/src/crawlee/request_loaders/_sitemap_request_loader.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 245, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/request_loaders/_sitemap_request_loader.py#L245" } ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, "signatures": [ { - "id": 358, - "name": "iterate_keys", - "modifiers": [ - "async" - ], - "kind": 4096, - "kindString": "Call signature", - "flags": {}, "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Iterate over the keys in the key-value store.\\n\", {\"" + "text": "Mark a request as successfully handled." } ] }, - "type": { - "type": "reference", - "name": "AsyncIterator[IterateKeysTuple]" - }, + "flags": {}, + "id": 88908, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "mark_request_as_handled", "parameters": [ { - "id": 359, - "name": "exclusive_start_key", + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 88909, "kind": 32768, "kindString": "Parameter", - "flags": {}, + "name": "request", "type": { + "name": "Request", + "type": "reference", + "target": "409", + "ref": { + "id": 86324, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_request.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 145 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/Request" + } + }, + "parentId": 2993 + } + ], + "type": { + "name": "ProcessedRequest | None", + "type": "union", + "types": [ + { "type": "reference", - "name": "str | None" + "name": "ProcessedRequest", + "target": "1340", + "ref": { + "id": 87255, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 140 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/ProcessedRequest" + } }, - "defaultValue": "None" + { + "type": "literal", + "value": {} + } + ] + }, + "overwrites": { + "name": "RequestLoader.mark_request_as_handled", + "target": 2945, + "type": "reference", + "ref": { + "id": 88860, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 53 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestLoader#mark_request_as_handled" } - ] + }, + "parentId": 2992 } - ] + ], + "overwrites": { + "name": "RequestLoader.mark_request_as_handled", + "target": 2945, + "type": "reference", + "ref": { + "id": 88860, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 53 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestLoader#mark_request_as_handled" + } + }, + "permalink": "/python/api/class/SitemapRequestLoader#mark_request_as_handled", + "parentId": 2963 }, { - "id": 360, - "name": "set_value", - "module": "storages.key_value_store", "kind": 2048, "kindString": "Method", - "flags": {}, + "children": [], "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Set or delete a value in the key-value store.\\n\", {\"Arguments\": [{\"param\": \"key\", \"type\": \"str\", \"desc\": \"The key under which the value should be saved.\"}, {\"param\": \"value\", \"type\": \"Any\", \"desc\": \"The value to save. If the value is `None`, the corresponding key-value pair will be deleted.\"}, {\"param\": \"content_type\", \"type\": \"str, optional\", \"desc\": \"The content type of the saved value.\"}]}]}" + "text": "Start the sitemap loading process." } ] }, - "children": [], + "decorations": [], + "flags": {}, "groups": [], + "id": 88910, + "module": "request_loaders._sitemap_request_loader", + "name": "start", + "parsedDocstring": { + "text": "Start the sitemap loading process." + }, "sources": [ { - "filename": "/tmp-old-source-v17/src/apify/storages/key_value_store.py", - "line": 178, "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/storages/key_value_store.py#L178" + "fileName": "/src/crawlee/request_loaders/_sitemap_request_loader.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 253, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/request_loaders/_sitemap_request_loader.py#L253" } ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, "signatures": [ { - "id": 361, - "name": "set_value", - "modifiers": [ - "async" - ], - "kind": 4096, - "kindString": "Call signature", - "flags": {}, "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Set or delete a value in the key-value store.\\n\", {\"" + "text": "Start the sitemap loading process." } ] }, + "flags": {}, + "id": 88911, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "start", + "parameters": [], "type": { - "type": "reference", - "name": "None" + "name": "None", + "type": "literal", + "value": {} }, - "parameters": [ - { - "id": 362, - "name": "key", - "kind": 32768, - "kindString": "Parameter", - "flags": {}, - "type": { - "type": "intrinsic", - "name": "str" - } - }, - { - "id": 363, - "name": "value", - "kind": 32768, - "kindString": "Parameter", - "flags": {}, - "type": { - "type": "reference", - "name": "Any" - } - }, - { - "id": 364, - "name": "content_type", - "kind": 32768, - "kindString": "Parameter", - "flags": {}, - "type": { - "type": "reference", - "name": "str | None" - }, - "defaultValue": "None" - } - ] + "parentId": 2995 } - ] + ], + "permalink": "/python/api/class/SitemapRequestLoader#start", + "parentId": 2963 }, { - "id": 365, - "name": "get_public_url", - "module": "storages.key_value_store", "kind": 2048, "kindString": "Method", - "flags": {}, + "children": [], "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Get a URL for the given key that may be used to publicly access the value in the remote key-value store.\\n\", {\"Arguments\": [{\"param\": \"key\", \"type\": \"str\", \"desc\": \"The key for which the URL should be generated.\"}]}]}" + "text": "Abort the sitemap loading process." } ] }, - "children": [], + "decorations": [], + "flags": {}, "groups": [], + "id": 88912, + "module": "request_loaders._sitemap_request_loader", + "name": "abort_loading", + "parsedDocstring": { + "text": "Abort the sitemap loading process." + }, "sources": [ { - "filename": "/tmp-old-source-v17/src/apify/storages/key_value_store.py", - "line": 206, "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/storages/key_value_store.py#L206" + "fileName": "/src/crawlee/request_loaders/_sitemap_request_loader.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 259, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/request_loaders/_sitemap_request_loader.py#L259" } ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, "signatures": [ { - "id": 366, - "name": "get_public_url", - "modifiers": [ - "async" - ], - "kind": 4096, - "kindString": "Call signature", - "flags": {}, "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Get a URL for the given key that may be used to publicly access the value in the remote key-value store.\\n\", {\"" + "text": "Abort the sitemap loading process." } ] }, + "flags": {}, + "id": 88913, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "abort_loading", + "parameters": [], "type": { - "type": "intrinsic", - "name": "str" + "name": "None", + "type": "literal", + "value": {} }, - "parameters": [ - { - "id": 367, - "name": "key", - "kind": 32768, - "kindString": "Parameter", - "flags": {}, - "type": { - "type": "intrinsic", - "name": "str" - } - } - ] + "parentId": 2997 } - ] + ], + "permalink": "/python/api/class/SitemapRequestLoader#abort_loading", + "parentId": 2963 }, { - "id": 368, - "name": "drop", - "module": "storages.key_value_store", "kind": 2048, "kindString": "Method", - "flags": {}, + "children": [], "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Remove the key-value store either from the Apify cloud storage or from the local directory.\"]}" + "text": "Close the request loader." } ] }, - "children": [], + "decorations": [], + "flags": {}, "groups": [], + "id": 88914, + "module": "request_loaders._sitemap_request_loader", + "name": "close", + "parsedDocstring": { + "text": "Close the request loader." + }, "sources": [ { - "filename": "/tmp-old-source-v17/src/apify/storages/key_value_store.py", - "line": 223, "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/storages/key_value_store.py#L223" + "fileName": "/src/crawlee/request_loaders/_sitemap_request_loader.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 266, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/request_loaders/_sitemap_request_loader.py#L266" } ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, "signatures": [ { - "id": 369, - "name": "drop", - "modifiers": [ - "async" - ], - "kind": 4096, - "kindString": "Call signature", - "flags": {}, "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Remove the key-value store either from the Apify cloud storage or from the local directory.\"]}" + "text": "Close the request loader." } ] }, + "flags": {}, + "id": 88915, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "close", + "parameters": [], "type": { - "type": "reference", - "name": "None" + "name": "None", + "type": "literal", + "value": {} }, - "parameters": [] + "parentId": 2999 } - ] + ], + "permalink": "/python/api/class/SitemapRequestLoader#close", + "parentId": 2963 }, { - "id": 370, - "name": "open", - "module": "storages.key_value_store", "kind": 2048, "kindString": "Method", - "flags": {}, + "children": [], "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Open a key-value store.\\n\\nKey-value stores are used to store records or files, along with their MIME content type.\\nThe records are stored and retrieved using a unique key.\\nThe actual data is stored either on a local filesystem or in the Apify cloud.\\n\", {\"Arguments\": [{\"param\": \"id\", \"type\": \"str, optional\", \"desc\": \"ID of the key-value store to be opened.\\nIf neither `id` nor `name` are provided, the method returns the default key-value store associated with the actor run.\\nIf the key-value store with the given ID does not exist, it raises an error.\"}, {\"param\": \"name\", \"type\": \"str, optional\", \"desc\": \"Name of the key-value store to be opened.\\nIf neither `id` nor `name` are provided, the method returns the default key-value store associated with the actor run.\\nIf the key-value store with the given name does not exist, it is created.\"}, {\"param\": \"force_cloud\", \"type\": \"bool, optional\", \"desc\": \"If set to True, it will open a key-value store on the Apify Platform even when running the actor locally.\\nDefaults to False.\"}, {\"param\": \"config\", \"type\": \"Configuration, optional\", \"desc\": \"A `Configuration` instance, uses global configuration if omitted.\\n\"}]}, {\"Returns\": [{\"param\": \"KeyValueStore\", \"desc\": \"An instance of the `KeyValueStore` class for the given ID or name.\"}]}]}" + "text": "Combine the loader with a request manager to support adding and reclaiming requests.\n" } ] }, - "children": [], + "decorations": [], + "flags": {}, "groups": [], + "id": 90057, + "module": "request_loaders._request_loader", + "name": "to_tandem", + "parsedDocstring": { + "text": "Combine the loader with a request manager to support adding and reclaiming requests.\n", + "args": { + "request_manager": "Request manager to combine the loader with.\nIf None is given, the default request queue is used." + } + }, "sources": [ { - "filename": "/tmp-old-source-v17/src/apify/storages/key_value_store.py", - "line": 229, "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/storages/key_value_store.py#L229" + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 56, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/request_loaders/_request_loader.py#L56" } ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, "signatures": [ { - "id": 371, - "name": "open", - "modifiers": [ - "async" - ], - "kind": 4096, - "kindString": "Call signature", - "flags": {}, "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Open a key-value store.\\n\\nKey-value stores are used to store records or files, along with their MIME content type.\\nThe records are stored and retrieved using a unique key.\\nThe actual data is stored either on a local filesystem or in the Apify cloud.\\n\", {\"" + "text": "Combine the loader with a request manager to support adding and reclaiming requests.\n" } ] }, - "type": { - "type": "reference", - "name": "KeyValueStore", - "id": 352 - }, + "flags": {}, + "id": 88864, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "to_tandem", "parameters": [ { - "id": 372, - "name": "id", - "kind": 32768, - "kindString": "Parameter", - "flags": { - "keyword-only": "true" - }, - "type": { - "type": "reference", - "name": "str | None" - }, - "defaultValue": "None" - }, - { - "id": 373, - "name": "name", - "kind": 32768, - "kindString": "Parameter", - "flags": { - "keyword-only": "true" - }, - "type": { - "type": "reference", - "name": "str | None" + "comment": { + "summary": [ + { + "kind": "text", + "text": "Request manager to combine the loader with.\nIf None is given, the default request queue is used." + } + ] }, - "defaultValue": "None" - }, - { - "id": 374, - "name": "force_cloud", - "kind": 32768, - "kindString": "Parameter", + "defaultValue": "None", "flags": { - "keyword-only": "true" - }, - "type": { - "type": "intrinsic", - "name": "bool" + "isOptional": true, + "keyword-only": false }, - "defaultValue": "False" - }, - { - "id": 375, - "name": "config", + "id": 88865, "kind": 32768, "kindString": "Parameter", - "flags": { - "keyword-only": "true" - }, + "name": "request_manager", "type": { + "name": "RequestManager | None", "type": "reference", - "name": "Configuration | None" + "target": 0, + "ref": {} }, - "defaultValue": "None" + "parentId": 2949 + } + ], + "type": { + "name": "RequestManagerTandem", + "type": "reference", + "target": "2803", + "ref": { + "id": 88718, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_manager_tandem.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 24 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/RequestManagerTandem" + } + }, + "inheritedFrom": { + "name": "RequestLoader.to_tandem", + "target": 2948, + "type": "reference", + "ref": { + "id": 88863, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 56 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestLoader#to_tandem" } - ] + }, + "parentId": 4142 } - ] + ], + "inheritedFrom": { + "name": "RequestLoader.to_tandem", + "target": 2948, + "type": "reference", + "ref": { + "id": 88863, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 56 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestLoader#to_tandem" + } + }, + "permalink": "/python/api/class/SitemapRequestLoader#to_tandem", + "parentId": 2963 + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "A request loader that reads URLs from sitemap(s).\n\nThe loader is designed to handle sitemaps that follow the format described in the Sitemaps protocol\n(https://www.sitemaps.org/protocol.html). It supports both XML and plain text sitemap formats.\nNote that HTML pages containing links are not supported - those should be handled by regular crawlers\nand the `enqueue_links` functionality.\n\nThe loader fetches and parses sitemaps in the background, allowing crawling to start\nbefore all URLs are loaded. It supports filtering URLs using glob and regex patterns.\n\nThe loader supports state persistence, allowing it to resume from where it left off\nafter interruption when a `persist_state_key` is provided during initialization." + } + ] + }, + "decorations": [ + { + "args": "('Request loaders')", + "name": "docs_group" + } + ], + "flags": {}, + "groups": [ + { + "children": [ + 88890, + 88892, + 88879, + 88912, + 88914, + 88905, + 88899, + 88897, + 88901, + 88903, + 88907, + 88910, + 90057 + ], + "title": "Methods" + } + ], + "id": 88878, + "module": "request_loaders._sitemap_request_loader", + "name": "SitemapRequestLoader", + "parsedDocstring": { + "text": "A request loader that reads URLs from sitemap(s).\n\nThe loader is designed to handle sitemaps that follow the format described in the Sitemaps protocol\n(https://www.sitemaps.org/protocol.html). It supports both XML and plain text sitemap formats.\nNote that HTML pages containing links are not supported - those should be handled by regular crawlers\nand the `enqueue_links` functionality.\n\nThe loader fetches and parses sitemaps in the background, allowing crawling to start\nbefore all URLs are loaded. It supports filtering URLs using glob and regex patterns.\n\nThe loader supports state persistence, allowing it to resume from where it left off\nafter interruption when a `persist_state_key` is provided during initialization." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_sitemap_request_loader.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 94, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/request_loaders/_sitemap_request_loader.py#L94" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "extendedTypes": [ + { + "name": "RequestLoader", + "target": "2934", + "type": "reference", + "ref": { + "id": 88849, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb", + "line": 17 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/RequestLoader" + } } ], - "groups": [ - { - "title": "Methods", - "children": [ - 368, - 365, - 353, - 357, - 370, - 360 - ] - } + "permalink": "/python/api/class/SitemapRequestLoader", + "categories": [] + } + ], + "flags": {}, + "groups": [ + { + "children": [ + 300 + ], + "title": "Async Resource Clients" + }, + { + "children": [ + 28, + 36, + 234, + 1, + 636, + 695, + 697, + 700, + 543, + 257, + 248, + 764 + ], + "title": "Classes" + }, + { + "children": [ + 346, + 276, + 217, + 339, + 193, + 273, + 336, + 453, + 482, + 495, + 548, + 582, + 612, + 732, + 745, + 349, + 351, + 342, + 598, + 359, + 198, + 367, + 375, + 185, + 460, + 458, + 188, + 506, + 559, + 593, + 579, + 449, + 609, + 209, + 280, + 304, + 326, + 501, + 554, + 588, + 618, + 383, + 284, + 532, + 286, + 476, + 488, + 622, + 629, + 565, + 568, + 602, + 388, + 737, + 269, + 316, + 196, + 357, + 245, + 355, + 521, + 465, + 463, + 474, + 486, + 620, + 627, + 595, + 508, + 561, + 181, + 319, + 288, + 291, + 294, + 297, + 371, + 205, + 201, + 751, + 394, + 540, + 311, + 307, + 470, + 468, + 740, + 734, + 574, + 536, + 571, + 425, + 429, + 323, + 503, + 556, + 590, + 438, + 605, + 443, + 353, + 213, + 332 + ], + "title": "Methods" + }, + { + "children": [ + 241, + 760, + 242, + 228, + 743, + 744, + 243, + 224, + 216, + 272, + 271, + 635, + 229, + 230, + 180, + 179, + 178, + 227, + 330, + 493, + 192, + 366, + 494, + 233, + 268, + 232, + 27, + 757, + 231, + 761, + 331, + 382, + 365, + 364, + 759, + 758, + 762, + 225, + 226, + 481, + 634, + 244, + 763, + 26, + 267, + 694 + ], + "title": "Properties" + }, + { + "children": [ + 220 + ], + "title": "Constants" + }, + { + "children": [ + 433, + 409, + 396, + 390 ], - "sources": [ - { - "filename": "/tmp-old-source-v17/src/apify/storages/key_value_store.py", - "line": 35, - "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/storages/key_value_store.py#L35" - } + "title": "Scrapy integration" + }, + { + "title": "Storages", + "children": [ + 3126, + 5600, + 8889, + 11313 + ] + }, + { + "title": "Storage data", + "children": [ + 14466, + 15802, + 17122, + 21338, + 25569, + 29772, + 31117, + 31531, + 32908, + 37119 + ] + }, + { + "title": "Event managers", + "children": [ + 39286, + 42779 + ] + }, + { + "title": "Events", + "children": [ + 46827 + ] + }, + { + "title": "Event data", + "children": [ + 48978, + 51120, + 53257, + 56571, + 58705 + ] + }, + { + "title": "Storage clients", + "children": [ + 61200, + 63477, + 67833, + 72303 + ] + }, + { + "title": "Request loaders", + "children": [ + 77547, + 80615, + 84566, + 88878 ] + } + ], + "id": 0, + "kind": 1, + "kindString": "Project", + "name": "apify-client", + "sources": [ + { + "character": 0, + "fileName": "src/index.ts", + "line": 1, + "gitRevision": "6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb" + } + ], + "symbolIdMap": { + "1": { + "qualifiedName": "Configuration", + "sourceFileName": "/src/apify/config.py" + }, + "2": { + "qualifiedName": "__init__", + "sourceFileName": "/src/apify/config.py" + }, + "24": { + "qualifiedName": "get_global_configuration", + "sourceFileName": "/src/apify/config.py" + }, + "26": { + "qualifiedName": "T", + "sourceFileName": "/src/apify/actor.py" + }, + "27": { + "qualifiedName": "MainReturnType", + "sourceFileName": "/src/apify/actor.py" + }, + "28": { + "qualifiedName": "_ActorContextManager", + "sourceFileName": "/src/apify/actor.py" + }, + "29": { + "qualifiedName": "__aenter__", + "sourceFileName": "/src/apify/actor.py" + }, + "31": { + "qualifiedName": "__aexit__", + "sourceFileName": "/src/apify/actor.py" + }, + "36": { + "qualifiedName": "Actor", + "sourceFileName": "/src/apify/actor.py" + }, + "37": { + "qualifiedName": "__init__", + "sourceFileName": "/src/apify/actor.py" + }, + "40": { + "qualifiedName": "apify_client", + "sourceFileName": "/src/apify/actor.py" + }, + "41": { + "qualifiedName": "config", + "sourceFileName": "/src/apify/actor.py" + }, + "42": { + "qualifiedName": "event_manager", + "sourceFileName": "/src/apify/actor.py" + }, + "43": { + "qualifiedName": "log", + "sourceFileName": "/src/apify/actor.py" + }, + "44": { + "qualifiedName": "init", + "sourceFileName": "/src/apify/actor.py" + }, + "46": { + "qualifiedName": "get_system_info", + "sourceFileName": "/src/apify/actor.py" + }, + "48": { + "qualifiedName": "exit", + "sourceFileName": "/src/apify/actor.py" + }, + "54": { + "qualifiedName": "fail", + "sourceFileName": "/src/apify/actor.py" + }, + "59": { + "qualifiedName": "main", + "sourceFileName": "/src/apify/actor.py" + }, + "62": { + "qualifiedName": "new_client", + "sourceFileName": "/src/apify/actor.py" + }, + "69": { + "qualifiedName": "open_dataset", + "sourceFileName": "/src/apify/actor.py" + }, + "74": { + "qualifiedName": "open_key_value_store", + "sourceFileName": "/src/apify/actor.py" + }, + "79": { + "qualifiedName": "open_request_queue", + "sourceFileName": "/src/apify/actor.py" + }, + "84": { + "qualifiedName": "push_data", + "sourceFileName": "/src/apify/actor.py" + }, + "87": { + "qualifiedName": "get_input", + "sourceFileName": "/src/apify/actor.py" + }, + "89": { + "qualifiedName": "get_value", + "sourceFileName": "/src/apify/actor.py" + }, + "93": { + "qualifiedName": "set_value", + "sourceFileName": "/src/apify/actor.py" + }, + "98": { + "qualifiedName": "on", + "sourceFileName": "/src/apify/actor.py" + }, + "102": { + "qualifiedName": "off", + "sourceFileName": "/src/apify/actor.py" + }, + "106": { + "qualifiedName": "is_at_home", + "sourceFileName": "/src/apify/actor.py" + }, + "108": { + "qualifiedName": "get_env", + "sourceFileName": "/src/apify/actor.py" + }, + "110": { + "qualifiedName": "start", + "sourceFileName": "/src/apify/actor.py" + }, + "121": { + "qualifiedName": "abort", + "sourceFileName": "/src/apify/actor.py" + }, + "126": { + "qualifiedName": "call", + "sourceFileName": "/src/apify/actor.py" + }, + "137": { + "qualifiedName": "call_task", + "sourceFileName": "/src/apify/actor.py" + }, + "147": { + "qualifiedName": "metamorph", + "sourceFileName": "/src/apify/actor.py" + }, + "154": { + "qualifiedName": "reboot", + "sourceFileName": "/src/apify/actor.py" + }, + "158": { + "qualifiedName": "add_webhook", + "sourceFileName": "/src/apify/actor.py" + }, + "166": { + "qualifiedName": "set_status_message", + "sourceFileName": "/src/apify/actor.py" + }, + "170": { + "qualifiedName": "create_proxy_configuration", + "sourceFileName": "/src/apify/actor.py" + }, + "178": { + "qualifiedName": "ENCRYPTION_KEY_LENGTH", + "sourceFileName": "/src/apify/_crypto.py" + }, + "179": { + "qualifiedName": "ENCRYPTION_IV_LENGTH", + "sourceFileName": "/src/apify/_crypto.py" + }, + "180": { + "qualifiedName": "ENCRYPTION_AUTH_TAG_LENGTH", + "sourceFileName": "/src/apify/_crypto.py" + }, + "181": { + "qualifiedName": "load_private_key", + "sourceFileName": "/src/apify/_crypto.py" + }, + "185": { + "qualifiedName": "crypto_random_object_id", + "sourceFileName": "/src/apify/_crypto.py" + }, + "188": { + "qualifiedName": "decrypt_input_secrets", + "sourceFileName": "/src/apify/_crypto.py" + }, + "192": { + "qualifiedName": "ListenerType", + "sourceFileName": "/src/apify/event_manager.py" + }, + "193": { + "qualifiedName": "__init__", + "sourceFileName": "/src/apify/event_manager.py" + }, + "196": { + "qualifiedName": "init", + "sourceFileName": "/src/apify/event_manager.py" + }, + "198": { + "qualifiedName": "close", + "sourceFileName": "/src/apify/event_manager.py" + }, + "201": { + "qualifiedName": "on", + "sourceFileName": "/src/apify/event_manager.py" + }, + "205": { + "qualifiedName": "off", + "sourceFileName": "/src/apify/event_manager.py" + }, + "209": { + "qualifiedName": "emit", + "sourceFileName": "/src/apify/event_manager.py" + }, + "213": { + "qualifiedName": "wait_for_all_listeners_to_complete", + "sourceFileName": "/src/apify/event_manager.py" + }, + "216": { + "qualifiedName": "DEPRECATED_NAMES", + "sourceFileName": "/src/apify/consts.py" + }, + "217": { + "qualifiedName": "__getattr__", + "sourceFileName": "/src/apify/consts.py" + }, + "220": { + "qualifiedName": "StorageTypes", + "sourceFileName": "/src/apify/consts.py" + }, + "221": { + "qualifiedName": "DATASET", + "sourceFileName": "/src/apify/consts.py" + }, + "222": { + "qualifiedName": "KEY_VALUE_STORE", + "sourceFileName": "/src/apify/consts.py" + }, + "223": { + "qualifiedName": "REQUEST_QUEUE", + "sourceFileName": "/src/apify/consts.py" + }, + "224": { + "qualifiedName": "DEFAULT_API_PARAM_LIMIT", + "sourceFileName": "/src/apify/consts.py" + }, + "225": { + "qualifiedName": "REQUEST_ID_LENGTH", + "sourceFileName": "/src/apify/consts.py" + }, + "226": { + "qualifiedName": "REQUEST_QUEUE_HEAD_MAX_LIMIT", + "sourceFileName": "/src/apify/consts.py" + }, + "227": { + "qualifiedName": "EVENT_LISTENERS_TIMEOUT_SECS", + "sourceFileName": "/src/apify/consts.py" + }, + "228": { + "qualifiedName": "BASE64_REGEXP", + "sourceFileName": "/src/apify/consts.py" + }, + "229": { + "qualifiedName": "ENCRYPTED_INPUT_VALUE_PREFIX", + "sourceFileName": "/src/apify/consts.py" + }, + "230": { + "qualifiedName": "ENCRYPTED_INPUT_VALUE_REGEXP", + "sourceFileName": "/src/apify/consts.py" + }, + "231": { + "qualifiedName": "MAX_PAYLOAD_SIZE_BYTES", + "sourceFileName": "/src/apify/consts.py" + }, + "232": { + "qualifiedName": "logger_name", + "sourceFileName": "/src/apify/log.py" + }, + "233": { + "qualifiedName": "logger", + "sourceFileName": "/src/apify/log.py" + }, + "234": { + "qualifiedName": "ActorLogFormatter", + "sourceFileName": "/src/apify/log.py" + }, + "235": { + "qualifiedName": "empty_record", + "sourceFileName": "/src/apify/log.py" + }, + "236": { + "qualifiedName": "__init__", + "sourceFileName": "/src/apify/log.py" + }, + "241": { + "qualifiedName": "__version__", + "sourceFileName": "/src/apify/__init__.py" + }, + "242": { + "qualifiedName": "APIFY_PROXY_VALUE_REGEX", + "sourceFileName": "/src/apify/proxy_configuration.py" + }, + "243": { + "qualifiedName": "COUNTRY_CODE_REGEX", + "sourceFileName": "/src/apify/proxy_configuration.py" + }, + "244": { + "qualifiedName": "SESSION_ID_MAX_LENGTH", + "sourceFileName": "/src/apify/proxy_configuration.py" + }, + "245": { + "qualifiedName": "is_url", + "sourceFileName": "/src/apify/proxy_configuration.py" + }, + "248": { + "qualifiedName": "ProxyInfo", + "sourceFileName": "/src/apify/proxy_configuration.py" + }, + "249": { + "qualifiedName": "url", + "sourceFileName": "/src/apify/proxy_configuration.py" + }, + "250": { + "qualifiedName": "hostname", + "sourceFileName": "/src/apify/proxy_configuration.py" + }, + "251": { + "qualifiedName": "port", + "sourceFileName": "/src/apify/proxy_configuration.py" + }, + "252": { + "qualifiedName": "username", + "sourceFileName": "/src/apify/proxy_configuration.py" + }, + "253": { + "qualifiedName": "password", + "sourceFileName": "/src/apify/proxy_configuration.py" + }, + "254": { + "qualifiedName": "groups", + "sourceFileName": "/src/apify/proxy_configuration.py" + }, + "255": { + "qualifiedName": "country_code", + "sourceFileName": "/src/apify/proxy_configuration.py" + }, + "256": { + "qualifiedName": "session_id", + "sourceFileName": "/src/apify/proxy_configuration.py" + }, + "257": { + "qualifiedName": "ProxyConfiguration", + "sourceFileName": "/src/apify/proxy_configuration.py" + }, + "258": { + "qualifiedName": "is_man_in_the_middle", + "sourceFileName": "/src/apify/proxy_configuration.py" + }, + "259": { + "qualifiedName": "initialize", + "sourceFileName": "/src/apify/proxy_configuration.py" + }, + "261": { + "qualifiedName": "new_url", + "sourceFileName": "/src/apify/proxy_configuration.py" + }, + "264": { + "qualifiedName": "new_proxy_info", + "sourceFileName": "/src/apify/proxy_configuration.py" + }, + "267": { + "qualifiedName": "T", + "sourceFileName": "/src/apify/_utils.py" + }, + "268": { + "qualifiedName": "logger", + "sourceFileName": "/src/apify/_utils.py" + }, + "269": { + "qualifiedName": "get_system_info", + "sourceFileName": "/src/apify/_utils.py" + }, + "271": { + "qualifiedName": "DualPropertyType", + "sourceFileName": "/src/apify/_utils.py" + }, + "272": { + "qualifiedName": "DualPropertyOwner", + "sourceFileName": "/src/apify/_utils.py" + }, + "273": { + "qualifiedName": "__init__", + "sourceFileName": "/src/apify/_utils.py" + }, + "276": { + "qualifiedName": "__get__", + "sourceFileName": "/src/apify/_utils.py" + }, + "280": { + "qualifiedName": "fetch_and_parse_env_var", + "sourceFileName": "/src/apify/_utils.py" + }, + "284": { + "qualifiedName": "get_cpu_usage_percent", + "sourceFileName": "/src/apify/_utils.py" + }, + "286": { + "qualifiedName": "get_memory_usage_bytes", + "sourceFileName": "/src/apify/_utils.py" + }, + "288": { + "qualifiedName": "maybe_parse_bool", + "sourceFileName": "/src/apify/_utils.py" + }, + "291": { + "qualifiedName": "maybe_parse_datetime", + "sourceFileName": "/src/apify/_utils.py" + }, + "294": { + "qualifiedName": "maybe_parse_float", + "sourceFileName": "/src/apify/_utils.py" + }, + "297": { + "qualifiedName": "maybe_parse_int", + "sourceFileName": "/src/apify/_utils.py" + }, + "300": { + "qualifiedName": "run_func_at_interval_async", + "sourceFileName": "/src/apify/_utils.py" + }, + "304": { + "qualifiedName": "force_remove", + "sourceFileName": "/src/apify/_utils.py" + }, + "307": { + "qualifiedName": "raise_on_non_existing_storage", + "sourceFileName": "/src/apify/_utils.py" + }, + "311": { + "qualifiedName": "raise_on_duplicate_storage", + "sourceFileName": "/src/apify/_utils.py" + }, + "316": { + "qualifiedName": "guess_file_extension", + "sourceFileName": "/src/apify/_utils.py" + }, + "319": { + "qualifiedName": "maybe_parse_body", + "sourceFileName": "/src/apify/_utils.py" + }, + "323": { + "qualifiedName": "unique_key_to_request_id", + "sourceFileName": "/src/apify/_utils.py" + }, + "326": { + "qualifiedName": "force_rename", + "sourceFileName": "/src/apify/_utils.py" + }, + "330": { + "qualifiedName": "ImplementationType", + "sourceFileName": "/src/apify/_utils.py" + }, + "331": { + "qualifiedName": "MetadataType", + "sourceFileName": "/src/apify/_utils.py" + }, + "332": { + "qualifiedName": "wrap_internal", + "sourceFileName": "/src/apify/_utils.py" + }, + "336": { + "qualifiedName": "__init__", + "sourceFileName": "/src/apify/_utils.py" + }, + "339": { + "qualifiedName": "__getitem__", + "sourceFileName": "/src/apify/_utils.py" + }, + "342": { + "qualifiedName": "__setitem__", + "sourceFileName": "/src/apify/_utils.py" + }, + "346": { + "qualifiedName": "__delitem__", + "sourceFileName": "/src/apify/_utils.py" + }, + "349": { + "qualifiedName": "__iter__", + "sourceFileName": "/src/apify/_utils.py" + }, + "351": { + "qualifiedName": "__len__", + "sourceFileName": "/src/apify/_utils.py" + }, + "353": { + "qualifiedName": "values", + "sourceFileName": "/src/apify/_utils.py" + }, + "355": { + "qualifiedName": "items", + "sourceFileName": "/src/apify/_utils.py" + }, + "357": { + "qualifiedName": "is_running_in_ipython", + "sourceFileName": "/src/apify/_utils.py" + }, + "359": { + "qualifiedName": "budget_ow", + "sourceFileName": "/src/apify/_utils.py" + }, + "364": { + "qualifiedName": "PARSE_DATE_FIELDS_MAX_DEPTH", + "sourceFileName": "/src/apify/_utils.py" + }, + "365": { + "qualifiedName": "PARSE_DATE_FIELDS_KEY_SUFFIX", + "sourceFileName": "/src/apify/_utils.py" + }, + "366": { + "qualifiedName": "ListOrDictOrAny", + "sourceFileName": "/src/apify/_utils.py" + }, + "367": { + "qualifiedName": "compute_short_hash", + "sourceFileName": "/src/apify/_utils.py" + }, + "371": { + "qualifiedName": "normalize_url", + "sourceFileName": "/src/apify/_utils.py" + }, + "375": { + "qualifiedName": "compute_unique_key", + "sourceFileName": "/src/apify/_utils.py" + }, + "382": { + "qualifiedName": "nested_event_loop", + "sourceFileName": "/src/apify/scrapy/utils.py" + }, + "383": { + "qualifiedName": "get_basic_auth_header", + "sourceFileName": "/src/apify/scrapy/utils.py" + }, + "388": { + "qualifiedName": "get_running_event_loop_id", + "sourceFileName": "/src/apify/scrapy/utils.py" + }, + "390": { + "qualifiedName": "apply_apify_settings", + "sourceFileName": "/src/apify/scrapy/utils.py" + }, + "394": { + "qualifiedName": "open_queue_with_custom_client", + "sourceFileName": "/src/apify/scrapy/utils.py" + }, + "396": { + "qualifiedName": "ApifyScheduler", + "sourceFileName": "/src/apify/scrapy/scheduler.py" + }, + "397": { + "qualifiedName": "__init__", + "sourceFileName": "/src/apify/scrapy/scheduler.py" + }, + "399": { + "qualifiedName": "open", + "sourceFileName": "/src/apify/scrapy/scheduler.py" + }, + "402": { + "qualifiedName": "has_pending_requests", + "sourceFileName": "/src/apify/scrapy/scheduler.py" + }, + "404": { + "qualifiedName": "enqueue_request", + "sourceFileName": "/src/apify/scrapy/scheduler.py" + }, + "407": { + "qualifiedName": "next_request", + "sourceFileName": "/src/apify/scrapy/scheduler.py" + }, + "409": { + "qualifiedName": "ApifyHttpProxyMiddleware", + "sourceFileName": "/src/apify/scrapy/middlewares/apify_proxy.py" + }, + "410": { + "qualifiedName": "__init__", + "sourceFileName": "/src/apify/scrapy/middlewares/apify_proxy.py" + }, + "413": { + "qualifiedName": "from_crawler", + "sourceFileName": "/src/apify/scrapy/middlewares/apify_proxy.py" + }, + "416": { + "qualifiedName": "process_request", + "sourceFileName": "/src/apify/scrapy/middlewares/apify_proxy.py" + }, + "420": { + "qualifiedName": "process_exception", + "sourceFileName": "/src/apify/scrapy/middlewares/apify_proxy.py" + }, + "425": { + "qualifiedName": "to_apify_request", + "sourceFileName": "/src/apify/scrapy/requests.py" + }, + "429": { + "qualifiedName": "to_scrapy_request", + "sourceFileName": "/src/apify/scrapy/requests.py" + }, + "433": { + "qualifiedName": "ActorDatasetPushPipeline", + "sourceFileName": "/src/apify/scrapy/pipelines/actor_dataset_push.py" + }, + "434": { + "qualifiedName": "process_item", + "sourceFileName": "/src/apify/scrapy/pipelines/actor_dataset_push.py" + }, + "438": { + "qualifiedName": "update_metadata", + "sourceFileName": "/src/apify/_memory_storage/file_storage_utils.py" + }, + "443": { + "qualifiedName": "update_request_queue_item", + "sourceFileName": "/src/apify/_memory_storage/file_storage_utils.py" + }, + "449": { + "qualifiedName": "delete_request", + "sourceFileName": "/src/apify/_memory_storage/file_storage_utils.py" + }, + "453": { + "qualifiedName": "__init__", + "sourceFileName": "/src/apify/_memory_storage/memory_storage_client.py" + }, + "458": { + "qualifiedName": "datasets", + "sourceFileName": "/src/apify/_memory_storage/memory_storage_client.py" + }, + "460": { + "qualifiedName": "dataset", + "sourceFileName": "/src/apify/_memory_storage/memory_storage_client.py" + }, + "463": { + "qualifiedName": "key_value_stores", + "sourceFileName": "/src/apify/_memory_storage/memory_storage_client.py" + }, + "465": { + "qualifiedName": "key_value_store", + "sourceFileName": "/src/apify/_memory_storage/memory_storage_client.py" + }, + "468": { + "qualifiedName": "request_queues", + "sourceFileName": "/src/apify/_memory_storage/memory_storage_client.py" + }, + "470": { + "qualifiedName": "request_queue", + "sourceFileName": "/src/apify/_memory_storage/memory_storage_client.py" + }, + "474": { + "qualifiedName": "list", + "sourceFileName": "/src/apify/_memory_storage/resource_clients/key_value_store_collection.py" + }, + "476": { + "qualifiedName": "get_or_create", + "sourceFileName": "/src/apify/_memory_storage/resource_clients/key_value_store_collection.py" + }, + "481": { + "qualifiedName": "ResourceClientType", + "sourceFileName": "/src/apify/_memory_storage/resource_clients/base_resource_collection_client.py" + }, + "482": { + "qualifiedName": "__init__", + "sourceFileName": "/src/apify/_memory_storage/resource_clients/base_resource_collection_client.py" + }, + "486": { + "qualifiedName": "list", + "sourceFileName": "/src/apify/_memory_storage/resource_clients/base_resource_collection_client.py" + }, + "488": { + "qualifiedName": "get_or_create", + "sourceFileName": "/src/apify/_memory_storage/resource_clients/base_resource_collection_client.py" + }, + "493": { + "qualifiedName": "LIST_ITEMS_LIMIT", + "sourceFileName": "/src/apify/_memory_storage/resource_clients/dataset.py" + }, + "494": { + "qualifiedName": "LOCAL_ENTRY_NAME_DIGITS", + "sourceFileName": "/src/apify/_memory_storage/resource_clients/dataset.py" + }, + "495": { + "qualifiedName": "__init__", + "sourceFileName": "/src/apify/_memory_storage/resource_clients/dataset.py" + }, + "501": { + "qualifiedName": "get", + "sourceFileName": "/src/apify/_memory_storage/resource_clients/dataset.py" + }, + "503": { + "qualifiedName": "update", + "sourceFileName": "/src/apify/_memory_storage/resource_clients/dataset.py" + }, + "506": { + "qualifiedName": "delete", + "sourceFileName": "/src/apify/_memory_storage/resource_clients/dataset.py" + }, + "508": { + "qualifiedName": "list_items", + "sourceFileName": "/src/apify/_memory_storage/resource_clients/dataset.py" + }, + "521": { + "qualifiedName": "iterate_items", + "sourceFileName": "/src/apify/_memory_storage/resource_clients/dataset.py" + }, + "532": { + "qualifiedName": "get_items_as_bytes", + "sourceFileName": "/src/apify/_memory_storage/resource_clients/dataset.py" + }, + "536": { + "qualifiedName": "stream_items", + "sourceFileName": "/src/apify/_memory_storage/resource_clients/dataset.py" + }, + "540": { + "qualifiedName": "push_items", + "sourceFileName": "/src/apify/_memory_storage/resource_clients/dataset.py" + }, + "543": { + "qualifiedName": "KeyValueStoreRecord", + "sourceFileName": "/src/apify/_memory_storage/resource_clients/key_value_store.py" + }, + "544": { + "qualifiedName": "key", + "sourceFileName": "/src/apify/_memory_storage/resource_clients/key_value_store.py" + }, + "545": { + "qualifiedName": "value", + "sourceFileName": "/src/apify/_memory_storage/resource_clients/key_value_store.py" + }, + "546": { + "qualifiedName": "contentType", + "sourceFileName": "/src/apify/_memory_storage/resource_clients/key_value_store.py" + }, + "547": { + "qualifiedName": "filename", + "sourceFileName": "/src/apify/_memory_storage/resource_clients/key_value_store.py" + }, + "548": { + "qualifiedName": "__init__", + "sourceFileName": "/src/apify/_memory_storage/resource_clients/key_value_store.py" + }, + "554": { + "qualifiedName": "get", + "sourceFileName": "/src/apify/_memory_storage/resource_clients/key_value_store.py" + }, + "556": { + "qualifiedName": "update", + "sourceFileName": "/src/apify/_memory_storage/resource_clients/key_value_store.py" + }, + "559": { + "qualifiedName": "delete", + "sourceFileName": "/src/apify/_memory_storage/resource_clients/key_value_store.py" + }, + "561": { + "qualifiedName": "list_keys", + "sourceFileName": "/src/apify/_memory_storage/resource_clients/key_value_store.py" + }, + "565": { + "qualifiedName": "get_record", + "sourceFileName": "/src/apify/_memory_storage/resource_clients/key_value_store.py" + }, + "568": { + "qualifiedName": "get_record_as_bytes", + "sourceFileName": "/src/apify/_memory_storage/resource_clients/key_value_store.py" + }, + "571": { + "qualifiedName": "stream_record", + "sourceFileName": "/src/apify/_memory_storage/resource_clients/key_value_store.py" + }, + "574": { + "qualifiedName": "set_record", + "sourceFileName": "/src/apify/_memory_storage/resource_clients/key_value_store.py" + }, + "579": { + "qualifiedName": "delete_record", + "sourceFileName": "/src/apify/_memory_storage/resource_clients/key_value_store.py" + }, + "582": { + "qualifiedName": "__init__", + "sourceFileName": "/src/apify/_memory_storage/resource_clients/request_queue.py" + }, + "588": { + "qualifiedName": "get", + "sourceFileName": "/src/apify/_memory_storage/resource_clients/request_queue.py" + }, + "590": { + "qualifiedName": "update", + "sourceFileName": "/src/apify/_memory_storage/resource_clients/request_queue.py" + }, + "593": { + "qualifiedName": "delete", + "sourceFileName": "/src/apify/_memory_storage/resource_clients/request_queue.py" + }, + "595": { + "qualifiedName": "list_head", + "sourceFileName": "/src/apify/_memory_storage/resource_clients/request_queue.py" + }, + "598": { + "qualifiedName": "add_request", + "sourceFileName": "/src/apify/_memory_storage/resource_clients/request_queue.py" + }, + "602": { + "qualifiedName": "get_request", + "sourceFileName": "/src/apify/_memory_storage/resource_clients/request_queue.py" + }, + "605": { + "qualifiedName": "update_request", + "sourceFileName": "/src/apify/_memory_storage/resource_clients/request_queue.py" }, + "609": { + "qualifiedName": "delete_request", + "sourceFileName": "/src/apify/_memory_storage/resource_clients/request_queue.py" + }, + "612": { + "qualifiedName": "__init__", + "sourceFileName": "/src/apify/_memory_storage/resource_clients/base_resource_client.py" + }, + "618": { + "qualifiedName": "get", + "sourceFileName": "/src/apify/_memory_storage/resource_clients/base_resource_client.py" + }, + "620": { + "qualifiedName": "list", + "sourceFileName": "/src/apify/_memory_storage/resource_clients/request_queue_collection.py" + }, + "622": { + "qualifiedName": "get_or_create", + "sourceFileName": "/src/apify/_memory_storage/resource_clients/request_queue_collection.py" + }, + "627": { + "qualifiedName": "list", + "sourceFileName": "/src/apify/_memory_storage/resource_clients/dataset_collection.py" + }, + "629": { + "qualifiedName": "get_or_create", + "sourceFileName": "/src/apify/_memory_storage/resource_clients/dataset_collection.py" + }, + "634": { + "qualifiedName": "SAFETY_BUFFER_PERCENT", + "sourceFileName": "/src/apify/storages/dataset.py" + }, + "635": { + "qualifiedName": "EFFECTIVE_LIMIT_BYTES", + "sourceFileName": "/src/apify/storages/dataset.py" + }, + "636": { + "qualifiedName": "Dataset", + "sourceFileName": "/src/apify/storages/dataset.py" + }, + "637": { + "qualifiedName": "push_data", + "sourceFileName": "/src/apify/storages/dataset.py" + }, + "640": { + "qualifiedName": "get_data", + "sourceFileName": "/src/apify/storages/dataset.py" + }, + "653": { + "qualifiedName": "export_to", + "sourceFileName": "/src/apify/storages/dataset.py" + }, + "659": { + "qualifiedName": "export_to_json", + "sourceFileName": "/src/apify/storages/dataset.py" + }, + "666": { + "qualifiedName": "export_to_csv", + "sourceFileName": "/src/apify/storages/dataset.py" + }, + "673": { + "qualifiedName": "get_info", + "sourceFileName": "/src/apify/storages/dataset.py" + }, + "675": { + "qualifiedName": "iterate_items", + "sourceFileName": "/src/apify/storages/dataset.py" + }, + "686": { + "qualifiedName": "drop", + "sourceFileName": "/src/apify/storages/dataset.py" + }, + "688": { + "qualifiedName": "open", + "sourceFileName": "/src/apify/storages/dataset.py" + }, + "694": { + "qualifiedName": "T", + "sourceFileName": "/src/apify/storages/key_value_store.py" + }, + "695": { + "qualifiedName": "IterateKeysInfo", + "sourceFileName": "/src/apify/storages/key_value_store.py" + }, + "696": { + "qualifiedName": "size", + "sourceFileName": "/src/apify/storages/key_value_store.py" + }, + "697": { + "qualifiedName": "IterateKeysTuple", + "sourceFileName": "/src/apify/storages/key_value_store.py" + }, + "698": { + "qualifiedName": "key", + "sourceFileName": "/src/apify/storages/key_value_store.py" + }, + "699": { + "qualifiedName": "info", + "sourceFileName": "/src/apify/storages/key_value_store.py" + }, + "700": { + "qualifiedName": "KeyValueStore", + "sourceFileName": "/src/apify/storages/key_value_store.py" + }, + "701": { + "qualifiedName": "get_value", + "sourceFileName": "/src/apify/storages/key_value_store.py" + }, + "705": { + "qualifiedName": "iterate_keys", + "sourceFileName": "/src/apify/storages/key_value_store.py" + }, + "708": { + "qualifiedName": "set_value", + "sourceFileName": "/src/apify/storages/key_value_store.py" + }, + "713": { + "qualifiedName": "get_public_url", + "sourceFileName": "/src/apify/storages/key_value_store.py" + }, + "716": { + "qualifiedName": "drop", + "sourceFileName": "/src/apify/storages/key_value_store.py" + }, + "718": { + "qualifiedName": "open", + "sourceFileName": "/src/apify/storages/key_value_store.py" + }, + "732": { + "qualifiedName": "__init__", + "sourceFileName": "/src/apify/storages/storage_client_manager.py" + }, + "734": { + "qualifiedName": "set_config", + "sourceFileName": "/src/apify/storages/storage_client_manager.py" + }, + "737": { + "qualifiedName": "get_storage_client", + "sourceFileName": "/src/apify/storages/storage_client_manager.py" + }, + "740": { + "qualifiedName": "set_cloud_client", + "sourceFileName": "/src/apify/storages/storage_client_manager.py" + }, + "743": { + "qualifiedName": "BaseResourceClientType", + "sourceFileName": "/src/apify/storages/base_storage.py" + }, + "744": { + "qualifiedName": "BaseResourceCollectionClientType", + "sourceFileName": "/src/apify/storages/base_storage.py" + }, + "745": { + "qualifiedName": "__init__", + "sourceFileName": "/src/apify/storages/base_storage.py" + }, + "751": { + "qualifiedName": "open", + "sourceFileName": "/src/apify/storages/base_storage.py" + }, + "757": { + "qualifiedName": "MAX_CACHED_REQUESTS", + "sourceFileName": "/src/apify/storages/request_queue.py" + }, + "758": { + "qualifiedName": "QUERY_HEAD_MIN_LENGTH", + "sourceFileName": "/src/apify/storages/request_queue.py" + }, + "759": { + "qualifiedName": "QUERY_HEAD_BUFFER", + "sourceFileName": "/src/apify/storages/request_queue.py" + }, + "760": { + "qualifiedName": "API_PROCESSED_REQUESTS_DELAY_MILLIS", + "sourceFileName": "/src/apify/storages/request_queue.py" + }, + "761": { + "qualifiedName": "MAX_QUERIES_FOR_CONSISTENCY", + "sourceFileName": "/src/apify/storages/request_queue.py" + }, + "762": { + "qualifiedName": "RECENTLY_HANDLED_CACHE_SIZE", + "sourceFileName": "/src/apify/storages/request_queue.py" + }, + "763": { + "qualifiedName": "STORAGE_CONSISTENCY_DELAY_MILLIS", + "sourceFileName": "/src/apify/storages/request_queue.py" + }, + "764": { + "qualifiedName": "RequestQueue", + "sourceFileName": "/src/apify/storages/request_queue.py" + }, + "765": { + "qualifiedName": "add_request", + "sourceFileName": "/src/apify/storages/request_queue.py" + }, + "771": { + "qualifiedName": "get_request", + "sourceFileName": "/src/apify/storages/request_queue.py" + }, + "774": { + "qualifiedName": "fetch_next_request", + "sourceFileName": "/src/apify/storages/request_queue.py" + }, + "776": { + "qualifiedName": "mark_request_as_handled", + "sourceFileName": "/src/apify/storages/request_queue.py" + }, + "779": { + "qualifiedName": "reclaim_request", + "sourceFileName": "/src/apify/storages/request_queue.py" + }, + "783": { + "qualifiedName": "is_empty", + "sourceFileName": "/src/apify/storages/request_queue.py" + }, + "785": { + "qualifiedName": "is_finished", + "sourceFileName": "/src/apify/storages/request_queue.py" + }, + "787": { + "qualifiedName": "drop", + "sourceFileName": "/src/apify/storages/request_queue.py" + }, + "789": { + "qualifiedName": "get_info", + "sourceFileName": "/src/apify/storages/request_queue.py" + }, + "791": { + "qualifiedName": "open", + "sourceFileName": "/src/apify/storages/request_queue.py" + } + }, + "overloads": [ { - "id": 376, - "name": "RequestQueue", - "module": "storages.request_queue", - "kind": 128, - "kindString": "Class", - "flags": {}, - "comment": { - "summary": [ - { - "kind": "text", - "text": "{\"content\": [\"Represents a queue of URLs to crawl.\\n\\nCan be used for deep crawling of websites where you start with several URLs and then recursively\\nfollow links to other pages. The data structure supports both breadth-first and depth-first crawling orders.\\n\\nEach URL is represented using an instance of the {@apilink Request} class.\\nThe queue can only contain unique URLs. More precisely, it can only contain request dictionaries\\nwith distinct `uniqueKey` properties. By default, `uniqueKey` is generated from the URL, but it can also be overridden.\\nTo add a single URL multiple times to the queue,\\ncorresponding request dictionary will need to have different `uniqueKey` properties.\\n\\nDo not instantiate this class directly, use the `Actor.open_request_queue()` function instead.\\n\\n`RequestQueue` stores its data either on local disk or in the Apify cloud,\\ndepending on whether the `APIFY_LOCAL_STORAGE_DIR` or `APIFY_TOKEN` environment variables are set.\\n\\nIf the `APIFY_LOCAL_STORAGE_DIR` environment variable is set, the data is stored in\\nthe local directory in the following files:\\n```\\n{APIFY_LOCAL_STORAGE_DIR}/request_queues/{QUEUE_ID}/{REQUEST_ID}.json\\n```\", \"Note that `{QUEUE_ID}` is the name or ID of the request queue. The default request queue has ID: `default`,\\nunless you override it by setting the `APIFY_DEFAULT_REQUEST_QUEUE_ID` environment variable.\\nThe `{REQUEST_ID}` is the id of the request.\\n\\nIf the `APIFY_TOKEN` environment variable is set but `APIFY_LOCAL_STORAGE_DIR` is not, the data is stored in the\\n[Apify Request Queue](https://docs.apify.com/storage/request-queue)\\ncloud storage.\"]}" - } - ] - }, - "children": [ - { - "id": 377, - "name": "add_request", - "module": "storages.request_queue", - "kind": 2048, - "kindString": "Method", - "flags": {}, - "comment": { - "summary": [ - { - "kind": "text", - "text": "{\"content\": [\"Adds a request to the `RequestQueue` while managing deduplication and positioning within the queue.\\n\\nThe deduplication of requests relies on the `uniqueKey` field within the request dictionary. If `uniqueKey`\\nexists, it remains unchanged; if it does not, it is generated based on the request's `url`, `method`,\\nand `payload` fields. The generation of `uniqueKey` can be influenced by the `keep_url_fragment` and\\n`use_extended_unique_key` flags, which dictate whether to include the URL fragment and the request's method\\nand payload, respectively, in its computation.\\n\\nThe request can be added to the forefront (beginning) or the back of the queue based on the `forefront`\\nparameter. Information about the request's addition to the queue, including whether it was already present or\\nhandled, is returned in an output dictionary.\\n\", {\"Arguments\": [{\"param\": \"request\", \"desc\": \"The request object to be added to the queue. Must include at least the `url` key.\\nOptionaly it can include the `method`, `payload` and `uniqueKey` keys.\\n\"}, {\"param\": \"forefront\", \"desc\": \"If True, adds the request to the forefront of the queue; otherwise, adds it to the end.\\n\"}, {\"param\": \"keep_url_fragment\", \"desc\": \"Determines whether the URL fragment (the part of the URL after '#') should be retained\\nin the unique key computation.\\n\"}, {\"param\": \"use_extended_unique_key\", \"desc\": \"Determines whether to use an extended unique key, incorporating the request's\\nmethod and payload into the unique key computation.\\n\"}]}, \"Returns: A dictionary containing information about the operation, including:\\n- `requestId` (str): The ID of the request.\\n- `uniqueKey` (str): The unique key associated with the request.\\n- `wasAlreadyPresent` (bool): Indicates whether the request was already in the queue.\\n- `wasAlreadyHandled` (bool): Indicates whether the request was already processed.\"]}" - } - ] - }, - "children": [], - "groups": [], - "sources": [ - { - "filename": "/tmp-old-source-v17/src/apify/storages/request_queue.py", - "line": 143, - "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/storages/request_queue.py#L143" - } - ], - "signatures": [ - { - "id": 378, - "name": "add_request", - "modifiers": [ - "async" - ], - "kind": 4096, - "kindString": "Call signature", - "flags": {}, - "comment": { - "summary": [ - { - "kind": "text", - "text": "{\"content\": [\"Adds a request to the `RequestQueue` while managing deduplication and positioning within the queue.\\n\\nThe deduplication of requests relies on the `uniqueKey` field within the request dictionary. If `uniqueKey`\\nexists, it remains unchanged; if it does not, it is generated based on the request's `url`, `method`,\\nand `payload` fields. The generation of `uniqueKey` can be influenced by the `keep_url_fragment` and\\n`use_extended_unique_key` flags, which dictate whether to include the URL fragment and the request's method\\nand payload, respectively, in its computation.\\n\\nThe request can be added to the forefront (beginning) or the back of the queue based on the `forefront`\\nparameter. Information about the request's addition to the queue, including whether it was already present or\\nhandled, is returned in an output dictionary.\\n\", {\"" - } - ], - "blockTags": [ - { - "tag": "@returns", - "content": [ - { - "kind": "text", - "text": "\\n- `requestId` (str)" - } - ] - } - ] - }, - "type": { - "type": "intrinsic", - "name": "dict" - }, - "parameters": [ - { - "id": 379, - "name": "request", - "kind": 32768, - "kindString": "Parameter", - "flags": {}, - "type": { - "type": "intrinsic", - "name": "dict" - } - }, - { - "id": 380, - "name": "forefront", - "kind": 32768, - "kindString": "Parameter", - "flags": { - "keyword-only": "true" - }, - "type": { - "type": "intrinsic", - "name": "bool" - }, - "defaultValue": "False" - }, - { - "id": 381, - "name": "keep_url_fragment", - "kind": 32768, - "kindString": "Parameter", - "flags": { - "keyword-only": "true" - }, - "type": { - "type": "intrinsic", - "name": "bool" - }, - "defaultValue": "False" - }, - { - "id": 382, - "name": "use_extended_unique_key", - "kind": 32768, - "kindString": "Parameter", - "flags": { - "keyword-only": "true" - }, - "type": { - "type": "intrinsic", - "name": "bool" - }, - "defaultValue": "False" - } - ] - } - ] - }, + "args": [ { - "id": 383, - "name": "get_request", - "module": "storages.request_queue", - "kind": 2048, - "kindString": "Method", - "flags": {}, - "comment": { - "summary": [ - { - "kind": "text", - "text": "{\"content\": [\"Retrieve a request from the queue.\\n\", {\"Arguments\": [{\"param\": \"request_id\", \"type\": \"str\", \"desc\": \"ID of the request to retrieve.\\n\"}]}, {\"Returns\": [\"dict, optional: The retrieved request, or `None`, if it does not exist.\"]}]}" - } - ] + "location": { + "filename": "REPO_ROOT_PLACEHOLDER/src/apify/_utils.py", + "lineno": 119 }, - "children": [], - "groups": [], - "sources": [ - { - "filename": "/tmp-old-source-v17/src/apify/storages/request_queue.py", - "line": 226, - "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/storages/request_queue.py#L226" - } - ], - "signatures": [ - { - "id": 384, - "name": "get_request", - "modifiers": [ - "async" - ], - "kind": 4096, - "kindString": "Call signature", - "flags": {}, - "comment": { - "summary": [ - { - "kind": "text", - "text": "{\"content\": [\"Retrieve a request from the queue.\\n\", {\"" - } - ] - }, - "type": { - "type": "reference", - "name": "dict | None" - }, - "parameters": [ - { - "id": 385, - "name": "request_id", - "kind": 32768, - "kindString": "Parameter", - "flags": {}, - "type": { - "type": "intrinsic", - "name": "str" - } - } - ] - } - ] - }, + "name": "env_var", + "type": "POSITIONAL", + "datatype": "BOOL_ENV_VARS_TYPE" + } + ], + "return_type": "bool | None", + "decorations": [ { - "id": 386, - "name": "fetch_next_request", - "module": "storages.request_queue", - "kind": 2048, - "kindString": "Method", - "flags": {}, - "comment": { - "summary": [ - { - "kind": "text", - "text": "{\"content\": [\"Return the next request in the queue to be processed.\\n\\nOnce you successfully finish processing of the request, you need to call\\n`RequestQueue.mark_request_as_handled` to mark the request as handled in the queue.\\nIf there was some error in processing the request, call `RequestQueue.reclaim_request` instead,\\nso that the queue will give the request to some other consumer in another call to the `fetch_next_request` method.\\n\\nNote that the `None` return value does not mean the queue processing finished, it means there are currently no pending requests.\\nTo check whether all requests in queue were finished, use `RequestQueue.is_finished` instead.\\n\", {\"Returns\": [\"dict, optional: The request or `None` if there are no more pending requests.\"]}]}" - } - ] + "location": { + "filename": "REPO_ROOT_PLACEHOLDER/src/apify/_utils.py", + "lineno": 118 }, - "children": [], - "groups": [], - "sources": [ - { - "filename": "/tmp-old-source-v17/src/apify/storages/request_queue.py", - "line": 238, - "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/storages/request_queue.py#L238" - } - ], - "signatures": [ - { - "id": 387, - "name": "fetch_next_request", - "modifiers": [ - "async" - ], - "kind": 4096, - "kindString": "Call signature", - "flags": {}, - "comment": { - "summary": [ - { - "kind": "text", - "text": "{\"content\": [\"Return the next request in the queue to be processed.\\n\\nOnce you successfully finish processing of the request, you need to call\\n`RequestQueue.mark_request_as_handled` to mark the request as handled in the queue.\\nIf there was some error in processing the request, call `RequestQueue.reclaim_request` instead,\\nso that the queue will give the request to some other consumer in another call to the `fetch_next_request` method.\\n\\nNote that the `None` return value does not mean the queue processing finished, it means there are currently no pending requests.\\nTo check whether all requests in queue were finished, use `RequestQueue.is_finished` instead.\\n\", {\"" - } - ] - }, - "type": { - "type": "reference", - "name": "dict | None" - }, - "parameters": [] - } - ] - }, + "name": "overload" + } + ], + "location": { + "filename": "REPO_ROOT_PLACEHOLDER/src/apify/_utils.py", + "lineno": 119 + }, + "name": "fetch_and_parse_env_var", + "type": "function", + "parsedDocstring": { + "text": "" + } + }, + { + "args": [ { - "id": 388, - "name": "mark_request_as_handled", - "module": "storages.request_queue", - "kind": 2048, - "kindString": "Method", - "flags": {}, - "comment": { - "summary": [ - { - "kind": "text", - "text": "{\"content\": [\"Mark a request as handled after successful processing.\\n\\nHandled requests will never again be returned by the `RequestQueue.fetch_next_request` method.\\n\", {\"Arguments\": [{\"param\": \"request\", \"type\": \"dict\", \"desc\": \"The request to mark as handled.\\n\"}]}, {\"Returns\": [\"dict, optional: Information about the queue operation with keys `requestId`, `uniqueKey`, `wasAlreadyPresent`, `wasAlreadyHandled`.\", \"`None` if the given request was not in progress.\"]}]}" - } - ] + "location": { + "filename": "REPO_ROOT_PLACEHOLDER/src/apify/_utils.py", + "lineno": 124 }, - "children": [], - "groups": [], - "sources": [ - { - "filename": "/tmp-old-source-v17/src/apify/storages/request_queue.py", - "line": 306, - "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/storages/request_queue.py#L306" - } - ], - "signatures": [ - { - "id": 389, - "name": "mark_request_as_handled", - "modifiers": [ - "async" - ], - "kind": 4096, - "kindString": "Call signature", - "flags": {}, - "comment": { - "summary": [ - { - "kind": "text", - "text": "{\"content\": [\"Mark a request as handled after successful processing.\\n\\nHandled requests will never again be returned by the `RequestQueue.fetch_next_request` method.\\n\", {\"" - } - ] - }, - "type": { - "type": "reference", - "name": "dict | None" - }, - "parameters": [ - { - "id": 390, - "name": "request", - "kind": 32768, - "kindString": "Parameter", - "flags": {}, - "type": { - "type": "intrinsic", - "name": "dict" - } - } - ] - } - ] + "name": "env_var", + "type": "POSITIONAL", + "datatype": "BOOL_ENV_VARS_TYPE" }, { - "id": 391, - "name": "reclaim_request", - "module": "storages.request_queue", - "kind": 2048, - "kindString": "Method", - "flags": {}, - "comment": { - "summary": [ - { - "kind": "text", - "text": "{\"content\": [\"Reclaim a failed request back to the queue.\\n\\nThe request will be returned for processing later again\\nby another call to `RequestQueue.fetchNextRequest`.\\n\", {\"Arguments\": [{\"param\": \"request\", \"type\": \"dict\", \"desc\": \"The request to return to the queue.\"}, {\"param\": \"forefront\", \"type\": \"bool, optional\", \"desc\": \"Whether to add the request to the head or the end of the queue\"}]}, {\"Returns\": [\"dict, optional: Information about the queue operation with keys `requestId`, `uniqueKey`, `wasAlreadyPresent`, `wasAlreadyHandled`.\", \"`None` if the given request was not in progress.\"]}]}" - } - ] + "location": { + "filename": "REPO_ROOT_PLACEHOLDER/src/apify/_utils.py", + "lineno": 124 }, - "children": [], - "groups": [], - "sources": [ - { - "filename": "/tmp-old-source-v17/src/apify/storages/request_queue.py", - "line": 345, - "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/storages/request_queue.py#L345" - } - ], - "signatures": [ - { - "id": 392, - "name": "reclaim_request", - "modifiers": [ - "async" - ], - "kind": 4096, - "kindString": "Call signature", - "flags": {}, - "comment": { - "summary": [ - { - "kind": "text", - "text": "{\"content\": [\"Reclaim a failed request back to the queue.\\n\\nThe request will be returned for processing later again\\nby another call to `RequestQueue.fetchNextRequest`.\\n\", {\"" - } - ] - }, - "type": { - "type": "reference", - "name": "dict | None" - }, - "parameters": [ - { - "id": 393, - "name": "request", - "kind": 32768, - "kindString": "Parameter", - "flags": {}, - "type": { - "type": "intrinsic", - "name": "dict" - } - }, - { - "id": 394, - "name": "forefront", - "kind": 32768, - "kindString": "Parameter", - "flags": {}, - "type": { - "type": "intrinsic", - "name": "bool" - }, - "defaultValue": "False" - } - ] - } - ] - }, + "name": "default", + "type": "POSITIONAL", + "datatype": "bool" + } + ], + "return_type": "bool", + "decorations": [ { - "id": 395, - "name": "is_empty", - "module": "storages.request_queue", - "kind": 2048, - "kindString": "Method", - "flags": {}, - "comment": { - "summary": [ - { - "kind": "text", - "text": "{\"content\": [\"Check whether the queue is empty.\\n\", {\"Returns\": [{\"param\": \"bool\", \"desc\": \"`True` if the next call to `RequestQueue.fetchNextRequest` would return `None`, otherwise `False`.\"}]}]}" - } - ] + "location": { + "filename": "REPO_ROOT_PLACEHOLDER/src/apify/_utils.py", + "lineno": 123 }, - "children": [], - "groups": [], - "sources": [ - { - "filename": "/tmp-old-source-v17/src/apify/storages/request_queue.py", - "line": 401, - "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/storages/request_queue.py#L401" - } - ], - "signatures": [ - { - "id": 396, - "name": "is_empty", - "modifiers": [ - "async" - ], - "kind": 4096, - "kindString": "Call signature", - "flags": {}, - "comment": { - "summary": [ - { - "kind": "text", - "text": "{\"content\": [\"Check whether the queue is empty.\\n\", {\"" - } - ] - }, - "type": { - "type": "intrinsic", - "name": "bool" - }, - "parameters": [] - } - ] + "name": "overload" + } + ], + "location": { + "filename": "REPO_ROOT_PLACEHOLDER/src/apify/_utils.py", + "lineno": 124 + }, + "name": "fetch_and_parse_env_var", + "type": "function", + "parsedDocstring": { + "text": "" + } + }, + { + "args": [ + { + "location": { + "filename": "REPO_ROOT_PLACEHOLDER/src/apify/_utils.py", + "lineno": 129 + }, + "name": "env_var", + "type": "POSITIONAL", + "datatype": "DATETIME_ENV_VARS_TYPE" + } + ], + "return_type": "datetime | str | None", + "decorations": [ + { + "location": { + "filename": "REPO_ROOT_PLACEHOLDER/src/apify/_utils.py", + "lineno": 128 + }, + "name": "overload" + } + ], + "location": { + "filename": "REPO_ROOT_PLACEHOLDER/src/apify/_utils.py", + "lineno": 129 + }, + "name": "fetch_and_parse_env_var", + "type": "function", + "parsedDocstring": { + "text": "" + } + }, + { + "args": [ + { + "location": { + "filename": "REPO_ROOT_PLACEHOLDER/src/apify/_utils.py", + "lineno": 134 + }, + "name": "env_var", + "type": "POSITIONAL", + "datatype": "DATETIME_ENV_VARS_TYPE" }, { - "id": 397, - "name": "is_finished", - "module": "storages.request_queue", - "kind": 2048, - "kindString": "Method", - "flags": {}, - "comment": { - "summary": [ - { - "kind": "text", - "text": "{\"content\": [\"Check whether the queue is finished.\\n\\nDue to the nature of distributed storage used by the queue,\\nthe function might occasionally return a false negative,\\nbut it will never return a false positive.\\n\", {\"Returns\": [{\"param\": \"bool\", \"desc\": \"`True` if all requests were already handled and there are no more left. `False` otherwise.\"}]}]}" - } - ] + "location": { + "filename": "REPO_ROOT_PLACEHOLDER/src/apify/_utils.py", + "lineno": 134 }, - "children": [], - "groups": [], - "sources": [ - { - "filename": "/tmp-old-source-v17/src/apify/storages/request_queue.py", - "line": 410, - "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/storages/request_queue.py#L410" - } - ], - "signatures": [ - { - "id": 398, - "name": "is_finished", - "modifiers": [ - "async" - ], - "kind": 4096, - "kindString": "Call signature", - "flags": {}, - "comment": { - "summary": [ - { - "kind": "text", - "text": "{\"content\": [\"Check whether the queue is finished.\\n\\nDue to the nature of distributed storage used by the queue,\\nthe function might occasionally return a false negative,\\nbut it will never return a false positive.\\n\", {\"" - } - ] - }, - "type": { - "type": "intrinsic", - "name": "bool" - }, - "parameters": [] - } - ] + "name": "default", + "type": "POSITIONAL", + "datatype": "datetime" + } + ], + "return_type": "datetime | str", + "decorations": [ + { + "location": { + "filename": "REPO_ROOT_PLACEHOLDER/src/apify/_utils.py", + "lineno": 133 + }, + "name": "overload" + } + ], + "location": { + "filename": "REPO_ROOT_PLACEHOLDER/src/apify/_utils.py", + "lineno": 134 + }, + "name": "fetch_and_parse_env_var", + "type": "function", + "parsedDocstring": { + "text": "" + } + }, + { + "args": [ + { + "location": { + "filename": "REPO_ROOT_PLACEHOLDER/src/apify/_utils.py", + "lineno": 139 + }, + "name": "env_var", + "type": "POSITIONAL", + "datatype": "FLOAT_ENV_VARS_TYPE" + } + ], + "return_type": "float | None", + "decorations": [ + { + "location": { + "filename": "REPO_ROOT_PLACEHOLDER/src/apify/_utils.py", + "lineno": 138 + }, + "name": "overload" + } + ], + "location": { + "filename": "REPO_ROOT_PLACEHOLDER/src/apify/_utils.py", + "lineno": 139 + }, + "name": "fetch_and_parse_env_var", + "type": "function", + "parsedDocstring": { + "text": "" + } + }, + { + "args": [ + { + "location": { + "filename": "REPO_ROOT_PLACEHOLDER/src/apify/_utils.py", + "lineno": 144 + }, + "name": "env_var", + "type": "POSITIONAL", + "datatype": "FLOAT_ENV_VARS_TYPE" }, { - "id": 399, - "name": "drop", - "module": "storages.request_queue", - "kind": 2048, - "kindString": "Method", - "flags": {}, - "comment": { - "summary": [ - { - "kind": "text", - "text": "{\"content\": [\"Remove the request queue either from the Apify cloud storage or from the local directory.\"]}" - } - ] + "location": { + "filename": "REPO_ROOT_PLACEHOLDER/src/apify/_utils.py", + "lineno": 144 + }, + "name": "default", + "type": "POSITIONAL", + "datatype": "float" + } + ], + "return_type": "float", + "decorations": [ + { + "location": { + "filename": "REPO_ROOT_PLACEHOLDER/src/apify/_utils.py", + "lineno": 143 + }, + "name": "overload" + } + ], + "location": { + "filename": "REPO_ROOT_PLACEHOLDER/src/apify/_utils.py", + "lineno": 144 + }, + "name": "fetch_and_parse_env_var", + "type": "function", + "parsedDocstring": { + "text": "" + } + }, + { + "args": [ + { + "location": { + "filename": "REPO_ROOT_PLACEHOLDER/src/apify/_utils.py", + "lineno": 149 + }, + "name": "env_var", + "type": "POSITIONAL", + "datatype": "INTEGER_ENV_VARS_TYPE" + } + ], + "return_type": "int | None", + "decorations": [ + { + "location": { + "filename": "REPO_ROOT_PLACEHOLDER/src/apify/_utils.py", + "lineno": 148 + }, + "name": "overload" + } + ], + "location": { + "filename": "REPO_ROOT_PLACEHOLDER/src/apify/_utils.py", + "lineno": 149 + }, + "name": "fetch_and_parse_env_var", + "type": "function", + "parsedDocstring": { + "text": "" + } + }, + { + "args": [ + { + "location": { + "filename": "REPO_ROOT_PLACEHOLDER/src/apify/_utils.py", + "lineno": 154 }, - "children": [], - "groups": [], - "sources": [ - { - "filename": "/tmp-old-source-v17/src/apify/storages/request_queue.py", - "line": 557, - "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/storages/request_queue.py#L557" - } - ], - "signatures": [ - { - "id": 400, - "name": "drop", - "modifiers": [ - "async" - ], - "kind": 4096, - "kindString": "Call signature", - "flags": {}, - "comment": { - "summary": [ - { - "kind": "text", - "text": "{\"content\": [\"Remove the request queue either from the Apify cloud storage or from the local directory.\"]}" - } - ] - }, - "type": { - "type": "reference", - "name": "None" - }, - "parameters": [] - } - ] + "name": "env_var", + "type": "POSITIONAL", + "datatype": "INTEGER_ENV_VARS_TYPE" }, { - "id": 401, - "name": "get_info", - "module": "storages.request_queue", - "kind": 2048, - "kindString": "Method", - "flags": {}, - "comment": { - "summary": [ - { - "kind": "text", - "text": "{\"content\": [\"Get an object containing general information about the request queue.\\n\", {\"Returns\": [{\"param\": \"dict\", \"desc\": \"Object returned by calling the GET request queue API endpoint.\"}]}]}" - } - ] + "location": { + "filename": "REPO_ROOT_PLACEHOLDER/src/apify/_utils.py", + "lineno": 154 }, - "children": [], - "groups": [], - "sources": [ - { - "filename": "/tmp-old-source-v17/src/apify/storages/request_queue.py", - "line": 562, - "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/storages/request_queue.py#L562" - } - ], - "signatures": [ - { - "id": 402, - "name": "get_info", - "modifiers": [ - "async" - ], - "kind": 4096, - "kindString": "Call signature", - "flags": {}, - "comment": { - "summary": [ - { - "kind": "text", - "text": "{\"content\": [\"Get an object containing general information about the request queue.\\n\", {\"" - } - ] - }, - "type": { - "type": "reference", - "name": "dict | None" - }, - "parameters": [] - } - ] - }, + "name": "default", + "type": "POSITIONAL", + "datatype": "int" + } + ], + "return_type": "int", + "decorations": [ { - "id": 403, - "name": "open", - "module": "storages.request_queue", - "kind": 2048, - "kindString": "Method", - "flags": {}, - "comment": { - "summary": [ - { - "kind": "text", - "text": "{\"content\": [\"Open a request queue.\\n\\nRequest queue represents a queue of URLs to crawl, which is stored either on local filesystem or in the Apify cloud.\\nThe queue is used for deep crawling of websites, where you start with several URLs and then\\nrecursively follow links to other pages. The data structure supports both breadth-first\\nand depth-first crawling orders.\\n\", {\"Arguments\": [{\"param\": \"id\", \"type\": \"str, optional\", \"desc\": \"ID of the request queue to be opened.\\nIf neither `id` nor `name` are provided, the method returns the default request queue associated with the actor run.\\nIf the request queue with the given ID does not exist, it raises an error.\"}, {\"param\": \"name\", \"type\": \"str, optional\", \"desc\": \"Name of the request queue to be opened.\\nIf neither `id` nor `name` are provided, the method returns the default request queue associated with the actor run.\\nIf the request queue with the given name does not exist, it is created.\"}, {\"param\": \"force_cloud\", \"type\": \"bool, optional\", \"desc\": \"If set to True, it will open a request queue on the Apify Platform even when running the actor locally.\\nDefaults to False.\"}, {\"param\": \"config\", \"type\": \"Configuration, optional\", \"desc\": \"A `Configuration` instance, uses global configuration if omitted.\\n\"}]}, {\"Returns\": [{\"param\": \"RequestQueue\", \"desc\": \"An instance of the `RequestQueue` class for the given ID or name.\"}]}]}" - } - ] + "location": { + "filename": "REPO_ROOT_PLACEHOLDER/src/apify/_utils.py", + "lineno": 153 }, - "children": [], - "groups": [], - "sources": [ - { - "filename": "/tmp-old-source-v17/src/apify/storages/request_queue.py", - "line": 571, - "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/storages/request_queue.py#L571" - } - ], - "signatures": [ - { - "id": 404, - "name": "open", - "modifiers": [ - "async" - ], - "kind": 4096, - "kindString": "Call signature", - "flags": {}, - "comment": { - "summary": [ - { - "kind": "text", - "text": "{\"content\": [\"Open a request queue.\\n\\nRequest queue represents a queue of URLs to crawl, which is stored either on local filesystem or in the Apify cloud.\\nThe queue is used for deep crawling of websites, where you start with several URLs and then\\nrecursively follow links to other pages. The data structure supports both breadth-first\\nand depth-first crawling orders.\\n\", {\"" - } - ] - }, - "type": { - "type": "reference", - "name": "RequestQueue", - "id": 376 - }, - "parameters": [ - { - "id": 405, - "name": "id", - "kind": 32768, - "kindString": "Parameter", - "flags": { - "keyword-only": "true" - }, - "type": { - "type": "reference", - "name": "str | None" - }, - "defaultValue": "None" - }, - { - "id": 406, - "name": "name", - "kind": 32768, - "kindString": "Parameter", - "flags": { - "keyword-only": "true" - }, - "type": { - "type": "reference", - "name": "str | None" - }, - "defaultValue": "None" - }, - { - "id": 407, - "name": "force_cloud", - "kind": 32768, - "kindString": "Parameter", - "flags": { - "keyword-only": "true" - }, - "type": { - "type": "intrinsic", - "name": "bool" - }, - "defaultValue": "False" - }, - { - "id": 408, - "name": "config", - "kind": 32768, - "kindString": "Parameter", - "flags": { - "keyword-only": "true" - }, - "type": { - "type": "reference", - "name": "Configuration | None" - }, - "defaultValue": "None" - } - ] - } - ] + "name": "overload" } ], - "groups": [ + "location": { + "filename": "REPO_ROOT_PLACEHOLDER/src/apify/_utils.py", + "lineno": 154 + }, + "name": "fetch_and_parse_env_var", + "type": "function", + "parsedDocstring": { + "text": "" + } + }, + { + "args": [ { - "title": "Methods", - "children": [ - 377, - 399, - 386, - 401, - 383, - 395, - 397, - 388, - 403, - 391 - ] + "location": { + "filename": "REPO_ROOT_PLACEHOLDER/src/apify/_utils.py", + "lineno": 159 + }, + "name": "env_var", + "type": "POSITIONAL", + "datatype": "STRING_ENV_VARS_TYPE" + }, + { + "location": { + "filename": "REPO_ROOT_PLACEHOLDER/src/apify/_utils.py", + "lineno": 159 + }, + "name": "default", + "type": "POSITIONAL", + "datatype": "str" } ], - "sources": [ + "return_type": "str", + "decorations": [ { - "filename": "/tmp-old-source-v17/src/apify/storages/request_queue.py", - "line": 49, - "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/storages/request_queue.py#L49" + "location": { + "filename": "REPO_ROOT_PLACEHOLDER/src/apify/_utils.py", + "lineno": 158 + }, + "name": "overload" } - ] - } - ], - "groups": [ - { - "title": "Methods", - "children": [ - 286, - 21, - 26, - 34, - 1, - 4, - 8, - 17, - 279, - 284, - 11, - 215, - 30, - 290, - 258, - 262, - 14 - ] + ], + "location": { + "filename": "REPO_ROOT_PLACEHOLDER/src/apify/_utils.py", + "lineno": 159 + }, + "name": "fetch_and_parse_env_var", + "type": "function", + "parsedDocstring": { + "text": "" + } }, { - "title": "Other", - "children": [ - 41, - 253, - 209, - 237, - 266, - 183, - 292, - 350, - 351, - 352, - 227, - 218, - 376 - ] + "args": [ + { + "location": { + "filename": "REPO_ROOT_PLACEHOLDER/src/apify/_utils.py", + "lineno": 164 + }, + "name": "env_var", + "type": "POSITIONAL", + "datatype": "STRING_ENV_VARS_TYPE" + } + ], + "return_type": "str | None", + "decorations": [ + { + "location": { + "filename": "REPO_ROOT_PLACEHOLDER/src/apify/_utils.py", + "lineno": 163 + }, + "name": "overload" + } + ], + "location": { + "filename": "REPO_ROOT_PLACEHOLDER/src/apify/_utils.py", + "lineno": 164 + }, + "name": "fetch_and_parse_env_var", + "type": "function", + "parsedDocstring": { + "text": "" + } }, { - "title": "Constants", - "children": [ - 208 - ] + "args": [ + { + "location": { + "filename": "REPO_ROOT_PLACEHOLDER/src/apify/_utils.py", + "lineno": 169 + }, + "name": "env_var", + "type": "POSITIONAL", + "datatype": "ActorEnvVars | ApifyEnvVars" + } + ], + "return_type": "Any", + "decorations": [ + { + "location": { + "filename": "REPO_ROOT_PLACEHOLDER/src/apify/_utils.py", + "lineno": 168 + }, + "name": "overload" + } + ], + "location": { + "filename": "REPO_ROOT_PLACEHOLDER/src/apify/_utils.py", + "lineno": 169 + }, + "name": "fetch_and_parse_env_var", + "type": "function", + "parsedDocstring": { + "text": "" + } }, { - "title": "Properties", - "children": [ - 236 - ] - } - ], - "sources": [ + "args": [ + { + "location": { + "filename": "REPO_ROOT_PLACEHOLDER/src/apify/_utils.py", + "lineno": 377 + }, + "name": "value", + "type": "POSITIONAL", + "datatype": "str | float | bool" + }, + { + "location": { + "filename": "REPO_ROOT_PLACEHOLDER/src/apify/_utils.py", + "lineno": 377 + }, + "name": "predicate", + "type": "POSITIONAL", + "datatype": "tuple[type, bool]" + }, + { + "location": { + "filename": "REPO_ROOT_PLACEHOLDER/src/apify/_utils.py", + "lineno": 377 + }, + "name": "value_name", + "type": "POSITIONAL", + "datatype": "str" + } + ], + "return_type": "None", + "decorations": [ + { + "location": { + "filename": "REPO_ROOT_PLACEHOLDER/src/apify/_utils.py", + "lineno": 376 + }, + "name": "overload" + } + ], + "location": { + "filename": "REPO_ROOT_PLACEHOLDER/src/apify/_utils.py", + "lineno": 377 + }, + "name": "budget_ow", + "type": "function", + "parsedDocstring": { + "text": "" + } + }, { - "fileName": "src/index.ts", - "line": 1, - "character": 0, - "url": "http://example.com/blob/123456/src/dummy.py" + "args": [ + { + "location": { + "filename": "REPO_ROOT_PLACEHOLDER/src/apify/_utils.py", + "lineno": 382 + }, + "name": "value", + "type": "POSITIONAL", + "datatype": "dict" + }, + { + "location": { + "filename": "REPO_ROOT_PLACEHOLDER/src/apify/_utils.py", + "lineno": 382 + }, + "name": "predicate", + "type": "POSITIONAL", + "datatype": "dict[str, tuple[type, bool]]" + } + ], + "return_type": "None", + "decorations": [ + { + "location": { + "filename": "REPO_ROOT_PLACEHOLDER/src/apify/_utils.py", + "lineno": 381 + }, + "name": "overload" + } + ], + "location": { + "filename": "REPO_ROOT_PLACEHOLDER/src/apify/_utils.py", + "lineno": 382 + }, + "name": "budget_ow", + "type": "function", + "parsedDocstring": { + "text": "" + } } ] } \ No newline at end of file diff --git a/website/versioned_docs/version-2.7/api-packages.json b/website/versioned_docs/version-2.7/api-packages.json index e2e41f0d..765ea7e7 100644 --- a/website/versioned_docs/version-2.7/api-packages.json +++ b/website/versioned_docs/version-2.7/api-packages.json @@ -1 +1 @@ -[{"entryPoints":{"index":{"label":"Index","path":"src/index.ts"}},"packageRoot":".","packagePath":".","packageSlug":".","packageName":"apify-sdk-python"}] +[{"entryPoints":{"index":{"label":"Index","path":"src/index.ts"}},"packageRoot":".","packagePath":".","packageSlug":".","packageName":"apify-sdk-python-website"}] \ No newline at end of file diff --git a/website/versioned_docs/version-2.7/api-typedoc.json b/website/versioned_docs/version-2.7/api-typedoc.json index 7561b6f2..26e2971c 100644 --- a/website/versioned_docs/version-2.7/api-typedoc.json +++ b/website/versioned_docs/version-2.7/api-typedoc.json @@ -1,6951 +1,61959 @@ { - "id": 0, - "name": "apify", - "kind": 1, - "kindString": "Project", - "flags": {}, - "originalName": "", "children": [ { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], "id": 1, - "name": "_ActorType", - "module": "_actor", - "docsGroup": "Classes", - "kind": 128, - "kindString": "Class", + "module": "_crypto", + "name": "ENCRYPTION_KEY_LENGTH", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_crypto.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 20 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, "flags": {}, + "groups": [], + "id": 2, + "module": "_crypto", + "name": "ENCRYPTION_IV_LENGTH", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_crypto.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 21 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"The class of `Actor`. Only make a new instance if you're absolutely sure you need to.\"]}" + "text": "" } ] }, - "children": [ + "flags": {}, + "groups": [], + "id": 3, + "module": "_crypto", + "name": "ENCRYPTION_AUTH_TAG_LENGTH", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_crypto.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 22 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 4, + "module": "_crypto", + "name": "CHARSET", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_crypto.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 168 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Encode the given number to base62." + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 5, + "module": "_crypto", + "name": "encode_base62", + "parsedDocstring": { + "text": "Encode the given number to base62." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_crypto.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 171 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ { - "id": 2, - "name": "__init__", - "module": "_actor", - "kind": 512, - "kindString": "Constructor", - "flags": {}, "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Create an Actor instance.\\n\\nNote that you don't have to do this, all the functionality is accessible using the default instance\\n(e.g. `Actor.open_dataset()`).\\n\", {\"Arguments\": [{\"param\": \"configuration\", \"desc\": \"The Actor configuration to be used. If not passed, a new Configuration instance will\\nbe created.\"}, {\"param\": \"configure_logging\", \"desc\": \"Should the default logging configuration be configured?\"}, {\"param\": \"exit_process\", \"desc\": \"Whether the Actor should call `sys.exit` when the context manager exits. The default is\\nTrue except for the IPython, Pytest and Scrapy environments.\"}]}]}" + "text": "Encode the given number to base62." } ] }, + "flags": {}, + "id": 6, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "encode_base62", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 7, + "kind": 32768, + "kindString": "Parameter", + "name": "num", + "type": { + "name": "int", + "type": "reference" + } + } + ], + "type": { + "name": "str", + "type": "reference" + } + } + ] + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 8, + "module": "_actor", + "name": "MainReturnType", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_actor.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 52 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 2048, + "kindString": "Method", "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Create an Actor instance.\n\nNote that you don't have to do this, all the functionality is accessible using the default instance\n(e.g. `Actor.open_dataset()`).\n" + } + ] + }, + "decorations": [], + "flags": {}, "groups": [], + "id": 10, + "module": "_actor", + "name": "__init__", + "parsedDocstring": { + "text": "Create an Actor instance.\n\nNote that you don't have to do this, all the functionality is accessible using the default instance\n(e.g. `Actor.open_dataset()`).\n", + "args": { + "configuration": "The Actor configuration to be used. If not passed, a new Configuration instance will\nbe created.", + "configure_logging": "Should the default logging configuration be configured?", + "exit_process": "Whether the Actor should call `sys.exit` when the context manager exits. The default is\nTrue except for the IPython, Pytest and Scrapy environments." + } + }, "sources": [ { - "filename": "/tmp-old-source-v27/src/apify/_actor.py", - "line": 63, "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/_actor.py#L63" + "fileName": "/src/apify/_actor.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 63 } ], + "type": { + "name": "Undefined", + "type": "reference" + }, "signatures": [ { - "id": 3, - "name": "__init__", - "modifiers": [], - "kind": 4096, - "kindString": "Call signature", - "flags": {}, "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Create an Actor instance.\\n\\nNote that you don't have to do this, all the functionality is accessible using the default instance\\n(e.g. `Actor.open_dataset()`).\\n\", {\"" + "text": "Create an Actor instance.\n\nNote that you don't have to do this, all the functionality is accessible using the default instance\n(e.g. `Actor.open_dataset()`).\n" } ] }, - "type": { - "type": "reference", - "name": "None" - }, + "flags": {}, + "id": 11, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "__init__", "parameters": [ { - "id": 4, - "name": "configuration", + "comment": { + "summary": [ + { + "kind": "text", + "text": "The Actor configuration to be used. If not passed, a new Configuration instance will\nbe created." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": false + }, + "id": 12, "kind": 32768, "kindString": "Parameter", - "flags": {}, + "name": "configuration", "type": { - "type": "reference", - "name": "Configuration | None" - }, - "defaultValue": "None" + "name": "Configuration | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "Configuration", + "target": "225" + }, + { + "type": "literal", + "value": null + } + ] + } }, { - "id": 5, - "name": "configure_logging", - "kind": 32768, - "kindString": "Parameter", + "comment": { + "summary": [ + { + "kind": "text", + "text": "Should the default logging configuration be configured?" + } + ] + }, + "defaultValue": "True", "flags": { - "keyword-only": "true" + "isOptional": true, + "keyword-only": true }, + "id": 13, + "kind": 32768, + "kindString": "Parameter", + "name": "configure_logging", "type": { - "type": "intrinsic", - "name": "bool" - }, - "defaultValue": "True" + "name": "bool", + "type": "reference" + } }, { - "id": 6, - "name": "exit_process", - "kind": 32768, - "kindString": "Parameter", + "comment": { + "summary": [ + { + "kind": "text", + "text": "Whether the Actor should call `sys.exit` when the context manager exits. The default is\nTrue except for the IPython, Pytest and Scrapy environments." + } + ] + }, + "defaultValue": "None", "flags": { - "keyword-only": "true" + "isOptional": true, + "keyword-only": true }, + "id": 14, + "kind": 32768, + "kindString": "Parameter", + "name": "exit_process", "type": { - "type": "reference", - "name": "bool | None" - }, - "defaultValue": "None" + "name": "bool | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "bool" + }, + { + "type": "literal", + "value": null + } + ] + } } - ] + ], + "type": { + "name": "None", + "type": "literal", + "value": null + } } ] }, { - "id": 7, - "name": "__call__", - "module": "_actor", "kind": 2048, "kindString": "Method", - "flags": {}, + "children": [], "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Make a new Actor instance with a non-default configuration.\"]}" + "text": "" } ] }, - "children": [], + "decorations": [], + "flags": {}, "groups": [], + "id": 15, + "module": "_actor", + "name": "__repr__", + "parsedDocstring": { + "text": "" + }, "sources": [ { - "filename": "/tmp-old-source-v27/src/apify/_actor.py", - "line": 149, "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/_actor.py#L149" + "fileName": "/src/apify/_actor.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 143 } ], + "type": { + "name": "Undefined", + "type": "reference" + }, "signatures": [ { - "id": 8, - "name": "__call__", - "modifiers": [], + "flags": {}, + "id": 16, "kind": 4096, "kindString": "Call signature", - "flags": {}, - "comment": { - "summary": [ - { - "kind": "text", - "text": "{\"content\": [\"Make a new Actor instance with a non-default configuration.\"]}" - } - ] - }, + "modifiers": [], + "name": "__repr__", + "parameters": [], "type": { - "type": "reference", - "name": "Self" - }, - "parameters": [ - { - "id": 9, - "name": "configuration", - "kind": 32768, - "kindString": "Parameter", - "flags": {}, - "type": { - "type": "reference", - "name": "Configuration | None" - }, - "defaultValue": "None" - }, - { - "id": 10, - "name": "configure_logging", - "kind": 32768, - "kindString": "Parameter", - "flags": { - "keyword-only": "true" - }, - "type": { - "type": "intrinsic", - "name": "bool" - }, - "defaultValue": "True" - }, - { - "id": 11, - "name": "exit_process", - "kind": 32768, - "kindString": "Parameter", - "flags": { - "keyword-only": "true" - }, - "type": { - "type": "reference", - "name": "bool | None" - }, - "defaultValue": "None" - } - ] + "name": "str", + "type": "reference" + } } ] }, { - "id": 12, - "name": "apify_client", - "module": "_actor", "kind": 2048, "kindString": "Method", - "flags": {}, + "children": [], "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"The ApifyClientAsync instance the Actor instance uses.\"]}" + "text": "Make a new Actor instance with a non-default configuration." } ] }, - "children": [], + "decorations": [], + "flags": {}, "groups": [], + "id": 17, + "module": "_actor", + "name": "__call__", + "parsedDocstring": { + "text": "Make a new Actor instance with a non-default configuration." + }, "sources": [ { - "filename": "/tmp-old-source-v27/src/apify/_actor.py", - "line": 164, "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/_actor.py#L164" + "fileName": "/src/apify/_actor.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 149 } ], + "type": { + "name": "Undefined", + "type": "reference" + }, "signatures": [ { - "id": 13, - "name": "apify_client", - "modifiers": [], - "kind": 4096, - "kindString": "Call signature", - "flags": {}, "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"The ApifyClientAsync instance the Actor instance uses.\"]}" + "text": "Make a new Actor instance with a non-default configuration." } ] }, + "flags": {}, + "id": 18, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "__call__", + "parameters": [ + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": false + }, + "id": 19, + "kind": 32768, + "kindString": "Parameter", + "name": "configuration", + "type": { + "name": "Configuration | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "Configuration", + "target": "225" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "defaultValue": "True", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 20, + "kind": 32768, + "kindString": "Parameter", + "name": "configure_logging", + "type": { + "name": "bool", + "type": "reference" + } + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 21, + "kind": 32768, + "kindString": "Parameter", + "name": "exit_process", + "type": { + "name": "bool | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "bool" + }, + { + "type": "literal", + "value": null + } + ] + } + } + ], "type": { - "type": "reference", - "name": "ApifyClientAsync" - }, - "parameters": [] + "name": "Self", + "type": "reference" + } } ] }, { - "id": 14, - "name": "configuration", - "module": "_actor", - "kind": 2048, - "kindString": "Method", - "flags": {}, + "kind": 1024, + "kindString": "Property", + "children": [], "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"The Configuration instance the Actor instance uses.\"]}" + "text": "The ApifyClientAsync instance the Actor instance uses." } ] }, - "children": [], + "decorations": [ + { + "name": "property" + } + ], + "flags": {}, "groups": [], + "id": 22, + "module": "_actor", + "name": "apify_client", + "parsedDocstring": { + "text": "The ApifyClientAsync instance the Actor instance uses." + }, "sources": [ { - "filename": "/tmp-old-source-v27/src/apify/_actor.py", - "line": 169, "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/_actor.py#L169" + "fileName": "/src/apify/_actor.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 164 } ], - "signatures": [ + "type": { + "name": "ApifyClientAsync", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "The Configuration instance the Actor instance uses." + } + ] + }, + "decorations": [ { - "id": 15, - "name": "configuration", - "modifiers": [], - "kind": 4096, - "kindString": "Call signature", - "flags": {}, - "comment": { - "summary": [ - { - "kind": "text", - "text": "{\"content\": [\"The Configuration instance the Actor instance uses.\"]}" - } - ] - }, - "type": { - "type": "reference", - "name": "Configuration", - "id": 180 - }, - "parameters": [] + "name": "property" } - ] + ], + "flags": {}, + "groups": [], + "id": 23, + "module": "_actor", + "name": "configuration", + "parsedDocstring": { + "text": "The Configuration instance the Actor instance uses." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_actor.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 169 + } + ], + "type": { + "name": "Configuration", + "type": "reference", + "target": "225" + } }, { - "id": 16, - "name": "config", - "module": "_actor", - "kind": 2048, - "kindString": "Method", - "flags": {}, + "kind": 1024, + "kindString": "Property", + "children": [], "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"The Configuration instance the Actor instance uses.\"]}" + "text": "The Configuration instance the Actor instance uses." } ] }, - "children": [], + "decorations": [ + { + "name": "property" + } + ], + "flags": {}, "groups": [], + "id": 24, + "module": "_actor", + "name": "config", + "parsedDocstring": { + "text": "The Configuration instance the Actor instance uses." + }, "sources": [ { - "filename": "/tmp-old-source-v27/src/apify/_actor.py", - "line": 174, "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/_actor.py#L174" + "fileName": "/src/apify/_actor.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 174 } ], - "signatures": [ + "type": { + "name": "Configuration", + "type": "reference", + "target": "225" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "The EventManager instance the Actor instance uses." + } + ] + }, + "decorations": [ { - "id": 17, - "name": "config", - "modifiers": [], - "kind": 4096, - "kindString": "Call signature", - "flags": {}, - "comment": { - "summary": [ - { - "kind": "text", - "text": "{\"content\": [\"The Configuration instance the Actor instance uses.\"]}" - } - ] - }, - "type": { - "type": "reference", - "name": "Configuration", - "id": 180 - }, - "parameters": [] + "name": "property" } - ] + ], + "flags": {}, + "groups": [], + "id": 25, + "module": "_actor", + "name": "event_manager", + "parsedDocstring": { + "text": "The EventManager instance the Actor instance uses." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_actor.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 179 + } + ], + "type": { + "name": "EventManager", + "type": "reference" + } }, { - "id": 18, - "name": "event_manager", + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "The logging.Logger instance the Actor uses." + } + ] + }, + "decorations": [ + { + "name": "property" + } + ], + "flags": {}, + "groups": [], + "id": 26, "module": "_actor", + "name": "log", + "parsedDocstring": { + "text": "The logging.Logger instance the Actor uses." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_actor.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 184 + } + ], + "type": { + "name": "logging.Logger", + "type": "reference" + } + }, + { "kind": 2048, "kindString": "Method", - "flags": {}, + "children": [], "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"The EventManager instance the Actor instance uses.\"]}" + "text": "Initialize the Actor instance.\n\nThis initializes the Actor instance. It configures the right storage client based on whether the Actor is\nrunning locally or on the Apify platform, it initializes the event manager for processing Actor events,\nand starts an interval for regularly sending `PERSIST_STATE` events, so that the Actor can regularly persist\nits state in response to these events.\n\nThis method should be called immediately before performing any additional Actor actions, and it should be\ncalled only once." } ] }, - "children": [], + "decorations": [], + "flags": {}, "groups": [], + "id": 27, + "module": "_actor", + "name": "init", + "parsedDocstring": { + "text": "Initialize the Actor instance.\n\nThis initializes the Actor instance. It configures the right storage client based on whether the Actor is\nrunning locally or on the Apify platform, it initializes the event manager for processing Actor events,\nand starts an interval for regularly sending `PERSIST_STATE` events, so that the Actor can regularly persist\nits state in response to these events.\n\nThis method should be called immediately before performing any additional Actor actions, and it should be\ncalled only once." + }, "sources": [ { - "filename": "/tmp-old-source-v27/src/apify/_actor.py", - "line": 179, "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/_actor.py#L179" + "fileName": "/src/apify/_actor.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 207 } ], + "type": { + "name": "Undefined", + "type": "reference" + }, "signatures": [ { - "id": 19, - "name": "event_manager", - "modifiers": [], - "kind": 4096, - "kindString": "Call signature", - "flags": {}, "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"The EventManager instance the Actor instance uses.\"]}" + "text": "Initialize the Actor instance.\n\nThis initializes the Actor instance. It configures the right storage client based on whether the Actor is\nrunning locally or on the Apify platform, it initializes the event manager for processing Actor events,\nand starts an interval for regularly sending `PERSIST_STATE` events, so that the Actor can regularly persist\nits state in response to these events.\n\nThis method should be called immediately before performing any additional Actor actions, and it should be\ncalled only once." } ] }, + "flags": {}, + "id": 28, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "init", + "parameters": [], "type": { - "type": "reference", - "name": "EventManager" - }, - "parameters": [] + "name": "None", + "type": "literal", + "value": null + } } ] }, { - "id": 20, - "name": "log", - "module": "_actor", "kind": 2048, "kindString": "Method", - "flags": {}, + "children": [], "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"The logging.Logger instance the Actor uses.\"]}" + "text": "Exit the Actor instance.\n\nThis stops the Actor instance. It cancels all the intervals for regularly sending `PERSIST_STATE` events,\nsends a final `PERSIST_STATE` event, waits for all the event listeners to finish, and stops the event manager.\n" } ] }, - "children": [], + "decorations": [], + "flags": {}, "groups": [], + "id": 29, + "module": "_actor", + "name": "exit", + "parsedDocstring": { + "text": "Exit the Actor instance.\n\nThis stops the Actor instance. It cancels all the intervals for regularly sending `PERSIST_STATE` events,\nsends a final `PERSIST_STATE` event, waits for all the event listeners to finish, and stops the event manager.\n", + "args": { + "exit_code": "The exit code with which the Actor should fail (defaults to `0`).", + "event_listeners_timeout": "How long should the Actor wait for Actor event listeners to finish before exiting.", + "status_message": "The final status message that the Actor should display.", + "cleanup_timeout": "How long we should wait for event listeners." + } + }, "sources": [ { - "filename": "/tmp-old-source-v27/src/apify/_actor.py", - "line": 184, "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/_actor.py#L184" + "fileName": "/src/apify/_actor.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 256 } ], + "type": { + "name": "Undefined", + "type": "reference" + }, "signatures": [ { - "id": 21, - "name": "log", - "modifiers": [], - "kind": 4096, - "kindString": "Call signature", - "flags": {}, "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"The logging.Logger instance the Actor uses.\"]}" + "text": "Exit the Actor instance.\n\nThis stops the Actor instance. It cancels all the intervals for regularly sending `PERSIST_STATE` events,\nsends a final `PERSIST_STATE` event, waits for all the event listeners to finish, and stops the event manager.\n" } ] }, + "flags": {}, + "id": 30, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "exit", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The exit code with which the Actor should fail (defaults to `0`)." + } + ] + }, + "defaultValue": "0", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 31, + "kind": 32768, + "kindString": "Parameter", + "name": "exit_code", + "type": { + "name": "int", + "type": "reference" + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "How long should the Actor wait for Actor event listeners to finish before exiting." + } + ] + }, + "defaultValue": "EVENT_LISTENERS_TIMEOUT", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 32, + "kind": 32768, + "kindString": "Parameter", + "name": "event_listeners_timeout", + "type": { + "name": "timedelta | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "timedelta" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The final status message that the Actor should display." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 33, + "kind": 32768, + "kindString": "Parameter", + "name": "status_message", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "How long we should wait for event listeners." + } + ] + }, + "defaultValue": "timedelta(seconds=30)", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 34, + "kind": 32768, + "kindString": "Parameter", + "name": "cleanup_timeout", + "type": { + "name": "timedelta", + "type": "reference" + } + } + ], "type": { - "type": "reference", - "name": "logging.Logger" - }, - "parameters": [] + "name": "None", + "type": "literal", + "value": null + } } ] }, { - "id": 22, - "name": "init", - "module": "_actor", "kind": 2048, "kindString": "Method", - "flags": {}, + "children": [], "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Initialize the Actor instance.\\n\\nThis initializes the Actor instance. It configures the right storage client based on whether the Actor is\\nrunning locally or on the Apify platform, it initializes the event manager for processing Actor events,\\nand starts an interval for regularly sending `PERSIST_STATE` events, so that the Actor can regularly persist\\nits state in response to these events.\\n\\nThis method should be called immediately before performing any additional Actor actions, and it should be\\ncalled only once.\"]}" + "text": "Fail the Actor instance.\n\nThis performs all the same steps as Actor.exit(), but it additionally sets the exit code to `1` (by default).\n" } ] }, - "children": [], + "decorations": [], + "flags": {}, "groups": [], + "id": 35, + "module": "_actor", + "name": "fail", + "parsedDocstring": { + "text": "Fail the Actor instance.\n\nThis performs all the same steps as Actor.exit(), but it additionally sets the exit code to `1` (by default).\n", + "args": { + "exit_code": "The exit code with which the Actor should fail (defaults to `1`).", + "exception": "The exception with which the Actor failed.", + "status_message": "The final status message that the Actor should display." + } + }, "sources": [ { - "filename": "/tmp-old-source-v27/src/apify/_actor.py", - "line": 207, "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/_actor.py#L207" + "fileName": "/src/apify/_actor.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 302 } ], + "type": { + "name": "Undefined", + "type": "reference" + }, "signatures": [ { - "id": 23, - "name": "init", - "modifiers": [ - "async" - ], - "kind": 4096, - "kindString": "Call signature", - "flags": {}, "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Initialize the Actor instance.\\n\\nThis initializes the Actor instance. It configures the right storage client based on whether the Actor is\\nrunning locally or on the Apify platform, it initializes the event manager for processing Actor events,\\nand starts an interval for regularly sending `PERSIST_STATE` events, so that the Actor can regularly persist\\nits state in response to these events.\\n\\nThis method should be called immediately before performing any additional Actor actions, and it should be\\ncalled only once.\"]}" + "text": "Fail the Actor instance.\n\nThis performs all the same steps as Actor.exit(), but it additionally sets the exit code to `1` (by default).\n" } ] }, + "flags": {}, + "id": 36, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "fail", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The exit code with which the Actor should fail (defaults to `1`)." + } + ] + }, + "defaultValue": "1", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 37, + "kind": 32768, + "kindString": "Parameter", + "name": "exit_code", + "type": { + "name": "int", + "type": "reference" + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The exception with which the Actor failed." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 38, + "kind": 32768, + "kindString": "Parameter", + "name": "exception", + "type": { + "name": "BaseException | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "BaseException" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The final status message that the Actor should display." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 39, + "kind": 32768, + "kindString": "Parameter", + "name": "status_message", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + } + ], "type": { - "type": "reference", - "name": "None" - }, - "parameters": [] + "name": "None", + "type": "literal", + "value": null + } } ] }, { - "id": 24, - "name": "exit", - "module": "_actor", "kind": 2048, "kindString": "Method", - "flags": {}, + "children": [], "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Exit the Actor instance.\\n\\nThis stops the Actor instance. It cancels all the intervals for regularly sending `PERSIST_STATE` events,\\nsends a final `PERSIST_STATE` event, waits for all the event listeners to finish, and stops the event manager.\\n\", {\"Arguments\": [{\"param\": \"exit_code\", \"desc\": \"The exit code with which the Actor should fail (defaults to `0`).\"}, {\"param\": \"event_listeners_timeout\", \"desc\": \"How long should the Actor wait for Actor event listeners to finish before exiting.\"}, {\"param\": \"status_message\", \"desc\": \"The final status message that the Actor should display.\"}, {\"param\": \"cleanup_timeout\", \"desc\": \"How long we should wait for event listeners.\"}]}]}" + "text": "Return a new instance of the Apify API client.\n\nThe `ApifyClientAsync` class is provided by the [apify-client](https://github.com/apify/apify-client-python)\npackage, and it is automatically configured using the `APIFY_API_BASE_URL` and `APIFY_TOKEN` environment\nvariables.\n\nYou can override the token via the available options. That's useful if you want to use the client\nas a different Apify user than the SDK internals are using.\n" } ] }, - "children": [], + "decorations": [], + "flags": {}, "groups": [], + "id": 40, + "module": "_actor", + "name": "new_client", + "parsedDocstring": { + "text": "Return a new instance of the Apify API client.\n\nThe `ApifyClientAsync` class is provided by the [apify-client](https://github.com/apify/apify-client-python)\npackage, and it is automatically configured using the `APIFY_API_BASE_URL` and `APIFY_TOKEN` environment\nvariables.\n\nYou can override the token via the available options. That's useful if you want to use the client\nas a different Apify user than the SDK internals are using.\n", + "args": { + "token": "The Apify API token.", + "api_url": "The URL of the Apify API server to which to connect to. Defaults to https://api.apify.com.", + "max_retries": "How many times to retry a failed request at most.", + "min_delay_between_retries": "How long will the client wait between retrying requests\n(increases exponentially from this value).", + "timeout": "The socket timeout of the HTTP requests sent to the Apify API." + } + }, "sources": [ { - "filename": "/tmp-old-source-v27/src/apify/_actor.py", - "line": 256, "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/_actor.py#L256" + "fileName": "/src/apify/_actor.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 327 } ], + "type": { + "name": "Undefined", + "type": "reference" + }, "signatures": [ { - "id": 25, - "name": "exit", - "modifiers": [ - "async" - ], - "kind": 4096, - "kindString": "Call signature", - "flags": {}, "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Exit the Actor instance.\\n\\nThis stops the Actor instance. It cancels all the intervals for regularly sending `PERSIST_STATE` events,\\nsends a final `PERSIST_STATE` event, waits for all the event listeners to finish, and stops the event manager.\\n\", {\"" + "text": "Return a new instance of the Apify API client.\n\nThe `ApifyClientAsync` class is provided by the [apify-client](https://github.com/apify/apify-client-python)\npackage, and it is automatically configured using the `APIFY_API_BASE_URL` and `APIFY_TOKEN` environment\nvariables.\n\nYou can override the token via the available options. That's useful if you want to use the client\nas a different Apify user than the SDK internals are using.\n" } ] }, - "type": { - "type": "reference", - "name": "None" - }, + "flags": {}, + "id": 41, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "new_client", "parameters": [ { - "id": 26, - "name": "exit_code", - "kind": 32768, - "kindString": "Parameter", + "comment": { + "summary": [ + { + "kind": "text", + "text": "The Apify API token." + } + ] + }, + "defaultValue": "None", "flags": { - "keyword-only": "true" + "isOptional": true, + "keyword-only": true }, + "id": 42, + "kind": 32768, + "kindString": "Parameter", + "name": "token", "type": { - "type": "intrinsic", - "name": "int" - }, - "defaultValue": "0" + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } }, { - "id": 27, - "name": "event_listeners_timeout", - "kind": 32768, - "kindString": "Parameter", + "comment": { + "summary": [ + { + "kind": "text", + "text": "The URL of the Apify API server to which to connect to. Defaults to https://api.apify.com." + } + ] + }, + "defaultValue": "None", "flags": { - "keyword-only": "true" + "isOptional": true, + "keyword-only": true }, + "id": 43, + "kind": 32768, + "kindString": "Parameter", + "name": "api_url", "type": { - "type": "reference", - "name": "timedelta | None" - }, - "defaultValue": "EVENT_LISTENERS_TIMEOUT" + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } }, { - "id": 28, - "name": "status_message", - "kind": 32768, - "kindString": "Parameter", + "comment": { + "summary": [ + { + "kind": "text", + "text": "How many times to retry a failed request at most." + } + ] + }, + "defaultValue": "None", "flags": { - "keyword-only": "true" + "isOptional": true, + "keyword-only": true }, + "id": 44, + "kind": 32768, + "kindString": "Parameter", + "name": "max_retries", "type": { - "type": "reference", - "name": "str | None" - }, - "defaultValue": "None" + "name": "int | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "int" + }, + { + "type": "literal", + "value": null + } + ] + } }, { - "id": 29, - "name": "cleanup_timeout", + "comment": { + "summary": [ + { + "kind": "text", + "text": "How long will the client wait between retrying requests\n(increases exponentially from this value)." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 45, "kind": 32768, "kindString": "Parameter", + "name": "min_delay_between_retries", + "type": { + "name": "timedelta | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "timedelta" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The socket timeout of the HTTP requests sent to the Apify API." + } + ] + }, + "defaultValue": "None", "flags": { - "keyword-only": "true" + "isOptional": true, + "keyword-only": true }, + "id": 46, + "kind": 32768, + "kindString": "Parameter", + "name": "timeout", "type": { - "type": "reference", - "name": "timedelta" - }, - "defaultValue": "timedelta(seconds=30)" + "name": "timedelta | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "timedelta" + }, + { + "type": "literal", + "value": null + } + ] + } } - ] + ], + "type": { + "name": "ApifyClientAsync", + "type": "reference" + } } ] }, { - "id": 30, - "name": "fail", - "module": "_actor", "kind": 2048, "kindString": "Method", - "flags": {}, + "children": [], "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Fail the Actor instance.\\n\\nThis performs all the same steps as Actor.exit(), but it additionally sets the exit code to `1` (by default).\\n\", {\"Arguments\": [{\"param\": \"exit_code\", \"desc\": \"The exit code with which the Actor should fail (defaults to `1`).\"}, {\"param\": \"exception\", \"desc\": \"The exception with which the Actor failed.\"}, {\"param\": \"status_message\", \"desc\": \"The final status message that the Actor should display.\"}]}]}" + "text": "Open a dataset.\n\nDatasets are used to store structured data where each object stored has the same attributes, such as online\nstore products or real estate offers. The actual data is stored either on the local filesystem or in\nthe Apify cloud.\n" } ] }, - "children": [], + "decorations": [], + "flags": {}, "groups": [], + "id": 47, + "module": "_actor", + "name": "open_dataset", + "parsedDocstring": { + "text": "Open a dataset.\n\nDatasets are used to store structured data where each object stored has the same attributes, such as online\nstore products or real estate offers. The actual data is stored either on the local filesystem or in\nthe Apify cloud.\n", + "args": { + "id": "ID of the dataset to be opened. If neither `id` nor `name` are provided, the method returns\nthe default dataset associated with the Actor run.", + "name": "Name of the dataset to be opened. If neither `id` nor `name` are provided, the method returns\nthe default dataset associated with the Actor run.", + "force_cloud": "If set to `True` then the Apify cloud storage is always used. This way it is possible\nto combine local and cloud storage.\n" + }, + "returns": "An instance of the `Dataset` class for the given ID or name." + }, "sources": [ { - "filename": "/tmp-old-source-v27/src/apify/_actor.py", - "line": 302, "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/_actor.py#L302" + "fileName": "/src/apify/_actor.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 365 } ], + "type": { + "name": "Undefined", + "type": "reference" + }, "signatures": [ { - "id": 31, - "name": "fail", - "modifiers": [ - "async" - ], - "kind": 4096, - "kindString": "Call signature", - "flags": {}, "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "An instance of the `Dataset` class for the given ID or name." + } + ], + "tag": "@returns" + } + ], "summary": [ { "kind": "text", - "text": "{\"content\": [\"Fail the Actor instance.\\n\\nThis performs all the same steps as Actor.exit(), but it additionally sets the exit code to `1` (by default).\\n\", {\"" + "text": "Open a dataset.\n\nDatasets are used to store structured data where each object stored has the same attributes, such as online\nstore products or real estate offers. The actual data is stored either on the local filesystem or in\nthe Apify cloud.\n" } ] }, - "type": { - "type": "reference", - "name": "None" - }, + "flags": {}, + "id": 48, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "open_dataset", "parameters": [ { - "id": 32, - "name": "exit_code", - "kind": 32768, - "kindString": "Parameter", + "comment": { + "summary": [ + { + "kind": "text", + "text": "ID of the dataset to be opened. If neither `id` nor `name` are provided, the method returns\nthe default dataset associated with the Actor run." + } + ] + }, + "defaultValue": "None", "flags": { - "keyword-only": "true" + "isOptional": true, + "keyword-only": true }, + "id": 49, + "kind": 32768, + "kindString": "Parameter", + "name": "id", "type": { - "type": "intrinsic", - "name": "int" - }, - "defaultValue": "1" + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } }, { - "id": 33, - "name": "exception", - "kind": 32768, - "kindString": "Parameter", + "comment": { + "summary": [ + { + "kind": "text", + "text": "Name of the dataset to be opened. If neither `id` nor `name` are provided, the method returns\nthe default dataset associated with the Actor run." + } + ] + }, + "defaultValue": "None", "flags": { - "keyword-only": "true" + "isOptional": true, + "keyword-only": true }, - "type": { - "type": "reference", - "name": "BaseException | None" - }, - "defaultValue": "None" - }, - { - "id": 34, - "name": "status_message", + "id": 50, "kind": 32768, "kindString": "Parameter", + "name": "name", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "If set to `True` then the Apify cloud storage is always used. This way it is possible\nto combine local and cloud storage.\n" + } + ] + }, + "defaultValue": "False", "flags": { - "keyword-only": "true" + "isOptional": true, + "keyword-only": true }, + "id": 51, + "kind": 32768, + "kindString": "Parameter", + "name": "force_cloud", "type": { - "type": "reference", - "name": "str | None" - }, - "defaultValue": "None" + "name": "bool", + "type": "reference" + } } - ] + ], + "type": { + "name": "Dataset", + "type": "reference" + } } ] }, { - "id": 35, - "name": "new_client", - "module": "_actor", "kind": 2048, "kindString": "Method", - "flags": {}, + "children": [], "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Return a new instance of the Apify API client.\\n\\nThe `ApifyClientAsync` class is provided by the [apify-client](https://github.com/apify/apify-client-python)\\npackage, and it is automatically configured using the `APIFY_API_BASE_URL` and `APIFY_TOKEN` environment\\nvariables.\\n\\nYou can override the token via the available options. That's useful if you want to use the client\\nas a different Apify user than the SDK internals are using.\\n\", {\"Arguments\": [{\"param\": \"token\", \"desc\": \"The Apify API token.\"}, {\"param\": \"api_url\", \"desc\": \"The URL of the Apify API server to which to connect to. Defaults to https://api.apify.com.\"}, {\"param\": \"max_retries\", \"desc\": \"How many times to retry a failed request at most.\"}, {\"param\": \"min_delay_between_retries\", \"desc\": \"How long will the client wait between retrying requests\\n(increases exponentially from this value).\"}, {\"param\": \"timeout\", \"desc\": \"The socket timeout of the HTTP requests sent to the Apify API.\"}]}]}" + "text": "Open a key-value store.\n\nKey-value stores are used to store records or files, along with their MIME content type. The records are stored\nand retrieved using a unique key. The actual data is stored either on a local filesystem or in the Apify cloud.\n" } ] }, - "children": [], + "decorations": [], + "flags": {}, "groups": [], + "id": 52, + "module": "_actor", + "name": "open_key_value_store", + "parsedDocstring": { + "text": "Open a key-value store.\n\nKey-value stores are used to store records or files, along with their MIME content type. The records are stored\nand retrieved using a unique key. The actual data is stored either on a local filesystem or in the Apify cloud.\n", + "args": { + "id": "ID of the key-value store to be opened. If neither `id` nor `name` are provided, the method returns\nthe default key-value store associated with the Actor run.", + "name": "Name of the key-value store to be opened. If neither `id` nor `name` are provided, the method\nreturns the default key-value store associated with the Actor run.", + "force_cloud": "If set to `True` then the Apify cloud storage is always used. This way it is possible\nto combine local and cloud storage.\n" + }, + "returns": "An instance of the `KeyValueStore` class for the given ID or name." + }, "sources": [ { - "filename": "/tmp-old-source-v27/src/apify/_actor.py", - "line": 327, "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/_actor.py#L327" + "fileName": "/src/apify/_actor.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 401 } ], + "type": { + "name": "Undefined", + "type": "reference" + }, "signatures": [ { - "id": 36, - "name": "new_client", - "modifiers": [], - "kind": 4096, - "kindString": "Call signature", - "flags": {}, "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "An instance of the `KeyValueStore` class for the given ID or name." + } + ], + "tag": "@returns" + } + ], "summary": [ { "kind": "text", - "text": "{\"content\": [\"Return a new instance of the Apify API client.\\n\\nThe `ApifyClientAsync` class is provided by the [apify-client](https://github.com/apify/apify-client-python)\\npackage, and it is automatically configured using the `APIFY_API_BASE_URL` and `APIFY_TOKEN` environment\\nvariables.\\n\\nYou can override the token via the available options. That's useful if you want to use the client\\nas a different Apify user than the SDK internals are using.\\n\", {\"" + "text": "Open a key-value store.\n\nKey-value stores are used to store records or files, along with their MIME content type. The records are stored\nand retrieved using a unique key. The actual data is stored either on a local filesystem or in the Apify cloud.\n" } ] }, - "type": { - "type": "reference", - "name": "ApifyClientAsync" - }, + "flags": {}, + "id": 53, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "open_key_value_store", "parameters": [ { - "id": 37, - "name": "token", - "kind": 32768, - "kindString": "Parameter", - "flags": { - "keyword-only": "true" + "comment": { + "summary": [ + { + "kind": "text", + "text": "ID of the key-value store to be opened. If neither `id` nor `name` are provided, the method returns\nthe default key-value store associated with the Actor run." + } + ] }, - "type": { - "type": "reference", - "name": "str | None" + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true }, - "defaultValue": "None" - }, - { - "id": 38, - "name": "api_url", + "id": 54, "kind": 32768, "kindString": "Parameter", - "flags": { - "keyword-only": "true" - }, + "name": "id", "type": { - "type": "reference", - "name": "str | None" - }, - "defaultValue": "None" + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } }, { - "id": 39, - "name": "max_retries", - "kind": 32768, - "kindString": "Parameter", - "flags": { - "keyword-only": "true" + "comment": { + "summary": [ + { + "kind": "text", + "text": "Name of the key-value store to be opened. If neither `id` nor `name` are provided, the method\nreturns the default key-value store associated with the Actor run." + } + ] }, - "type": { - "type": "reference", - "name": "int | None" + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true }, - "defaultValue": "None" - }, - { - "id": 40, - "name": "min_delay_between_retries", + "id": 55, "kind": 32768, "kindString": "Parameter", - "flags": { - "keyword-only": "true" - }, + "name": "name", "type": { - "type": "reference", - "name": "timedelta | None" - }, - "defaultValue": "None" + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } }, { - "id": 41, - "name": "timeout", - "kind": 32768, - "kindString": "Parameter", + "comment": { + "summary": [ + { + "kind": "text", + "text": "If set to `True` then the Apify cloud storage is always used. This way it is possible\nto combine local and cloud storage.\n" + } + ] + }, + "defaultValue": "False", "flags": { - "keyword-only": "true" + "isOptional": true, + "keyword-only": true }, + "id": 56, + "kind": 32768, + "kindString": "Parameter", + "name": "force_cloud", "type": { - "type": "reference", - "name": "timedelta | None" - }, - "defaultValue": "None" + "name": "bool", + "type": "reference" + } } - ] + ], + "type": { + "name": "KeyValueStore", + "type": "reference" + } } ] }, { - "id": 42, - "name": "open_dataset", - "module": "_actor", "kind": 2048, "kindString": "Method", - "flags": {}, + "children": [], "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Open a dataset.\\n\\nDatasets are used to store structured data where each object stored has the same attributes, such as online\\nstore products or real estate offers. The actual data is stored either on the local filesystem or in\\nthe Apify cloud.\\n\", {\"Arguments\": [{\"param\": \"id\", \"desc\": \"ID of the dataset to be opened. If neither `id` nor `name` are provided, the method returns\\nthe default dataset associated with the Actor run.\"}, {\"param\": \"name\", \"desc\": \"Name of the dataset to be opened. If neither `id` nor `name` are provided, the method returns\\nthe default dataset associated with the Actor run.\"}, {\"param\": \"force_cloud\", \"desc\": \"If set to `True` then the Apify cloud storage is always used. This way it is possible\\nto combine local and cloud storage.\\n\"}]}, {\"Returns\": [\"An instance of the `Dataset` class for the given ID or name.\"]}]}" + "text": "Open a request queue.\n\nRequest queue represents a queue of URLs to crawl, which is stored either on local filesystem or in\nthe Apify cloud. The queue is used for deep crawling of websites, where you start with several URLs and then\nrecursively follow links to other pages. The data structure supports both breadth-first and depth-first\ncrawling orders.\n" } ] }, - "children": [], + "decorations": [], + "flags": {}, "groups": [], + "id": 57, + "module": "_actor", + "name": "open_request_queue", + "parsedDocstring": { + "text": "Open a request queue.\n\nRequest queue represents a queue of URLs to crawl, which is stored either on local filesystem or in\nthe Apify cloud. The queue is used for deep crawling of websites, where you start with several URLs and then\nrecursively follow links to other pages. The data structure supports both breadth-first and depth-first\ncrawling orders.\n", + "args": { + "id": "ID of the request queue to be opened. If neither `id` nor `name` are provided, the method returns\nthe default request queue associated with the Actor run.", + "name": "Name of the request queue to be opened. If neither `id` nor `name` are provided, the method returns\nthe default request queue associated with the Actor run.", + "force_cloud": "If set to `True` then the Apify cloud storage is always used. This way it is possible\nto combine local and cloud storage.\n" + }, + "returns": "An instance of the `RequestQueue` class for the given ID or name." + }, "sources": [ { - "filename": "/tmp-old-source-v27/src/apify/_actor.py", - "line": 365, "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/_actor.py#L365" + "fileName": "/src/apify/_actor.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 435 } ], + "type": { + "name": "Undefined", + "type": "reference" + }, "signatures": [ { - "id": 43, - "name": "open_dataset", - "modifiers": [ - "async" - ], - "kind": 4096, - "kindString": "Call signature", - "flags": {}, "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "An instance of the `RequestQueue` class for the given ID or name." + } + ], + "tag": "@returns" + } + ], "summary": [ { "kind": "text", - "text": "{\"content\": [\"Open a dataset.\\n\\nDatasets are used to store structured data where each object stored has the same attributes, such as online\\nstore products or real estate offers. The actual data is stored either on the local filesystem or in\\nthe Apify cloud.\\n\", {\"" + "text": "Open a request queue.\n\nRequest queue represents a queue of URLs to crawl, which is stored either on local filesystem or in\nthe Apify cloud. The queue is used for deep crawling of websites, where you start with several URLs and then\nrecursively follow links to other pages. The data structure supports both breadth-first and depth-first\ncrawling orders.\n" } ] }, - "type": { - "type": "reference", - "name": "Dataset" - }, + "flags": {}, + "id": 58, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "open_request_queue", "parameters": [ { - "id": 44, - "name": "id", - "kind": 32768, - "kindString": "Parameter", + "comment": { + "summary": [ + { + "kind": "text", + "text": "ID of the request queue to be opened. If neither `id` nor `name` are provided, the method returns\nthe default request queue associated with the Actor run." + } + ] + }, + "defaultValue": "None", "flags": { - "keyword-only": "true" + "isOptional": true, + "keyword-only": true }, + "id": 59, + "kind": 32768, + "kindString": "Parameter", + "name": "id", "type": { - "type": "reference", - "name": "str | None" - }, - "defaultValue": "None" + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } }, { - "id": 45, - "name": "name", - "kind": 32768, - "kindString": "Parameter", + "comment": { + "summary": [ + { + "kind": "text", + "text": "Name of the request queue to be opened. If neither `id` nor `name` are provided, the method returns\nthe default request queue associated with the Actor run." + } + ] + }, + "defaultValue": "None", "flags": { - "keyword-only": "true" + "isOptional": true, + "keyword-only": true }, + "id": 60, + "kind": 32768, + "kindString": "Parameter", + "name": "name", "type": { - "type": "reference", - "name": "str | None" - }, - "defaultValue": "None" + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } }, { - "id": 46, - "name": "force_cloud", - "kind": 32768, - "kindString": "Parameter", + "comment": { + "summary": [ + { + "kind": "text", + "text": "If set to `True` then the Apify cloud storage is always used. This way it is possible\nto combine local and cloud storage.\n" + } + ] + }, + "defaultValue": "False", "flags": { - "keyword-only": "true" + "isOptional": true, + "keyword-only": true }, + "id": 61, + "kind": 32768, + "kindString": "Parameter", + "name": "force_cloud", "type": { - "type": "intrinsic", - "name": "bool" - }, - "defaultValue": "False" + "name": "bool", + "type": "reference" + } } - ] + ], + "type": { + "name": "RequestQueue", + "type": "reference" + } } ] }, { - "id": 47, - "name": "open_key_value_store", - "module": "_actor", "kind": 2048, "kindString": "Method", - "flags": {}, + "children": [], "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Open a key-value store.\\n\\nKey-value stores are used to store records or files, along with their MIME content type. The records are stored\\nand retrieved using a unique key. The actual data is stored either on a local filesystem or in the Apify cloud.\\n\", {\"Arguments\": [{\"param\": \"id\", \"desc\": \"ID of the key-value store to be opened. If neither `id` nor `name` are provided, the method returns\\nthe default key-value store associated with the Actor run.\"}, {\"param\": \"name\", \"desc\": \"Name of the key-value store to be opened. If neither `id` nor `name` are provided, the method\\nreturns the default key-value store associated with the Actor run.\"}, {\"param\": \"force_cloud\", \"desc\": \"If set to `True` then the Apify cloud storage is always used. This way it is possible\\nto combine local and cloud storage.\\n\"}]}, {\"Returns\": [\"An instance of the `KeyValueStore` class for the given ID or name.\"]}]}" + "text": "Store an object or a list of objects to the default dataset of the current Actor run.\n" } ] }, - "children": [], + "decorations": [], + "flags": {}, "groups": [], + "id": 62, + "module": "_actor", + "name": "push_data", + "parsedDocstring": { + "text": "Store an object or a list of objects to the default dataset of the current Actor run.\n", + "args": { + "data": "The data to push to the default dataset.", + "charged_event_name": "If provided and if the Actor uses the pay-per-event pricing model,\nthe method will attempt to charge for the event for each pushed item." + } + }, "sources": [ { - "filename": "/tmp-old-source-v27/src/apify/_actor.py", - "line": 401, "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/_actor.py#L401" + "fileName": "/src/apify/_actor.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 476 } ], + "type": { + "name": "Undefined", + "type": "reference" + }, "signatures": [ { - "id": 48, - "name": "open_key_value_store", - "modifiers": [ - "async" - ], - "kind": 4096, - "kindString": "Call signature", - "flags": {}, "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Open a key-value store.\\n\\nKey-value stores are used to store records or files, along with their MIME content type. The records are stored\\nand retrieved using a unique key. The actual data is stored either on a local filesystem or in the Apify cloud.\\n\", {\"" + "text": "Store an object or a list of objects to the default dataset of the current Actor run.\n" } ] }, - "type": { - "type": "reference", - "name": "KeyValueStore" - }, + "flags": {}, + "id": 63, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "push_data", "parameters": [ { - "id": 49, - "name": "id", - "kind": 32768, - "kindString": "Parameter", - "flags": { - "keyword-only": "true" + "comment": { + "summary": [ + { + "kind": "text", + "text": "The data to push to the default dataset." + } + ] }, - "type": { - "type": "reference", - "name": "str | None" + "flags": { + "isOptional": false, + "keyword-only": false }, - "defaultValue": "None" - }, - { - "id": 50, - "name": "name", + "id": 64, "kind": 32768, "kindString": "Parameter", - "flags": { - "keyword-only": "true" - }, + "name": "data", "type": { - "type": "reference", - "name": "str | None" - }, - "defaultValue": "None" + "name": "dict | list[dict]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "dict" + }, + { + "type": "reference", + "name": "list", + "typeArguments": [ + { + "type": "reference", + "name": "dict" + } + ], + "target": "596" + } + ] + } }, { - "id": 51, - "name": "force_cloud", - "kind": 32768, - "kindString": "Parameter", + "comment": { + "summary": [ + { + "kind": "text", + "text": "If provided and if the Actor uses the pay-per-event pricing model,\nthe method will attempt to charge for the event for each pushed item." + } + ] + }, + "defaultValue": "None", "flags": { - "keyword-only": "true" + "isOptional": true, + "keyword-only": false }, + "id": 65, + "kind": 32768, + "kindString": "Parameter", + "name": "charged_event_name", "type": { - "type": "intrinsic", - "name": "bool" - }, - "defaultValue": "False" + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } } - ] - } - ] - }, - { - "id": 52, - "name": "open_request_queue", - "module": "_actor", - "kind": 2048, - "kindString": "Method", - "flags": {}, - "comment": { - "summary": [ - { - "kind": "text", - "text": "{\"content\": [\"Open a request queue.\\n\\nRequest queue represents a queue of URLs to crawl, which is stored either on local filesystem or in\\nthe Apify cloud. The queue is used for deep crawling of websites, where you start with several URLs and then\\nrecursively follow links to other pages. The data structure supports both breadth-first and depth-first\\ncrawling orders.\\n\", {\"Arguments\": [{\"param\": \"id\", \"desc\": \"ID of the request queue to be opened. If neither `id` nor `name` are provided, the method returns\\nthe default request queue associated with the Actor run.\"}, {\"param\": \"name\", \"desc\": \"Name of the request queue to be opened. If neither `id` nor `name` are provided, the method returns\\nthe default request queue associated with the Actor run.\"}, {\"param\": \"force_cloud\", \"desc\": \"If set to `True` then the Apify cloud storage is always used. This way it is possible\\nto combine local and cloud storage.\\n\"}]}, {\"Returns\": [\"An instance of the `RequestQueue` class for the given ID or name.\"]}]}" - } - ] - }, - "children": [], - "groups": [], - "sources": [ - { - "filename": "/tmp-old-source-v27/src/apify/_actor.py", - "line": 435, - "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/_actor.py#L435" - } - ], - "signatures": [ - { - "id": 53, - "name": "open_request_queue", - "modifiers": [ - "async" ], - "kind": 4096, - "kindString": "Call signature", - "flags": {}, + "type": { + "name": "ChargeResult | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "ChargeResult", + "target": "379" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Open a request queue.\\n\\nRequest queue represents a queue of URLs to crawl, which is stored either on local filesystem or in\\nthe Apify cloud. The queue is used for deep crawling of websites, where you start with several URLs and then\\nrecursively follow links to other pages. The data structure supports both breadth-first and depth-first\\ncrawling orders.\\n\", {\"" + "text": "Store an object or a list of objects to the default dataset of the current Actor run.\n" } ] }, - "type": { - "type": "reference", - "name": "RequestQueue" - }, + "flags": {}, + "id": 163, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "push_data", "parameters": [ { - "id": 54, - "name": "id", - "kind": 32768, - "kindString": "Parameter", - "flags": { - "keyword-only": "true" + "comment": { + "summary": [ + { + "kind": "text", + "text": "The data to push to the default dataset." + } + ] }, - "type": { - "type": "reference", - "name": "str | None" - }, - "defaultValue": "None" - }, - { - "id": 55, - "name": "name", - "kind": 32768, - "kindString": "Parameter", "flags": { - "keyword-only": "true" - }, - "type": { - "type": "reference", - "name": "str | None" + "isOptional": false, + "keyword-only": false }, - "defaultValue": "None" - }, - { - "id": 56, - "name": "force_cloud", + "id": 164, "kind": 32768, "kindString": "Parameter", - "flags": { - "keyword-only": "true" - }, + "name": "data", "type": { - "type": "intrinsic", - "name": "bool" - }, - "defaultValue": "False" + "name": "dict | list[dict]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "dict" + }, + { + "type": "reference", + "name": "list", + "typeArguments": [ + { + "type": "reference", + "name": "dict" + } + ], + "target": "596" + } + ] + } } - ] - } - ] - }, - { - "id": 57, - "name": "push_data", - "module": "_actor", - "kind": 2048, - "kindString": "Method", - "flags": {}, - "comment": { - "summary": [ - { - "kind": "text", - "text": "{\"content\": [\"Store an object or a list of objects to the default dataset of the current Actor run.\\n\", {\"Arguments\": [{\"param\": \"data\", \"desc\": \"The data to push to the default dataset.\"}, {\"param\": \"charged_event_name\", \"desc\": \"If provided and if the Actor uses the pay-per-event pricing model,\\nthe method will attempt to charge for the event for each pushed item.\"}]}]}" + ], + "type": { + "name": "None", + "type": "literal", + "value": null } - ] - }, - "children": [], - "groups": [], - "sources": [ - { - "filename": "/tmp-old-source-v27/src/apify/_actor.py", - "line": 476, - "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/_actor.py#L476" - } - ], - "signatures": [ + }, { - "id": 58, - "name": "push_data", - "modifiers": [ - "async" - ], - "kind": 4096, - "kindString": "Call signature", - "flags": {}, "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Store an object or a list of objects to the default dataset of the current Actor run.\\n\", {\"" + "text": "Store an object or a list of objects to the default dataset of the current Actor run.\n" } ] }, - "type": { - "type": "reference", - "name": "ChargeResult | None" - }, + "flags": {}, + "id": 165, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "push_data", "parameters": [ { - "id": 59, - "name": "data", + "comment": { + "summary": [ + { + "kind": "text", + "text": "The data to push to the default dataset." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 166, "kind": 32768, "kindString": "Parameter", - "flags": {}, + "name": "data", "type": { - "type": "reference", - "name": "dict | list[dict]" + "name": "dict | list[dict]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "dict" + }, + { + "type": "reference", + "name": "list", + "typeArguments": [ + { + "type": "reference", + "name": "dict" + } + ], + "target": "596" + } + ] } }, { - "id": 60, - "name": "charged_event_name", + "comment": { + "summary": [ + { + "kind": "text", + "text": "If provided and if the Actor uses the pay-per-event pricing model,\nthe method will attempt to charge for the event for each pushed item." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 167, "kind": 32768, "kindString": "Parameter", - "flags": {}, + "name": "charged_event_name", "type": { - "type": "reference", - "name": "str | None" - }, - "defaultValue": "None" + "name": "str", + "type": "reference" + } } - ] + ], + "type": { + "name": "ChargeResult", + "type": "reference", + "target": "379" + } } ] }, { - "id": 61, - "name": "get_input", - "module": "_actor", "kind": 2048, "kindString": "Method", - "flags": {}, + "children": [], "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Get the Actor input value from the default key-value store associated with the current Actor run.\"]}" + "text": "Get the Actor input value from the default key-value store associated with the current Actor run." } ] }, - "children": [], + "decorations": [], + "flags": {}, "groups": [], + "id": 66, + "module": "_actor", + "name": "get_input", + "parsedDocstring": { + "text": "Get the Actor input value from the default key-value store associated with the current Actor run." + }, "sources": [ { - "filename": "/tmp-old-source-v27/src/apify/_actor.py", - "line": 513, "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/_actor.py#L513" + "fileName": "/src/apify/_actor.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 513 } ], + "type": { + "name": "Undefined", + "type": "reference" + }, "signatures": [ { - "id": 62, - "name": "get_input", - "modifiers": [ - "async" - ], - "kind": 4096, - "kindString": "Call signature", - "flags": {}, "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Get the Actor input value from the default key-value store associated with the current Actor run.\"]}" + "text": "Get the Actor input value from the default key-value store associated with the current Actor run." } ] }, + "flags": {}, + "id": 67, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "get_input", + "parameters": [], "type": { - "type": "reference", - "name": "Any" - }, - "parameters": [] + "name": "Any", + "type": "reference" + } } ] }, { - "id": 63, - "name": "get_value", - "module": "_actor", "kind": 2048, "kindString": "Method", - "flags": {}, + "children": [], "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Get a value from the default key-value store associated with the current Actor run.\\n\", {\"Arguments\": [{\"param\": \"key\", \"desc\": \"The key of the record which to retrieve.\"}, {\"param\": \"default_value\", \"desc\": \"Default value returned in case the record does not exist.\"}]}]}" + "text": "Get a value from the default key-value store associated with the current Actor run.\n" } ] }, - "children": [], + "decorations": [], + "flags": {}, "groups": [], + "id": 68, + "module": "_actor", + "name": "get_value", + "parsedDocstring": { + "text": "Get a value from the default key-value store associated with the current Actor run.\n", + "args": { + "key": "The key of the record which to retrieve.", + "default_value": "Default value returned in case the record does not exist." + } + }, "sources": [ { - "filename": "/tmp-old-source-v27/src/apify/_actor.py", - "line": 529, "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/_actor.py#L529" + "fileName": "/src/apify/_actor.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 529 } ], + "type": { + "name": "Undefined", + "type": "reference" + }, "signatures": [ { - "id": 64, - "name": "get_value", - "modifiers": [ - "async" - ], - "kind": 4096, - "kindString": "Call signature", - "flags": {}, "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Get a value from the default key-value store associated with the current Actor run.\\n\", {\"" + "text": "Get a value from the default key-value store associated with the current Actor run.\n" } ] }, - "type": { - "type": "reference", - "name": "Any" - }, + "flags": {}, + "id": 69, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "get_value", "parameters": [ { - "id": 65, - "name": "key", + "comment": { + "summary": [ + { + "kind": "text", + "text": "The key of the record which to retrieve." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 70, "kind": 32768, "kindString": "Parameter", - "flags": {}, + "name": "key", "type": { - "type": "intrinsic", - "name": "str" + "name": "str", + "type": "reference" } }, { - "id": 66, - "name": "default_value", + "comment": { + "summary": [ + { + "kind": "text", + "text": "Default value returned in case the record does not exist." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": false + }, + "id": 71, "kind": 32768, "kindString": "Parameter", - "flags": {}, + "name": "default_value", "type": { - "type": "reference", - "name": "Any" - }, - "defaultValue": "None" + "name": "Any", + "type": "reference" + } } - ] + ], + "type": { + "name": "Any", + "type": "reference" + } } ] }, { - "id": 67, - "name": "set_value", - "module": "_actor", "kind": 2048, "kindString": "Method", - "flags": {}, + "children": [], "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Set or delete a value in the default key-value store associated with the current Actor run.\\n\", {\"Arguments\": [{\"param\": \"key\", \"desc\": \"The key of the record which to set.\"}, {\"param\": \"value\", \"desc\": \"The value of the record which to set, or None, if the record should be deleted.\"}, {\"param\": \"content_type\", \"desc\": \"The content type which should be set to the value.\"}]}]}" + "text": "Set or delete a value in the default key-value store associated with the current Actor run.\n" } ] }, - "children": [], + "decorations": [], + "flags": {}, "groups": [], + "id": 72, + "module": "_actor", + "name": "set_value", + "parsedDocstring": { + "text": "Set or delete a value in the default key-value store associated with the current Actor run.\n", + "args": { + "key": "The key of the record which to set.", + "value": "The value of the record which to set, or None, if the record should be deleted.", + "content_type": "The content type which should be set to the value." + } + }, "sources": [ { - "filename": "/tmp-old-source-v27/src/apify/_actor.py", - "line": 541, "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/_actor.py#L541" + "fileName": "/src/apify/_actor.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 541 } ], + "type": { + "name": "Undefined", + "type": "reference" + }, "signatures": [ { - "id": 68, - "name": "set_value", - "modifiers": [ - "async" - ], - "kind": 4096, - "kindString": "Call signature", - "flags": {}, "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Set or delete a value in the default key-value store associated with the current Actor run.\\n\", {\"" + "text": "Set or delete a value in the default key-value store associated with the current Actor run.\n" } ] }, - "type": { - "type": "reference", - "name": "None" - }, + "flags": {}, + "id": 73, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "set_value", "parameters": [ { - "id": 69, - "name": "key", + "comment": { + "summary": [ + { + "kind": "text", + "text": "The key of the record which to set." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 74, "kind": 32768, "kindString": "Parameter", - "flags": {}, + "name": "key", "type": { - "type": "intrinsic", - "name": "str" + "name": "str", + "type": "reference" } }, { - "id": 70, - "name": "value", + "comment": { + "summary": [ + { + "kind": "text", + "text": "The value of the record which to set, or None, if the record should be deleted." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 75, "kind": 32768, "kindString": "Parameter", - "flags": {}, + "name": "value", "type": { - "type": "reference", - "name": "Any" + "name": "Any", + "type": "reference" } }, { - "id": 71, - "name": "content_type", - "kind": 32768, - "kindString": "Parameter", + "comment": { + "summary": [ + { + "kind": "text", + "text": "The content type which should be set to the value." + } + ] + }, + "defaultValue": "None", "flags": { - "keyword-only": "true" + "isOptional": true, + "keyword-only": true }, + "id": 76, + "kind": 32768, + "kindString": "Parameter", + "name": "content_type", "type": { - "type": "reference", - "name": "str | None" - }, - "defaultValue": "None" + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } } - ] + ], + "type": { + "name": "None", + "type": "literal", + "value": null + } } ] }, { - "id": 72, - "name": "get_charging_manager", - "module": "_actor", "kind": 2048, "kindString": "Method", - "flags": {}, + "children": [], "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Retrieve the charging manager to access granular pricing information.\"]}" + "text": "Retrieve the charging manager to access granular pricing information." } ] }, - "children": [], + "decorations": [], + "flags": {}, "groups": [], + "id": 77, + "module": "_actor", + "name": "get_charging_manager", + "parsedDocstring": { + "text": "Retrieve the charging manager to access granular pricing information." + }, "sources": [ { - "filename": "/tmp-old-source-v27/src/apify/_actor.py", - "line": 560, "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/_actor.py#L560" + "fileName": "/src/apify/_actor.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 560 } ], + "type": { + "name": "Undefined", + "type": "reference" + }, "signatures": [ { - "id": 73, - "name": "get_charging_manager", - "modifiers": [], - "kind": 4096, - "kindString": "Call signature", - "flags": {}, "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Retrieve the charging manager to access granular pricing information.\"]}" + "text": "Retrieve the charging manager to access granular pricing information." } ] }, + "flags": {}, + "id": 78, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "get_charging_manager", + "parameters": [], "type": { - "type": "reference", "name": "ChargingManager", - "id": 159 - }, - "parameters": [] + "type": "reference", + "target": "367" + } } ] }, { - "id": 74, - "name": "charge", - "module": "_actor", "kind": 2048, "kindString": "Method", - "flags": {}, + "children": [], "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Charge for a specified number of events - sub-operations of the Actor.\\n\\nThis is relevant only for the pay-per-event pricing model.\\n\", {\"Arguments\": [{\"param\": \"event_name\", \"desc\": \"Name of the event to be charged for.\"}, {\"param\": \"count\", \"desc\": \"Number of events to charge for.\"}]}]}" + "text": "Charge for a specified number of events - sub-operations of the Actor.\n\nThis is relevant only for the pay-per-event pricing model.\n" } ] }, - "children": [], + "decorations": [], + "flags": {}, "groups": [], + "id": 79, + "module": "_actor", + "name": "charge", + "parsedDocstring": { + "text": "Charge for a specified number of events - sub-operations of the Actor.\n\nThis is relevant only for the pay-per-event pricing model.\n", + "args": { + "event_name": "Name of the event to be charged for.", + "count": "Number of events to charge for." + } + }, "sources": [ { - "filename": "/tmp-old-source-v27/src/apify/_actor.py", - "line": 565, "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/_actor.py#L565" + "fileName": "/src/apify/_actor.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 565 } ], + "type": { + "name": "Undefined", + "type": "reference" + }, "signatures": [ { - "id": 75, - "name": "charge", - "modifiers": [ - "async" - ], - "kind": 4096, - "kindString": "Call signature", - "flags": {}, "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Charge for a specified number of events - sub-operations of the Actor.\\n\\nThis is relevant only for the pay-per-event pricing model.\\n\", {\"" + "text": "Charge for a specified number of events - sub-operations of the Actor.\n\nThis is relevant only for the pay-per-event pricing model.\n" } ] }, - "type": { - "type": "reference", - "name": "ChargeResult", - "id": 171 - }, + "flags": {}, + "id": 80, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "charge", "parameters": [ { - "id": 76, - "name": "event_name", + "comment": { + "summary": [ + { + "kind": "text", + "text": "Name of the event to be charged for." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 81, "kind": 32768, "kindString": "Parameter", - "flags": {}, + "name": "event_name", "type": { - "type": "intrinsic", - "name": "str" + "name": "str", + "type": "reference" } }, { - "id": 77, - "name": "count", + "comment": { + "summary": [ + { + "kind": "text", + "text": "Number of events to charge for." + } + ] + }, + "defaultValue": "1", + "flags": { + "isOptional": true, + "keyword-only": false + }, + "id": 82, "kind": 32768, "kindString": "Parameter", - "flags": {}, + "name": "count", "type": { - "type": "intrinsic", - "name": "int" - }, - "defaultValue": "1" + "name": "int", + "type": "reference" + } } - ] + ], + "type": { + "name": "ChargeResult", + "type": "reference", + "target": "379" + } } ] }, { - "id": 78, - "name": "on", - "module": "_actor", "kind": 2048, "kindString": "Method", - "flags": {}, + "children": [], "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Add an event listener to the Actor's event manager.\\n\\nThe following events can be emitted:\\n\\n- `Event.SYSTEM_INFO`: Emitted every minute; the event data contains information about the Actor's resource\\nusage.\\n\\n- `Event.MIGRATING`: Emitted when the Actor on the Apify platform is about to be migrated to another worker\\nserver. Use this event to persist the Actor's state and gracefully stop in-progress tasks, preventing\\ndisruption.\\n\\n- `Event.PERSIST_STATE`: Emitted regularly (default: 60 seconds) to notify the Actor to persist its state,\\npreventing work repetition after a restart. This event is emitted together with the `MIGRATING` event, where\\nthe `isMigrating` flag in the event data is `True`; otherwise, the flag is `False`. This event is for\\nconvenience; the same effect can be achieved by setting an interval and listening for the `MIGRATING` event.\\n\\n- `Event.ABORTING`: Emitted when a user aborts an Actor run on the Apify platform, allowing the Actor time\\nto clean up its state if the abort is graceful.\\n\", {\"Arguments\": [{\"param\": \"event_name\", \"desc\": \"The Actor event to listen for.\"}, {\"param\": \"listener\", \"desc\": \"The function to be called when the event is emitted (can be async).\"}]}]}" + "text": "Add an event listener to the Actor's event manager.\n\nThe following events can be emitted:\n\n- `Event.SYSTEM_INFO`: Emitted every minute; the event data contains information about the Actor's resource\nusage.\n\n- `Event.MIGRATING`: Emitted when the Actor on the Apify platform is about to be migrated to another worker\nserver. Use this event to persist the Actor's state and gracefully stop in-progress tasks, preventing\ndisruption.\n\n- `Event.PERSIST_STATE`: Emitted regularly (default: 60 seconds) to notify the Actor to persist its state,\npreventing work repetition after a restart. This event is emitted together with the `MIGRATING` event, where\nthe `isMigrating` flag in the event data is `True`; otherwise, the flag is `False`. This event is for\nconvenience; the same effect can be achieved by setting an interval and listening for the `MIGRATING` event.\n\n- `Event.ABORTING`: Emitted when a user aborts an Actor run on the Apify platform, allowing the Actor time\nto clean up its state if the abort is graceful.\n" } ] }, - "children": [], + "decorations": [], + "flags": {}, "groups": [], + "id": 83, + "module": "_actor", + "name": "on", + "parsedDocstring": { + "text": "Add an event listener to the Actor's event manager.\n\nThe following events can be emitted:\n\n- `Event.SYSTEM_INFO`: Emitted every minute; the event data contains information about the Actor's resource\nusage.\n\n- `Event.MIGRATING`: Emitted when the Actor on the Apify platform is about to be migrated to another worker\nserver. Use this event to persist the Actor's state and gracefully stop in-progress tasks, preventing\ndisruption.\n\n- `Event.PERSIST_STATE`: Emitted regularly (default: 60 seconds) to notify the Actor to persist its state,\npreventing work repetition after a restart. This event is emitted together with the `MIGRATING` event, where\nthe `isMigrating` flag in the event data is `True`; otherwise, the flag is `False`. This event is for\nconvenience; the same effect can be achieved by setting an interval and listening for the `MIGRATING` event.\n\n- `Event.ABORTING`: Emitted when a user aborts an Actor run on the Apify platform, allowing the Actor time\nto clean up its state if the abort is graceful.\n", + "args": { + "event_name": "The Actor event to listen for.", + "listener": "The function to be called when the event is emitted (can be async)." + } + }, "sources": [ { - "filename": "/tmp-old-source-v27/src/apify/_actor.py", - "line": 600, "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/_actor.py#L600" + "fileName": "/src/apify/_actor.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 600 } ], + "type": { + "name": "Undefined", + "type": "reference" + }, "signatures": [ { - "id": 79, - "name": "on", - "modifiers": [], - "kind": 4096, - "kindString": "Call signature", - "flags": {}, "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Add an event listener to the Actor's event manager.\\n\\nThe following events can be emitted:\\n\\n- `Event.SYSTEM_INFO`: Emitted every minute; the event data contains information about the Actor's resource\\nusage.\\n\\n- `Event.MIGRATING`: Emitted when the Actor on the Apify platform is about to be migrated to another worker\\nserver. Use this event to persist the Actor's state and gracefully stop in-progress tasks, preventing\\ndisruption.\\n\\n- `Event.PERSIST_STATE`: Emitted regularly (default: 60 seconds) to notify the Actor to persist its state,\\npreventing work repetition after a restart. This event is emitted together with the `MIGRATING` event, where\\nthe `isMigrating` flag in the event data is `True`; otherwise, the flag is `False`. This event is for\\nconvenience; the same effect can be achieved by setting an interval and listening for the `MIGRATING` event.\\n\\n- `Event.ABORTING`: Emitted when a user aborts an Actor run on the Apify platform, allowing the Actor time\\nto clean up its state if the abort is graceful.\\n\", {\"" + "text": "Add an event listener to the Actor's event manager.\n\nThe following events can be emitted:\n\n- `Event.SYSTEM_INFO`: Emitted every minute; the event data contains information about the Actor's resource\nusage.\n\n- `Event.MIGRATING`: Emitted when the Actor on the Apify platform is about to be migrated to another worker\nserver. Use this event to persist the Actor's state and gracefully stop in-progress tasks, preventing\ndisruption.\n\n- `Event.PERSIST_STATE`: Emitted regularly (default: 60 seconds) to notify the Actor to persist its state,\npreventing work repetition after a restart. This event is emitted together with the `MIGRATING` event, where\nthe `isMigrating` flag in the event data is `True`; otherwise, the flag is `False`. This event is for\nconvenience; the same effect can be achieved by setting an interval and listening for the `MIGRATING` event.\n\n- `Event.ABORTING`: Emitted when a user aborts an Actor run on the Apify platform, allowing the Actor time\nto clean up its state if the abort is graceful.\n" } ] }, - "type": { - "type": "reference", - "name": "EventListener[Any]" - }, + "flags": {}, + "id": 84, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "on", "parameters": [ { - "id": 80, - "name": "event_name", + "comment": { + "summary": [ + { + "kind": "text", + "text": "The Actor event to listen for." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 85, "kind": 32768, "kindString": "Parameter", - "flags": {}, + "name": "event_name", "type": { - "type": "reference", - "name": "Event" + "name": "Event", + "type": "reference" } }, { - "id": 81, - "name": "listener", + "comment": { + "summary": [ + { + "kind": "text", + "text": "The function to be called when the event is emitted (can be async)." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 86, "kind": 32768, "kindString": "Parameter", - "flags": {}, + "name": "listener", "type": { + "name": "EventListener", "type": "reference", - "name": "EventListener[Any]" + "typeArguments": [ + { + "type": "reference", + "name": "Any" + } + ] } } - ] - } - ] - }, - { - "id": 82, - "name": "off", - "module": "_actor", - "kind": 2048, - "kindString": "Method", - "flags": {}, - "comment": { - "summary": [ - { - "kind": "text", - "text": "{\"content\": [\"Remove a listener, or all listeners, from an Actor event.\\n\", {\"Arguments\": [{\"param\": \"event_name\", \"desc\": \"The Actor event for which to remove listeners.\"}, {\"param\": \"listener\", \"desc\": \"The listener which is supposed to be removed. If not passed, all listeners of this event\\nare removed.\"}]}]}" + ], + "type": { + "name": "EventListener", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "Any" + } + ] } - ] - }, - "children": [], - "groups": [], - "sources": [ - { - "filename": "/tmp-old-source-v27/src/apify/_actor.py", - "line": 642, - "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/_actor.py#L642" - } - ], - "signatures": [ + }, { - "id": 83, - "name": "off", - "modifiers": [], - "kind": 4096, - "kindString": "Call signature", - "flags": {}, "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Remove a listener, or all listeners, from an Actor event.\\n\", {\"" + "text": "Add an event listener to the Actor's event manager.\n\nThe following events can be emitted:\n\n- `Event.SYSTEM_INFO`: Emitted every minute; the event data contains information about the Actor's resource\nusage.\n\n- `Event.MIGRATING`: Emitted when the Actor on the Apify platform is about to be migrated to another worker\nserver. Use this event to persist the Actor's state and gracefully stop in-progress tasks, preventing\ndisruption.\n\n- `Event.PERSIST_STATE`: Emitted regularly (default: 60 seconds) to notify the Actor to persist its state,\npreventing work repetition after a restart. This event is emitted together with the `MIGRATING` event, where\nthe `isMigrating` flag in the event data is `True`; otherwise, the flag is `False`. This event is for\nconvenience; the same effect can be achieved by setting an interval and listening for the `MIGRATING` event.\n\n- `Event.ABORTING`: Emitted when a user aborts an Actor run on the Apify platform, allowing the Actor time\nto clean up its state if the abort is graceful.\n" } ] }, - "type": { - "type": "reference", - "name": "None" - }, + "flags": {}, + "id": 168, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "on", "parameters": [ { - "id": 84, - "name": "event_name", + "comment": { + "summary": [ + { + "kind": "text", + "text": "The Actor event to listen for." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 169, "kind": 32768, "kindString": "Parameter", - "flags": {}, + "name": "event_name", "type": { + "name": "Literal", "type": "reference", - "name": "Event" + "typeArguments": [ + { + "type": "reference", + "name": "Event.PERSIST_STATE" + } + ] } }, { - "id": 85, - "name": "listener", + "comment": { + "summary": [ + { + "kind": "text", + "text": "The function to be called when the event is emitted (can be async)." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 170, "kind": 32768, "kindString": "Parameter", - "flags": {}, + "name": "listener", "type": { + "name": "EventListener", "type": "reference", - "name": "Callable | None" - }, - "defaultValue": "None" + "typeArguments": [ + { + "type": "reference", + "name": "EventPersistStateData" + } + ] + } } - ] - } - ] - }, - { - "id": 86, - "name": "is_at_home", - "module": "_actor", - "kind": 2048, - "kindString": "Method", - "flags": {}, - "comment": { - "summary": [ - { - "kind": "text", - "text": "{\"content\": [\"Return `True` when the Actor is running on the Apify platform, and `False` otherwise (e.g. local run).\"]}" - } - ] - }, - "children": [], - "groups": [], - "sources": [ - { - "filename": "/tmp-old-source-v27/src/apify/_actor.py", - "line": 654, - "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/_actor.py#L654" - } - ], - "signatures": [ - { - "id": 87, - "name": "is_at_home", - "modifiers": [], - "kind": 4096, - "kindString": "Call signature", - "flags": {}, - "comment": { - "summary": [ + ], + "type": { + "name": "EventListener", + "type": "reference", + "typeArguments": [ { - "kind": "text", - "text": "{\"content\": [\"Return `True` when the Actor is running on the Apify platform, and `False` otherwise (e.g. local run).\"]}" + "type": "reference", + "name": "EventPersistStateData" } ] - }, - "type": { - "type": "intrinsic", - "name": "bool" - }, - "parameters": [] - } - ] - }, - { - "id": 88, - "name": "get_env", - "module": "_actor", - "kind": 2048, - "kindString": "Method", - "flags": {}, - "comment": { - "summary": [ - { - "kind": "text", - "text": "{\"content\": [\"Return a dictionary with information parsed from all the `APIFY_XXX` environment variables.\\n\\nFor a list of all the environment variables, see the\\n[Actor documentation](https://docs.apify.com/actors/development/environment-variables). If some variables\\nare not defined or are invalid, the corresponding value in the resulting dictionary will be None.\"]}" } - ] - }, - "children": [], - "groups": [], - "sources": [ - { - "filename": "/tmp-old-source-v27/src/apify/_actor.py", - "line": 658, - "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/_actor.py#L658" - } - ], - "signatures": [ + }, { - "id": 89, - "name": "get_env", - "modifiers": [], - "kind": 4096, - "kindString": "Call signature", - "flags": {}, "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Return a dictionary with information parsed from all the `APIFY_XXX` environment variables.\\n\\nFor a list of all the environment variables, see the\\n[Actor documentation](https://docs.apify.com/actors/development/environment-variables). If some variables\\nare not defined or are invalid, the corresponding value in the resulting dictionary will be None.\"]}" + "text": "Add an event listener to the Actor's event manager.\n\nThe following events can be emitted:\n\n- `Event.SYSTEM_INFO`: Emitted every minute; the event data contains information about the Actor's resource\nusage.\n\n- `Event.MIGRATING`: Emitted when the Actor on the Apify platform is about to be migrated to another worker\nserver. Use this event to persist the Actor's state and gracefully stop in-progress tasks, preventing\ndisruption.\n\n- `Event.PERSIST_STATE`: Emitted regularly (default: 60 seconds) to notify the Actor to persist its state,\npreventing work repetition after a restart. This event is emitted together with the `MIGRATING` event, where\nthe `isMigrating` flag in the event data is `True`; otherwise, the flag is `False`. This event is for\nconvenience; the same effect can be achieved by setting an interval and listening for the `MIGRATING` event.\n\n- `Event.ABORTING`: Emitted when a user aborts an Actor run on the Apify platform, allowing the Actor time\nto clean up its state if the abort is graceful.\n" } ] }, + "flags": {}, + "id": 171, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "on", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The Actor event to listen for." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 172, + "kind": 32768, + "kindString": "Parameter", + "name": "event_name", + "type": { + "name": "Literal", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "Event.SYSTEM_INFO" + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The function to be called when the event is emitted (can be async)." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 173, + "kind": 32768, + "kindString": "Parameter", + "name": "listener", + "type": { + "name": "EventListener", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "EventSystemInfoData" + } + ] + } + } + ], "type": { - "type": "intrinsic", - "name": "dict" - }, - "parameters": [] - } - ] - }, - { - "id": 90, - "name": "start", - "module": "_actor", - "kind": 2048, - "kindString": "Method", - "flags": {}, - "comment": { - "summary": [ - { - "kind": "text", - "text": "{\"content\": [\"Run an Actor on the Apify platform.\\n\\nUnlike `Actor.call`, this method just starts the run without waiting for finish.\\n\", {\"Arguments\": [{\"param\": \"actor_id\", \"desc\": \"The ID of the Actor to be run.\"}, {\"param\": \"run_input\", \"desc\": \"The input to pass to the Actor run.\"}, {\"param\": \"token\", \"desc\": \"The Apify API token to use for this request (defaults to the `APIFY_TOKEN` environment variable).\"}, {\"param\": \"content_type\", \"desc\": \"The content type of the input.\"}, {\"param\": \"build\", \"desc\": \"Specifies the Actor build to run. It can be either a build tag or build number. By default,\\nthe run uses the build specified in the default run configuration for the Actor (typically latest).\"}, {\"param\": \"memory_mbytes\", \"desc\": \"Memory limit for the run, in megabytes. By default, the run uses a memory limit specified\\nin the default run configuration for the Actor.\"}, {\"param\": \"timeout\", \"desc\": \"Optional timeout for the run, in seconds. By default, the run uses timeout specified in\\nthe default run configuration for the Actor. Using `RemainingTime` will set timeout of the other Actor\\nto the time remaining from this Actor timeout.\"}, {\"param\": \"wait_for_finish\", \"desc\": \"The maximum number of seconds the server waits for the run to finish. By default,\\nit is 0, the maximum value is 300.\"}, {\"param\": \"webhooks\", \"desc\": \"Optional ad-hoc webhooks (https://docs.apify.com/webhooks/ad-hoc-webhooks) associated with\\nthe Actor run which can be used to receive a notification, e.g. when the Actor finished or failed.\\nIf you already have a webhook set up for the Actor or task, you do not have to add it again here.\\n\"}]}, {\"Returns\": [\"Info about the started Actor run\"]}]}" + "name": "EventListener", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "EventSystemInfoData" + } + ] } - ] - }, - "children": [], - "groups": [], - "sources": [ - { - "filename": "/tmp-old-source-v27/src/apify/_actor.py", - "line": 687, - "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/_actor.py#L687" - } - ], - "signatures": [ + }, { - "id": 91, - "name": "start", - "modifiers": [ - "async" - ], - "kind": 4096, - "kindString": "Call signature", - "flags": {}, "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Run an Actor on the Apify platform.\\n\\nUnlike `Actor.call`, this method just starts the run without waiting for finish.\\n\", {\"" + "text": "Add an event listener to the Actor's event manager.\n\nThe following events can be emitted:\n\n- `Event.SYSTEM_INFO`: Emitted every minute; the event data contains information about the Actor's resource\nusage.\n\n- `Event.MIGRATING`: Emitted when the Actor on the Apify platform is about to be migrated to another worker\nserver. Use this event to persist the Actor's state and gracefully stop in-progress tasks, preventing\ndisruption.\n\n- `Event.PERSIST_STATE`: Emitted regularly (default: 60 seconds) to notify the Actor to persist its state,\npreventing work repetition after a restart. This event is emitted together with the `MIGRATING` event, where\nthe `isMigrating` flag in the event data is `True`; otherwise, the flag is `False`. This event is for\nconvenience; the same effect can be achieved by setting an interval and listening for the `MIGRATING` event.\n\n- `Event.ABORTING`: Emitted when a user aborts an Actor run on the Apify platform, allowing the Actor time\nto clean up its state if the abort is graceful.\n" } ] }, - "type": { - "type": "reference", - "name": "ActorRun", - "id": 193 - }, + "flags": {}, + "id": 174, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "on", "parameters": [ { - "id": 92, - "name": "actor_id", + "comment": { + "summary": [ + { + "kind": "text", + "text": "The Actor event to listen for." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 175, "kind": 32768, "kindString": "Parameter", - "flags": {}, + "name": "event_name", "type": { - "type": "intrinsic", - "name": "str" + "name": "Literal", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "Event.MIGRATING" + } + ] } }, { - "id": 93, - "name": "run_input", + "comment": { + "summary": [ + { + "kind": "text", + "text": "The function to be called when the event is emitted (can be async)." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 176, "kind": 32768, "kindString": "Parameter", - "flags": {}, + "name": "listener", "type": { + "name": "EventListener", "type": "reference", - "name": "Any" - }, - "defaultValue": "None" - }, + "typeArguments": [ + { + "type": "reference", + "name": "EventMigratingData" + } + ] + } + } + ], + "type": { + "name": "EventListener", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "EventMigratingData" + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Add an event listener to the Actor's event manager.\n\nThe following events can be emitted:\n\n- `Event.SYSTEM_INFO`: Emitted every minute; the event data contains information about the Actor's resource\nusage.\n\n- `Event.MIGRATING`: Emitted when the Actor on the Apify platform is about to be migrated to another worker\nserver. Use this event to persist the Actor's state and gracefully stop in-progress tasks, preventing\ndisruption.\n\n- `Event.PERSIST_STATE`: Emitted regularly (default: 60 seconds) to notify the Actor to persist its state,\npreventing work repetition after a restart. This event is emitted together with the `MIGRATING` event, where\nthe `isMigrating` flag in the event data is `True`; otherwise, the flag is `False`. This event is for\nconvenience; the same effect can be achieved by setting an interval and listening for the `MIGRATING` event.\n\n- `Event.ABORTING`: Emitted when a user aborts an Actor run on the Apify platform, allowing the Actor time\nto clean up its state if the abort is graceful.\n" + } + ] + }, + "flags": {}, + "id": 177, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "on", + "parameters": [ { - "id": 94, - "name": "token", - "kind": 32768, - "kindString": "Parameter", - "flags": { - "keyword-only": "true" + "comment": { + "summary": [ + { + "kind": "text", + "text": "The Actor event to listen for." + } + ] }, - "type": { - "type": "reference", - "name": "str | None" + "flags": { + "isOptional": false, + "keyword-only": false }, - "defaultValue": "None" - }, - { - "id": 95, - "name": "content_type", + "id": 178, "kind": 32768, "kindString": "Parameter", - "flags": { - "keyword-only": "true" - }, + "name": "event_name", "type": { + "name": "Literal", "type": "reference", - "name": "str | None" - }, - "defaultValue": "None" + "typeArguments": [ + { + "type": "reference", + "name": "Event.ABORTING" + } + ] + } }, { - "id": 96, - "name": "build", - "kind": 32768, - "kindString": "Parameter", + "comment": { + "summary": [ + { + "kind": "text", + "text": "The function to be called when the event is emitted (can be async)." + } + ] + }, "flags": { - "keyword-only": "true" + "isOptional": false, + "keyword-only": false }, + "id": 179, + "kind": 32768, + "kindString": "Parameter", + "name": "listener", "type": { + "name": "EventListener", "type": "reference", - "name": "str | None" - }, - "defaultValue": "None" - }, + "typeArguments": [ + { + "type": "reference", + "name": "EventAbortingData" + } + ] + } + } + ], + "type": { + "name": "EventListener", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "EventAbortingData" + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Add an event listener to the Actor's event manager.\n\nThe following events can be emitted:\n\n- `Event.SYSTEM_INFO`: Emitted every minute; the event data contains information about the Actor's resource\nusage.\n\n- `Event.MIGRATING`: Emitted when the Actor on the Apify platform is about to be migrated to another worker\nserver. Use this event to persist the Actor's state and gracefully stop in-progress tasks, preventing\ndisruption.\n\n- `Event.PERSIST_STATE`: Emitted regularly (default: 60 seconds) to notify the Actor to persist its state,\npreventing work repetition after a restart. This event is emitted together with the `MIGRATING` event, where\nthe `isMigrating` flag in the event data is `True`; otherwise, the flag is `False`. This event is for\nconvenience; the same effect can be achieved by setting an interval and listening for the `MIGRATING` event.\n\n- `Event.ABORTING`: Emitted when a user aborts an Actor run on the Apify platform, allowing the Actor time\nto clean up its state if the abort is graceful.\n" + } + ] + }, + "flags": {}, + "id": 180, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "on", + "parameters": [ { - "id": 97, - "name": "memory_mbytes", - "kind": 32768, - "kindString": "Parameter", + "comment": { + "summary": [ + { + "kind": "text", + "text": "The Actor event to listen for." + } + ] + }, "flags": { - "keyword-only": "true" + "isOptional": false, + "keyword-only": false }, + "id": 181, + "kind": 32768, + "kindString": "Parameter", + "name": "event_name", "type": { + "name": "Literal", "type": "reference", - "name": "int | None" - }, - "defaultValue": "None" + "typeArguments": [ + { + "type": "reference", + "name": "Event.EXIT" + } + ] + } }, { - "id": 98, - "name": "timeout", - "kind": 32768, - "kindString": "Parameter", + "comment": { + "summary": [ + { + "kind": "text", + "text": "The function to be called when the event is emitted (can be async)." + } + ] + }, "flags": { - "keyword-only": "true" + "isOptional": false, + "keyword-only": false }, + "id": 182, + "kind": 32768, + "kindString": "Parameter", + "name": "listener", "type": { + "name": "EventListener", "type": "reference", - "name": "timedelta | None | Literal['RemainingTime']" - }, - "defaultValue": "None" - }, + "typeArguments": [ + { + "type": "reference", + "name": "EventExitData" + } + ] + } + } + ], + "type": { + "name": "EventListener", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "EventExitData" + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Add an event listener to the Actor's event manager.\n\nThe following events can be emitted:\n\n- `Event.SYSTEM_INFO`: Emitted every minute; the event data contains information about the Actor's resource\nusage.\n\n- `Event.MIGRATING`: Emitted when the Actor on the Apify platform is about to be migrated to another worker\nserver. Use this event to persist the Actor's state and gracefully stop in-progress tasks, preventing\ndisruption.\n\n- `Event.PERSIST_STATE`: Emitted regularly (default: 60 seconds) to notify the Actor to persist its state,\npreventing work repetition after a restart. This event is emitted together with the `MIGRATING` event, where\nthe `isMigrating` flag in the event data is `True`; otherwise, the flag is `False`. This event is for\nconvenience; the same effect can be achieved by setting an interval and listening for the `MIGRATING` event.\n\n- `Event.ABORTING`: Emitted when a user aborts an Actor run on the Apify platform, allowing the Actor time\nto clean up its state if the abort is graceful.\n" + } + ] + }, + "flags": {}, + "id": 183, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "on", + "parameters": [ { - "id": 99, - "name": "wait_for_finish", - "kind": 32768, - "kindString": "Parameter", + "comment": { + "summary": [ + { + "kind": "text", + "text": "The Actor event to listen for." + } + ] + }, "flags": { - "keyword-only": "true" + "isOptional": false, + "keyword-only": false }, + "id": 184, + "kind": 32768, + "kindString": "Parameter", + "name": "event_name", "type": { - "type": "reference", - "name": "int | None" - }, - "defaultValue": "None" + "name": "Event", + "type": "reference" + } }, { - "id": 100, - "name": "webhooks", - "kind": 32768, - "kindString": "Parameter", + "comment": { + "summary": [ + { + "kind": "text", + "text": "The function to be called when the event is emitted (can be async)." + } + ] + }, "flags": { - "keyword-only": "true" + "isOptional": false, + "keyword-only": false }, + "id": 185, + "kind": 32768, + "kindString": "Parameter", + "name": "listener", "type": { + "name": "EventListener", "type": "reference", - "name": "list[Webhook] | None" - }, - "defaultValue": "None" + "typeArguments": [ + { + "type": "literal", + "value": null + } + ] + } } - ] + ], + "type": { + "name": "EventListener", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "Any" + } + ] + } } ] }, { - "id": 101, - "name": "abort", - "module": "_actor", "kind": 2048, "kindString": "Method", - "flags": {}, + "children": [], "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Abort given Actor run on the Apify platform using the current user account.\\n\\nThe user account is determined by the `APIFY_TOKEN` environment variable.\\n\", {\"Arguments\": [{\"param\": \"run_id\", \"desc\": \"The ID of the Actor run to be aborted.\"}, {\"param\": \"token\", \"desc\": \"The Apify API token to use for this request (defaults to the `APIFY_TOKEN` environment variable).\"}, {\"param\": \"status_message\", \"desc\": \"Status message of the Actor to be set on the platform.\"}, {\"param\": \"gracefully\", \"desc\": \"If True, the Actor run will abort gracefully. It will send `aborting` and `persistState`\\nevents into the run and force-stop the run after 30 seconds. It is helpful in cases where you plan\\nto resurrect the run later.\\n\"}]}, {\"Returns\": [\"Info about the aborted Actor run.\"]}]}" + "text": "Remove a listener, or all listeners, from an Actor event.\n" } ] }, - "children": [], + "decorations": [], + "flags": {}, "groups": [], + "id": 87, + "module": "_actor", + "name": "off", + "parsedDocstring": { + "text": "Remove a listener, or all listeners, from an Actor event.\n", + "args": { + "event_name": "The Actor event for which to remove listeners.", + "listener": "The listener which is supposed to be removed. If not passed, all listeners of this event\nare removed." + } + }, "sources": [ { - "filename": "/tmp-old-source-v27/src/apify/_actor.py", - "line": 769, "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/_actor.py#L769" + "fileName": "/src/apify/_actor.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 642 } ], + "type": { + "name": "Undefined", + "type": "reference" + }, "signatures": [ { - "id": 102, - "name": "abort", - "modifiers": [ - "async" - ], - "kind": 4096, - "kindString": "Call signature", - "flags": {}, "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Abort given Actor run on the Apify platform using the current user account.\\n\\nThe user account is determined by the `APIFY_TOKEN` environment variable.\\n\", {\"" + "text": "Remove a listener, or all listeners, from an Actor event.\n" } ] }, - "type": { - "type": "reference", - "name": "ActorRun", - "id": 193 - }, + "flags": {}, + "id": 88, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "off", "parameters": [ { - "id": 103, - "name": "run_id", + "comment": { + "summary": [ + { + "kind": "text", + "text": "The Actor event for which to remove listeners." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 89, "kind": 32768, "kindString": "Parameter", - "flags": {}, + "name": "event_name", "type": { - "type": "intrinsic", - "name": "str" + "name": "Event", + "type": "reference" } }, { - "id": 104, - "name": "token", - "kind": 32768, - "kindString": "Parameter", - "flags": { - "keyword-only": "true" - }, - "type": { - "type": "reference", - "name": "str | None" + "comment": { + "summary": [ + { + "kind": "text", + "text": "The listener which is supposed to be removed. If not passed, all listeners of this event\nare removed." + } + ] }, - "defaultValue": "None" - }, - { - "id": 105, - "name": "status_message", - "kind": 32768, - "kindString": "Parameter", + "defaultValue": "None", "flags": { - "keyword-only": "true" + "isOptional": true, + "keyword-only": false }, - "type": { - "type": "reference", - "name": "str | None" - }, - "defaultValue": "None" - }, - { - "id": 106, - "name": "gracefully", + "id": 90, "kind": 32768, "kindString": "Parameter", - "flags": { - "keyword-only": "true" - }, + "name": "listener", "type": { - "type": "reference", - "name": "bool | None" - }, - "defaultValue": "None" + "name": "Callable | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "Callable" + }, + { + "type": "literal", + "value": null + } + ] + } } - ] - } - ] - }, - { - "id": 107, - "name": "call", - "module": "_actor", - "kind": 2048, - "kindString": "Method", - "flags": {}, - "comment": { - "summary": [ - { - "kind": "text", - "text": "{\"content\": [\"Start an Actor on the Apify Platform and wait for it to finish before returning.\\n\\nIt waits indefinitely, unless the wait argument is provided.\\n\", {\"Arguments\": [{\"param\": \"actor_id\", \"desc\": \"The ID of the Actor to be run.\"}, {\"param\": \"run_input\", \"desc\": \"The input to pass to the Actor run.\"}, {\"param\": \"token\", \"desc\": \"The Apify API token to use for this request (defaults to the `APIFY_TOKEN` environment variable).\"}, {\"param\": \"content_type\", \"desc\": \"The content type of the input.\"}, {\"param\": \"build\", \"desc\": \"Specifies the Actor build to run. It can be either a build tag or build number. By default,\\nthe run uses the build specified in the default run configuration for the Actor (typically latest).\"}, {\"param\": \"memory_mbytes\", \"desc\": \"Memory limit for the run, in megabytes. By default, the run uses a memory limit specified\\nin the default run configuration for the Actor.\"}, {\"param\": \"timeout\", \"desc\": \"Optional timeout for the run, in seconds. By default, the run uses timeout specified in\\nthe default run configuration for the Actor. Using `RemainingTime` will set timeout of the other Actor\\nto the time remaining from this Actor timeout.\"}, {\"param\": \"webhooks\", \"desc\": \"Optional webhooks (https://docs.apify.com/webhooks) associated with the Actor run, which can\\nbe used to receive a notification, e.g. when the Actor finished or failed. If you already have\\na webhook set up for the Actor, you do not have to add it again here.\"}, {\"param\": \"wait\", \"desc\": \"The maximum number of seconds the server waits for the run to finish. If not provided,\\nwaits indefinitely.\"}, {\"param\": \"logger\", \"desc\": \"Logger used to redirect logs from the Actor run. Using \\\"default\\\" literal means that a predefined\\ndefault logger will be used. Setting `None` will disable any log propagation. Passing custom logger\\nwill redirect logs to the provided logger.\\n\"}]}, {\"Returns\": [\"Info about the started Actor run.\"]}]}" + ], + "type": { + "name": "None", + "type": "literal", + "value": null } - ] - }, - "children": [], - "groups": [], - "sources": [ - { - "filename": "/tmp-old-source-v27/src/apify/_actor.py", - "line": 803, - "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/_actor.py#L803" - } - ], - "signatures": [ + }, { - "id": 108, - "name": "call", - "modifiers": [ - "async" - ], - "kind": 4096, - "kindString": "Call signature", - "flags": {}, "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Start an Actor on the Apify Platform and wait for it to finish before returning.\\n\\nIt waits indefinitely, unless the wait argument is provided.\\n\", {\"" + "text": "Remove a listener, or all listeners, from an Actor event.\n" } ] }, - "type": { - "type": "reference", - "name": "ActorRun | None" - }, + "flags": {}, + "id": 186, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "off", "parameters": [ { - "id": 109, - "name": "actor_id", + "comment": { + "summary": [ + { + "kind": "text", + "text": "The Actor event for which to remove listeners." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 187, "kind": 32768, "kindString": "Parameter", - "flags": {}, + "name": "event_name", "type": { - "type": "intrinsic", - "name": "str" + "name": "Literal", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "Event.PERSIST_STATE" + } + ] } }, { - "id": 110, - "name": "run_input", + "comment": { + "summary": [ + { + "kind": "text", + "text": "The listener which is supposed to be removed. If not passed, all listeners of this event\nare removed." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 188, "kind": 32768, "kindString": "Parameter", - "flags": {}, + "name": "listener", "type": { + "name": "EventListener", "type": "reference", - "name": "Any" - }, - "defaultValue": "None" - }, + "typeArguments": [ + { + "type": "reference", + "name": "EventPersistStateData" + } + ] + } + } + ], + "type": { + "name": "None", + "type": "literal", + "value": null + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Remove a listener, or all listeners, from an Actor event.\n" + } + ] + }, + "flags": {}, + "id": 189, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "off", + "parameters": [ { - "id": 111, - "name": "token", - "kind": 32768, - "kindString": "Parameter", - "flags": { - "keyword-only": "true" + "comment": { + "summary": [ + { + "kind": "text", + "text": "The Actor event for which to remove listeners." + } + ] }, - "type": { - "type": "reference", - "name": "str | None" + "flags": { + "isOptional": false, + "keyword-only": false }, - "defaultValue": "None" - }, - { - "id": 112, - "name": "content_type", + "id": 190, "kind": 32768, "kindString": "Parameter", - "flags": { - "keyword-only": "true" - }, + "name": "event_name", "type": { + "name": "Literal", "type": "reference", - "name": "str | None" - }, - "defaultValue": "None" + "typeArguments": [ + { + "type": "reference", + "name": "Event.SYSTEM_INFO" + } + ] + } }, { - "id": 113, - "name": "build", - "kind": 32768, - "kindString": "Parameter", - "flags": { - "keyword-only": "true" + "comment": { + "summary": [ + { + "kind": "text", + "text": "The listener which is supposed to be removed. If not passed, all listeners of this event\nare removed." + } + ] }, - "type": { - "type": "reference", - "name": "str | None" + "flags": { + "isOptional": false, + "keyword-only": false }, - "defaultValue": "None" - }, - { - "id": 114, - "name": "memory_mbytes", + "id": 191, "kind": 32768, "kindString": "Parameter", - "flags": { - "keyword-only": "true" - }, + "name": "listener", "type": { + "name": "EventListener", "type": "reference", - "name": "int | None" - }, - "defaultValue": "None" - }, + "typeArguments": [ + { + "type": "reference", + "name": "EventSystemInfoData" + } + ] + } + } + ], + "type": { + "name": "None", + "type": "literal", + "value": null + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Remove a listener, or all listeners, from an Actor event.\n" + } + ] + }, + "flags": {}, + "id": 192, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "off", + "parameters": [ { - "id": 115, - "name": "timeout", - "kind": 32768, - "kindString": "Parameter", - "flags": { - "keyword-only": "true" + "comment": { + "summary": [ + { + "kind": "text", + "text": "The Actor event for which to remove listeners." + } + ] }, - "type": { - "type": "reference", - "name": "timedelta | None | Literal['RemainingTime']" + "flags": { + "isOptional": false, + "keyword-only": false }, - "defaultValue": "None" - }, - { - "id": 116, - "name": "webhooks", + "id": 193, "kind": 32768, "kindString": "Parameter", - "flags": { - "keyword-only": "true" - }, + "name": "event_name", "type": { + "name": "Literal", "type": "reference", - "name": "list[Webhook] | None" - }, - "defaultValue": "None" + "typeArguments": [ + { + "type": "reference", + "name": "Event.MIGRATING" + } + ] + } }, { - "id": 117, - "name": "wait", - "kind": 32768, - "kindString": "Parameter", - "flags": { - "keyword-only": "true" + "comment": { + "summary": [ + { + "kind": "text", + "text": "The listener which is supposed to be removed. If not passed, all listeners of this event\nare removed." + } + ] }, - "type": { - "type": "reference", - "name": "timedelta | None" + "flags": { + "isOptional": false, + "keyword-only": false }, - "defaultValue": "None" - }, - { - "id": 118, - "name": "logger", + "id": 194, "kind": 32768, "kindString": "Parameter", - "flags": { - "keyword-only": "true" - }, + "name": "listener", "type": { + "name": "EventListener", "type": "reference", - "name": "logging.Logger | None | Literal['default']" - }, - "defaultValue": "'default'" + "typeArguments": [ + { + "type": "reference", + "name": "EventMigratingData" + } + ] + } } - ] - } - ] - }, - { - "id": 119, - "name": "call_task", - "module": "_actor", - "kind": 2048, - "kindString": "Method", - "flags": {}, - "comment": { - "summary": [ - { - "kind": "text", - "text": "{\"content\": [\"Start an Actor task on the Apify Platform and wait for it to finish before returning.\\n\\nIt waits indefinitely, unless the wait argument is provided.\\n\\nNote that an Actor task is a saved input configuration and options for an Actor. If you want to run an Actor\\ndirectly rather than an Actor task, please use the `Actor.call`\\n\", {\"Arguments\": [{\"param\": \"task_id\", \"desc\": \"The ID of the Actor to be run.\"}, {\"param\": \"task_input\", \"desc\": \"Overrides the input to pass to the Actor run.\"}, {\"param\": \"token\", \"desc\": \"The Apify API token to use for this request (defaults to the `APIFY_TOKEN` environment variable).\"}, {\"param\": \"content_type\", \"desc\": \"The content type of the input.\"}, {\"param\": \"build\", \"desc\": \"Specifies the Actor build to run. It can be either a build tag or build number. By default,\\nthe run uses the build specified in the default run configuration for the Actor (typically latest).\"}, {\"param\": \"memory_mbytes\", \"desc\": \"Memory limit for the run, in megabytes. By default, the run uses a memory limit specified\\nin the default run configuration for the Actor.\"}, {\"param\": \"timeout\", \"desc\": \"Optional timeout for the run, in seconds. By default, the run uses timeout specified in\\nthe default run configuration for the Actor.\"}, {\"param\": \"webhooks\", \"desc\": \"Optional webhooks (https://docs.apify.com/webhooks) associated with the Actor run, which can\\nbe used to receive a notification, e.g. when the Actor finished or failed. If you already have\\na webhook set up for the Actor, you do not have to add it again here.\"}, {\"param\": \"wait\", \"desc\": \"The maximum number of seconds the server waits for the run to finish. If not provided, waits\\nindefinitely.\\n\"}]}, {\"Returns\": [\"Info about the started Actor run.\"]}]}" + ], + "type": { + "name": "None", + "type": "literal", + "value": null } - ] - }, - "children": [], - "groups": [], - "sources": [ - { - "filename": "/tmp-old-source-v27/src/apify/_actor.py", - "line": 878, - "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/_actor.py#L878" - } - ], - "signatures": [ + }, { - "id": 120, - "name": "call_task", - "modifiers": [ - "async" - ], - "kind": 4096, - "kindString": "Call signature", - "flags": {}, "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Start an Actor task on the Apify Platform and wait for it to finish before returning.\\n\\nIt waits indefinitely, unless the wait argument is provided.\\n\\nNote that an Actor task is a saved input configuration and options for an Actor. If you want to run an Actor\\ndirectly rather than an Actor task, please use the `Actor.call`\\n\", {\"" + "text": "Remove a listener, or all listeners, from an Actor event.\n" } ] }, - "type": { - "type": "reference", - "name": "ActorRun | None" - }, + "flags": {}, + "id": 195, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "off", "parameters": [ { - "id": 121, - "name": "task_id", + "comment": { + "summary": [ + { + "kind": "text", + "text": "The Actor event for which to remove listeners." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 196, "kind": 32768, "kindString": "Parameter", - "flags": {}, + "name": "event_name", "type": { - "type": "intrinsic", - "name": "str" + "name": "Literal", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "Event.ABORTING" + } + ] } }, { - "id": 122, - "name": "task_input", + "comment": { + "summary": [ + { + "kind": "text", + "text": "The listener which is supposed to be removed. If not passed, all listeners of this event\nare removed." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 197, "kind": 32768, "kindString": "Parameter", - "flags": {}, + "name": "listener", "type": { + "name": "EventListener", "type": "reference", - "name": "dict | None" - }, - "defaultValue": "None" - }, + "typeArguments": [ + { + "type": "reference", + "name": "EventAbortingData" + } + ] + } + } + ], + "type": { + "name": "None", + "type": "literal", + "value": null + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Remove a listener, or all listeners, from an Actor event.\n" + } + ] + }, + "flags": {}, + "id": 198, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "off", + "parameters": [ { - "id": 123, - "name": "build", - "kind": 32768, - "kindString": "Parameter", - "flags": { - "keyword-only": "true" + "comment": { + "summary": [ + { + "kind": "text", + "text": "The Actor event for which to remove listeners." + } + ] }, - "type": { - "type": "reference", - "name": "str | None" + "flags": { + "isOptional": false, + "keyword-only": false }, - "defaultValue": "None" - }, - { - "id": 124, - "name": "memory_mbytes", + "id": 199, "kind": 32768, "kindString": "Parameter", - "flags": { - "keyword-only": "true" - }, + "name": "event_name", "type": { + "name": "Literal", "type": "reference", - "name": "int | None" - }, - "defaultValue": "None" + "typeArguments": [ + { + "type": "reference", + "name": "Event.EXIT" + } + ] + } }, { - "id": 125, - "name": "timeout", - "kind": 32768, - "kindString": "Parameter", - "flags": { - "keyword-only": "true" + "comment": { + "summary": [ + { + "kind": "text", + "text": "The listener which is supposed to be removed. If not passed, all listeners of this event\nare removed." + } + ] }, - "type": { - "type": "reference", - "name": "timedelta | None" + "flags": { + "isOptional": false, + "keyword-only": false }, - "defaultValue": "None" - }, - { - "id": 126, - "name": "webhooks", + "id": 200, "kind": 32768, "kindString": "Parameter", - "flags": { - "keyword-only": "true" - }, + "name": "listener", "type": { + "name": "EventListener", "type": "reference", - "name": "list[Webhook] | None" - }, - "defaultValue": "None" - }, + "typeArguments": [ + { + "type": "reference", + "name": "EventExitData" + } + ] + } + } + ], + "type": { + "name": "None", + "type": "literal", + "value": null + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Remove a listener, or all listeners, from an Actor event.\n" + } + ] + }, + "flags": {}, + "id": 201, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "off", + "parameters": [ { - "id": 127, - "name": "wait", - "kind": 32768, - "kindString": "Parameter", + "comment": { + "summary": [ + { + "kind": "text", + "text": "The Actor event for which to remove listeners." + } + ] + }, "flags": { - "keyword-only": "true" + "isOptional": false, + "keyword-only": false }, + "id": 202, + "kind": 32768, + "kindString": "Parameter", + "name": "event_name", "type": { - "type": "reference", - "name": "timedelta | None" - }, - "defaultValue": "None" + "name": "Event", + "type": "reference" + } }, { - "id": 128, - "name": "token", - "kind": 32768, - "kindString": "Parameter", + "comment": { + "summary": [ + { + "kind": "text", + "text": "The listener which is supposed to be removed. If not passed, all listeners of this event\nare removed." + } + ] + }, "flags": { - "keyword-only": "true" + "isOptional": false, + "keyword-only": false }, + "id": 203, + "kind": 32768, + "kindString": "Parameter", + "name": "listener", "type": { + "name": "EventListener", "type": "reference", - "name": "str | None" - }, - "defaultValue": "None" + "typeArguments": [ + { + "type": "literal", + "value": null + } + ] + } } - ] + ], + "type": { + "name": "None", + "type": "literal", + "value": null + } } ] }, { - "id": 129, - "name": "metamorph", - "module": "_actor", "kind": 2048, "kindString": "Method", - "flags": {}, + "children": [], "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Transform this Actor run to an Actor run of a different Actor.\\n\\nThe platform stops the current Actor container and starts a new container with the new Actor instead. All\\nthe default storages are preserved, and the new input is stored under the `INPUT-METAMORPH-1` key in the same\\ndefault key-value store.\\n\", {\"Arguments\": [{\"param\": \"target_actor_id\", \"desc\": \"ID of the target Actor that the run should be transformed into\"}, {\"param\": \"run_input\", \"desc\": \"The input to pass to the new run.\"}, {\"param\": \"target_actor_build\", \"desc\": \"The build of the target Actor. It can be either a build tag or build number.\\nBy default, the run uses the build specified in the default run configuration for the target Actor\\n(typically the latest build).\"}, {\"param\": \"content_type\", \"desc\": \"The content type of the input.\"}, {\"param\": \"custom_after_sleep\", \"desc\": \"How long to sleep for after the metamorph, to wait for the container to be stopped.\"}]}]}" + "text": "Return `True` when the Actor is running on the Apify platform, and `False` otherwise (e.g. local run)." } ] }, - "children": [], + "decorations": [], + "flags": {}, "groups": [], + "id": 91, + "module": "_actor", + "name": "is_at_home", + "parsedDocstring": { + "text": "Return `True` when the Actor is running on the Apify platform, and `False` otherwise (e.g. local run)." + }, "sources": [ { - "filename": "/tmp-old-source-v27/src/apify/_actor.py", - "line": 939, "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/_actor.py#L939" + "fileName": "/src/apify/_actor.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 654 } ], + "type": { + "name": "Undefined", + "type": "reference" + }, "signatures": [ { - "id": 130, - "name": "metamorph", - "modifiers": [ - "async" - ], - "kind": 4096, - "kindString": "Call signature", - "flags": {}, "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Transform this Actor run to an Actor run of a different Actor.\\n\\nThe platform stops the current Actor container and starts a new container with the new Actor instead. All\\nthe default storages are preserved, and the new input is stored under the `INPUT-METAMORPH-1` key in the same\\ndefault key-value store.\\n\", {\"" + "text": "Return `True` when the Actor is running on the Apify platform, and `False` otherwise (e.g. local run)." } ] }, + "flags": {}, + "id": 92, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "is_at_home", + "parameters": [], "type": { - "type": "reference", - "name": "None" - }, - "parameters": [ - { - "id": 131, - "name": "target_actor_id", - "kind": 32768, - "kindString": "Parameter", - "flags": {}, - "type": { - "type": "intrinsic", - "name": "str" - } - }, - { - "id": 132, - "name": "run_input", - "kind": 32768, - "kindString": "Parameter", - "flags": {}, - "type": { - "type": "reference", - "name": "Any" - }, - "defaultValue": "None" - }, - { - "id": 133, - "name": "target_actor_build", - "kind": 32768, - "kindString": "Parameter", - "flags": { - "keyword-only": "true" - }, - "type": { - "type": "reference", - "name": "str | None" - }, - "defaultValue": "None" - }, - { - "id": 134, - "name": "content_type", - "kind": 32768, - "kindString": "Parameter", - "flags": { - "keyword-only": "true" - }, - "type": { - "type": "reference", - "name": "str | None" - }, - "defaultValue": "None" - }, - { - "id": 135, - "name": "custom_after_sleep", - "kind": 32768, - "kindString": "Parameter", - "flags": { - "keyword-only": "true" - }, - "type": { - "type": "reference", - "name": "timedelta | None" - }, - "defaultValue": "None" - } - ] + "name": "bool", + "type": "reference" + } } ] }, { - "id": 136, - "name": "reboot", - "module": "_actor", "kind": 2048, "kindString": "Method", - "flags": {}, + "children": [], "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Internally reboot this Actor.\\n\\nThe system stops the current container and starts a new one, with the same run ID and default storages.\\n\", {\"Arguments\": [{\"param\": \"event_listeners_timeout\", \"desc\": \"How long should the Actor wait for Actor event listeners to finish before exiting.\"}, {\"param\": \"custom_after_sleep\", \"desc\": \"How long to sleep for after the reboot, to wait for the container to be stopped.\"}]}]}" + "text": "Return a dictionary with information parsed from all the `APIFY_XXX` environment variables.\n\nFor a list of all the environment variables, see the\n[Actor documentation](https://docs.apify.com/actors/development/environment-variables). If some variables\nare not defined or are invalid, the corresponding value in the resulting dictionary will be None." } ] }, - "children": [], + "decorations": [], + "flags": {}, "groups": [], + "id": 93, + "module": "_actor", + "name": "get_env", + "parsedDocstring": { + "text": "Return a dictionary with information parsed from all the `APIFY_XXX` environment variables.\n\nFor a list of all the environment variables, see the\n[Actor documentation](https://docs.apify.com/actors/development/environment-variables). If some variables\nare not defined or are invalid, the corresponding value in the resulting dictionary will be None." + }, "sources": [ { - "filename": "/tmp-old-source-v27/src/apify/_actor.py", - "line": 986, "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/_actor.py#L986" + "fileName": "/src/apify/_actor.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 658 } ], + "type": { + "name": "Undefined", + "type": "reference" + }, "signatures": [ { - "id": 137, - "name": "reboot", - "modifiers": [ - "async" - ], - "kind": 4096, - "kindString": "Call signature", - "flags": {}, "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Internally reboot this Actor.\\n\\nThe system stops the current container and starts a new one, with the same run ID and default storages.\\n\", {\"" + "text": "Return a dictionary with information parsed from all the `APIFY_XXX` environment variables.\n\nFor a list of all the environment variables, see the\n[Actor documentation](https://docs.apify.com/actors/development/environment-variables). If some variables\nare not defined or are invalid, the corresponding value in the resulting dictionary will be None." } ] }, + "flags": {}, + "id": 94, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "get_env", + "parameters": [], "type": { - "type": "reference", - "name": "None" - }, - "parameters": [ - { - "id": 138, - "name": "event_listeners_timeout", - "kind": 32768, - "kindString": "Parameter", - "flags": { - "keyword-only": "true" - }, - "type": { - "type": "reference", - "name": "timedelta | None" - }, - "defaultValue": "EVENT_LISTENERS_TIMEOUT" - }, - { - "id": 139, - "name": "custom_after_sleep", - "kind": 32768, - "kindString": "Parameter", - "flags": { - "keyword-only": "true" - }, - "type": { - "type": "reference", - "name": "timedelta | None" - }, - "defaultValue": "None" - } - ] + "name": "dict", + "type": "reference" + } } ] }, { - "id": 140, - "name": "add_webhook", - "module": "_actor", "kind": 2048, "kindString": "Method", - "flags": {}, + "children": [], "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Create an ad-hoc webhook for the current Actor run.\\n\\nThis webhook lets you receive a notification when the Actor run finished or failed.\\n\\nNote that webhooks are only supported for Actors running on the Apify platform. When running the Actor locally,\\nthe function will print a warning and have no effect.\\n\\nFor more information about Apify Actor webhooks, please see the [documentation](https://docs.apify.com/webhooks).\\n\", {\"Arguments\": [{\"param\": \"webhook\", \"desc\": \"The webhook to be added\"}, {\"param\": \"ignore_ssl_errors\", \"desc\": \"Whether the webhook should ignore SSL errors returned by request_url\"}, {\"param\": \"do_not_retry\", \"desc\": \"Whether the webhook should retry sending the payload to request_url upon failure.\"}, {\"param\": \"idempotency_key\", \"desc\": \"A unique identifier of a webhook. You can use it to ensure that you won't create\\nthe same webhook multiple times.\\n\"}]}, {\"Returns\": [\"The created webhook.\"]}]}" + "text": "Run an Actor on the Apify platform.\n\nUnlike `Actor.call`, this method just starts the run without waiting for finish.\n" } ] }, - "children": [], + "decorations": [], + "flags": {}, "groups": [], + "id": 95, + "module": "_actor", + "name": "start", + "parsedDocstring": { + "text": "Run an Actor on the Apify platform.\n\nUnlike `Actor.call`, this method just starts the run without waiting for finish.\n", + "args": { + "actor_id": "The ID of the Actor to be run.", + "run_input": "The input to pass to the Actor run.", + "token": "The Apify API token to use for this request (defaults to the `APIFY_TOKEN` environment variable).", + "content_type": "The content type of the input.", + "build": "Specifies the Actor build to run. It can be either a build tag or build number. By default,\nthe run uses the build specified in the default run configuration for the Actor (typically latest).", + "memory_mbytes": "Memory limit for the run, in megabytes. By default, the run uses a memory limit specified\nin the default run configuration for the Actor.", + "timeout": "Optional timeout for the run, in seconds. By default, the run uses timeout specified in\nthe default run configuration for the Actor. Using `RemainingTime` will set timeout of the other Actor\nto the time remaining from this Actor timeout.", + "wait_for_finish": "The maximum number of seconds the server waits for the run to finish. By default,\nit is 0, the maximum value is 300.", + "webhooks": "Optional ad-hoc webhooks (https://docs.apify.com/webhooks/ad-hoc-webhooks) associated with\nthe Actor run which can be used to receive a notification, e.g. when the Actor finished or failed.\nIf you already have a webhook set up for the Actor or task, you do not have to add it again here.\n" + }, + "returns": "Info about the started Actor run" + }, "sources": [ { - "filename": "/tmp-old-source-v27/src/apify/_actor.py", - "line": 1041, "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/_actor.py#L1041" + "fileName": "/src/apify/_actor.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 687 } ], + "type": { + "name": "Undefined", + "type": "reference" + }, "signatures": [ { - "id": 141, - "name": "add_webhook", - "modifiers": [ - "async" - ], - "kind": 4096, - "kindString": "Call signature", - "flags": {}, "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "Info about the started Actor run" + } + ], + "tag": "@returns" + } + ], "summary": [ { "kind": "text", - "text": "{\"content\": [\"Create an ad-hoc webhook for the current Actor run.\\n\\nThis webhook lets you receive a notification when the Actor run finished or failed.\\n\\nNote that webhooks are only supported for Actors running on the Apify platform. When running the Actor locally,\\nthe function will print a warning and have no effect.\\n\\nFor more information about Apify Actor webhooks, please see the [documentation](https://docs.apify.com/webhooks).\\n\", {\"" + "text": "Run an Actor on the Apify platform.\n\nUnlike `Actor.call`, this method just starts the run without waiting for finish.\n" } ] }, - "type": { - "type": "reference", - "name": "None" - }, - "parameters": [ + "flags": {}, + "id": 96, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "start", + "parameters": [ { - "id": 142, - "name": "webhook", + "comment": { + "summary": [ + { + "kind": "text", + "text": "The ID of the Actor to be run." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 97, "kind": 32768, "kindString": "Parameter", - "flags": {}, + "name": "actor_id", "type": { - "type": "reference", - "name": "Webhook", - "id": 188 + "name": "str", + "type": "reference" } }, { - "id": 143, - "name": "ignore_ssl_errors", + "comment": { + "summary": [ + { + "kind": "text", + "text": "The input to pass to the Actor run." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": false + }, + "id": 98, "kind": 32768, "kindString": "Parameter", + "name": "run_input", + "type": { + "name": "Any", + "type": "reference" + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The Apify API token to use for this request (defaults to the `APIFY_TOKEN` environment variable)." + } + ] + }, + "defaultValue": "None", "flags": { - "keyword-only": "true" + "isOptional": true, + "keyword-only": true }, + "id": 99, + "kind": 32768, + "kindString": "Parameter", + "name": "token", "type": { - "type": "reference", - "name": "bool | None" - }, - "defaultValue": "None" + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } }, { - "id": 144, - "name": "do_not_retry", + "comment": { + "summary": [ + { + "kind": "text", + "text": "The content type of the input." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 100, "kind": 32768, "kindString": "Parameter", + "name": "content_type", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Specifies the Actor build to run. It can be either a build tag or build number. By default,\nthe run uses the build specified in the default run configuration for the Actor (typically latest)." + } + ] + }, + "defaultValue": "None", "flags": { - "keyword-only": "true" + "isOptional": true, + "keyword-only": true }, + "id": 101, + "kind": 32768, + "kindString": "Parameter", + "name": "build", "type": { - "type": "reference", - "name": "bool | None" + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Memory limit for the run, in megabytes. By default, the run uses a memory limit specified\nin the default run configuration for the Actor." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true }, - "defaultValue": "None" + "id": 102, + "kind": 32768, + "kindString": "Parameter", + "name": "memory_mbytes", + "type": { + "name": "int | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "int" + }, + { + "type": "literal", + "value": null + } + ] + } }, { - "id": 145, - "name": "idempotency_key", + "comment": { + "summary": [ + { + "kind": "text", + "text": "Optional timeout for the run, in seconds. By default, the run uses timeout specified in\nthe default run configuration for the Actor. Using `RemainingTime` will set timeout of the other Actor\nto the time remaining from this Actor timeout." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 103, "kind": 32768, "kindString": "Parameter", + "name": "timeout", + "type": { + "name": "timedelta | None | Literal['RemainingTime']", + "type": "union", + "types": [ + { + "type": "union", + "types": [ + { + "type": "reference", + "name": "timedelta" + }, + { + "type": "literal", + "value": null + } + ] + }, + { + "type": "reference", + "name": "Literal", + "typeArguments": [ + { + "type": "literal", + "value": "RemainingTime" + } + ] + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The maximum number of seconds the server waits for the run to finish. By default,\nit is 0, the maximum value is 300." + } + ] + }, + "defaultValue": "None", "flags": { - "keyword-only": "true" + "isOptional": true, + "keyword-only": true }, + "id": 104, + "kind": 32768, + "kindString": "Parameter", + "name": "wait_for_finish", "type": { - "type": "reference", - "name": "str | None" + "name": "int | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "int" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Optional ad-hoc webhooks (https://docs.apify.com/webhooks/ad-hoc-webhooks) associated with\nthe Actor run which can be used to receive a notification, e.g. when the Actor finished or failed.\nIf you already have a webhook set up for the Actor or task, you do not have to add it again here.\n" + } + ] }, - "defaultValue": "None" + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 105, + "kind": 32768, + "kindString": "Parameter", + "name": "webhooks", + "type": { + "name": "list[Webhook] | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "list", + "typeArguments": [ + { + "type": "reference", + "name": "Webhook", + "target": "270" + } + ], + "target": "596" + }, + { + "type": "literal", + "value": null + } + ] + } } - ] + ], + "type": { + "name": "ActorRun", + "type": "reference", + "target": "316" + } } ] }, { - "id": 146, - "name": "set_status_message", - "module": "_actor", "kind": 2048, "kindString": "Method", - "flags": {}, + "children": [], "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Set the status message for the current Actor run.\\n\", {\"Arguments\": [{\"param\": \"status_message\", \"desc\": \"The status message to set to the run.\"}, {\"param\": \"is_terminal\", \"desc\": \"Set this flag to True if this is the final status message of the Actor run.\\n\"}]}, {\"Returns\": [\"The updated Actor run object.\"]}]}" + "text": "Abort given Actor run on the Apify platform using the current user account.\n\nThe user account is determined by the `APIFY_TOKEN` environment variable.\n" } ] }, - "children": [], + "decorations": [], + "flags": {}, "groups": [], + "id": 106, + "module": "_actor", + "name": "abort", + "parsedDocstring": { + "text": "Abort given Actor run on the Apify platform using the current user account.\n\nThe user account is determined by the `APIFY_TOKEN` environment variable.\n", + "args": { + "run_id": "The ID of the Actor run to be aborted.", + "token": "The Apify API token to use for this request (defaults to the `APIFY_TOKEN` environment variable).", + "status_message": "Status message of the Actor to be set on the platform.", + "gracefully": "If True, the Actor run will abort gracefully. It will send `aborting` and `persistState`\nevents into the run and force-stop the run after 30 seconds. It is helpful in cases where you plan\nto resurrect the run later.\n" + }, + "returns": "Info about the aborted Actor run." + }, "sources": [ { - "filename": "/tmp-old-source-v27/src/apify/_actor.py", - "line": 1088, "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/_actor.py#L1088" + "fileName": "/src/apify/_actor.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 769 } ], + "type": { + "name": "Undefined", + "type": "reference" + }, "signatures": [ { - "id": 147, - "name": "set_status_message", - "modifiers": [ - "async" - ], - "kind": 4096, - "kindString": "Call signature", - "flags": {}, "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "Info about the aborted Actor run." + } + ], + "tag": "@returns" + } + ], "summary": [ { "kind": "text", - "text": "{\"content\": [\"Set the status message for the current Actor run.\\n\", {\"" + "text": "Abort given Actor run on the Apify platform using the current user account.\n\nThe user account is determined by the `APIFY_TOKEN` environment variable.\n" } ] }, - "type": { - "type": "reference", - "name": "ActorRun | None" - }, + "flags": {}, + "id": 107, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "abort", "parameters": [ { - "id": 148, - "name": "status_message", + "comment": { + "summary": [ + { + "kind": "text", + "text": "The ID of the Actor run to be aborted." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 108, "kind": 32768, "kindString": "Parameter", - "flags": {}, + "name": "run_id", "type": { - "type": "intrinsic", - "name": "str" + "name": "str", + "type": "reference" } }, { - "id": 149, - "name": "is_terminal", + "comment": { + "summary": [ + { + "kind": "text", + "text": "The Apify API token to use for this request (defaults to the `APIFY_TOKEN` environment variable)." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 109, "kind": 32768, "kindString": "Parameter", + "name": "token", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Status message of the Actor to be set on the platform." + } + ] + }, + "defaultValue": "None", "flags": { - "keyword-only": "true" + "isOptional": true, + "keyword-only": true }, + "id": 110, + "kind": 32768, + "kindString": "Parameter", + "name": "status_message", "type": { - "type": "reference", - "name": "bool | None" + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "If True, the Actor run will abort gracefully. It will send `aborting` and `persistState`\nevents into the run and force-stop the run after 30 seconds. It is helpful in cases where you plan\nto resurrect the run later.\n" + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true }, - "defaultValue": "None" + "id": 111, + "kind": 32768, + "kindString": "Parameter", + "name": "gracefully", + "type": { + "name": "bool | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "bool" + }, + { + "type": "literal", + "value": null + } + ] + } } - ] + ], + "type": { + "name": "ActorRun", + "type": "reference", + "target": "316" + } } ] }, { - "id": 150, - "name": "create_proxy_configuration", - "module": "_actor", "kind": 2048, "kindString": "Method", - "flags": {}, + "children": [], "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Create a ProxyConfiguration object with the passed proxy configuration.\\n\\nConfigures connection to a proxy server with the provided options. Proxy servers are used to prevent target\\nwebsites from blocking your crawlers based on IP address rate limits or blacklists.\\n\\nFor more details and code examples, see the `ProxyConfiguration` class.\\n\", {\"Arguments\": [{\"param\": \"actor_proxy_input\", \"desc\": \"Proxy configuration field from the Actor input, if input has such input field. If you\\npass this argument, all the other arguments will be inferred from it.\"}, {\"param\": \"password\", \"desc\": \"Password for the Apify Proxy. If not provided, will use os.environ['APIFY_PROXY_PASSWORD'],\\nif available.\"}, {\"param\": \"groups\", \"desc\": \"Proxy groups which the Apify Proxy should use, if provided.\"}, {\"param\": \"country_code\", \"desc\": \"Country which the Apify Proxy should use, if provided.\"}, {\"param\": \"proxy_urls\", \"desc\": \"Custom proxy server URLs which should be rotated through.\"}, {\"param\": \"new_url_function\", \"desc\": \"Function which returns a custom proxy URL to be used.\\n\"}]}, {\"Returns\": [\"ProxyConfiguration object with the passed configuration, or None, if no proxy should be used based\", \"on the configuration.\"]}]}" + "text": "Start an Actor on the Apify Platform and wait for it to finish before returning.\n\nIt waits indefinitely, unless the wait argument is provided.\n" } ] }, - "children": [], + "decorations": [], + "flags": {}, "groups": [], + "id": 112, + "module": "_actor", + "name": "call", + "parsedDocstring": { + "text": "Start an Actor on the Apify Platform and wait for it to finish before returning.\n\nIt waits indefinitely, unless the wait argument is provided.\n", + "args": { + "actor_id": "The ID of the Actor to be run.", + "run_input": "The input to pass to the Actor run.", + "token": "The Apify API token to use for this request (defaults to the `APIFY_TOKEN` environment variable).", + "content_type": "The content type of the input.", + "build": "Specifies the Actor build to run. It can be either a build tag or build number. By default,\nthe run uses the build specified in the default run configuration for the Actor (typically latest).", + "memory_mbytes": "Memory limit for the run, in megabytes. By default, the run uses a memory limit specified\nin the default run configuration for the Actor.", + "timeout": "Optional timeout for the run, in seconds. By default, the run uses timeout specified in\nthe default run configuration for the Actor. Using `RemainingTime` will set timeout of the other Actor\nto the time remaining from this Actor timeout.", + "webhooks": "Optional webhooks (https://docs.apify.com/webhooks) associated with the Actor run, which can\nbe used to receive a notification, e.g. when the Actor finished or failed. If you already have\na webhook set up for the Actor, you do not have to add it again here.", + "wait": "The maximum number of seconds the server waits for the run to finish. If not provided,\nwaits indefinitely.", + "logger": "Logger used to redirect logs from the Actor run. Using \"default\" literal means that a predefined\ndefault logger will be used. Setting `None` will disable any log propagation. Passing custom logger\nwill redirect logs to the provided logger.\n" + }, + "returns": "Info about the started Actor run." + }, "sources": [ { - "filename": "/tmp-old-source-v27/src/apify/_actor.py", - "line": 1120, "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/_actor.py#L1120" + "fileName": "/src/apify/_actor.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 803 } ], + "type": { + "name": "Undefined", + "type": "reference" + }, "signatures": [ { - "id": 151, - "name": "create_proxy_configuration", - "modifiers": [ - "async" - ], - "kind": 4096, - "kindString": "Call signature", - "flags": {}, "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "Info about the started Actor run." + } + ], + "tag": "@returns" + } + ], "summary": [ { "kind": "text", - "text": "{\"content\": [\"Create a ProxyConfiguration object with the passed proxy configuration.\\n\\nConfigures connection to a proxy server with the provided options. Proxy servers are used to prevent target\\nwebsites from blocking your crawlers based on IP address rate limits or blacklists.\\n\\nFor more details and code examples, see the `ProxyConfiguration` class.\\n\", {\"" + "text": "Start an Actor on the Apify Platform and wait for it to finish before returning.\n\nIt waits indefinitely, unless the wait argument is provided.\n" } ] }, - "type": { - "type": "reference", - "name": "ProxyConfiguration | None" - }, + "flags": {}, + "id": 113, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "call", "parameters": [ { - "id": 152, - "name": "actor_proxy_input", - "kind": 32768, - "kindString": "Parameter", - "flags": { - "keyword-only": "true" + "comment": { + "summary": [ + { + "kind": "text", + "text": "The ID of the Actor to be run." + } + ] }, - "type": { - "type": "reference", - "name": "dict\n | None" + "flags": { + "isOptional": false, + "keyword-only": false }, - "defaultValue": "None" - }, - { - "id": 153, - "name": "password", + "id": 114, "kind": 32768, "kindString": "Parameter", - "flags": { - "keyword-only": "true" - }, + "name": "actor_id", "type": { - "type": "reference", - "name": "str | None" - }, - "defaultValue": "None" + "name": "str", + "type": "reference" + } }, { - "id": 154, - "name": "groups", - "kind": 32768, - "kindString": "Parameter", + "comment": { + "summary": [ + { + "kind": "text", + "text": "The input to pass to the Actor run." + } + ] + }, + "defaultValue": "None", "flags": { - "keyword-only": "true" + "isOptional": true, + "keyword-only": false }, + "id": 115, + "kind": 32768, + "kindString": "Parameter", + "name": "run_input", "type": { - "type": "reference", - "name": "list[str] | None" - }, - "defaultValue": "None" + "name": "Any", + "type": "reference" + } }, { - "id": 155, - "name": "country_code", - "kind": 32768, - "kindString": "Parameter", + "comment": { + "summary": [ + { + "kind": "text", + "text": "The Apify API token to use for this request (defaults to the `APIFY_TOKEN` environment variable)." + } + ] + }, + "defaultValue": "None", "flags": { - "keyword-only": "true" + "isOptional": true, + "keyword-only": true }, + "id": 116, + "kind": 32768, + "kindString": "Parameter", + "name": "token", "type": { - "type": "reference", - "name": "str | None" - }, - "defaultValue": "None" + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } }, { - "id": 156, - "name": "proxy_urls", - "kind": 32768, - "kindString": "Parameter", + "comment": { + "summary": [ + { + "kind": "text", + "text": "The content type of the input." + } + ] + }, + "defaultValue": "None", "flags": { - "keyword-only": "true" + "isOptional": true, + "keyword-only": true }, + "id": 117, + "kind": 32768, + "kindString": "Parameter", + "name": "content_type", "type": { - "type": "reference", - "name": "list[str | None] | None" - }, - "defaultValue": "None" + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } }, { - "id": 157, - "name": "new_url_function", - "kind": 32768, - "kindString": "Parameter", + "comment": { + "summary": [ + { + "kind": "text", + "text": "Specifies the Actor build to run. It can be either a build tag or build number. By default,\nthe run uses the build specified in the default run configuration for the Actor (typically latest)." + } + ] + }, + "defaultValue": "None", "flags": { - "keyword-only": "true" + "isOptional": true, + "keyword-only": true }, + "id": 118, + "kind": 32768, + "kindString": "Parameter", + "name": "build", "type": { - "type": "reference", - "name": "_NewUrlFunction | None" + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Memory limit for the run, in megabytes. By default, the run uses a memory limit specified\nin the default run configuration for the Actor." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 119, + "kind": 32768, + "kindString": "Parameter", + "name": "memory_mbytes", + "type": { + "name": "int | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "int" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Optional timeout for the run, in seconds. By default, the run uses timeout specified in\nthe default run configuration for the Actor. Using `RemainingTime` will set timeout of the other Actor\nto the time remaining from this Actor timeout." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 120, + "kind": 32768, + "kindString": "Parameter", + "name": "timeout", + "type": { + "name": "timedelta | None | Literal['RemainingTime']", + "type": "union", + "types": [ + { + "type": "union", + "types": [ + { + "type": "reference", + "name": "timedelta" + }, + { + "type": "literal", + "value": null + } + ] + }, + { + "type": "reference", + "name": "Literal", + "typeArguments": [ + { + "type": "literal", + "value": "RemainingTime" + } + ] + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Optional webhooks (https://docs.apify.com/webhooks) associated with the Actor run, which can\nbe used to receive a notification, e.g. when the Actor finished or failed. If you already have\na webhook set up for the Actor, you do not have to add it again here." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 121, + "kind": 32768, + "kindString": "Parameter", + "name": "webhooks", + "type": { + "name": "list[Webhook] | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "list", + "typeArguments": [ + { + "type": "reference", + "name": "Webhook", + "target": "270" + } + ], + "target": "596" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The maximum number of seconds the server waits for the run to finish. If not provided,\nwaits indefinitely." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 122, + "kind": 32768, + "kindString": "Parameter", + "name": "wait", + "type": { + "name": "timedelta | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "timedelta" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Logger used to redirect logs from the Actor run. Using \"default\" literal means that a predefined\ndefault logger will be used. Setting `None` will disable any log propagation. Passing custom logger\nwill redirect logs to the provided logger.\n" + } + ] + }, + "defaultValue": "'default'", + "flags": { + "isOptional": true, + "keyword-only": true }, - "defaultValue": "None" + "id": 123, + "kind": 32768, + "kindString": "Parameter", + "name": "logger", + "type": { + "name": "logging.Logger | None | Literal['default']", + "type": "union", + "types": [ + { + "type": "union", + "types": [ + { + "type": "reference", + "name": "logging.Logger" + }, + { + "type": "literal", + "value": null + } + ] + }, + { + "type": "reference", + "name": "Literal", + "typeArguments": [ + { + "type": "literal", + "value": "default" + } + ] + } + ] + } } - ] + ], + "type": { + "name": "ActorRun | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "ActorRun", + "target": "316" + }, + { + "type": "literal", + "value": null + } + ] + } } ] - } - ], - "groups": [ - { - "title": "Constructors", - "children": [ - 2 - ] }, { - "title": "Methods", - "children": [ - 7, - 101, - 140, - 12, - 107, - 119, - 74, - 16, - 14, - 150, - 18, - 24, - 30, - 72, - 88, - 61, - 63, - 22, - 86, - 20, - 129, - 35, - 82, - 78, - 42, - 47, - 52, - 57, - 136, - 146, - 67, - 90 - ] - } - ], - "sources": [ - { - "filename": "/tmp-old-source-v27/src/apify/_actor.py", - "line": 57, - "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/_actor.py#L57" - } - ] - }, - { - "id": 158, - "name": "Actor", - "module": "_actor", - "kind": 1024, - "kindString": "Property", - "flags": {}, - "comment": { - "summary": [ - { - "kind": "text", - "text": "{\"content\": [\"The entry point of the SDK, through which all the Actor operations should be done.\"]}" - } - ] - }, - "children": [], - "groups": [], - "sources": [ - { - "filename": "/tmp-old-source-v27/src/apify/_actor.py", - "line": 1198, - "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/_actor.py#L1198" - } - ] - }, - { - "id": 159, - "name": "ChargingManager", - "module": "_charging", - "docsGroup": "Interfaces", - "kind": 128, - "kindString": "Class", - "flags": {}, - "comment": { - "summary": [ - { - "kind": "text", - "text": "{\"content\": [\"Provides fine-grained access to pay-per-event functionality.\"]}" - } - ] - }, - "children": [ - { - "id": 160, - "name": "charge", - "module": "_charging", "kind": 2048, "kindString": "Method", - "flags": {}, + "children": [], "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Charge for a specified number of events - sub-operations of the Actor.\\n\\nThis is relevant only for the pay-per-event pricing model.\\n\", {\"Arguments\": [{\"param\": \"event_name\", \"desc\": \"Name of the event to be charged for.\"}, {\"param\": \"count\", \"desc\": \"Number of events to charge for.\"}]}]}" + "text": "Start an Actor task on the Apify Platform and wait for it to finish before returning.\n\nIt waits indefinitely, unless the wait argument is provided.\n\nNote that an Actor task is a saved input configuration and options for an Actor. If you want to run an Actor\ndirectly rather than an Actor task, please use the `Actor.call`\n" } ] }, - "children": [], + "decorations": [], + "flags": {}, "groups": [], + "id": 124, + "module": "_actor", + "name": "call_task", + "parsedDocstring": { + "text": "Start an Actor task on the Apify Platform and wait for it to finish before returning.\n\nIt waits indefinitely, unless the wait argument is provided.\n\nNote that an Actor task is a saved input configuration and options for an Actor. If you want to run an Actor\ndirectly rather than an Actor task, please use the `Actor.call`\n", + "args": { + "task_id": "The ID of the Actor to be run.", + "task_input": "Overrides the input to pass to the Actor run.", + "token": "The Apify API token to use for this request (defaults to the `APIFY_TOKEN` environment variable).", + "content_type": "The content type of the input.", + "build": "Specifies the Actor build to run. It can be either a build tag or build number. By default,\nthe run uses the build specified in the default run configuration for the Actor (typically latest).", + "memory_mbytes": "Memory limit for the run, in megabytes. By default, the run uses a memory limit specified\nin the default run configuration for the Actor.", + "timeout": "Optional timeout for the run, in seconds. By default, the run uses timeout specified in\nthe default run configuration for the Actor.", + "webhooks": "Optional webhooks (https://docs.apify.com/webhooks) associated with the Actor run, which can\nbe used to receive a notification, e.g. when the Actor finished or failed. If you already have\na webhook set up for the Actor, you do not have to add it again here.", + "wait": "The maximum number of seconds the server waits for the run to finish. If not provided, waits\nindefinitely.\n" + }, + "returns": "Info about the started Actor run." + }, "sources": [ { - "filename": "/tmp-old-source-v27/src/apify/_charging.py", - "line": 34, "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/_charging.py#L34" + "fileName": "/src/apify/_actor.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 878 } ], + "type": { + "name": "Undefined", + "type": "reference" + }, "signatures": [ { - "id": 161, - "name": "charge", - "modifiers": [ - "async" - ], - "kind": 4096, - "kindString": "Call signature", - "flags": {}, "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "Info about the started Actor run." + } + ], + "tag": "@returns" + } + ], "summary": [ { "kind": "text", - "text": "{\"content\": [\"Charge for a specified number of events - sub-operations of the Actor.\\n\\nThis is relevant only for the pay-per-event pricing model.\\n\", {\"" + "text": "Start an Actor task on the Apify Platform and wait for it to finish before returning.\n\nIt waits indefinitely, unless the wait argument is provided.\n\nNote that an Actor task is a saved input configuration and options for an Actor. If you want to run an Actor\ndirectly rather than an Actor task, please use the `Actor.call`\n" } ] }, - "type": { - "type": "reference", - "name": "ChargeResult", - "id": 171 - }, + "flags": {}, + "id": 125, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "call_task", "parameters": [ { - "id": 162, - "name": "event_name", + "comment": { + "summary": [ + { + "kind": "text", + "text": "The ID of the Actor to be run." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 126, "kind": 32768, "kindString": "Parameter", - "flags": {}, + "name": "task_id", "type": { - "type": "intrinsic", - "name": "str" + "name": "str", + "type": "reference" } }, { - "id": 163, - "name": "count", + "comment": { + "summary": [ + { + "kind": "text", + "text": "Overrides the input to pass to the Actor run." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": false + }, + "id": 127, "kind": 32768, "kindString": "Parameter", - "flags": {}, + "name": "task_input", "type": { - "type": "intrinsic", - "name": "int" + "name": "dict | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "dict" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Specifies the Actor build to run. It can be either a build tag or build number. By default,\nthe run uses the build specified in the default run configuration for the Actor (typically latest)." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 128, + "kind": 32768, + "kindString": "Parameter", + "name": "build", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Memory limit for the run, in megabytes. By default, the run uses a memory limit specified\nin the default run configuration for the Actor." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 129, + "kind": 32768, + "kindString": "Parameter", + "name": "memory_mbytes", + "type": { + "name": "int | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "int" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Optional timeout for the run, in seconds. By default, the run uses timeout specified in\nthe default run configuration for the Actor." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 130, + "kind": 32768, + "kindString": "Parameter", + "name": "timeout", + "type": { + "name": "timedelta | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "timedelta" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Optional webhooks (https://docs.apify.com/webhooks) associated with the Actor run, which can\nbe used to receive a notification, e.g. when the Actor finished or failed. If you already have\na webhook set up for the Actor, you do not have to add it again here." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 131, + "kind": 32768, + "kindString": "Parameter", + "name": "webhooks", + "type": { + "name": "list[Webhook] | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "list", + "typeArguments": [ + { + "type": "reference", + "name": "Webhook", + "target": "270" + } + ], + "target": "596" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The maximum number of seconds the server waits for the run to finish. If not provided, waits\nindefinitely.\n" + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 132, + "kind": 32768, + "kindString": "Parameter", + "name": "wait", + "type": { + "name": "timedelta | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "timedelta" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The Apify API token to use for this request (defaults to the `APIFY_TOKEN` environment variable)." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true }, - "defaultValue": "1" + "id": 133, + "kind": 32768, + "kindString": "Parameter", + "name": "token", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } } - ] + ], + "type": { + "name": "ActorRun | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "ActorRun", + "target": "316" + }, + { + "type": "literal", + "value": null + } + ] + } } ] }, { - "id": 164, - "name": "calculate_total_charged_amount", - "module": "_charging", "kind": 2048, "kindString": "Method", - "flags": {}, + "children": [], "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Calculate the total amount of money charged for pay-per-event events so far.\"]}" + "text": "Transform this Actor run to an Actor run of a different Actor.\n\nThe platform stops the current Actor container and starts a new container with the new Actor instead. All\nthe default storages are preserved, and the new input is stored under the `INPUT-METAMORPH-1` key in the same\ndefault key-value store.\n" } ] }, - "children": [], + "decorations": [], + "flags": {}, "groups": [], + "id": 134, + "module": "_actor", + "name": "metamorph", + "parsedDocstring": { + "text": "Transform this Actor run to an Actor run of a different Actor.\n\nThe platform stops the current Actor container and starts a new container with the new Actor instead. All\nthe default storages are preserved, and the new input is stored under the `INPUT-METAMORPH-1` key in the same\ndefault key-value store.\n", + "args": { + "target_actor_id": "ID of the target Actor that the run should be transformed into", + "run_input": "The input to pass to the new run.", + "target_actor_build": "The build of the target Actor. It can be either a build tag or build number.\nBy default, the run uses the build specified in the default run configuration for the target Actor\n(typically the latest build).", + "content_type": "The content type of the input.", + "custom_after_sleep": "How long to sleep for after the metamorph, to wait for the container to be stopped." + } + }, "sources": [ { - "filename": "/tmp-old-source-v27/src/apify/_charging.py", - "line": 44, "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/_charging.py#L44" + "fileName": "/src/apify/_actor.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 939 } ], + "type": { + "name": "Undefined", + "type": "reference" + }, "signatures": [ { - "id": 165, - "name": "calculate_total_charged_amount", - "modifiers": [], - "kind": 4096, - "kindString": "Call signature", - "flags": {}, "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Calculate the total amount of money charged for pay-per-event events so far.\"]}" + "text": "Transform this Actor run to an Actor run of a different Actor.\n\nThe platform stops the current Actor container and starts a new container with the new Actor instead. All\nthe default storages are preserved, and the new input is stored under the `INPUT-METAMORPH-1` key in the same\ndefault key-value store.\n" } ] }, + "flags": {}, + "id": 135, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "metamorph", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "ID of the target Actor that the run should be transformed into" + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 136, + "kind": 32768, + "kindString": "Parameter", + "name": "target_actor_id", + "type": { + "name": "str", + "type": "reference" + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The input to pass to the new run." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": false + }, + "id": 137, + "kind": 32768, + "kindString": "Parameter", + "name": "run_input", + "type": { + "name": "Any", + "type": "reference" + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The build of the target Actor. It can be either a build tag or build number.\nBy default, the run uses the build specified in the default run configuration for the target Actor\n(typically the latest build)." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 138, + "kind": 32768, + "kindString": "Parameter", + "name": "target_actor_build", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The content type of the input." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 139, + "kind": 32768, + "kindString": "Parameter", + "name": "content_type", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "How long to sleep for after the metamorph, to wait for the container to be stopped." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 140, + "kind": 32768, + "kindString": "Parameter", + "name": "custom_after_sleep", + "type": { + "name": "timedelta | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "timedelta" + }, + { + "type": "literal", + "value": null + } + ] + } + } + ], "type": { - "type": "reference", - "name": "Decimal" - }, - "parameters": [] + "name": "None", + "type": "literal", + "value": null + } } ] }, { - "id": 166, - "name": "calculate_max_event_charge_count_within_limit", - "module": "_charging", "kind": 2048, "kindString": "Method", - "flags": {}, + "children": [], "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Calculate how many instances of an event can be charged before we reach the configured limit.\\n\", {\"Arguments\": [{\"param\": \"event_name\", \"desc\": \"Name of the inspected event.\"}]}]}" + "text": "Internally reboot this Actor.\n\nThe system stops the current container and starts a new one, with the same run ID and default storages.\n" } ] }, - "children": [], + "decorations": [], + "flags": {}, "groups": [], + "id": 141, + "module": "_actor", + "name": "reboot", + "parsedDocstring": { + "text": "Internally reboot this Actor.\n\nThe system stops the current container and starts a new one, with the same run ID and default storages.\n", + "args": { + "event_listeners_timeout": "How long should the Actor wait for Actor event listeners to finish before exiting.", + "custom_after_sleep": "How long to sleep for after the reboot, to wait for the container to be stopped." + } + }, "sources": [ { - "filename": "/tmp-old-source-v27/src/apify/_charging.py", - "line": 47, "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/_charging.py#L47" + "fileName": "/src/apify/_actor.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 986 } ], + "type": { + "name": "Undefined", + "type": "reference" + }, "signatures": [ { - "id": 167, - "name": "calculate_max_event_charge_count_within_limit", - "modifiers": [], - "kind": 4096, - "kindString": "Call signature", - "flags": {}, "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Calculate how many instances of an event can be charged before we reach the configured limit.\\n\", {\"" + "text": "Internally reboot this Actor.\n\nThe system stops the current container and starts a new one, with the same run ID and default storages.\n" } ] }, - "type": { - "type": "reference", - "name": "int | None" - }, + "flags": {}, + "id": 142, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "reboot", "parameters": [ { - "id": 168, - "name": "event_name", + "comment": { + "summary": [ + { + "kind": "text", + "text": "How long should the Actor wait for Actor event listeners to finish before exiting." + } + ] + }, + "defaultValue": "EVENT_LISTENERS_TIMEOUT", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 143, "kind": 32768, "kindString": "Parameter", - "flags": {}, + "name": "event_listeners_timeout", "type": { - "type": "intrinsic", - "name": "str" + "name": "timedelta | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "timedelta" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "How long to sleep for after the reboot, to wait for the container to be stopped." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 144, + "kind": 32768, + "kindString": "Parameter", + "name": "custom_after_sleep", + "type": { + "name": "timedelta | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "timedelta" + }, + { + "type": "literal", + "value": null + } + ] } } - ] + ], + "type": { + "name": "None", + "type": "literal", + "value": null + } } ] }, { - "id": 169, - "name": "get_pricing_info", - "module": "_charging", "kind": 2048, "kindString": "Method", - "flags": {}, + "children": [], "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Retrieve detailed information about the effective pricing of the current Actor run.\\n\\nThis can be used for instance when your code needs to support multiple pricing models in transition periods.\"]}" + "text": "Create an ad-hoc webhook for the current Actor run.\n\nThis webhook lets you receive a notification when the Actor run finished or failed.\n\nNote that webhooks are only supported for Actors running on the Apify platform. When running the Actor locally,\nthe function will print a warning and have no effect.\n\nFor more information about Apify Actor webhooks, please see the [documentation](https://docs.apify.com/webhooks).\n" } ] }, - "children": [], + "decorations": [], + "flags": {}, "groups": [], + "id": 145, + "module": "_actor", + "name": "add_webhook", + "parsedDocstring": { + "text": "Create an ad-hoc webhook for the current Actor run.\n\nThis webhook lets you receive a notification when the Actor run finished or failed.\n\nNote that webhooks are only supported for Actors running on the Apify platform. When running the Actor locally,\nthe function will print a warning and have no effect.\n\nFor more information about Apify Actor webhooks, please see the [documentation](https://docs.apify.com/webhooks).\n", + "args": { + "webhook": "The webhook to be added", + "ignore_ssl_errors": "Whether the webhook should ignore SSL errors returned by request_url", + "do_not_retry": "Whether the webhook should retry sending the payload to request_url upon failure.", + "idempotency_key": "A unique identifier of a webhook. You can use it to ensure that you won't create\nthe same webhook multiple times.\n" + }, + "returns": "The created webhook." + }, "sources": [ { - "filename": "/tmp-old-source-v27/src/apify/_charging.py", - "line": 54, "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/_charging.py#L54" + "fileName": "/src/apify/_actor.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 1041 } ], + "type": { + "name": "Undefined", + "type": "reference" + }, "signatures": [ { - "id": 170, - "name": "get_pricing_info", - "modifiers": [], - "kind": 4096, - "kindString": "Call signature", - "flags": {}, "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "The created webhook." + } + ], + "tag": "@returns" + } + ], "summary": [ { "kind": "text", - "text": "{\"content\": [\"Retrieve detailed information about the effective pricing of the current Actor run.\\n\\nThis can be used for instance when your code needs to support multiple pricing models in transition periods.\"]}" + "text": "Create an ad-hoc webhook for the current Actor run.\n\nThis webhook lets you receive a notification when the Actor run finished or failed.\n\nNote that webhooks are only supported for Actors running on the Apify platform. When running the Actor locally,\nthe function will print a warning and have no effect.\n\nFor more information about Apify Actor webhooks, please see the [documentation](https://docs.apify.com/webhooks).\n" } ] }, + "flags": {}, + "id": 146, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "add_webhook", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The webhook to be added" + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 147, + "kind": 32768, + "kindString": "Parameter", + "name": "webhook", + "type": { + "name": "Webhook", + "type": "reference", + "target": "270" + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Whether the webhook should ignore SSL errors returned by request_url" + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 148, + "kind": 32768, + "kindString": "Parameter", + "name": "ignore_ssl_errors", + "type": { + "name": "bool | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "bool" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Whether the webhook should retry sending the payload to request_url upon failure." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 149, + "kind": 32768, + "kindString": "Parameter", + "name": "do_not_retry", + "type": { + "name": "bool | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "bool" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "A unique identifier of a webhook. You can use it to ensure that you won't create\nthe same webhook multiple times.\n" + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 150, + "kind": 32768, + "kindString": "Parameter", + "name": "idempotency_key", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + } + ], "type": { - "type": "reference", - "name": "ActorPricingInfo", - "id": 175 - }, - "parameters": [] + "name": "None", + "type": "literal", + "value": null + } } ] - } - ], - "groups": [ - { - "title": "Methods", - "children": [ - 166, - 164, - 160, - 169 - ] - } - ], - "sources": [ - { - "filename": "/tmp-old-source-v27/src/apify/_charging.py", - "line": 31, - "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/_charging.py#L31" - } - ] - }, - { - "id": 171, - "name": "ChargeResult", - "module": "_charging", - "docsGroup": "Data structures", - "kind": 128, - "kindString": "Class", - "flags": {}, - "comment": { - "summary": [ - { - "kind": "text", - "text": "{\"content\": [\"Result of the `ChargingManager.charge` method.\"]}" - } - ] - }, - "children": [ + }, { - "id": 172, - "name": "event_charge_limit_reached", - "module": "_charging", - "kind": 1024, - "kindString": "Property", - "flags": {}, + "kind": 2048, + "kindString": "Method", + "children": [], "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"If true, no more events of this type can be charged within the limit.\"]}" + "text": "Set the status message for the current Actor run.\n" } ] }, - "type": { - "type": "intrinsic", - "name": "bool" - }, - "children": [], + "decorations": [], + "flags": {}, "groups": [], + "id": 151, + "module": "_actor", + "name": "set_status_message", + "parsedDocstring": { + "text": "Set the status message for the current Actor run.\n", + "args": { + "status_message": "The status message to set to the run.", + "is_terminal": "Set this flag to True if this is the final status message of the Actor run.\n" + }, + "returns": "The updated Actor run object." + }, "sources": [ { - "filename": "/tmp-old-source-v27/src/apify/_charging.py", - "line": 66, "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/_charging.py#L66" + "fileName": "/src/apify/_actor.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 1088 } - ] - }, - { - "id": 173, - "name": "charged_count", - "module": "_charging", - "kind": 1024, - "kindString": "Property", - "flags": {}, - "comment": { - "summary": [ - { - "kind": "text", - "text": "{\"content\": [\"Total amount of charged events - may be lower than the requested amount.\"]}" - } - ] - }, + ], "type": { - "type": "intrinsic", - "name": "int" + "name": "Undefined", + "type": "reference" }, - "children": [], - "groups": [], - "sources": [ + "signatures": [ { - "filename": "/tmp-old-source-v27/src/apify/_charging.py", - "line": 69, - "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/_charging.py#L69" + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "The updated Actor run object." + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Set the status message for the current Actor run.\n" + } + ] + }, + "flags": {}, + "id": 152, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "set_status_message", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The status message to set to the run." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 153, + "kind": 32768, + "kindString": "Parameter", + "name": "status_message", + "type": { + "name": "str", + "type": "reference" + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Set this flag to True if this is the final status message of the Actor run.\n" + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 154, + "kind": 32768, + "kindString": "Parameter", + "name": "is_terminal", + "type": { + "name": "bool | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "bool" + }, + { + "type": "literal", + "value": null + } + ] + } + } + ], + "type": { + "name": "ActorRun | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "ActorRun", + "target": "316" + }, + { + "type": "literal", + "value": null + } + ] + } } ] }, { - "id": 174, - "name": "chargeable_within_limit", - "module": "_charging", - "kind": 1024, - "kindString": "Property", - "flags": {}, + "kind": 2048, + "kindString": "Method", + "children": [], "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"How many events of each known type can still be charged within the limit.\"]}" + "text": "Create a ProxyConfiguration object with the passed proxy configuration.\n\nConfigures connection to a proxy server with the provided options. Proxy servers are used to prevent target\nwebsites from blocking your crawlers based on IP address rate limits or blacklists.\n\nFor more details and code examples, see the `ProxyConfiguration` class.\n" } ] }, - "type": { - "type": "reference", - "name": "dict[str, int | None]" - }, - "children": [], + "decorations": [], + "flags": {}, "groups": [], + "id": 155, + "module": "_actor", + "name": "create_proxy_configuration", + "parsedDocstring": { + "text": "Create a ProxyConfiguration object with the passed proxy configuration.\n\nConfigures connection to a proxy server with the provided options. Proxy servers are used to prevent target\nwebsites from blocking your crawlers based on IP address rate limits or blacklists.\n\nFor more details and code examples, see the `ProxyConfiguration` class.\n", + "args": { + "actor_proxy_input": "Proxy configuration field from the Actor input, if input has such input field. If you\npass this argument, all the other arguments will be inferred from it.", + "password": "Password for the Apify Proxy. If not provided, will use os.environ['APIFY_PROXY_PASSWORD'],\nif available.", + "groups": "Proxy groups which the Apify Proxy should use, if provided.", + "country_code": "Country which the Apify Proxy should use, if provided.", + "proxy_urls": "Custom proxy server URLs which should be rotated through.", + "new_url_function": "Function which returns a custom proxy URL to be used.\n" + }, + "returns": "ProxyConfiguration object with the passed configuration, or None, if no proxy should be used based\non the configuration." + }, "sources": [ { - "filename": "/tmp-old-source-v27/src/apify/_charging.py", - "line": 72, "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/_charging.py#L72" + "fileName": "/src/apify/_actor.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 1120 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "ProxyConfiguration object with the passed configuration, or None, if no proxy should be used based\non the configuration." + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Create a ProxyConfiguration object with the passed proxy configuration.\n\nConfigures connection to a proxy server with the provided options. Proxy servers are used to prevent target\nwebsites from blocking your crawlers based on IP address rate limits or blacklists.\n\nFor more details and code examples, see the `ProxyConfiguration` class.\n" + } + ] + }, + "flags": {}, + "id": 156, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "create_proxy_configuration", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Proxy configuration field from the Actor input, if input has such input field. If you\npass this argument, all the other arguments will be inferred from it." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 157, + "kind": 32768, + "kindString": "Parameter", + "name": "actor_proxy_input", + "type": { + "name": "dict | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "dict" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Password for the Apify Proxy. If not provided, will use os.environ['APIFY_PROXY_PASSWORD'],\nif available." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 158, + "kind": 32768, + "kindString": "Parameter", + "name": "password", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Proxy groups which the Apify Proxy should use, if provided." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 159, + "kind": 32768, + "kindString": "Parameter", + "name": "groups", + "type": { + "name": "list[str] | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "list", + "typeArguments": [ + { + "type": "reference", + "name": "str" + } + ], + "target": "596" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Country which the Apify Proxy should use, if provided." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 160, + "kind": 32768, + "kindString": "Parameter", + "name": "country_code", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Custom proxy server URLs which should be rotated through." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 161, + "kind": 32768, + "kindString": "Parameter", + "name": "proxy_urls", + "type": { + "name": "list[str | None] | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "list", + "typeArguments": [ + { + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + ], + "target": "596" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Function which returns a custom proxy URL to be used.\n" + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 162, + "kind": 32768, + "kindString": "Parameter", + "name": "new_url_function", + "type": { + "name": "_NewUrlFunction | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "_NewUrlFunction" + }, + { + "type": "literal", + "value": null + } + ] + } + } + ], + "type": { + "name": "ProxyConfiguration | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "ProxyConfiguration", + "target": "211" + }, + { + "type": "literal", + "value": null + } + ] + } } ] } ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "The class of `Actor`. Only make a new instance if you're absolutely sure you need to." + } + ] + }, + "decorations": [ + { + "args": "('Actor')", + "name": "docs_name" + }, + { + "args": "('Classes')", + "name": "docs_group" + } + ], + "flags": {}, "groups": [ { - "title": "Properties", "children": [ - 174, - 173, - 172 - ] + 17, + 10, + 15, + 106, + 145, + 112, + 124, + 79, + 155, + 29, + 35, + 77, + 93, + 66, + 68, + 27, + 91, + 134, + 40, + 87, + 83, + 47, + 52, + 57, + 62, + 141, + 151, + 72, + 95 + ], + "title": "Methods" + }, + { + "children": [ + 22, + 24, + 23, + 25, + 26 + ], + "title": "Properties" } ], + "id": 9, + "module": "_actor", + "name": "Actor", + "parsedDocstring": { + "text": "The class of `Actor`. Only make a new instance if you're absolutely sure you need to." + }, "sources": [ { - "filename": "/tmp-old-source-v27/src/apify/_charging.py", - "line": 63, "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/_charging.py#L63" + "fileName": "/src/apify/_actor.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 57 } - ] + ], + "type": { + "name": "Undefined", + "type": "reference" + } }, { - "id": 175, - "name": "ActorPricingInfo", - "module": "_charging", - "docsGroup": "Data structures", - "kind": 128, - "kindString": "Class", + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "The entry point of the SDK, through which all the Actor operations should be done." + } + ] + }, "flags": {}, + "groups": [], + "id": 204, + "module": "_actor", + "name": "Actor", + "parsedDocstring": { + "text": "The entry point of the SDK, through which all the Actor operations should be done." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_actor.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 1198 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Result of the `ChargingManager.get_pricing_info` method.\"]}" + "text": "" } ] }, - "children": [ + "flags": {}, + "groups": [], + "id": 205, + "module": "_proxy_configuration", + "name": "APIFY_PROXY_VALUE_REGEX", + "parsedDocstring": { + "text": "" + }, + "sources": [ { - "id": 176, - "name": "pricing_model", - "module": "_charging", - "kind": 1024, - "kindString": "Property", - "flags": {}, - "comment": { - "summary": [ - { - "kind": "text", - "text": "{\"content\": [\"The currently effective pricing model.\"]}" - } - ] - }, - "type": { - "type": "reference", - "name": "PricingModel | None" - }, - "children": [], - "groups": [], - "sources": [ - { - "filename": "/tmp-old-source-v27/src/apify/_charging.py", - "line": 81, - "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/_charging.py#L81" - } - ] - }, + "character": 1, + "fileName": "/src/apify/_proxy_configuration.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 26 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 206, + "module": "_proxy_configuration", + "name": "COUNTRY_CODE_REGEX", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_proxy_configuration.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 27 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 207, + "module": "_proxy_configuration", + "name": "SESSION_ID_MAX_LENGTH", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_proxy_configuration.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 28 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 128, + "kindString": "Class", + "children": [ { - "id": 177, - "name": "max_total_charge_usd", - "module": "_charging", "kind": 1024, "kindString": "Property", - "flags": {}, + "children": [], "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"A configured limit for the total charged amount - if you exceed it, you won't receive more money than this.\"]}" + "text": "An array of proxy groups to be used by the [Apify Proxy](https://docs.apify.com/proxy). If not provided,\nthe proxy will select the groups automatically." } ] }, - "type": { - "type": "reference", - "name": "Decimal" - }, - "children": [], + "flags": {}, "groups": [], + "id": 209, + "module": "_proxy_configuration", + "name": "groups", + "parsedDocstring": { + "text": "An array of proxy groups to be used by the [Apify Proxy](https://docs.apify.com/proxy). If not provided,\nthe proxy will select the groups automatically." + }, "sources": [ { - "filename": "/tmp-old-source-v27/src/apify/_charging.py", - "line": 84, "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/_charging.py#L84" + "fileName": "/src/apify/_proxy_configuration.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 77 } - ] + ], + "type": { + "name": "list", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "str" + } + ], + "target": "596" + } }, { - "id": 178, - "name": "is_pay_per_event", - "module": "_charging", "kind": 1024, "kindString": "Property", - "flags": {}, + "children": [], "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"A shortcut - true if the Actor runs with the pay-per-event pricing model.\"]}" + "text": "If set and relevant proxies are available in your Apify account, all proxied requests will use IP addresses\nthat are geolocated to the specified country. For example `GB` for IPs from Great Britain. Note that online\nservices often have their own rules for handling geolocation and thus the country selection is a best attempt\nat geolocation, rather than a guaranteed hit. This parameter is optional, by default, each proxied request is\nassigned an IP address from a random country. The country code needs to be a two letter ISO country code.\nSee the [full list of available country codes](https://en.wikipedia.org/wiki/ISO_3166-1_alpha-2#Officially_assigned_code_elements).\nThis parameter is optional, by default, the proxy uses all available proxy servers from all countries." } ] }, - "type": { - "type": "intrinsic", - "name": "bool" - }, - "children": [], + "flags": {}, "groups": [], + "id": 210, + "module": "_proxy_configuration", + "name": "country_code", + "parsedDocstring": { + "text": "If set and relevant proxies are available in your Apify account, all proxied requests will use IP addresses\nthat are geolocated to the specified country. For example `GB` for IPs from Great Britain. Note that online\nservices often have their own rules for handling geolocation and thus the country selection is a best attempt\nat geolocation, rather than a guaranteed hit. This parameter is optional, by default, each proxied request is\nassigned an IP address from a random country. The country code needs to be a two letter ISO country code.\nSee the [full list of available country codes](https://en.wikipedia.org/wiki/ISO_3166-1_alpha-2#Officially_assigned_code_elements).\nThis parameter is optional, by default, the proxy uses all available proxy servers from all countries." + }, "sources": [ { - "filename": "/tmp-old-source-v27/src/apify/_charging.py", - "line": 87, "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/_charging.py#L87" + "fileName": "/src/apify/_proxy_configuration.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 81 } - ] - }, - { - "id": 179, - "name": "per_event_prices", - "module": "_charging", - "kind": 1024, - "kindString": "Property", - "flags": {}, - "comment": { - "summary": [ + ], + "type": { + "name": "str | None", + "type": "union", + "types": [ { - "kind": "text", - "text": "{\"content\": [\"Price of every known event type.\"]}" + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null } ] - }, - "type": { - "type": "reference", - "name": "dict[str, Decimal]" - }, - "children": [], - "groups": [], - "sources": [ - { - "filename": "/tmp-old-source-v27/src/apify/_charging.py", - "line": 90, - "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/_charging.py#L90" - } - ] + } + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Provides information about a proxy connection that is used for requests." + } + ] + }, + "decorations": [ + { + "args": "('Classes')", + "name": "docs_group" + }, + { + "name": "dataclass" } ], + "flags": {}, "groups": [ { - "title": "Properties", "children": [ - 178, - 177, - 179, - 176 - ] + 210, + 209 + ], + "title": "Properties" } ], + "id": 208, + "module": "_proxy_configuration", + "name": "ProxyInfo", + "parsedDocstring": { + "text": "Provides information about a proxy connection that is used for requests." + }, "sources": [ { - "filename": "/tmp-old-source-v27/src/apify/_charging.py", - "line": 78, "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/_charging.py#L78" + "fileName": "/src/apify/_proxy_configuration.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 74 } - ] + ], + "type": { + "name": "Undefined", + "type": "reference" + } }, { - "id": 180, - "name": "Configuration", - "module": "_configuration", - "docsGroup": "Classes", "kind": 128, "kindString": "Class", - "flags": {}, - "comment": { - "summary": [ - { - "kind": "text", - "text": "{\"content\": [\"A class for specifying the configuration of an Actor.\\n\\nCan be used either globally via `Configuration.get_global_configuration()`,\\nor it can be specific to each `Actor` instance on the `actor.config` property.\"]}" - } - ] - }, "children": [ { - "id": 181, - "name": "disable_browser_sandbox_on_platform", - "module": "_configuration", "kind": 2048, "kindString": "Method", - "flags": {}, + "children": [], "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Disable the browser sandbox mode when running on the Apify platform.\\n\\nRunning in environment where `is_at_home` is True does not benefit from browser sandbox as it is already running\\nin a container. It can be on the contrary undesired as the process in the container might be running as root and\\nthis will crash chromium that was started with browser sandbox mode.\"]}" + "text": "Check if using proxy, if so, check the access.\n\nLoad the Apify Proxy password from API (only if not passed to constructor or through env var).\n\nOnly called if Apify Proxy configuration is used. Also checks if country has access to Apify Proxy groups\nif the country code is provided.\n\nYou should use the Actor.create_proxy_configuration function to create a pre-initialized\n`ProxyConfiguration` instance instead of calling this manually." } ] }, - "children": [], + "decorations": [], + "flags": {}, "groups": [], + "id": 212, + "module": "_proxy_configuration", + "name": "initialize", + "parsedDocstring": { + "text": "Check if using proxy, if so, check the access.\n\nLoad the Apify Proxy password from API (only if not passed to constructor or through env var).\n\nOnly called if Apify Proxy configuration is used. Also checks if country has access to Apify Proxy groups\nif the country code is provided.\n\nYou should use the Actor.create_proxy_configuration function to create a pre-initialized\n`ProxyConfiguration` instance instead of calling this manually." + }, "sources": [ { - "filename": "/tmp-old-source-v27/src/apify/_configuration.py", - "line": 378, "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/_configuration.py#L378" + "fileName": "/src/apify/_proxy_configuration.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 180 } ], + "type": { + "name": "Undefined", + "type": "reference" + }, "signatures": [ { - "id": 182, - "name": "disable_browser_sandbox_on_platform", - "modifiers": [], - "kind": 4096, - "kindString": "Call signature", - "flags": {}, "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Disable the browser sandbox mode when running on the Apify platform.\\n\\nRunning in environment where `is_at_home` is True does not benefit from browser sandbox as it is already running\\nin a container. It can be on the contrary undesired as the process in the container might be running as root and\\nthis will crash chromium that was started with browser sandbox mode.\"]}" + "text": "Check if using proxy, if so, check the access.\n\nLoad the Apify Proxy password from API (only if not passed to constructor or through env var).\n\nOnly called if Apify Proxy configuration is used. Also checks if country has access to Apify Proxy groups\nif the country code is provided.\n\nYou should use the Actor.create_proxy_configuration function to create a pre-initialized\n`ProxyConfiguration` instance instead of calling this manually." } ] }, + "flags": {}, + "id": 213, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "initialize", + "parameters": [], "type": { - "type": "reference", - "name": "Self" - }, - "parameters": [] + "name": "None", + "type": "literal", + "value": null + } } ] }, { - "id": 183, - "name": "get_global_configuration", - "module": "_configuration", "kind": 2048, "kindString": "Method", - "flags": {}, + "children": [], "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Retrieve the global instance of the configuration.\\n\\nMostly for the backwards compatibility. It is recommended to use the `service_locator.get_configuration()`\\ninstead.\"]}" + "text": "Create a new ProxyInfo object.\n\nUse it if you want to work with a rich representation of a proxy URL. If you need the URL string only,\nuse `ProxyConfiguration.new_url`.\n" } ] }, - "children": [], + "decorations": [], + "flags": {}, "groups": [], + "id": 214, + "module": "_proxy_configuration", + "name": "new_proxy_info", + "parsedDocstring": { + "text": "Create a new ProxyInfo object.\n\nUse it if you want to work with a rich representation of a proxy URL. If you need the URL string only,\nuse `ProxyConfiguration.new_url`.\n", + "args": { + "session_id": "Represents the identifier of a proxy session (https://docs.apify.com/proxy#sessions).\nAll the HTTP requests going through the proxy with the same session identifier will use the same\ntarget proxy server (i.e. the same IP address). The identifier must not be longer than 50 characters\nand include only the following: `0-9`, `a-z`, `A-Z`, `\".\"`, `\"_\"` and `\"~\"`.", + "request": "request for which the proxy info is being issued, used in proxy tier handling.", + "proxy_tier": "allows forcing the proxy tier to be used.\n" + }, + "returns": "Dictionary that represents information about the proxy and its configuration." + }, "sources": [ { - "filename": "/tmp-old-source-v27/src/apify/_configuration.py", - "line": 391, "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/_configuration.py#L391" + "fileName": "/src/apify/_proxy_configuration.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 205 } ], + "type": { + "name": "Undefined", + "type": "reference" + }, "signatures": [ { - "id": 184, - "name": "get_global_configuration", - "modifiers": [], - "kind": 4096, - "kindString": "Call signature", - "flags": {}, "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "Dictionary that represents information about the proxy and its configuration." + } + ], + "tag": "@returns" + } + ], "summary": [ { "kind": "text", - "text": "{\"content\": [\"Retrieve the global instance of the configuration.\\n\\nMostly for the backwards compatibility. It is recommended to use the `service_locator.get_configuration()`\\ninstead.\"]}" + "text": "Create a new ProxyInfo object.\n\nUse it if you want to work with a rich representation of a proxy URL. If you need the URL string only,\nuse `ProxyConfiguration.new_url`.\n" } ] }, + "flags": {}, + "id": 215, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "new_proxy_info", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Represents the identifier of a proxy session (https://docs.apify.com/proxy#sessions).\nAll the HTTP requests going through the proxy with the same session identifier will use the same\ntarget proxy server (i.e. the same IP address). The identifier must not be longer than 50 characters\nand include only the following: `0-9`, `a-z`, `A-Z`, `\".\"`, `\"_\"` and `\"~\"`." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": false + }, + "id": 216, + "kind": 32768, + "kindString": "Parameter", + "name": "session_id", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "request for which the proxy info is being issued, used in proxy tier handling." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": false + }, + "id": 217, + "kind": 32768, + "kindString": "Parameter", + "name": "request", + "type": { + "name": "Request | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "Request" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "allows forcing the proxy tier to be used.\n" + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": false + }, + "id": 218, + "kind": 32768, + "kindString": "Parameter", + "name": "proxy_tier", + "type": { + "name": "int | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "int" + }, + { + "type": "literal", + "value": null + } + ] + } + } + ], "type": { - "type": "reference", - "name": "Configuration", - "id": 180 - }, - "parameters": [] + "name": "ProxyInfo | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "ProxyInfo", + "target": "208" + }, + { + "type": "literal", + "value": null + } + ] + } } ] } ], - "groups": [ - { - "title": "Methods", - "children": [ - 181, - 183 - ] - } - ], - "sources": [ - { - "filename": "/tmp-old-source-v27/src/apify/_configuration.py", - "line": 29, - "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/_configuration.py#L29" - } - ] - }, - { - "id": 185, - "name": "encode_base62", - "module": "_crypto", - "kind": 2048, - "kindString": "Method", - "flags": {}, "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Encode the given number to base62.\"]}" + "text": "Configures a connection to a proxy server with the provided options.\n\nProxy servers are used to prevent target websites from blocking your crawlers based on IP address rate limits or\nblacklists. The default servers used by this class are managed by [Apify Proxy](https://docs.apify.com/proxy).\nTo be able to use Apify Proxy, you need an Apify account and access to the selected proxies. If you provide\nno configuration option, the proxies will be managed automatically using a smart algorithm.\n\nIf you want to use your own proxies, use the `proxy_urls` or `new_url_function` constructor options. Your list\nof proxy URLs will be rotated by the configuration, if this option is provided." } ] }, - "children": [], - "groups": [], - "sources": [ + "decorations": [ { - "filename": "/tmp-old-source-v27/src/apify/_crypto.py", - "line": 171, - "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/_crypto.py#L171" + "args": "('Classes')", + "name": "docs_group" } ], - "signatures": [ - { - "id": 186, - "name": "encode_base62", - "modifiers": [], - "kind": 4096, - "kindString": "Call signature", - "flags": {}, - "comment": { - "summary": [ - { - "kind": "text", - "text": "{\"content\": [\"Encode the given number to base62.\"]}" - } - ] - }, - "type": { - "type": "intrinsic", - "name": "str" - }, - "parameters": [ - { - "id": 187, - "name": "num", - "kind": 32768, - "kindString": "Parameter", - "flags": {}, - "type": { - "type": "intrinsic", - "name": "int" - } - } - ] - } - ] - }, - { - "id": 188, - "name": "Webhook", - "module": "_models", - "docsGroup": "Data structures", - "kind": 128, - "kindString": "Class", "flags": {}, - "children": [], - "groups": [], - "sources": [ + "groups": [ { - "filename": "/tmp-old-source-v27/src/apify/_models.py", - "line": 20, - "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/_models.py#L20" + "children": [ + 212, + 214 + ], + "title": "Methods" } - ] - }, - { - "id": 189, - "name": "ActorRunMeta", - "module": "_models", - "docsGroup": "Data structures", - "kind": 128, - "kindString": "Class", - "flags": {}, - "children": [], - "groups": [], + ], + "id": 211, + "module": "_proxy_configuration", + "name": "ProxyConfiguration", + "parsedDocstring": { + "text": "Configures a connection to a proxy server with the provided options.\n\nProxy servers are used to prevent target websites from blocking your crawlers based on IP address rate limits or\nblacklists. The default servers used by this class are managed by [Apify Proxy](https://docs.apify.com/proxy).\nTo be able to use Apify Proxy, you need an Apify account and access to the selected proxies. If you provide\nno configuration option, the proxies will be managed automatically using a smart algorithm.\n\nIf you want to use your own proxies, use the `proxy_urls` or `new_url_function` constructor options. Your list\nof proxy URLs will be rotated by the configuration, if this option is provided." + }, "sources": [ { - "filename": "/tmp-old-source-v27/src/apify/_models.py", - "line": 39, "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/_models.py#L39" + "fileName": "/src/apify/_proxy_configuration.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 93 } - ] + ], + "type": { + "name": "Undefined", + "type": "reference" + } }, { - "id": 190, - "name": "ActorRunStats", - "module": "_models", - "docsGroup": "Data structures", - "kind": 128, - "kindString": "Class", - "flags": {}, + "kind": 1024, + "kindString": "Property", "children": [], - "groups": [], - "sources": [ - { - "filename": "/tmp-old-source-v27/src/apify/_models.py", - "line": 46, - "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/_models.py#L46" - } - ] - }, - { - "id": 191, - "name": "ActorRunOptions", - "module": "_models", - "docsGroup": "Data structures", - "kind": 128, - "kindString": "Class", + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, "flags": {}, - "children": [], "groups": [], + "id": 219, + "module": "_consts", + "name": "EVENT_LISTENERS_TIMEOUT", + "parsedDocstring": { + "text": "" + }, "sources": [ { - "filename": "/tmp-old-source-v27/src/apify/_models.py", - "line": 67, "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/_models.py#L67" + "fileName": "/src/apify/_consts.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 6 } - ] + ], + "type": { + "name": "Undefined", + "type": "reference" + } }, { - "id": 192, - "name": "ActorRunUsage", - "module": "_models", - "docsGroup": "Data structures", - "kind": 128, - "kindString": "Class", - "flags": {}, + "kind": 1024, + "kindString": "Property", "children": [], - "groups": [], - "sources": [ - { - "filename": "/tmp-old-source-v27/src/apify/_models.py", - "line": 78, - "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/_models.py#L78" - } - ] - }, - { - "id": 193, - "name": "ActorRun", - "module": "_models", - "docsGroup": "Data structures", - "kind": 128, - "kindString": "Class", + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, "flags": {}, - "children": [], "groups": [], + "id": 220, + "module": "_consts", + "name": "BASE64_REGEXP", + "parsedDocstring": { + "text": "" + }, "sources": [ { - "filename": "/tmp-old-source-v27/src/apify/_models.py", - "line": 96, "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/_models.py#L96" + "fileName": "/src/apify/_consts.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 8 } - ] + ], + "type": { + "name": "Undefined", + "type": "reference" + } }, { - "id": 194, - "name": "PersistStateEvent", - "module": "_platform_event_manager", - "docsGroup": "Data structures", - "kind": 128, - "kindString": "Class", - "flags": {}, + "kind": 1024, + "kindString": "Property", "children": [], - "groups": [], - "sources": [ - { - "filename": "/tmp-old-source-v27/src/apify/_platform_event_manager.py", - "line": 35, - "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/_platform_event_manager.py#L35" - } - ] - }, - { - "id": 195, - "name": "SystemInfoEventData", - "module": "_platform_event_manager", - "docsGroup": "Data structures", - "kind": 128, - "kindString": "Class", + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, "flags": {}, - "children": [], "groups": [], + "id": 221, + "module": "_consts", + "name": "ENCRYPTED_STRING_VALUE_PREFIX", + "parsedDocstring": { + "text": "" + }, "sources": [ { - "filename": "/tmp-old-source-v27/src/apify/_platform_event_manager.py", - "line": 41, "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/_platform_event_manager.py#L41" + "fileName": "/src/apify/_consts.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 9 } - ] + ], + "type": { + "name": "Undefined", + "type": "reference" + } }, { - "id": 196, - "name": "SystemInfoEvent", - "module": "_platform_event_manager", - "docsGroup": "Data structures", - "kind": 128, - "kindString": "Class", - "flags": {}, + "kind": 1024, + "kindString": "Property", "children": [], - "groups": [], - "sources": [ - { - "filename": "/tmp-old-source-v27/src/apify/_platform_event_manager.py", - "line": 68, - "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/_platform_event_manager.py#L68" - } - ] - }, - { - "id": 197, - "name": "MigratingEvent", - "module": "_platform_event_manager", - "docsGroup": "Data structures", - "kind": 128, - "kindString": "Class", + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, "flags": {}, - "children": [], "groups": [], + "id": 222, + "module": "_consts", + "name": "ENCRYPTED_JSON_VALUE_PREFIX", + "parsedDocstring": { + "text": "" + }, "sources": [ { - "filename": "/tmp-old-source-v27/src/apify/_platform_event_manager.py", - "line": 74, "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/_platform_event_manager.py#L74" + "fileName": "/src/apify/_consts.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 10 } - ] + ], + "type": { + "name": "Undefined", + "type": "reference" + } }, { - "id": 198, - "name": "AbortingEvent", - "module": "_platform_event_manager", - "docsGroup": "Data structures", - "kind": 128, - "kindString": "Class", - "flags": {}, + "kind": 1024, + "kindString": "Property", "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, "groups": [], + "id": 223, + "module": "_consts", + "name": "ENCRYPTED_INPUT_VALUE_REGEXP", + "parsedDocstring": { + "text": "" + }, "sources": [ { - "filename": "/tmp-old-source-v27/src/apify/_platform_event_manager.py", - "line": 80, "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/_platform_event_manager.py#L80" + "fileName": "/src/apify/_consts.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 11 } - ] + ], + "type": { + "name": "Undefined", + "type": "reference" + } }, { - "id": 199, - "name": "ExitEvent", - "module": "_platform_event_manager", - "docsGroup": "Data structures", - "kind": 128, - "kindString": "Class", - "flags": {}, + "kind": 1024, + "kindString": "Property", "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, "groups": [], + "id": 224, + "module": "_configuration", + "name": "logger", + "parsedDocstring": { + "text": "" + }, "sources": [ { - "filename": "/tmp-old-source-v27/src/apify/_platform_event_manager.py", - "line": 86, "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/_platform_event_manager.py#L86" + "fileName": "/src/apify/_configuration.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 17 } - ] + ], + "type": { + "name": "Undefined", + "type": "reference" + } }, { - "id": 200, - "name": "EventWithoutData", - "module": "_platform_event_manager", - "docsGroup": "Data structures", "kind": 128, "kindString": "Class", - "flags": {}, - "children": [], - "groups": [], - "sources": [ + "children": [ { - "filename": "/tmp-old-source-v27/src/apify/_platform_event_manager.py", - "line": 92, - "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/_platform_event_manager.py#L92" - } - ] - }, - { - "id": 201, - "name": "DeprecatedEvent", - "module": "_platform_event_manager", - "docsGroup": "Data structures", - "kind": 128, - "kindString": "Class", - "flags": {}, - "children": [], - "groups": [], - "sources": [ - { - "filename": "/tmp-old-source-v27/src/apify/_platform_event_manager.py", - "line": 105, - "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/_platform_event_manager.py#L105" - } - ] - }, - { - "id": 202, - "name": "UnknownEvent", - "module": "_platform_event_manager", - "docsGroup": "Data structures", - "kind": 128, - "kindString": "Class", - "flags": {}, - "children": [], - "groups": [], - "sources": [ - { - "filename": "/tmp-old-source-v27/src/apify/_platform_event_manager.py", - "line": 111, - "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/_platform_event_manager.py#L111" - } - ] - }, - { - "id": 203, - "name": "PlatformEventManager", - "module": "_platform_event_manager", - "docsGroup": "Classes", - "kind": 128, - "kindString": "Class", - "flags": {}, - "comment": { - "summary": [ - { - "kind": "text", - "text": "{\"content\": [\"A class for managing Actor events.\\n\\nYou shouldn't use this class directly,\\nbut instead use it via the `Actor.on()` and `Actor.off()` methods.\"]}" - } - ] - }, - "children": [ - { - "id": 204, - "name": "__init__", - "module": "_platform_event_manager", - "kind": 512, - "kindString": "Constructor", - "flags": {}, + "kind": 1024, + "kindString": "Property", + "children": [], "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Create an instance of the EventManager.\\n\", {\"Arguments\": [{\"param\": \"config\", \"desc\": \"The Actor configuration to be used in this event manager.\"}, {\"param\": \"kwargs\", \"desc\": \"Event manager options - forwarded to the base class\"}]}]}" + "text": "" } ] }, - "children": [], + "flags": {}, "groups": [], + "id": 226, + "module": "_configuration", + "name": "actor_id", + "parsedDocstring": { + "text": "" + }, "sources": [ { - "filename": "/tmp-old-source-v27/src/apify/_platform_event_manager.py", - "line": 151, "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/_platform_event_manager.py#L151" + "fileName": "/src/apify/_configuration.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 36 } ], - "signatures": [ - { - "id": 205, - "name": "__init__", - "modifiers": [], - "kind": 4096, - "kindString": "Call signature", - "flags": {}, - "comment": { - "summary": [ - { - "kind": "text", - "text": "{\"content\": [\"Create an instance of the EventManager.\\n\", {\"" - } - ] - }, - "type": { + "type": { + "name": "Annotated[ str | None, Field( validation_alias=AliasChoices( 'actor_id', 'apify_actor_id', 'apify_act_id', ), description='ID of the Actor', ), ]", + "type": "union", + "types": [ + { "type": "reference", - "name": "None" + "name": "str" }, - "parameters": [ - { - "id": 206, - "name": "config", - "kind": 32768, - "kindString": "Parameter", - "flags": {}, - "type": { - "type": "reference", - "name": "Configuration", - "id": 180 - } - }, - { - "id": 207, - "name": "kwargs", - "kind": 32768, - "kindString": "Parameter", - "flags": {}, - "type": { - "type": "reference", - "name": "Unpack[EventManagerOptions]" - } - } - ] - } - ] - } - ], - "groups": [ - { - "title": "Constructors", - "children": [ - 204 - ] - } - ], - "sources": [ - { - "filename": "/tmp-old-source-v27/src/apify/_platform_event_manager.py", - "line": 139, - "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/_platform_event_manager.py#L139" - } - ] - }, - { - "id": 208, - "name": "ProxyInfo", - "module": "_proxy_configuration", - "docsGroup": "Classes", - "kind": 128, - "kindString": "Class", - "flags": {}, - "comment": { - "summary": [ - { - "kind": "text", - "text": "{\"content\": [\"Provides information about a proxy connection that is used for requests.\"]}" + { + "type": "literal", + "value": null + } + ] } - ] - }, - "children": [ + }, { - "id": 209, - "name": "groups", - "module": "_proxy_configuration", "kind": 1024, "kindString": "Property", - "flags": {}, + "children": [], "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"An array of proxy groups to be used by the [Apify Proxy](https://docs.apify.com/proxy). If not provided,\\nthe proxy will select the groups automatically.\"]}" + "text": "" } ] }, - "type": { - "type": "reference", - "name": "list[str]" - }, - "children": [], + "flags": {}, "groups": [], + "id": 227, + "module": "_configuration", + "name": "actor_full_name", + "parsedDocstring": { + "text": "" + }, "sources": [ { - "filename": "/tmp-old-source-v27/src/apify/_proxy_configuration.py", - "line": 77, "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/_proxy_configuration.py#L77" + "fileName": "/src/apify/_configuration.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 48 } - ] + ], + "type": { + "name": "Annotated[ str | None, Field( description='Full name of the Actor', ), ]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } }, { - "id": 210, - "name": "country_code", - "module": "_proxy_configuration", "kind": 1024, "kindString": "Property", - "flags": {}, + "children": [], "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"If set and relevant proxies are available in your Apify account, all proxied requests will use IP addresses\\nthat are geolocated to the specified country. For example `GB` for IPs from Great Britain. Note that online\\nservices often have their own rules for handling geolocation and thus the country selection is a best attempt\\nat geolocation, rather than a guaranteed hit. This parameter is optional, by default, each proxied request is\\nassigned an IP address from a random country. The country code needs to be a two letter ISO country code.\\nSee the [full list of available country codes](https://en.wikipedia.org/wiki/ISO_3166-1_alpha-2#Officially_assigned_code_elements).\\nThis parameter is optional, by default, the proxy uses all available proxy servers from all countries.\"]}" + "text": "" } ] }, - "type": { - "type": "reference", - "name": "str | None" - }, - "children": [], + "flags": {}, "groups": [], + "id": 228, + "module": "_configuration", + "name": "actor_run_id", + "parsedDocstring": { + "text": "" + }, "sources": [ { - "filename": "/tmp-old-source-v27/src/apify/_proxy_configuration.py", - "line": 81, "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/_proxy_configuration.py#L81" + "fileName": "/src/apify/_configuration.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 55 } - ] - } - ], - "groups": [ - { - "title": "Properties", - "children": [ - 210, - 209 - ] - } - ], - "sources": [ - { - "filename": "/tmp-old-source-v27/src/apify/_proxy_configuration.py", - "line": 74, - "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/_proxy_configuration.py#L74" - } - ] - }, - { - "id": 211, - "name": "ProxyConfiguration", - "module": "_proxy_configuration", - "docsGroup": "Classes", - "kind": 128, - "kindString": "Class", - "flags": {}, - "comment": { - "summary": [ - { - "kind": "text", - "text": "{\"content\": [\"Configures a connection to a proxy server with the provided options.\\n\\nProxy servers are used to prevent target websites from blocking your crawlers based on IP address rate limits or\\nblacklists. The default servers used by this class are managed by [Apify Proxy](https://docs.apify.com/proxy).\\nTo be able to use Apify Proxy, you need an Apify account and access to the selected proxies. If you provide\\nno configuration option, the proxies will be managed automatically using a smart algorithm.\\n\\nIf you want to use your own proxies, use the `proxy_urls` or `new_url_function` constructor options. Your list\\nof proxy URLs will be rotated by the configuration, if this option is provided.\"]}" + ], + "type": { + "name": "Annotated[ str | None, Field( validation_alias=AliasChoices( 'actor_run_id', 'apify_actor_run_id', 'apify_act_run_id', ), description='ID of the Actor run', ), ]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] } - ] - }, - "children": [ + }, { - "id": 212, - "name": "initialize", - "module": "_proxy_configuration", - "kind": 2048, - "kindString": "Method", - "flags": {}, + "kind": 1024, + "kindString": "Property", + "children": [], "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Check if using proxy, if so, check the access.\\n\\nLoad the Apify Proxy password from API (only if not passed to constructor or through env var).\\n\\nOnly called if Apify Proxy configuration is used. Also checks if country has access to Apify Proxy groups\\nif the country code is provided.\\n\\nYou should use the Actor.create_proxy_configuration function to create a pre-initialized\\n`ProxyConfiguration` instance instead of calling this manually.\"]}" + "text": "" } ] }, - "children": [], + "flags": {}, "groups": [], + "id": 229, + "module": "_configuration", + "name": "actor_build_id", + "parsedDocstring": { + "text": "" + }, "sources": [ { - "filename": "/tmp-old-source-v27/src/apify/_proxy_configuration.py", - "line": 180, "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/_proxy_configuration.py#L180" + "fileName": "/src/apify/_configuration.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 67 } ], - "signatures": [ - { - "id": 213, - "name": "initialize", - "modifiers": [ - "async" - ], - "kind": 4096, - "kindString": "Call signature", - "flags": {}, - "comment": { - "summary": [ - { - "kind": "text", - "text": "{\"content\": [\"Check if using proxy, if so, check the access.\\n\\nLoad the Apify Proxy password from API (only if not passed to constructor or through env var).\\n\\nOnly called if Apify Proxy configuration is used. Also checks if country has access to Apify Proxy groups\\nif the country code is provided.\\n\\nYou should use the Actor.create_proxy_configuration function to create a pre-initialized\\n`ProxyConfiguration` instance instead of calling this manually.\"]}" - } - ] - }, - "type": { + "type": { + "name": "Annotated[ str | None, Field( validation_alias=AliasChoices( 'actor_build_id', 'apify_actor_build_id', ), description='ID of the Actor build used in the run', ), ]", + "type": "union", + "types": [ + { "type": "reference", - "name": "None" + "name": "str" }, - "parameters": [] - } - ] + { + "type": "literal", + "value": null + } + ] + } }, { - "id": 214, - "name": "new_proxy_info", - "module": "_proxy_configuration", - "kind": 2048, - "kindString": "Method", - "flags": {}, + "kind": 1024, + "kindString": "Property", + "children": [], "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Create a new ProxyInfo object.\\n\\nUse it if you want to work with a rich representation of a proxy URL. If you need the URL string only,\\nuse `ProxyConfiguration.new_url`.\\n\", {\"Arguments\": [{\"param\": \"session_id\", \"desc\": \"Represents the identifier of a proxy session (https://docs.apify.com/proxy#sessions).\\nAll the HTTP requests going through the proxy with the same session identifier will use the same\\ntarget proxy server (i.e. the same IP address). The identifier must not be longer than 50 characters\\nand include only the following: `0-9`, `a-z`, `A-Z`, `\\\".\\\"`, `\\\"_\\\"` and `\\\"~\\\"`.\"}, {\"param\": \"request\", \"desc\": \"request for which the proxy info is being issued, used in proxy tier handling.\"}, {\"param\": \"proxy_tier\", \"desc\": \"allows forcing the proxy tier to be used.\\n\"}]}, {\"Returns\": [\"Dictionary that represents information about the proxy and its configuration.\"]}]}" + "text": "" } ] }, - "children": [], + "flags": {}, "groups": [], + "id": 230, + "module": "_configuration", + "name": "actor_build_number", + "parsedDocstring": { + "text": "" + }, "sources": [ { - "filename": "/tmp-old-source-v27/src/apify/_proxy_configuration.py", - "line": 205, "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/_proxy_configuration.py#L205" + "fileName": "/src/apify/_configuration.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 78 } ], - "signatures": [ - { - "id": 215, - "name": "new_proxy_info", - "modifiers": [ - "async" - ], - "kind": 4096, - "kindString": "Call signature", - "flags": {}, - "comment": { - "summary": [ - { - "kind": "text", - "text": "{\"content\": [\"Create a new ProxyInfo object.\\n\\nUse it if you want to work with a rich representation of a proxy URL. If you need the URL string only,\\nuse `ProxyConfiguration.new_url`.\\n\", {\"" - } - ] - }, - "type": { + "type": { + "name": "Annotated[ str | None, Field( validation_alias=AliasChoices( 'actor_build_number', 'apify_actor_build_number', ), description='Build number of the Actor build used in the run', ), ]", + "type": "union", + "types": [ + { "type": "reference", - "name": "ProxyInfo | None" + "name": "str" }, - "parameters": [ - { - "id": 216, - "name": "session_id", - "kind": 32768, - "kindString": "Parameter", - "flags": {}, - "type": { - "type": "reference", - "name": "str | None" - }, - "defaultValue": "None" - }, - { - "id": 217, - "name": "request", - "kind": 32768, - "kindString": "Parameter", - "flags": {}, - "type": { - "type": "reference", - "name": "Request | None" - }, - "defaultValue": "None" - }, - { - "id": 218, - "name": "proxy_tier", - "kind": 32768, - "kindString": "Parameter", - "flags": {}, - "type": { - "type": "reference", - "name": "int | None" - }, - "defaultValue": "None" - } - ] - } - ] - } - ], - "groups": [ - { - "title": "Methods", - "children": [ - 212, - 214 - ] - } - ], - "sources": [ - { - "filename": "/tmp-old-source-v27/src/apify/_proxy_configuration.py", - "line": 93, - "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/_proxy_configuration.py#L93" - } - ] - }, - { - "id": 219, - "name": "docs_group", - "module": "_utils", - "kind": 2048, - "kindString": "Method", - "flags": {}, - "comment": { - "summary": [ - { - "kind": "text", - "text": "{\"content\": [\"Mark a symbol for rendering and grouping in documentation.\\n\\nThis decorator is used solely for documentation purposes and does not modify the behavior\\nof the decorated callable.\\n\", {\"Arguments\": [{\"param\": \"group_name\", \"desc\": \"The documentation group to which the symbol belongs.\\n\"}]}, {\"Returns\": [\"The original callable without modification.\"]}]}" + { + "type": "literal", + "value": null + } + ] } - ] - }, - "children": [], - "groups": [], - "sources": [ - { - "filename": "/tmp-old-source-v27/src/apify/_utils.py", - "line": 33, - "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/_utils.py#L33" - } - ], - "signatures": [ + }, { - "id": 220, - "name": "docs_group", - "modifiers": [], - "kind": 4096, - "kindString": "Call signature", - "flags": {}, + "kind": 1024, + "kindString": "Property", + "children": [], "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Mark a symbol for rendering and grouping in documentation.\\n\\nThis decorator is used solely for documentation purposes and does not modify the behavior\\nof the decorated callable.\\n\", {\"" + "text": "" } ] }, - "type": { - "type": "reference", - "name": "Callable" + "flags": {}, + "groups": [], + "id": 231, + "module": "_configuration", + "name": "actor_build_tags", + "parsedDocstring": { + "text": "" }, - "parameters": [ + "sources": [ { - "id": 221, - "name": "group_name", - "kind": 32768, - "kindString": "Parameter", - "flags": {}, - "type": { + "character": 1, + "fileName": "/src/apify/_configuration.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 89 + } + ], + "type": { + "name": "Annotated[ list[str] | None, Field( description='Build tags of the Actor build used in the run', ), BeforeValidator(_transform_to_list), ]", + "type": "union", + "types": [ + { "type": "reference", - "name": "GroupName" + "name": "list", + "typeArguments": [ + { + "type": "reference", + "name": "str" + } + ], + "target": "596" + }, + { + "type": "literal", + "value": null } - } - ] - } - ] - }, - { - "id": 222, - "name": "docs_name", - "module": "_utils", - "kind": 2048, - "kindString": "Method", - "flags": {}, - "comment": { - "summary": [ - { - "kind": "text", - "text": "{\"content\": [\"Rename a symbol for documentation rendering.\\n\\nThis decorator modifies only the displayed name of the symbol in the generated documentation\\nand does not affect its runtime behavior.\\n\", {\"Arguments\": [{\"param\": \"symbol_name\", \"desc\": \"The name to be used in the documentation.\\n\"}]}, {\"Returns\": [\"The original callable without modification.\"]}]}" + ] } - ] - }, - "children": [], - "groups": [], - "sources": [ - { - "filename": "/tmp-old-source-v27/src/apify/_utils.py", - "line": 52, - "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/_utils.py#L52" - } - ], - "signatures": [ + }, { - "id": 223, - "name": "docs_name", - "modifiers": [], - "kind": 4096, - "kindString": "Call signature", - "flags": {}, + "kind": 1024, + "kindString": "Property", + "children": [], "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Rename a symbol for documentation rendering.\\n\\nThis decorator modifies only the displayed name of the symbol in the generated documentation\\nand does not affect its runtime behavior.\\n\", {\"" + "text": "" } ] }, - "type": { - "type": "reference", - "name": "Callable" + "flags": {}, + "groups": [], + "id": 232, + "module": "_configuration", + "name": "actor_task_id", + "parsedDocstring": { + "text": "" }, - "parameters": [ + "sources": [ { - "id": 224, - "name": "symbol_name", - "kind": 32768, - "kindString": "Parameter", - "flags": {}, - "type": { - "type": "intrinsic", + "character": 1, + "fileName": "/src/apify/_configuration.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 97 + } + ], + "type": { + "name": "Annotated[ str | None, Field( validation_alias=AliasChoices( 'actor_task_id', 'apify_actor_task_id', ), description='ID of the Actor task. Empty if Actor is run outside of any task, e.g. directly using the API', ), ]", + "type": "union", + "types": [ + { + "type": "reference", "name": "str" + }, + { + "type": "literal", + "value": null } - } - ] - } - ] - }, - { - "id": 225, - "name": "ApifyStorageClient", - "module": "apify_storage_client._apify_storage_client", - "docsGroup": "Classes", - "kind": 128, - "kindString": "Class", - "flags": {}, - "comment": { - "summary": [ - { - "kind": "text", - "text": "{\"content\": [\"A storage client implementation based on the Apify platform storage.\"]}" + ] } - ] - }, - "children": [], - "groups": [], - "sources": [ + }, { - "filename": "/tmp-old-source-v27/src/apify/apify_storage_client/_apify_storage_client.py", - "line": 24, - "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/apify_storage_client/_apify_storage_client.py#L24" - } - ] - }, - { - "id": 226, - "name": "DatasetClient", - "module": "apify_storage_client._dataset_client", - "kind": 128, - "kindString": "Class", - "flags": {}, - "comment": { - "summary": [ - { - "kind": "text", - "text": "{\"content\": [\"Dataset resource client implementation based on the Apify platform storage.\"]}" + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 233, + "module": "_configuration", + "name": "actor_events_ws_url", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_configuration.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 108 + } + ], + "type": { + "name": "Annotated[ str | None, Field( validation_alias=AliasChoices( 'actor_events_websocket_url', 'apify_actor_events_ws_url', ), description='Websocket URL where Actor may listen for events from Actor platform', ), ]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] } - ] - }, - "children": [], - "groups": [], - "sources": [ + }, { - "filename": "/tmp-old-source-v27/src/apify/apify_storage_client/_dataset_client.py", - "line": 20, - "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/apify_storage_client/_dataset_client.py#L20" - } - ] - }, - { - "id": 227, - "name": "DatasetCollectionClient", - "module": "apify_storage_client._dataset_collection_client", - "kind": 128, - "kindString": "Class", - "flags": {}, - "comment": { - "summary": [ - { - "kind": "text", - "text": "{\"content\": [\"Dataset collection resource client implementation based on the Apify platform storage.\"]}" - } - ] - }, - "children": [], - "groups": [], - "sources": [ - { - "filename": "/tmp-old-source-v27/src/apify/apify_storage_client/_dataset_collection_client.py", - "line": 14, - "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/apify_storage_client/_dataset_collection_client.py#L14" - } - ] - }, - { - "id": 228, - "name": "KeyValueStoreClient", - "module": "apify_storage_client._key_value_store_client", - "kind": 128, - "kindString": "Class", - "flags": {}, - "comment": { - "summary": [ - { - "kind": "text", - "text": "{\"content\": [\"Key-value store resource client implementation based on the Apify platform storage.\"]}" - } - ] - }, - "children": [ - { - "id": 229, - "name": "get_public_url", - "module": "apify_storage_client._key_value_store_client", - "kind": 2048, - "kindString": "Method", - "flags": {}, + "kind": 1024, + "kindString": "Property", + "children": [], "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Get a URL for the given key that may be used to publicly access the value in the remote key-value store.\\n\", {\"Arguments\": [{\"param\": \"key\", \"desc\": \"The key for which the URL should be generated.\"}]}]}" + "text": "" } ] }, - "children": [], + "flags": {}, "groups": [], + "id": 234, + "module": "_configuration", + "name": "api_base_url", + "parsedDocstring": { + "text": "" + }, "sources": [ { - "filename": "/tmp-old-source-v27/src/apify/apify_storage_client/_key_value_store_client.py", - "line": 89, "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/apify_storage_client/_key_value_store_client.py#L89" + "fileName": "/src/apify/_configuration.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 119 } ], - "signatures": [ - { - "id": 230, - "name": "get_public_url", - "modifiers": [ - "async" - ], - "kind": 4096, - "kindString": "Call signature", - "flags": {}, - "comment": { - "summary": [ - { - "kind": "text", - "text": "{\"content\": [\"Get a URL for the given key that may be used to publicly access the value in the remote key-value store.\\n\", {\"" - } - ] - }, - "type": { - "type": "intrinsic", - "name": "str" - }, - "parameters": [ - { - "id": 231, - "name": "key", - "kind": 32768, - "kindString": "Parameter", - "flags": {}, - "type": { - "type": "intrinsic", - "name": "str" - } - } - ] - } - ] - } - ], - "groups": [ - { - "title": "Methods", - "children": [ - 229 - ] - } - ], - "sources": [ - { - "filename": "/tmp-old-source-v27/src/apify/apify_storage_client/_key_value_store_client.py", - "line": 23, - "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/apify_storage_client/_key_value_store_client.py#L23" - } - ] - }, - { - "id": 232, - "name": "KeyValueStoreCollectionClient", - "module": "apify_storage_client._key_value_store_collection_client", - "kind": 128, - "kindString": "Class", - "flags": {}, - "comment": { - "summary": [ - { - "kind": "text", - "text": "{\"content\": [\"Key-value store collection resource client implementation based on the Apify platform storage.\"]}" - } - ] - }, - "children": [], - "groups": [], - "sources": [ - { - "filename": "/tmp-old-source-v27/src/apify/apify_storage_client/_key_value_store_collection_client.py", - "line": 14, - "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/apify_storage_client/_key_value_store_collection_client.py#L14" - } - ] - }, - { - "id": 233, - "name": "RequestQueueClient", - "module": "apify_storage_client._request_queue_client", - "kind": 128, - "kindString": "Class", - "flags": {}, - "comment": { - "summary": [ - { - "kind": "text", - "text": "{\"content\": [\"Request queue resource client implementation based on the Apify platform storage.\"]}" - } - ] - }, - "children": [], - "groups": [], - "sources": [ - { - "filename": "/tmp-old-source-v27/src/apify/apify_storage_client/_request_queue_client.py", - "line": 24, - "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/apify_storage_client/_request_queue_client.py#L24" - } - ] - }, - { - "id": 234, - "name": "RequestQueueCollectionClient", - "module": "apify_storage_client._request_queue_collection_client", - "kind": 128, - "kindString": "Class", - "flags": {}, - "comment": { - "summary": [ - { - "kind": "text", - "text": "{\"content\": [\"Request queue collection resource client implementation based on the Apify platform storage.\"]}" - } - ] - }, - "children": [], - "groups": [], - "sources": [ - { - "filename": "/tmp-old-source-v27/src/apify/apify_storage_client/_request_queue_collection_client.py", - "line": 14, - "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/apify_storage_client/_request_queue_collection_client.py#L14" - } - ] - }, - { - "id": 235, - "name": "run_scrapy_actor", - "module": "scrapy._actor_runner", - "kind": 2048, - "kindString": "Method", - "flags": {}, - "comment": { - "summary": [ - { - "kind": "text", - "text": "{\"content\": [\"Start Twisted's reactor and execute the provided Actor coroutine.\\n\\nThis function initiates the Twisted reactor and runs the given asyncio coroutine (typically the\\nActor's main) by converting it to a Deferred. This bridges the asyncio and Twisted event loops,\\nenabling the Apify and Scrapy integration to work together.\"]}" + "type": { + "name": "str", + "type": "reference" } - ] - }, - "children": [], - "groups": [], - "sources": [ - { - "filename": "/tmp-old-source-v27/src/apify/scrapy/_actor_runner.py", - "line": 19, - "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/scrapy/_actor_runner.py#L19" - } - ], - "signatures": [ + }, { - "id": 236, - "name": "run_scrapy_actor", - "modifiers": [], - "kind": 4096, - "kindString": "Call signature", - "flags": {}, + "kind": 1024, + "kindString": "Property", + "children": [], "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Start Twisted's reactor and execute the provided Actor coroutine.\\n\\nThis function initiates the Twisted reactor and runs the given asyncio coroutine (typically the\\nActor's main) by converting it to a Deferred. This bridges the asyncio and Twisted event loops,\\nenabling the Apify and Scrapy integration to work together.\"]}" + "text": "" } ] }, - "type": { - "type": "reference", - "name": "None" + "flags": {}, + "groups": [], + "id": 235, + "module": "_configuration", + "name": "api_public_base_url", + "parsedDocstring": { + "text": "" }, - "parameters": [ + "sources": [ { - "id": 237, - "name": "coro", - "kind": 32768, - "kindString": "Parameter", - "flags": {}, - "type": { - "type": "reference", - "name": "Coroutine" - } + "character": 1, + "fileName": "/src/apify/_configuration.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 127 } - ] - } - ] - }, - { - "id": 238, - "name": "AsyncThread", - "module": "scrapy._async_thread", - "kind": 128, - "kindString": "Class", - "flags": {}, - "comment": { - "summary": [ - { - "kind": "text", - "text": "{\"content\": [\"Class for running an asyncio event loop in a separate thread.\\n\\nThis allows running asynchronous coroutines from synchronous code by executingthem on an event loop\\nthat runs in its own dedicated thread.\"]}" + ], + "type": { + "name": "str", + "type": "reference" } - ] - }, - "children": [ + }, { - "id": 239, - "name": "run_coro", - "module": "scrapy._async_thread", - "kind": 2048, - "kindString": "Method", - "flags": {}, + "kind": 1024, + "kindString": "Property", + "children": [], "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Run a coroutine on an event loop running in a separate thread.\\n\\nThis method schedules the coroutine to run on the event loop and blocks until the coroutine completes\\nor the specified timeout is reached.\\n\", {\"Arguments\": [{\"param\": \"coro\", \"desc\": \"The coroutine to run.\"}, {\"param\": \"timeout\", \"desc\": \"The maximum number of seconds to wait for the coroutine to finish.\\n\"}]}, {\"Returns\": [\"The result returned by the coroutine.\", \"\"]}, {\"Raises\": [{\"param\": \"RuntimeError\", \"desc\": \"If the event loop is not running.\"}, {\"param\": \"TimeoutError\", \"desc\": \"If the coroutine does not complete within the timeout.\"}, {\"param\": \"Exception\", \"desc\": \"Any exception raised during coroutine execution.\"}]}]}" + "text": "" } ] }, - "children": [], + "flags": {}, "groups": [], + "id": 236, + "module": "_configuration", + "name": "dedicated_cpus", + "parsedDocstring": { + "text": "" + }, "sources": [ { - "filename": "/tmp-old-source-v27/src/apify/scrapy/_async_thread.py", - "line": 33, "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/scrapy/_async_thread.py#L33" + "fileName": "/src/apify/_configuration.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 135 } ], - "signatures": [ - { - "id": 240, - "name": "run_coro", - "modifiers": [], - "kind": 4096, - "kindString": "Call signature", - "flags": {}, - "comment": { - "summary": [ - { - "kind": "text", - "text": "{\"content\": [\"Run a coroutine on an event loop running in a separate thread.\\n\\nThis method schedules the coroutine to run on the event loop and blocks until the coroutine completes\\nor the specified timeout is reached.\\n\", {\"" - } - ] - }, - "type": { + "type": { + "name": "Annotated[ float | None, Field( alias='apify_dedicated_cpus', description='Number of CPU cores reserved for the actor, based on allocated memory', ), ]", + "type": "union", + "types": [ + { "type": "reference", - "name": "Any" + "name": "float" }, - "parameters": [ - { - "id": 241, - "name": "coro", - "kind": 32768, - "kindString": "Parameter", - "flags": {}, - "type": { - "type": "reference", - "name": "Coroutine" - } - }, - { - "id": 242, - "name": "timeout", - "kind": 32768, - "kindString": "Parameter", - "flags": {}, - "type": { - "type": "reference", - "name": "timedelta" - }, - "defaultValue": "timedelta(seconds=60)" - } - ] - } - ] + { + "type": "literal", + "value": null + } + ] + } }, { - "id": 243, - "name": "close", - "module": "scrapy._async_thread", - "kind": 2048, - "kindString": "Method", - "flags": {}, + "kind": 1024, + "kindString": "Property", + "children": [], "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Close the event loop and its thread gracefully.\\n\\nThis method cancels all pending tasks, stops the event loop, and waits for the thread to exit.\\nIf the thread does not exit within the given timeout, a forced shutdown is attempted.\\n\", {\"Arguments\": [{\"param\": \"timeout\", \"desc\": \"The maximum number of seconds to wait for the event loop thread to exit.\"}]}]}" + "text": "" } ] }, - "children": [], + "flags": {}, "groups": [], + "id": 237, + "module": "_configuration", + "name": "disable_outdated_warning", + "parsedDocstring": { + "text": "" + }, "sources": [ { - "filename": "/tmp-old-source-v27/src/apify/scrapy/_async_thread.py", - "line": 70, "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/scrapy/_async_thread.py#L70" + "fileName": "/src/apify/_configuration.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 143 } ], - "signatures": [ - { - "id": 244, - "name": "close", - "modifiers": [], - "kind": 4096, - "kindString": "Call signature", - "flags": {}, - "comment": { - "summary": [ - { - "kind": "text", - "text": "{\"content\": [\"Close the event loop and its thread gracefully.\\n\\nThis method cancels all pending tasks, stops the event loop, and waits for the thread to exit.\\nIf the thread does not exit within the given timeout, a forced shutdown is attempted.\\n\", {\"" - } - ] - }, - "type": { - "type": "reference", - "name": "None" - }, - "parameters": [ - { - "id": 245, - "name": "timeout", - "kind": 32768, - "kindString": "Parameter", - "flags": {}, - "type": { - "type": "reference", - "name": "timedelta" - }, - "defaultValue": "timedelta(seconds=60)" - } - ] - } - ] - } - ], - "groups": [ - { - "title": "Methods", - "children": [ - 243, - 239 - ] - } - ], - "sources": [ - { - "filename": "/tmp-old-source-v27/src/apify/scrapy/_async_thread.py", - "line": 16, - "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/scrapy/_async_thread.py#L16" - } - ] - }, - { - "id": 246, - "name": "initialize_logging", - "module": "scrapy._logging_config", - "kind": 2048, - "kindString": "Method", - "flags": {}, - "comment": { - "summary": [ - { - "kind": "text", - "text": "{\"content\": [\"Configure logging for Apify Actors and adjust Scrapy's logging settings.\"]}" + "type": { + "name": "bool", + "type": "reference" } - ] - }, - "children": [], - "groups": [], - "sources": [ - { - "filename": "/tmp-old-source-v27/src/apify/scrapy/_logging_config.py", - "line": 26, - "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/scrapy/_logging_config.py#L26" - } - ], - "signatures": [ + }, { - "id": 247, - "name": "initialize_logging", - "modifiers": [], - "kind": 4096, - "kindString": "Call signature", - "flags": {}, + "kind": 1024, + "kindString": "Property", + "children": [], "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Configure logging for Apify Actors and adjust Scrapy's logging settings.\"]}" + "text": "" } ] }, - "type": { - "type": "reference", - "name": "None" + "flags": {}, + "groups": [], + "id": 238, + "module": "_configuration", + "name": "fact", + "parsedDocstring": { + "text": "" }, - "parameters": [] - } - ] - }, - { - "id": 248, - "name": "ApifyCacheStorage", - "module": "scrapy.extensions._httpcache", - "kind": 128, - "kindString": "Class", - "flags": {}, - "comment": { - "summary": [ - { - "kind": "text", - "text": "{\"content\": [\"A Scrapy cache storage that uses the Apify `KeyValueStore` to store responses.\\n\\nIt can be set as a storage for Scrapy's built-in `HttpCacheMiddleware`, which caches\\nresponses to requests. See HTTPCache middleware settings (prefixed with `HTTPCACHE_`)\\nin the Scrapy documentation for more information. Requires the asyncio Twisted reactor\\nto be installed.\"]}" + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_configuration.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 152 + } + ], + "type": { + "name": "Annotated[str | None, Field(alias='apify_fact')]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] } - ] - }, - "children": [ + }, { - "id": 249, - "name": "open_spider", - "module": "scrapy.extensions._httpcache", - "kind": 2048, - "kindString": "Method", - "flags": {}, + "kind": 1024, + "kindString": "Property", + "children": [], "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Open the cache storage for a spider.\"]}" + "text": "" } ] }, - "children": [], + "flags": {}, "groups": [], + "id": 239, + "module": "_configuration", + "name": "input_key", + "parsedDocstring": { + "text": "" + }, "sources": [ { - "filename": "/tmp-old-source-v27/src/apify/scrapy/extensions/_httpcache.py", - "line": 46, "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/scrapy/extensions/_httpcache.py#L46" + "fileName": "/src/apify/_configuration.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 154 } ], - "signatures": [ - { - "id": 250, - "name": "open_spider", - "modifiers": [], - "kind": 4096, - "kindString": "Call signature", - "flags": {}, - "comment": { - "summary": [ - { - "kind": "text", - "text": "{\"content\": [\"Open the cache storage for a spider.\"]}" - } - ] - }, - "type": { - "type": "reference", - "name": "None" - }, - "parameters": [ - { - "id": 251, - "name": "spider", - "kind": 32768, - "kindString": "Parameter", - "flags": {}, - "type": { - "type": "reference", - "name": "Spider" - } - } - ] - } - ] + "type": { + "name": "str", + "type": "reference" + } }, { - "id": 252, - "name": "close_spider", - "module": "scrapy.extensions._httpcache", - "kind": 2048, - "kindString": "Method", - "flags": {}, + "kind": 1024, + "kindString": "Property", + "children": [], "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Close the cache storage for a spider.\"]}" + "text": "" } ] }, - "children": [], + "flags": {}, "groups": [], + "id": 240, + "module": "_configuration", + "name": "input_secrets_private_key_file", + "parsedDocstring": { + "text": "" + }, "sources": [ { - "filename": "/tmp-old-source-v27/src/apify/scrapy/extensions/_httpcache.py", - "line": 65, "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/scrapy/extensions/_httpcache.py#L65" + "fileName": "/src/apify/_configuration.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 166 } ], - "signatures": [ - { - "id": 253, - "name": "close_spider", - "modifiers": [], - "kind": 4096, - "kindString": "Call signature", - "flags": {}, - "comment": { - "summary": [ - { - "kind": "text", - "text": "{\"content\": [\"Close the cache storage for a spider.\"]}" - } - ] - }, - "type": { + "type": { + "name": "Annotated[ str | None, Field( alias='apify_input_secrets_private_key_file', description='Path to the secret key used to decrypt Secret inputs.', ), ]", + "type": "union", + "types": [ + { "type": "reference", - "name": "None" + "name": "str" }, - "parameters": [ - { - "id": 254, - "name": "_", - "kind": 32768, - "kindString": "Parameter", - "flags": {}, - "type": { - "type": "reference", - "name": "Spider" - } - }, - { - "id": 255, - "name": "current_time", - "kind": 32768, - "kindString": "Parameter", - "flags": {}, - "type": { - "type": "reference", - "name": "int | None" - }, - "defaultValue": "None" - } - ] - } - ] + { + "type": "literal", + "value": null + } + ] + } }, { - "id": 256, - "name": "retrieve_response", - "module": "scrapy.extensions._httpcache", - "kind": 2048, - "kindString": "Method", - "flags": {}, + "kind": 1024, + "kindString": "Property", + "children": [], "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Retrieve a response from the cache storage.\"]}" + "text": "" } ] }, - "children": [], + "flags": {}, "groups": [], + "id": 241, + "module": "_configuration", + "name": "input_secrets_private_key_passphrase", + "parsedDocstring": { + "text": "" + }, "sources": [ { - "filename": "/tmp-old-source-v27/src/apify/scrapy/extensions/_httpcache.py", - "line": 108, "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/scrapy/extensions/_httpcache.py#L108" + "fileName": "/src/apify/_configuration.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 174 } ], - "signatures": [ - { - "id": 257, - "name": "retrieve_response", - "modifiers": [], - "kind": 4096, - "kindString": "Call signature", - "flags": {}, - "comment": { - "summary": [ - { - "kind": "text", - "text": "{\"content\": [\"Retrieve a response from the cache storage.\"]}" - } - ] - }, - "type": { + "type": { + "name": "Annotated[ str | None, Field( alias='apify_input_secrets_private_key_passphrase', description='Passphrase for the input secret key', ), ]", + "type": "union", + "types": [ + { "type": "reference", - "name": "Response | None" + "name": "str" }, - "parameters": [ - { - "id": 258, - "name": "_", - "kind": 32768, - "kindString": "Parameter", - "flags": {}, - "type": { - "type": "reference", - "name": "Spider" - } - }, - { - "id": 259, - "name": "request", - "kind": 32768, - "kindString": "Parameter", - "flags": {}, - "type": { - "type": "reference", - "name": "Request" - } - }, - { - "id": 260, - "name": "current_time", - "kind": 32768, - "kindString": "Parameter", - "flags": {}, - "type": { - "type": "reference", - "name": "int | None" - }, - "defaultValue": "None" - } - ] - } - ] + { + "type": "literal", + "value": null + } + ] + } }, { - "id": 261, - "name": "store_response", - "module": "scrapy.extensions._httpcache", - "kind": 2048, - "kindString": "Method", - "flags": {}, + "kind": 1024, + "kindString": "Property", + "children": [], "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Store a response in the cache storage.\"]}" + "text": "" } ] }, - "children": [], + "flags": {}, "groups": [], + "id": 242, + "module": "_configuration", + "name": "is_at_home", + "parsedDocstring": { + "text": "" + }, "sources": [ { - "filename": "/tmp-old-source-v27/src/apify/scrapy/extensions/_httpcache.py", - "line": 140, "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/scrapy/extensions/_httpcache.py#L140" + "fileName": "/src/apify/_configuration.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 182 } ], - "signatures": [ - { - "id": 262, - "name": "store_response", - "modifiers": [], - "kind": 4096, - "kindString": "Call signature", - "flags": {}, - "comment": { - "summary": [ - { - "kind": "text", - "text": "{\"content\": [\"Store a response in the cache storage.\"]}" - } - ] - }, - "type": { - "type": "reference", - "name": "None" - }, - "parameters": [ - { - "id": 263, - "name": "_", - "kind": 32768, - "kindString": "Parameter", - "flags": {}, - "type": { - "type": "reference", - "name": "Spider" - } - }, - { - "id": 264, - "name": "request", - "kind": 32768, - "kindString": "Parameter", - "flags": {}, - "type": { - "type": "reference", - "name": "Request" - } - }, - { - "id": 265, - "name": "response", - "kind": 32768, - "kindString": "Parameter", - "flags": {}, - "type": { - "type": "reference", - "name": "Response" - } - } - ] - } - ] - } - ], - "groups": [ - { - "title": "Methods", - "children": [ - 252, - 249, - 256, - 261 - ] - } - ], - "sources": [ - { - "filename": "/tmp-old-source-v27/src/apify/scrapy/extensions/_httpcache.py", - "line": 29, - "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/scrapy/extensions/_httpcache.py#L29" - } - ] - }, - { - "id": 266, - "name": "to_gzip", - "module": "scrapy.extensions._httpcache", - "kind": 2048, - "kindString": "Method", - "flags": {}, - "comment": { - "summary": [ - { - "kind": "text", - "text": "{\"content\": [\"Dump a dictionary to a gzip-compressed byte stream.\"]}" + "type": { + "name": "bool", + "type": "reference" } - ] - }, - "children": [], - "groups": [], - "sources": [ - { - "filename": "/tmp-old-source-v27/src/apify/scrapy/extensions/_httpcache.py", - "line": 160, - "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/scrapy/extensions/_httpcache.py#L160" - } - ], - "signatures": [ + }, { - "id": 267, - "name": "to_gzip", - "modifiers": [], - "kind": 4096, - "kindString": "Call signature", - "flags": {}, + "kind": 1024, + "kindString": "Property", + "children": [], "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Dump a dictionary to a gzip-compressed byte stream.\"]}" + "text": "" } ] }, - "type": { - "type": "reference", - "name": "bytes" + "flags": {}, + "groups": [], + "id": 243, + "module": "_configuration", + "name": "latest_sdk_version", + "parsedDocstring": { + "text": "" }, - "parameters": [ - { - "id": 268, - "name": "data", - "kind": 32768, - "kindString": "Parameter", - "flags": {}, - "type": { - "type": "intrinsic", - "name": "dict" - } - }, + "sources": [ { - "id": 269, - "name": "mtime", - "kind": 32768, - "kindString": "Parameter", - "flags": {}, - "type": { + "character": 1, + "fileName": "/src/apify/_configuration.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 190 + } + ], + "type": { + "name": "Annotated[ str | None, Field( alias='apify_sdk_latest_version', description='Specifies the most recent release version of the Apify SDK for Javascript. Used for ' 'checking for updates.', ), deprecated('SDK version checking is not supported for the Python SDK'), ]", + "type": "union", + "types": [ + { "type": "reference", - "name": "int | None" + "name": "str" }, - "defaultValue": "None" - } - ] - } - ] - }, - { - "id": 270, - "name": "from_gzip", - "module": "scrapy.extensions._httpcache", - "kind": 2048, - "kindString": "Method", - "flags": {}, - "comment": { - "summary": [ - { - "kind": "text", - "text": "{\"content\": [\"Load a dictionary from a gzip-compressed byte stream.\"]}" + { + "type": "literal", + "value": null + } + ] } - ] - }, - "children": [], - "groups": [], - "sources": [ - { - "filename": "/tmp-old-source-v27/src/apify/scrapy/extensions/_httpcache.py", - "line": 168, - "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/scrapy/extensions/_httpcache.py#L168" - } - ], - "signatures": [ + }, { - "id": 271, - "name": "from_gzip", - "modifiers": [], - "kind": 4096, - "kindString": "Call signature", - "flags": {}, + "kind": 1024, + "kindString": "Property", + "children": [], "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Load a dictionary from a gzip-compressed byte stream.\"]}" + "text": "" } ] }, - "type": { - "type": "intrinsic", - "name": "dict" + "flags": {}, + "groups": [], + "id": 244, + "module": "_configuration", + "name": "log_format", + "parsedDocstring": { + "text": "" }, - "parameters": [ + "sources": [ { - "id": 272, - "name": "gzip_bytes", - "kind": 32768, - "kindString": "Parameter", - "flags": {}, - "type": { + "character": 1, + "fileName": "/src/apify/_configuration.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 200 + } + ], + "type": { + "name": "Annotated[ str | None, Field(alias='apify_log_format'), deprecated('Adjust the log format in code instead'), ]", + "type": "union", + "types": [ + { "type": "reference", - "name": "bytes" + "name": "str" + }, + { + "type": "literal", + "value": null } - } - ] - } - ] - }, - { - "id": 273, - "name": "read_gzip_time", - "module": "scrapy.extensions._httpcache", - "kind": 2048, - "kindString": "Method", - "flags": {}, - "comment": { - "summary": [ - { - "kind": "text", - "text": "{\"content\": [\"Read the modification time from a gzip-compressed byte stream without decompressing the data.\"]}" + ] } - ] - }, - "children": [], - "groups": [], - "sources": [ - { - "filename": "/tmp-old-source-v27/src/apify/scrapy/extensions/_httpcache.py", - "line": 175, - "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/scrapy/extensions/_httpcache.py#L175" - } - ], - "signatures": [ + }, { - "id": 274, - "name": "read_gzip_time", - "modifiers": [], - "kind": 4096, - "kindString": "Call signature", - "flags": {}, + "kind": 1024, + "kindString": "Property", + "children": [], "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Read the modification time from a gzip-compressed byte stream without decompressing the data.\"]}" + "text": "" } ] }, - "type": { - "type": "intrinsic", - "name": "int" + "flags": {}, + "groups": [], + "id": 245, + "module": "_configuration", + "name": "max_paid_dataset_items", + "parsedDocstring": { + "text": "" }, - "parameters": [ + "sources": [ { - "id": 275, - "name": "gzip_bytes", - "kind": 32768, - "kindString": "Parameter", - "flags": {}, - "type": { + "character": 1, + "fileName": "/src/apify/_configuration.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 206 + } + ], + "type": { + "name": "Annotated[ int | None, Field( alias='actor_max_paid_dataset_items', description='For paid-per-result Actors, the user-set limit on returned results. Do not exceed this limit', ), BeforeValidator(lambda val: val or None), ]", + "type": "union", + "types": [ + { "type": "reference", - "name": "bytes" + "name": "int" + }, + { + "type": "literal", + "value": null } - } - ] - } - ] - }, - { - "id": 276, - "name": "get_kvs_name", - "module": "scrapy.extensions._httpcache", - "kind": 2048, - "kindString": "Method", - "flags": {}, - "comment": { - "summary": [ - { - "kind": "text", - "text": "{\"content\": [\"Get the key value store name for a spider.\\n\\nThe key value store name is derived from the spider name by replacing all special characters\\nwith hyphens and trimming leading and trailing hyphens. The resulting name is prefixed with\\n'httpcache-' and truncated to the maximum length.\\n\\nThe documentation\\n[about storages](https://docs.apify.com/platform/storage/usage#named-and-unnamed-storages)\\nmentions that names can be up to 63 characters long, so the default max length is set to 60.\\n\\nSuch naming isn't unique per spider, but should be sufficiently unique for most use cases.\\nThe name of the key value store should indicate to which spider it belongs, e.g. in\\nthe listing in the Apify's console.\\n\", {\"Arguments\": [{\"param\": \"spider_name\", \"desc\": \"Value of the Spider instance's name attribute.\"}, {\"param\": \"max_length\", \"desc\": \"Maximum length of the key value store name.\\n\"}]}, \"Returns: Key value store name.\\n\", {\"Raises\": [{\"param\": \"ValueError\", \"desc\": \"If the spider name contains only special characters.\"}]}]}" + ] } - ] - }, - "children": [], - "groups": [], - "sources": [ - { - "filename": "/tmp-old-source-v27/src/apify/scrapy/extensions/_httpcache.py", - "line": 183, - "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/scrapy/extensions/_httpcache.py#L183" - } - ], - "signatures": [ + }, { - "id": 277, - "name": "get_kvs_name", - "modifiers": [], - "kind": 4096, - "kindString": "Call signature", - "flags": {}, + "kind": 1024, + "kindString": "Property", + "children": [], "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Get the key value store name for a spider.\\n\\nThe key value store name is derived from the spider name by replacing all special characters\\nwith hyphens and trimming leading and trailing hyphens. The resulting name is prefixed with\\n'httpcache-' and truncated to the maximum length.\\n\\nThe documentation\\n[about storages](https://docs.apify.com/platform/storage/usage#named-and-unnamed-storages)\\nmentions that names can be up to 63 characters long, so the default max length is set to 60.\\n\\nSuch naming isn't unique per spider, but should be sufficiently unique for most use cases.\\nThe name of the key value store should indicate to which spider it belongs, e.g. in\\nthe listing in the Apify's console.\\n\", {\"" - } - ], - "blockTags": [ - { - "tag": "@returns", - "content": [ - { - "kind": "text", - "text": "[{\"param\"" - } - ] + "text": "" } ] }, - "type": { - "type": "intrinsic", - "name": "str" + "flags": {}, + "groups": [], + "id": 246, + "module": "_configuration", + "name": "max_total_charge_usd", + "parsedDocstring": { + "text": "" }, - "parameters": [ - { - "id": 278, - "name": "spider_name", - "kind": 32768, - "kindString": "Parameter", - "flags": {}, - "type": { - "type": "intrinsic", - "name": "str" - } - }, + "sources": [ { - "id": 279, - "name": "max_length", - "kind": 32768, - "kindString": "Parameter", - "flags": {}, - "type": { - "type": "intrinsic", - "name": "int" - }, - "defaultValue": "60" + "character": 1, + "fileName": "/src/apify/_configuration.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 215 } - ] - } - ] - }, - { - "id": 280, - "name": "ApifyHttpProxyMiddleware", - "module": "scrapy.middlewares.apify_proxy", - "kind": 128, - "kindString": "Class", - "flags": {}, - "comment": { - "summary": [ - { - "kind": "text", - "text": "{\"content\": [\"Apify HTTP proxy middleware for Scrapy.\\n\\nThis middleware enhances request processing by adding a 'proxy' field to the request's meta and an authentication\\nheader. It draws inspiration from the `HttpProxyMiddleware` included by default in Scrapy projects. The proxy URL\\nis sourced from the settings under the `APIFY_PROXY_SETTINGS` key. The value of this key, a dictionary, should be\\nprovided by the Actor input. An example of the proxy settings:\\n\\nproxy_settings = {'useApifyProxy': true, 'apifyProxyGroups': []}\"]}" + ], + "type": { + "name": "Annotated[ Decimal | None, Field( alias='actor_max_total_charge_usd', description='For pay-per-event Actors, the user-set limit on total charges. Do not exceed this limit', ), BeforeValidator(lambda val: val or None), ]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "Decimal" + }, + { + "type": "literal", + "value": null + } + ] } - ] - }, - "children": [ + }, { - "id": 281, - "name": "__init__", - "module": "scrapy.middlewares.apify_proxy", - "kind": 512, - "kindString": "Constructor", - "flags": {}, + "kind": 1024, + "kindString": "Property", + "children": [], "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Create a new instance.\\n\", {\"Arguments\": [{\"param\": \"proxy_settings\", \"desc\": \"Dictionary containing proxy settings, provided by the Actor input.\"}, {\"param\": \"auth_encoding\", \"desc\": \"Encoding for basic authentication (default is 'latin-1').\"}]}]}" + "text": "" } ] }, - "children": [], + "flags": {}, "groups": [], + "id": 247, + "module": "_configuration", + "name": "test_pay_per_event", + "parsedDocstring": { + "text": "" + }, "sources": [ { - "filename": "/tmp-old-source-v27/src/apify/scrapy/middlewares/apify_proxy.py", - "line": 28, "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/scrapy/middlewares/apify_proxy.py#L28" + "fileName": "/src/apify/_configuration.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 224 } ], - "signatures": [ - { - "id": 282, - "name": "__init__", - "modifiers": [], - "kind": 4096, - "kindString": "Call signature", - "flags": {}, - "comment": { - "summary": [ - { - "kind": "text", - "text": "{\"content\": [\"Create a new instance.\\n\", {\"" - } - ] - }, - "type": { - "type": "reference", - "name": "None" - }, - "parameters": [ - { - "id": 283, - "name": "proxy_settings", - "kind": 32768, - "kindString": "Parameter", - "flags": {}, - "type": { - "type": "intrinsic", - "name": "dict" - } - } - ] - } - ] + "type": { + "name": "bool", + "type": "reference" + } }, { - "id": 284, - "name": "from_crawler", - "module": "scrapy.middlewares.apify_proxy", - "kind": 2048, - "kindString": "Method", - "flags": {}, + "kind": 1024, + "kindString": "Property", + "children": [], "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Create an instance of ApifyHttpProxyMiddleware from a Scrapy Crawler.\\n\", {\"Arguments\": [{\"param\": \"crawler\", \"desc\": \"Scrapy Crawler object.\\n\"}]}, \"Returns: Instance of the class.\"]}" + "text": "" } ] }, - "children": [], + "flags": {}, "groups": [], + "id": 248, + "module": "_configuration", + "name": "meta_origin", + "parsedDocstring": { + "text": "" + }, "sources": [ { - "filename": "/tmp-old-source-v27/src/apify/scrapy/middlewares/apify_proxy.py", - "line": 39, "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/scrapy/middlewares/apify_proxy.py#L39" + "fileName": "/src/apify/_configuration.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 232 } ], - "signatures": [ - { - "id": 285, - "name": "from_crawler", - "modifiers": [], - "kind": 4096, - "kindString": "Call signature", - "flags": {}, - "comment": { - "summary": [ - { - "kind": "text", - "text": "{\"content\": [\"Create an instance of ApifyHttpProxyMiddleware from a Scrapy Crawler.\\n\", {\"" - } - ] - }, - "type": { + "type": { + "name": "Annotated[ str | None, Field( alias='apify_meta_origin', description='Specifies how an Actor run was started', ), ]", + "type": "union", + "types": [ + { "type": "reference", - "name": "ApifyHttpProxyMiddleware", - "id": 280 + "name": "str" }, - "parameters": [ - { - "id": 286, - "name": "crawler", - "kind": 32768, - "kindString": "Parameter", - "flags": {}, - "type": { - "type": "reference", - "name": "Crawler" - } - } - ] - } - ] + { + "type": "literal", + "value": null + } + ] + } }, { - "id": 287, - "name": "process_request", - "module": "scrapy.middlewares.apify_proxy", - "kind": 2048, - "kindString": "Method", - "flags": {}, + "kind": 1024, + "kindString": "Property", + "children": [], "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Process a Scrapy request by assigning a new proxy.\\n\", {\"Arguments\": [{\"param\": \"request\", \"desc\": \"Scrapy Request object.\"}, {\"param\": \"spider\", \"desc\": \"Scrapy Spider object.\\n\"}]}, {\"Raises\": [{\"param\": \"ValueError\", \"desc\": \"If username and password are not provided in the proxy URL.\"}]}]}" + "text": "" } ] }, - "children": [], + "flags": {}, "groups": [], + "id": 249, + "module": "_configuration", + "name": "metamorph_after_sleep", + "parsedDocstring": { + "text": "" + }, "sources": [ { - "filename": "/tmp-old-source-v27/src/apify/scrapy/middlewares/apify_proxy.py", - "line": 67, "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/scrapy/middlewares/apify_proxy.py#L67" + "fileName": "/src/apify/_configuration.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 240 } ], - "signatures": [ - { - "id": 288, - "name": "process_request", - "modifiers": [ - "async" - ], - "kind": 4096, - "kindString": "Call signature", - "flags": {}, - "comment": { - "summary": [ - { - "kind": "text", - "text": "{\"content\": [\"Process a Scrapy request by assigning a new proxy.\\n\", {\"" - } - ] - }, - "type": { - "type": "reference", - "name": "None" - }, - "parameters": [ - { - "id": 289, - "name": "request", - "kind": 32768, - "kindString": "Parameter", - "flags": {}, - "type": { - "type": "reference", - "name": "Request" - } - }, - { - "id": 290, - "name": "spider", - "kind": 32768, - "kindString": "Parameter", - "flags": {}, - "type": { - "type": "reference", - "name": "Spider" - } - } - ] - } - ] + "type": { + "name": "timedelta_ms", + "type": "reference" + } }, { - "id": 291, - "name": "process_exception", - "module": "scrapy.middlewares.apify_proxy", - "kind": 2048, - "kindString": "Method", - "flags": {}, + "kind": 1024, + "kindString": "Property", + "children": [], "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Process an exception that occurs during request processing.\\n\", {\"Arguments\": [{\"param\": \"request\", \"desc\": \"Scrapy Request object.\"}, {\"param\": \"exception\", \"desc\": \"Exception object.\"}, {\"param\": \"spider\", \"desc\": \"Scrapy Spider object.\\n\"}]}, {\"Returns\": [\"Returning None, meaning Scrapy will continue processing this exception, executing any other\", \"process_exception() methods of installed middleware, until no middleware is left and the default\", \"exception handling kicks in.\"]}]}" + "text": "" } ] }, - "children": [], + "flags": {}, "groups": [], + "id": 250, + "module": "_configuration", + "name": "proxy_hostname", + "parsedDocstring": { + "text": "" + }, "sources": [ { - "filename": "/tmp-old-source-v27/src/apify/scrapy/middlewares/apify_proxy.py", - "line": 89, "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/scrapy/middlewares/apify_proxy.py#L89" + "fileName": "/src/apify/_configuration.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 248 } ], - "signatures": [ - { - "id": 292, - "name": "process_exception", - "modifiers": [], - "kind": 4096, - "kindString": "Call signature", - "flags": {}, - "comment": { - "summary": [ - { - "kind": "text", - "text": "{\"content\": [\"Process an exception that occurs during request processing.\\n\", {\"" - } - ] - }, - "type": { - "type": "reference", - "name": "None" - }, - "parameters": [ - { - "id": 293, - "name": "request", - "kind": 32768, - "kindString": "Parameter", - "flags": {}, - "type": { - "type": "reference", - "name": "Request" - } - }, - { - "id": 294, - "name": "exception", - "kind": 32768, - "kindString": "Parameter", - "flags": {}, - "type": { - "type": "reference", - "name": "Exception" - } - }, - { - "id": 295, - "name": "spider", - "kind": 32768, - "kindString": "Parameter", - "flags": {}, - "type": { - "type": "reference", - "name": "Spider" - } - } - ] - } - ] - } - ], - "groups": [ - { - "title": "Constructors", - "children": [ - 281 - ] - }, - { - "title": "Methods", - "children": [ - 284, - 291, - 287 - ] - } - ], - "sources": [ - { - "filename": "/tmp-old-source-v27/src/apify/scrapy/middlewares/apify_proxy.py", - "line": 17, - "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/scrapy/middlewares/apify_proxy.py#L17" - } - ] - }, - { - "id": 296, - "name": "ActorDatasetPushPipeline", - "module": "scrapy.pipelines.actor_dataset_push", - "kind": 128, - "kindString": "Class", - "flags": {}, - "comment": { - "summary": [ - { - "kind": "text", - "text": "{\"content\": [\"A Scrapy pipeline for pushing items to an Actor's default dataset.\\n\\nThis pipeline is designed to be enabled only when the Scrapy project is run as an Actor.\"]}" + "type": { + "name": "str", + "type": "reference" } - ] - }, - "children": [ + }, { - "id": 297, - "name": "process_item", - "module": "scrapy.pipelines.actor_dataset_push", - "kind": 2048, - "kindString": "Method", - "flags": {}, + "kind": 1024, + "kindString": "Property", + "children": [], "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Pushes the provided Scrapy item to the Actor's default dataset.\"]}" + "text": "" } ] }, - "children": [], + "flags": {}, "groups": [], + "id": 251, + "module": "_configuration", + "name": "proxy_password", + "parsedDocstring": { + "text": "" + }, "sources": [ { - "filename": "/tmp-old-source-v27/src/apify/scrapy/pipelines/actor_dataset_push.py", - "line": 22, "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/scrapy/pipelines/actor_dataset_push.py#L22" + "fileName": "/src/apify/_configuration.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 256 } ], - "signatures": [ - { - "id": 298, - "name": "process_item", - "modifiers": [ - "async" - ], - "kind": 4096, - "kindString": "Call signature", - "flags": {}, - "comment": { - "summary": [ - { - "kind": "text", - "text": "{\"content\": [\"Pushes the provided Scrapy item to the Actor's default dataset.\"]}" - } - ] - }, - "type": { + "type": { + "name": "Annotated[ str | None, Field( alias='apify_proxy_password', description='Password to the Apify proxy', ), ]", + "type": "union", + "types": [ + { "type": "reference", - "name": "Item" + "name": "str" }, - "parameters": [ - { - "id": 299, - "name": "item", - "kind": 32768, - "kindString": "Parameter", - "flags": {}, - "type": { - "type": "reference", - "name": "Item" - } - }, - { - "id": 300, - "name": "spider", - "kind": 32768, - "kindString": "Parameter", - "flags": {}, - "type": { - "type": "reference", - "name": "Spider" - } - } - ] - } - ] - } - ], - "groups": [ - { - "title": "Methods", - "children": [ - 297 - ] - } - ], - "sources": [ - { - "filename": "/tmp-old-source-v27/src/apify/scrapy/pipelines/actor_dataset_push.py", - "line": 16, - "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/scrapy/pipelines/actor_dataset_push.py#L16" - } - ] - }, - { - "id": 301, - "name": "to_apify_request", - "module": "scrapy.requests", - "kind": 2048, - "kindString": "Method", - "flags": {}, - "comment": { - "summary": [ - { - "kind": "text", - "text": "{\"content\": [\"Convert a Scrapy request to an Apify request.\\n\", {\"Arguments\": [{\"param\": \"scrapy_request\", \"desc\": \"The Scrapy request to be converted.\"}, {\"param\": \"spider\", \"desc\": \"The Scrapy spider that the request is associated with.\\n\"}]}, {\"Returns\": [\"The converted Apify request if the conversion was successful, otherwise None.\"]}]}" + { + "type": "literal", + "value": null + } + ] } - ] - }, - "children": [], - "groups": [], - "sources": [ - { - "filename": "/tmp-old-source-v27/src/apify/scrapy/requests.py", - "line": 19, - "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/scrapy/requests.py#L19" - } - ], - "signatures": [ + }, { - "id": 302, - "name": "to_apify_request", - "modifiers": [], - "kind": 4096, - "kindString": "Call signature", - "flags": {}, + "kind": 1024, + "kindString": "Property", + "children": [], "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Convert a Scrapy request to an Apify request.\\n\", {\"" + "text": "" } ] }, - "type": { - "type": "reference", - "name": "ApifyRequest | None" + "flags": {}, + "groups": [], + "id": 252, + "module": "_configuration", + "name": "proxy_port", + "parsedDocstring": { + "text": "" }, - "parameters": [ - { - "id": 303, - "name": "scrapy_request", - "kind": 32768, - "kindString": "Parameter", - "flags": {}, - "type": { - "type": "reference", - "name": "ScrapyRequest" - } - }, + "sources": [ { - "id": 304, - "name": "spider", - "kind": 32768, - "kindString": "Parameter", - "flags": {}, - "type": { - "type": "reference", - "name": "Spider" - } + "character": 1, + "fileName": "/src/apify/_configuration.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 264 } - ] - } - ] - }, - { - "id": 305, - "name": "to_scrapy_request", - "module": "scrapy.requests", - "kind": 2048, - "kindString": "Method", - "flags": {}, - "comment": { - "summary": [ - { - "kind": "text", - "text": "{\"content\": [\"Convert an Apify request to a Scrapy request.\\n\", {\"Arguments\": [{\"param\": \"apify_request\", \"desc\": \"The Apify request to be converted.\"}, {\"param\": \"spider\", \"desc\": \"The Scrapy spider that the request is associated with.\\n\"}]}, {\"Raises\": [{\"param\": \"TypeError\", \"desc\": \"If the Apify request is not an instance of the `ApifyRequest` class.\"}, {\"param\": \"ValueError\", \"desc\": \"If the Apify request does not contain the required keys.\\n\"}]}, {\"Returns\": [\"The converted Scrapy request.\"]}]}" + ], + "type": { + "name": "int", + "type": "reference" } - ] - }, - "children": [], - "groups": [], - "sources": [ - { - "filename": "/tmp-old-source-v27/src/apify/scrapy/requests.py", - "line": 82, - "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/scrapy/requests.py#L82" - } - ], - "signatures": [ + }, { - "id": 306, - "name": "to_scrapy_request", - "modifiers": [], - "kind": 4096, - "kindString": "Call signature", - "flags": {}, + "kind": 1024, + "kindString": "Property", + "children": [], "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Convert an Apify request to a Scrapy request.\\n\", {\"" + "text": "" } ] }, - "type": { - "type": "reference", - "name": "ScrapyRequest" + "flags": {}, + "groups": [], + "id": 253, + "module": "_configuration", + "name": "proxy_status_url", + "parsedDocstring": { + "text": "" }, - "parameters": [ - { - "id": 307, - "name": "apify_request", - "kind": 32768, - "kindString": "Parameter", - "flags": {}, - "type": { - "type": "reference", - "name": "ApifyRequest" - } - }, + "sources": [ { - "id": 308, - "name": "spider", - "kind": 32768, - "kindString": "Parameter", - "flags": {}, - "type": { - "type": "reference", - "name": "Spider" - } + "character": 1, + "fileName": "/src/apify/_configuration.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 272 } - ] - } - ] - }, - { - "id": 309, - "name": "ApifyScheduler", - "module": "scrapy.scheduler", - "kind": 128, - "kindString": "Class", - "flags": {}, - "comment": { - "summary": [ - { - "kind": "text", - "text": "{\"content\": [\"A Scrapy scheduler that uses the Apify `RequestQueue` to manage requests.\\n\\nThis scheduler requires the asyncio Twisted reactor to be installed.\"]}" + ], + "type": { + "name": "str", + "type": "reference" } - ] - }, - "children": [ + }, { - "id": 310, - "name": "open", - "module": "scrapy.scheduler", - "kind": 2048, - "kindString": "Method", - "flags": {}, + "kind": 1024, + "kindString": "Property", + "children": [], "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Open the scheduler.\\n\", {\"Arguments\": [{\"param\": \"spider\", \"desc\": \"The spider that the scheduler is associated with.\"}]}]}" + "text": "" } ] }, - "children": [], + "flags": {}, "groups": [], + "id": 254, + "module": "_configuration", + "name": "started_at", + "parsedDocstring": { + "text": "" + }, "sources": [ { - "filename": "/tmp-old-source-v27/src/apify/scrapy/scheduler.py", - "line": 43, "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/scrapy/scheduler.py#L43" + "fileName": "/src/apify/_configuration.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 280 } ], - "signatures": [ - { - "id": 311, - "name": "open", - "modifiers": [], - "kind": 4096, - "kindString": "Call signature", - "flags": {}, - "comment": { - "summary": [ - { - "kind": "text", - "text": "{\"content\": [\"Open the scheduler.\\n\", {\"" - } - ] - }, - "type": { + "type": { + "name": "Annotated[ datetime | None, Field( validation_alias=AliasChoices( 'actor_started_at', 'apify_started_at', ), description='Date when the Actor was started', ), ]", + "type": "union", + "types": [ + { "type": "reference", - "name": "Deferred[None] | None" + "name": "datetime" }, - "parameters": [ - { - "id": 312, - "name": "spider", - "kind": 32768, - "kindString": "Parameter", - "flags": {}, - "type": { - "type": "reference", - "name": "Spider" - } - } - ] - } - ] + { + "type": "literal", + "value": null + } + ] + } }, { - "id": 313, - "name": "close", - "module": "scrapy.scheduler", - "kind": 2048, - "kindString": "Method", - "flags": {}, + "kind": 1024, + "kindString": "Property", + "children": [], "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Close the scheduler.\\n\\nShut down the event loop and its thread gracefully.\\n\", {\"Arguments\": [{\"param\": \"reason\", \"desc\": \"The reason for closing the spider.\"}]}]}" + "text": "" } ] }, - "children": [], + "flags": {}, "groups": [], + "id": 255, + "module": "_configuration", + "name": "timeout_at", + "parsedDocstring": { + "text": "" + }, "sources": [ { - "filename": "/tmp-old-source-v27/src/apify/scrapy/scheduler.py", - "line": 66, "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/scrapy/scheduler.py#L66" + "fileName": "/src/apify/_configuration.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 291 } ], - "signatures": [ - { - "id": 314, - "name": "close", - "modifiers": [], - "kind": 4096, - "kindString": "Call signature", - "flags": {}, - "comment": { - "summary": [ - { - "kind": "text", - "text": "{\"content\": [\"Close the scheduler.\\n\\nShut down the event loop and its thread gracefully.\\n\", {\"" - } - ] - }, - "type": { + "type": { + "name": "Annotated[ datetime | None, Field( validation_alias=AliasChoices( 'actor_timeout_at', 'apify_timeout_at', ), description='Date when the Actor will time out', ), BeforeValidator(lambda val: val if val != '' else None), ]", + "type": "union", + "types": [ + { "type": "reference", - "name": "None" + "name": "datetime" }, - "parameters": [ - { - "id": 315, - "name": "reason", - "kind": 32768, - "kindString": "Parameter", - "flags": {}, - "type": { - "type": "intrinsic", - "name": "str" - } - } - ] - } - ] + { + "type": "literal", + "value": null + } + ] + } }, { - "id": 316, - "name": "has_pending_requests", - "module": "scrapy.scheduler", - "kind": 2048, - "kindString": "Method", - "flags": {}, + "kind": 1024, + "kindString": "Property", + "children": [], "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Check if the scheduler has any pending requests.\\n\", {\"Returns\": [\"True if the scheduler has any pending requests, False otherwise.\"]}]}" + "text": "" } ] }, - "children": [], + "flags": {}, "groups": [], + "id": 256, + "module": "_configuration", + "name": "standby_port", + "parsedDocstring": { + "text": "" + }, "sources": [ { - "filename": "/tmp-old-source-v27/src/apify/scrapy/scheduler.py", - "line": 87, "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/scrapy/scheduler.py#L87" + "fileName": "/src/apify/_configuration.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 303 } ], - "signatures": [ + "type": { + "name": "int", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 257, + "module": "_configuration", + "name": "standby_url", + "parsedDocstring": { + "text": "" + }, + "sources": [ { - "id": 317, - "name": "has_pending_requests", - "modifiers": [], - "kind": 4096, - "kindString": "Call signature", - "flags": {}, - "comment": { - "summary": [ - { - "kind": "text", - "text": "{\"content\": [\"Check if the scheduler has any pending requests.\\n\", {\"" - } - ] - }, - "type": { - "type": "intrinsic", - "name": "bool" - }, - "parameters": [] + "character": 1, + "fileName": "/src/apify/_configuration.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 312 } - ] + ], + "type": { + "name": "str", + "type": "reference" + } }, { - "id": 318, - "name": "enqueue_request", - "module": "scrapy.scheduler", - "kind": 2048, - "kindString": "Method", - "flags": {}, + "kind": 1024, + "kindString": "Property", + "children": [], "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Add a request to the scheduler.\\n\\nThis could be called from either from a spider or a downloader middleware (e.g. redirect, retry, ...).\\n\", {\"Arguments\": [{\"param\": \"request\", \"desc\": \"The request to add to the scheduler.\\n\"}]}, {\"Returns\": [\"True if the request was successfully enqueued, False otherwise.\"]}]}" + "text": "" } ] }, - "children": [], + "flags": {}, "groups": [], + "id": 258, + "module": "_configuration", + "name": "token", + "parsedDocstring": { + "text": "" + }, "sources": [ { - "filename": "/tmp-old-source-v27/src/apify/scrapy/scheduler.py", - "line": 104, "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/scrapy/scheduler.py#L104" + "fileName": "/src/apify/_configuration.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 321 } ], - "signatures": [ - { - "id": 319, - "name": "enqueue_request", - "modifiers": [], - "kind": 4096, - "kindString": "Call signature", - "flags": {}, - "comment": { - "summary": [ - { - "kind": "text", - "text": "{\"content\": [\"Add a request to the scheduler.\\n\\nThis could be called from either from a spider or a downloader middleware (e.g. redirect, retry, ...).\\n\", {\"" - } - ] - }, - "type": { - "type": "intrinsic", - "name": "bool" + "type": { + "name": "Annotated[ str | None, Field( alias='apify_token', description='API token of the user who started the Actor', ), ]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" }, - "parameters": [ - { - "id": 320, - "name": "request", - "kind": 32768, - "kindString": "Parameter", - "flags": {}, - "type": { - "type": "reference", - "name": "Request" - } - } - ] + { + "type": "literal", + "value": null + } + ] + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 259, + "module": "_configuration", + "name": "user_id", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_configuration.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 329 } - ] + ], + "type": { + "name": "Annotated[ str | None, Field( alias='apify_user_id', description='ID of the user who started the Actor. May differ from the Actor owner', ), ]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } }, { - "id": 321, - "name": "next_request", - "module": "scrapy.scheduler", - "kind": 2048, - "kindString": "Method", + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 260, + "module": "_configuration", + "name": "user_is_paying", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_configuration.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 337 + } + ], + "type": { + "name": "bool", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 261, + "module": "_configuration", + "name": "web_server_port", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_configuration.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 346 + } + ], + "type": { + "name": "int", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, "flags": {}, + "groups": [], + "id": 262, + "module": "_configuration", + "name": "web_server_url", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_configuration.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 358 + } + ], + "type": { + "name": "str", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Fetch the next request from the scheduler.\\n\", {\"Returns\": [\"The next request, or None if there are no more requests.\"]}]}" + "text": "" } ] }, + "flags": {}, + "groups": [], + "id": 263, + "module": "_configuration", + "name": "workflow_key", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_configuration.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 369 + } + ], + "type": { + "name": "Annotated[ str | None, Field( alias='apify_workflow_key', description='Identifier used for grouping related runs and API calls together', ), ]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "kind": 2048, + "kindString": "Method", "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Disable the browser sandbox mode when running on the Apify platform.\n\nRunning in environment where `is_at_home` is True does not benefit from browser sandbox as it is already running\nin a container. It can be on the contrary undesired as the process in the container might be running as root and\nthis will crash chromium that was started with browser sandbox mode." + } + ] + }, + "decorations": [ + { + "args": "(mode='after')", + "name": "model_validator" + } + ], + "flags": {}, "groups": [], + "id": 264, + "module": "_configuration", + "name": "disable_browser_sandbox_on_platform", + "parsedDocstring": { + "text": "Disable the browser sandbox mode when running on the Apify platform.\n\nRunning in environment where `is_at_home` is True does not benefit from browser sandbox as it is already running\nin a container. It can be on the contrary undesired as the process in the container might be running as root and\nthis will crash chromium that was started with browser sandbox mode." + }, "sources": [ { - "filename": "/tmp-old-source-v27/src/apify/scrapy/scheduler.py", - "line": 138, "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/scrapy/scheduler.py#L138" + "fileName": "/src/apify/_configuration.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 378 } ], + "type": { + "name": "Undefined", + "type": "reference" + }, "signatures": [ { - "id": 322, - "name": "next_request", - "modifiers": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Disable the browser sandbox mode when running on the Apify platform.\n\nRunning in environment where `is_at_home` is True does not benefit from browser sandbox as it is already running\nin a container. It can be on the contrary undesired as the process in the container might be running as root and\nthis will crash chromium that was started with browser sandbox mode." + } + ] + }, + "flags": {}, + "id": 265, "kind": 4096, "kindString": "Call signature", - "flags": {}, + "modifiers": [], + "name": "disable_browser_sandbox_on_platform", + "parameters": [], + "type": { + "name": "Self", + "type": "reference" + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Retrieve the global instance of the configuration.\n\nMostly for the backwards compatibility. It is recommended to use the `service_locator.get_configuration()`\ninstead." + } + ] + }, + "decorations": [ + { + "name": "classmethod" + } + ], + "flags": {}, + "groups": [], + "id": 266, + "module": "_configuration", + "name": "get_global_configuration", + "parsedDocstring": { + "text": "Retrieve the global instance of the configuration.\n\nMostly for the backwards compatibility. It is recommended to use the `service_locator.get_configuration()`\ninstead." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_configuration.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 391 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Fetch the next request from the scheduler.\\n\", {\"" + "text": "Retrieve the global instance of the configuration.\n\nMostly for the backwards compatibility. It is recommended to use the `service_locator.get_configuration()`\ninstead." } ] }, + "flags": {}, + "id": 267, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "get_global_configuration", + "parameters": [], "type": { + "name": "Configuration", "type": "reference", - "name": "Request | None" - }, - "parameters": [] + "target": "225" + } } ] } ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "A class for specifying the configuration of an Actor.\n\nCan be used either globally via `Configuration.get_global_configuration()`,\nor it can be specific to each `Actor` instance on the `actor.config` property." + } + ] + }, + "decorations": [ + { + "args": "('Classes')", + "name": "docs_group" + } + ], + "flags": {}, "groups": [ { - "title": "Methods", "children": [ - 313, - 318, - 316, - 321, - 310 - ] + 264, + 266 + ], + "title": "Methods" + }, + { + "children": [ + 229, + 230, + 231, + 233, + 227, + 226, + 228, + 232, + 234, + 235, + 236, + 237, + 238, + 239, + 240, + 241, + 242, + 243, + 244, + 245, + 246, + 248, + 249, + 250, + 251, + 252, + 253, + 256, + 257, + 254, + 247, + 255, + 258, + 259, + 260, + 261, + 262, + 263 + ], + "title": "Properties" } ], + "id": 225, + "module": "_configuration", + "name": "Configuration", + "parsedDocstring": { + "text": "A class for specifying the configuration of an Actor.\n\nCan be used either globally via `Configuration.get_global_configuration()`,\nor it can be specific to each `Actor` instance on the `actor.config` property." + }, "sources": [ { - "filename": "/tmp-old-source-v27/src/apify/scrapy/scheduler.py", - "line": 24, "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/scrapy/scheduler.py#L24" + "fileName": "/src/apify/_configuration.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 29 } - ] + ], + "type": { + "name": "Undefined", + "type": "reference" + } }, { - "id": 323, - "name": "get_basic_auth_header", - "module": "scrapy.utils", - "kind": 2048, - "kindString": "Method", - "flags": {}, + "kind": 1024, + "kindString": "Property", + "children": [], "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Generate a basic authentication header for the given username and password.\"]}" + "text": "" } ] }, - "children": [], + "flags": {}, "groups": [], + "id": 268, + "module": "log", + "name": "logger_name", + "parsedDocstring": { + "text": "" + }, "sources": [ { - "filename": "/tmp-old-source-v27/src/apify/scrapy/utils.py", - "line": 14, "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/scrapy/utils.py#L14" + "fileName": "/src/apify/log.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 9 } ], - "signatures": [ - { - "id": 324, - "name": "get_basic_auth_header", - "modifiers": [], - "kind": 4096, - "kindString": "Call signature", - "flags": {}, - "comment": { - "summary": [ - { - "kind": "text", - "text": "{\"content\": [\"Generate a basic authentication header for the given username and password.\"]}" - } - ] - }, - "type": { - "type": "reference", - "name": "bytes" - }, - "parameters": [ - { - "id": 325, - "name": "username", - "kind": 32768, - "kindString": "Parameter", - "flags": {}, - "type": { - "type": "intrinsic", - "name": "str" - } - }, - { - "id": 326, - "name": "password", - "kind": 32768, - "kindString": "Parameter", - "flags": {}, - "type": { - "type": "intrinsic", - "name": "str" - } - }, - { - "id": 327, - "name": "auth_encoding", - "kind": 32768, - "kindString": "Parameter", - "flags": {}, - "type": { - "type": "intrinsic", - "name": "str" - }, - "defaultValue": "'latin-1'" - } - ] - } - ] + "type": { + "name": "Undefined", + "type": "reference" + } }, { - "id": 328, - "name": "apply_apify_settings", - "module": "scrapy.utils", - "kind": 2048, - "kindString": "Method", - "flags": {}, + "kind": 1024, + "kindString": "Property", + "children": [], "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Integrates Apify configuration into a Scrapy project settings.\\n\\nNote: The function directly modifies the passed `settings` object and also returns it.\\n\", {\"Arguments\": [{\"param\": \"settings\", \"desc\": \"Scrapy project settings to be modified.\"}, {\"param\": \"proxy_config\", \"desc\": \"Proxy configuration to be stored in the settings.\\n\"}]}, {\"Returns\": [\"Scrapy project settings with custom configurations.\"]}]}" + "text": "" } ] }, - "children": [], + "flags": {}, "groups": [], + "id": 269, + "module": "log", + "name": "logger", + "parsedDocstring": { + "text": "" + }, "sources": [ { - "filename": "/tmp-old-source-v27/src/apify/scrapy/utils.py", - "line": 21, "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/scrapy/utils.py#L21" + "fileName": "/src/apify/log.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 12 } ], - "signatures": [ + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 128, + "kindString": "Class", + "children": [ { - "id": 329, - "name": "apply_apify_settings", - "modifiers": [], - "kind": 4096, - "kindString": "Call signature", - "flags": {}, + "kind": 1024, + "kindString": "Property", + "children": [], "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Integrates Apify configuration into a Scrapy project settings.\\n\\nNote: The function directly modifies the passed `settings` object and also returns it.\\n\", {\"" + "text": "" } ] }, + "flags": {}, + "groups": [], + "id": 271, + "module": "_models", + "name": "__model_config__", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 21 + } + ], "type": { - "type": "reference", - "name": "Settings" + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] }, - "parameters": [ + "flags": {}, + "groups": [], + "id": 272, + "module": "_models", + "name": "event_types", + "parsedDocstring": { + "text": "" + }, + "sources": [ { - "id": 330, - "name": "settings", - "kind": 32768, - "kindString": "Parameter", - "flags": { - "keyword-only": "true" - }, - "type": { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 23 + } + ], + "type": { + "name": "list", + "type": "reference", + "typeArguments": [ + { "type": "reference", - "name": "Settings | None" - }, - "defaultValue": "None" - }, + "name": "WebhookEventType" + } + ], + "target": "596" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 273, + "module": "_models", + "name": "request_url", + "parsedDocstring": { + "text": "" + }, + "sources": [ { - "id": 331, - "name": "proxy_config", - "kind": 32768, - "kindString": "Parameter", - "flags": { - "keyword-only": "true" - }, - "type": { - "type": "reference", - "name": "dict | None" - }, - "defaultValue": "None" + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 27 } - ] - } - ] - }, - { - "id": 332, - "name": "RequestList", - "module": "storages._request_list", - "docsGroup": "Classes", - "kind": 128, - "kindString": "Class", - "flags": {}, - "comment": { - "summary": [ - { - "kind": "text", - "text": "{\"content\": [\"Extends crawlee RequestList.\\n\\nMethod open is used to create RequestList from actor's requestListSources input.\"]}" + ], + "type": { + "name": "str", + "type": "reference" } - ] - }, - "children": [ + }, { - "id": 333, - "name": "open", - "module": "storages._request_list", - "kind": 2048, - "kindString": "Method", - "flags": {}, + "kind": 1024, + "kindString": "Property", + "children": [], "comment": { "summary": [ { "kind": "text", - "text": "{\"content\": [\"Initialize a new instance from request list source input.\\n\", {\"Arguments\": [{\"param\": \"name\", \"desc\": \"Name of the returned RequestList.\"}, {\"param\": \"request_list_sources_input\", \"desc\": \"List of dicts with either url key or requestsFromUrl key.\"}, {\"param\": \"http_client\", \"desc\": \"Client that will be used to send get request to urls defined by value of requestsFromUrl keys.\\n\"}]}, {\"Returns\": [\"RequestList created from request_list_sources_input.\", \"\"]}, \"### Usage\\n\\n```python\\nexample_input = [\\n # Gather urls from response body.\\n {'requestsFromUrl': 'https://crawlee.dev/file.txt', 'method': 'GET'},\\n # Directly include this url.\\n {'url': 'https://crawlee.dev', 'method': 'GET'}\\n]\\nrequest_list = await RequestList.open(request_list_sources_input=example_input)\\n```\"]}" + "text": "" } ] }, - "children": [], + "flags": {}, "groups": [], + "id": 274, + "module": "_models", + "name": "payload_template", + "parsedDocstring": { + "text": "" + }, "sources": [ { - "filename": "/tmp-old-source-v27/src/apify/storages/_request_list.py", - "line": 49, "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/storages/_request_list.py#L49" + "fileName": "/src/apify/_models.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 32 } ], - "signatures": [ - { - "id": 334, - "name": "open", - "modifiers": [ - "async" - ], - "kind": 4096, - "kindString": "Call signature", - "flags": {}, - "comment": { - "summary": [ - { - "kind": "text", - "text": "{\"content\": [\"Initialize a new instance from request list source input.\\n\", {\"" - } - ] - }, - "type": { + "type": { + "name": "Annotated[ str | None, Field(description='Template for the payload sent by the webook'), ]", + "type": "union", + "types": [ + { "type": "reference", - "name": "RequestList", - "id": 332 + "name": "str" }, - "parameters": [ - { - "id": 335, - "name": "name", - "kind": 32768, - "kindString": "Parameter", - "flags": {}, - "type": { - "type": "reference", - "name": "str | None" - }, - "defaultValue": "None" - }, - { - "id": 336, - "name": "request_list_sources_input", - "kind": 32768, - "kindString": "Parameter", - "flags": {}, - "type": { - "type": "reference", - "name": "list[dict[str, Any]] | None" - }, - "defaultValue": "None" - }, - { - "id": 337, - "name": "http_client", - "kind": 32768, - "kindString": "Parameter", - "flags": {}, - "type": { - "type": "reference", - "name": "HttpClient | None" - }, - "defaultValue": "None" - } - ] - } - ] + { + "type": "literal", + "value": null + } + ] + } + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "decorations": [ + { + "args": "('Data structures')", + "name": "docs_group" } ], + "flags": {}, "groups": [ { - "title": "Methods", "children": [ - 333 - ] + 271, + 272, + 274, + 273 + ], + "title": "Properties" } ], + "id": 270, + "module": "_models", + "name": "Webhook", + "parsedDocstring": { + "text": "" + }, "sources": [ { - "filename": "/tmp-old-source-v27/src/apify/storages/_request_list.py", - "line": 42, "character": 1, - "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/storages/_request_list.py#L42" + "fileName": "/src/apify/_models.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 20 } - ] - } - ], - "groups": [ + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, { - "title": "Classes", + "kind": 128, + "kindString": "Class", "children": [ - 1, - 225, - 180, - 203, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 276, + "module": "_models", + "name": "__model_config__", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 40 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 277, + "module": "_models", + "name": "origin", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 42 + } + ], + "type": { + "name": "MetaOrigin", + "type": "reference" + } + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "decorations": [ + { + "args": "('Data structures')", + "name": "docs_group" + } + ], + "flags": {}, + "groups": [ + { + "children": [ + 276, + 277 + ], + "title": "Properties" + } + ], + "id": 275, + "module": "_models", + "name": "ActorRunMeta", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 39 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 279, + "module": "_models", + "name": "__model_config__", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 47 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 280, + "module": "_models", + "name": "input_body_len", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 49 + } + ], + "type": { + "name": "Annotated[int | None, Field(alias='inputBodyLen')]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "int" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 281, + "module": "_models", + "name": "restart_count", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 50 + } + ], + "type": { + "name": "int", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 282, + "module": "_models", + "name": "resurrect_count", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 51 + } + ], + "type": { + "name": "int", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 283, + "module": "_models", + "name": "mem_avg_bytes", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 52 + } + ], + "type": { + "name": "Annotated[float | None, Field(alias='memAvgBytes')]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "float" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 284, + "module": "_models", + "name": "mem_max_bytes", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 53 + } + ], + "type": { + "name": "Annotated[int | None, Field(alias='memMaxBytes')]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "int" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 285, + "module": "_models", + "name": "mem_current_bytes", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 54 + } + ], + "type": { + "name": "Annotated[int | None, Field(alias='memCurrentBytes')]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "int" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 286, + "module": "_models", + "name": "cpu_avg_usage", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 55 + } + ], + "type": { + "name": "Annotated[float | None, Field(alias='cpuAvgUsage')]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "float" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 287, + "module": "_models", + "name": "cpu_max_usage", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 56 + } + ], + "type": { + "name": "Annotated[float | None, Field(alias='cpuMaxUsage')]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "float" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 288, + "module": "_models", + "name": "cpu_current_usage", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 57 + } + ], + "type": { + "name": "Annotated[float | None, Field(alias='cpuCurrentUsage')]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "float" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 289, + "module": "_models", + "name": "net_rx_bytes", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 58 + } + ], + "type": { + "name": "Annotated[int | None, Field(alias='netRxBytes')]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "int" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 290, + "module": "_models", + "name": "net_tx_bytes", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 59 + } + ], + "type": { + "name": "Annotated[int | None, Field(alias='netTxBytes')]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "int" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 291, + "module": "_models", + "name": "duration", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 60 + } + ], + "type": { + "name": "Annotated[timedelta_ms | None, Field(alias='durationMillis')]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "timedelta_ms" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 292, + "module": "_models", + "name": "run_time", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 61 + } + ], + "type": { + "name": "Annotated[timedelta | None, Field(alias='runTimeSecs')]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "timedelta" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 293, + "module": "_models", + "name": "metamorph", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 62 + } + ], + "type": { + "name": "Annotated[int | None, Field(alias='metamorph')]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "int" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 294, + "module": "_models", + "name": "compute_units", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 63 + } + ], + "type": { + "name": "float", + "type": "reference" + } + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "decorations": [ + { + "args": "('Data structures')", + "name": "docs_group" + } + ], + "flags": {}, + "groups": [ + { + "children": [ + 279, + 294, + 286, + 288, + 287, + 291, + 280, + 283, + 285, + 284, + 293, + 289, + 290, + 281, + 282, + 292 + ], + "title": "Properties" + } + ], + "id": 278, + "module": "_models", + "name": "ActorRunStats", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 46 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 296, + "module": "_models", + "name": "__model_config__", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 68 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 297, + "module": "_models", + "name": "build", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 70 + } + ], + "type": { + "name": "str", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 298, + "module": "_models", + "name": "timeout", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 71 + } + ], + "type": { + "name": "timedelta", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 299, + "module": "_models", + "name": "memory_mbytes", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 72 + } + ], + "type": { + "name": "int", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 300, + "module": "_models", + "name": "disk_mbytes", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 73 + } + ], + "type": { + "name": "int", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 301, + "module": "_models", + "name": "max_total_charge_usd", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 74 + } + ], + "type": { + "name": "Annotated[Decimal | None, Field(alias='maxTotalChargeUsd')]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "Decimal" + }, + { + "type": "literal", + "value": null + } + ] + } + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "decorations": [ + { + "args": "('Data structures')", + "name": "docs_group" + } + ], + "flags": {}, + "groups": [ + { + "children": [ + 296, + 297, + 300, + 301, + 299, + 298 + ], + "title": "Properties" + } + ], + "id": 295, + "module": "_models", + "name": "ActorRunOptions", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 67 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 303, + "module": "_models", + "name": "__model_config__", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 79 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 304, + "module": "_models", + "name": "actor_compute_units", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 81 + } + ], + "type": { + "name": "Annotated[float | None, Field(alias='ACTOR_COMPUTE_UNITS')]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "float" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 305, + "module": "_models", + "name": "dataset_reads", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 82 + } + ], + "type": { + "name": "Annotated[float | None, Field(alias='DATASET_READS')]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "float" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 306, + "module": "_models", + "name": "dataset_writes", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 83 + } + ], + "type": { + "name": "Annotated[float | None, Field(alias='DATASET_WRITES')]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "float" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 307, + "module": "_models", + "name": "key_value_store_reads", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 84 + } + ], + "type": { + "name": "Annotated[float | None, Field(alias='KEY_VALUE_STORE_READS')]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "float" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 308, + "module": "_models", + "name": "key_value_store_writes", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 85 + } + ], + "type": { + "name": "Annotated[float | None, Field(alias='KEY_VALUE_STORE_WRITES')]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "float" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 309, + "module": "_models", + "name": "key_value_store_lists", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 86 + } + ], + "type": { + "name": "Annotated[float | None, Field(alias='KEY_VALUE_STORE_LISTS')]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "float" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 310, + "module": "_models", + "name": "request_queue_reads", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 87 + } + ], + "type": { + "name": "Annotated[float | None, Field(alias='REQUEST_QUEUE_READS')]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "float" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 311, + "module": "_models", + "name": "request_queue_writes", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 88 + } + ], + "type": { + "name": "Annotated[float | None, Field(alias='REQUEST_QUEUE_WRITES')]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "float" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 312, + "module": "_models", + "name": "data_transfer_internal_gbytes", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 89 + } + ], + "type": { + "name": "Annotated[float | None, Field(alias='DATA_TRANSFER_INTERNAL_GBYTES')]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "float" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 313, + "module": "_models", + "name": "data_transfer_external_gbytes", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 90 + } + ], + "type": { + "name": "Annotated[float | None, Field(alias='DATA_TRANSFER_EXTERNAL_GBYTES')]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "float" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 314, + "module": "_models", + "name": "proxy_residential_transfer_gbytes", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 91 + } + ], + "type": { + "name": "Annotated[float | None, Field(alias='PROXY_RESIDENTIAL_TRANSFER_GBYTES')]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "float" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 315, + "module": "_models", + "name": "proxy_serps", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 92 + } + ], + "type": { + "name": "Annotated[float | None, Field(alias='PROXY_SERPS')]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "float" + }, + { + "type": "literal", + "value": null + } + ] + } + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "decorations": [ + { + "args": "('Data structures')", + "name": "docs_group" + } + ], + "flags": {}, + "groups": [ + { + "children": [ + 303, + 304, + 313, + 312, + 305, + 306, + 309, + 307, + 308, + 314, + 315, + 310, + 311 + ], + "title": "Properties" + } + ], + "id": 302, + "module": "_models", + "name": "ActorRunUsage", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 78 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 317, + "module": "_models", + "name": "__model_config__", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 97 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 318, + "module": "_models", + "name": "id", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 99 + } + ], + "type": { + "name": "str", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 319, + "module": "_models", + "name": "act_id", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 100 + } + ], + "type": { + "name": "str", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 320, + "module": "_models", + "name": "user_id", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 101 + } + ], + "type": { + "name": "str", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 321, + "module": "_models", + "name": "actor_task_id", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 102 + } + ], + "type": { + "name": "Annotated[str | None, Field(alias='actorTaskId')]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 322, + "module": "_models", + "name": "started_at", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 103 + } + ], + "type": { + "name": "datetime", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 323, + "module": "_models", + "name": "finished_at", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 104 + } + ], + "type": { + "name": "Annotated[datetime | None, Field(alias='finishedAt')]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "datetime" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 324, + "module": "_models", + "name": "status", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 105 + } + ], + "type": { + "name": "ActorJobStatus", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 325, + "module": "_models", + "name": "status_message", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 106 + } + ], + "type": { + "name": "Annotated[str | None, Field(alias='statusMessage')]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 326, + "module": "_models", + "name": "is_status_message_terminal", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 107 + } + ], + "type": { + "name": "Annotated[bool | None, Field(alias='isStatusMessageTerminal')]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "bool" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 327, + "module": "_models", + "name": "meta", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 108 + } + ], + "type": { + "name": "ActorRunMeta", + "type": "reference", + "target": "275" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 328, + "module": "_models", + "name": "stats", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 109 + } + ], + "type": { + "name": "ActorRunStats", + "type": "reference", + "target": "278" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 329, + "module": "_models", + "name": "options", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 110 + } + ], + "type": { + "name": "ActorRunOptions", + "type": "reference", + "target": "295" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 330, + "module": "_models", + "name": "build_id", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 111 + } + ], + "type": { + "name": "str", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 331, + "module": "_models", + "name": "exit_code", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 112 + } + ], + "type": { + "name": "Annotated[int | None, Field(alias='exitCode')]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "int" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 332, + "module": "_models", + "name": "default_key_value_store_id", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 113 + } + ], + "type": { + "name": "str", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 333, + "module": "_models", + "name": "default_dataset_id", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 114 + } + ], + "type": { + "name": "str", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 334, + "module": "_models", + "name": "default_request_queue_id", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 115 + } + ], + "type": { + "name": "str", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 335, + "module": "_models", + "name": "build_number", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 116 + } + ], + "type": { + "name": "Annotated[str | None, Field(alias='buildNumber')]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 336, + "module": "_models", + "name": "container_url", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 117 + } + ], + "type": { + "name": "str", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 337, + "module": "_models", + "name": "is_container_server_ready", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 118 + } + ], + "type": { + "name": "Annotated[bool | None, Field(alias='isContainerServerReady')]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "bool" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 338, + "module": "_models", + "name": "git_branch_name", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 119 + } + ], + "type": { + "name": "Annotated[str | None, Field(alias='gitBranchName')]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 339, + "module": "_models", + "name": "usage", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 120 + } + ], + "type": { + "name": "Annotated[ActorRunUsage | None, Field(alias='usage')]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "ActorRunUsage", + "target": "302" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 340, + "module": "_models", + "name": "usage_total_usd", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 121 + } + ], + "type": { + "name": "Annotated[float | None, Field(alias='usageTotalUsd')]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "float" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 341, + "module": "_models", + "name": "usage_usd", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 122 + } + ], + "type": { + "name": "Annotated[ActorRunUsage | None, Field(alias='usageUsd')]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "ActorRunUsage", + "target": "302" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 342, + "module": "_models", + "name": "pricing_info", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 123 + } + ], + "type": { + "name": "Annotated[ FreeActorPricingInfo | FlatPricePerMonthActorPricingInfo | PricePerDatasetItemActorPricingInfo | PayPerEventActorPricingInfo | None, Field(alias='pricingInfo', discriminator='pricing_model'), ]", + "type": "union", + "types": [ + { + "type": "union", + "types": [ + { + "type": "union", + "types": [ + { + "type": "union", + "types": [ + { + "type": "reference", + "name": "FreeActorPricingInfo", + "target": "344" + }, + { + "type": "reference", + "name": "FlatPricePerMonthActorPricingInfo", + "target": "346" + } + ] + }, + { + "type": "reference", + "name": "PricePerDatasetItemActorPricingInfo", + "target": "350" + } + ] + }, + { + "type": "reference", + "name": "PayPerEventActorPricingInfo", + "target": "360" + } + ] + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 343, + "module": "_models", + "name": "charged_event_counts", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 131 + } + ], + "type": { + "name": "Annotated[ dict[str, int] | None, Field(alias='chargedEventCounts'), ]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "dict", + "typeArguments": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "reference", + "name": "int" + } + ] + }, + { + "type": "literal", + "value": null + } + ] + } + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "decorations": [ + { + "args": "('Data structures')", + "name": "docs_group" + } + ], + "flags": {}, + "groups": [ + { + "children": [ + 317, + 319, + 321, + 330, + 335, + 343, + 336, + 333, + 332, + 334, + 331, + 323, + 338, + 318, + 337, + 326, + 327, + 329, + 342, + 322, + 328, + 324, + 325, + 339, + 340, + 341, + 320 + ], + "title": "Properties" + } + ], + "id": 316, + "module": "_models", + "name": "ActorRun", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 96 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 345, + "module": "_models", + "name": "pricing_model", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 138 + } + ], + "type": { + "name": "Literal", + "type": "reference", + "typeArguments": [ + { + "type": "literal", + "value": "FREE" + } + ] + } + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [ + { + "children": [ + 345 + ], + "title": "Properties" + } + ], + "id": 344, + "module": "_models", + "name": "FreeActorPricingInfo", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 137 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 347, + "module": "_models", + "name": "pricing_model", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 142 + } + ], + "type": { + "name": "Literal", + "type": "reference", + "typeArguments": [ + { + "type": "literal", + "value": "FLAT_PRICE_PER_MONTH" + } + ] + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 348, + "module": "_models", + "name": "trial_minutes", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 143 + } + ], + "type": { + "name": "Annotated[int | None, Field(alias='trialMinutes')]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "int" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 349, + "module": "_models", + "name": "price_per_unit_usd", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 144 + } + ], + "type": { + "name": "Decimal", + "type": "reference" + } + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [ + { + "children": [ + 349, + 347, + 348 + ], + "title": "Properties" + } + ], + "id": 346, + "module": "_models", + "name": "FlatPricePerMonthActorPricingInfo", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 141 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 351, + "module": "_models", + "name": "pricing_model", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 148 + } + ], + "type": { + "name": "Literal", + "type": "reference", + "typeArguments": [ + { + "type": "literal", + "value": "PRICE_PER_DATASET_ITEM" + } + ] + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 352, + "module": "_models", + "name": "unit_name", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 149 + } + ], + "type": { + "name": "Annotated[str | None, Field(alias='unitName')]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 353, + "module": "_models", + "name": "price_per_unit_usd", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 150 + } + ], + "type": { + "name": "Decimal", + "type": "reference" + } + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [ + { + "children": [ + 353, + 351, + 352 + ], + "title": "Properties" + } + ], + "id": 350, + "module": "_models", + "name": "PricePerDatasetItemActorPricingInfo", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 147 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 355, + "module": "_models", + "name": "event_price_usd", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 154 + } + ], + "type": { + "name": "Decimal", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 356, + "module": "_models", + "name": "event_title", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 155 + } + ], + "type": { + "name": "str", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 357, + "module": "_models", + "name": "event_description", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 156 + } + ], + "type": { + "name": "Annotated[str | None, Field(alias='eventDescription')]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [ + { + "children": [ + 357, + 355, + 356 + ], + "title": "Properties" + } + ], + "id": 354, + "module": "_models", + "name": "ActorChargeEvent", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 153 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 359, + "module": "_models", + "name": "actor_charge_events", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 160 + } + ], + "type": { + "name": "dict", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "reference", + "name": "ActorChargeEvent", + "target": "354" + } + ] + } + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [ + { + "children": [ + 359 + ], + "title": "Properties" + } + ], + "id": 358, + "module": "_models", + "name": "PricingPerEvent", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 159 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 361, + "module": "_models", + "name": "pricing_model", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 164 + } + ], + "type": { + "name": "Literal", + "type": "reference", + "typeArguments": [ + { + "type": "literal", + "value": "PAY_PER_EVENT" + } + ] + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 362, + "module": "_models", + "name": "pricing_per_event", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 165 + } + ], + "type": { + "name": "PricingPerEvent", + "type": "reference", + "target": "358" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 363, + "module": "_models", + "name": "minimal_max_total_charge_usd", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 166 + } + ], + "type": { + "name": "Annotated[Decimal | None, Field(alias='minimalMaxTotalChargeUsd')]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "Decimal" + }, + { + "type": "literal", + "value": null + } + ] + } + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [ + { + "children": [ + 363, + 361, + 362 + ], + "title": "Properties" + } + ], + "id": 360, + "module": "_models", + "name": "PayPerEventActorPricingInfo", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 163 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 364, + "module": "_models", + "name": "PricingModel", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 169 + } + ], + "type": { + "name": "TypeAlias", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 365, + "module": "__init__", + "name": "__version__", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/__init__.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 20 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 366, + "module": "_charging", + "name": "run_validator", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_charging.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 27 + } + ], + "type": { + "name": "TypeAdapter", + "type": "reference", + "typeArguments": [ + { + "type": "union", + "types": [ + { + "type": "reference", + "name": "ActorRun", + "target": "316" + }, + { + "type": "literal", + "value": null + } + ] + } + ] + } + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Charge for a specified number of events - sub-operations of the Actor.\n\nThis is relevant only for the pay-per-event pricing model.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 368, + "module": "_charging", + "name": "charge", + "parsedDocstring": { + "text": "Charge for a specified number of events - sub-operations of the Actor.\n\nThis is relevant only for the pay-per-event pricing model.\n", + "args": { + "event_name": "Name of the event to be charged for.", + "count": "Number of events to charge for." + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_charging.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 34 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Charge for a specified number of events - sub-operations of the Actor.\n\nThis is relevant only for the pay-per-event pricing model.\n" + } + ] + }, + "flags": {}, + "id": 369, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "charge", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Name of the event to be charged for." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 370, + "kind": 32768, + "kindString": "Parameter", + "name": "event_name", + "type": { + "name": "str", + "type": "reference" + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Number of events to charge for." + } + ] + }, + "defaultValue": "1", + "flags": { + "isOptional": true, + "keyword-only": false + }, + "id": 371, + "kind": 32768, + "kindString": "Parameter", + "name": "count", + "type": { + "name": "int", + "type": "reference" + } + } + ], + "type": { + "name": "ChargeResult", + "type": "reference", + "target": "379" + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Calculate the total amount of money charged for pay-per-event events so far." + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 372, + "module": "_charging", + "name": "calculate_total_charged_amount", + "parsedDocstring": { + "text": "Calculate the total amount of money charged for pay-per-event events so far." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_charging.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 44 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Calculate the total amount of money charged for pay-per-event events so far." + } + ] + }, + "flags": {}, + "id": 373, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "calculate_total_charged_amount", + "parameters": [], + "type": { + "name": "Decimal", + "type": "reference" + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Calculate how many instances of an event can be charged before we reach the configured limit.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 374, + "module": "_charging", + "name": "calculate_max_event_charge_count_within_limit", + "parsedDocstring": { + "text": "Calculate how many instances of an event can be charged before we reach the configured limit.\n", + "args": { + "event_name": "Name of the inspected event." + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_charging.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 47 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Calculate how many instances of an event can be charged before we reach the configured limit.\n" + } + ] + }, + "flags": {}, + "id": 375, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "calculate_max_event_charge_count_within_limit", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Name of the inspected event." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 376, + "kind": 32768, + "kindString": "Parameter", + "name": "event_name", + "type": { + "name": "str", + "type": "reference" + } + } + ], + "type": { + "name": "int | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "int" + }, + { + "type": "literal", + "value": null + } + ] + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Retrieve detailed information about the effective pricing of the current Actor run.\n\nThis can be used for instance when your code needs to support multiple pricing models in transition periods." + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 377, + "module": "_charging", + "name": "get_pricing_info", + "parsedDocstring": { + "text": "Retrieve detailed information about the effective pricing of the current Actor run.\n\nThis can be used for instance when your code needs to support multiple pricing models in transition periods." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_charging.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 54 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Retrieve detailed information about the effective pricing of the current Actor run.\n\nThis can be used for instance when your code needs to support multiple pricing models in transition periods." + } + ] + }, + "flags": {}, + "id": 378, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "get_pricing_info", + "parameters": [], + "type": { + "name": "ActorPricingInfo", + "type": "reference", + "target": "383" + } + } + ] + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Provides fine-grained access to pay-per-event functionality." + } + ] + }, + "decorations": [ + { + "args": "('Interfaces')", + "name": "docs_group" + } + ], + "flags": {}, + "groups": [ + { + "children": [ + 374, + 372, + 368, + 377 + ], + "title": "Methods" + } + ], + "id": 367, + "module": "_charging", + "name": "ChargingManager", + "parsedDocstring": { + "text": "Provides fine-grained access to pay-per-event functionality." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_charging.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 31 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "If true, no more events of this type can be charged within the limit." + } + ] + }, + "flags": {}, + "groups": [], + "id": 380, + "module": "_charging", + "name": "event_charge_limit_reached", + "parsedDocstring": { + "text": "If true, no more events of this type can be charged within the limit." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_charging.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 66 + } + ], + "type": { + "name": "bool", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Total amount of charged events - may be lower than the requested amount." + } + ] + }, + "flags": {}, + "groups": [], + "id": 381, + "module": "_charging", + "name": "charged_count", + "parsedDocstring": { + "text": "Total amount of charged events - may be lower than the requested amount." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_charging.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 69 + } + ], + "type": { + "name": "int", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "How many events of each known type can still be charged within the limit." + } + ] + }, + "flags": {}, + "groups": [], + "id": 382, + "module": "_charging", + "name": "chargeable_within_limit", + "parsedDocstring": { + "text": "How many events of each known type can still be charged within the limit." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_charging.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 72 + } + ], + "type": { + "name": "dict", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "union", + "types": [ + { + "type": "reference", + "name": "int" + }, + { + "type": "literal", + "value": null + } + ] + } + ] + } + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Result of the `ChargingManager.charge` method." + } + ] + }, + "decorations": [ + { + "args": "('Data structures')", + "name": "docs_group" + }, + { + "args": "(frozen=True)", + "name": "dataclass" + } + ], + "flags": {}, + "groups": [ + { + "children": [ + 382, + 381, + 380 + ], + "title": "Properties" + } + ], + "id": 379, + "module": "_charging", + "name": "ChargeResult", + "parsedDocstring": { + "text": "Result of the `ChargingManager.charge` method." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_charging.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 63 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "The currently effective pricing model." + } + ] + }, + "flags": {}, + "groups": [], + "id": 384, + "module": "_charging", + "name": "pricing_model", + "parsedDocstring": { + "text": "The currently effective pricing model." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_charging.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 81 + } + ], + "type": { + "name": "PricingModel | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "Literal[ 'FREE', 'FLAT_PRICE_PER_MONTH', 'PRICE_PER_DATASET_ITEM', 'PAY_PER_EVENT',]" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "A configured limit for the total charged amount - if you exceed it, you won't receive more money than this." + } + ] + }, + "flags": {}, + "groups": [], + "id": 385, + "module": "_charging", + "name": "max_total_charge_usd", + "parsedDocstring": { + "text": "A configured limit for the total charged amount - if you exceed it, you won't receive more money than this." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_charging.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 84 + } + ], + "type": { + "name": "Decimal", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "A shortcut - true if the Actor runs with the pay-per-event pricing model." + } + ] + }, + "flags": {}, + "groups": [], + "id": 386, + "module": "_charging", + "name": "is_pay_per_event", + "parsedDocstring": { + "text": "A shortcut - true if the Actor runs with the pay-per-event pricing model." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_charging.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 87 + } + ], + "type": { + "name": "bool", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Price of every known event type." + } + ] + }, + "flags": {}, + "groups": [], + "id": 387, + "module": "_charging", + "name": "per_event_prices", + "parsedDocstring": { + "text": "Price of every known event type." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_charging.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 90 + } + ], + "type": { + "name": "dict", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "reference", + "name": "Decimal" + } + ] + } + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Result of the `ChargingManager.get_pricing_info` method." + } + ] + }, + "decorations": [ + { + "args": "('Data structures')", + "name": "docs_group" + }, + { + "name": "dataclass" + } + ], + "flags": {}, + "groups": [ + { + "children": [ + 386, + 385, + 387, + 384 + ], + "title": "Properties" + } + ], + "id": 383, + "module": "_charging", + "name": "ActorPricingInfo", + "parsedDocstring": { + "text": "Result of the `ChargingManager.get_pricing_info` method." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_charging.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 78 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 388, + "module": "_charging", + "name": "LOCAL_CHARGING_LOG_DATASET_NAME", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_charging.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 98 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 389, + "module": "_charging", + "name": "__init__", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_charging.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 100 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "flags": {}, + "id": 390, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "__init__", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 391, + "kind": 32768, + "kindString": "Parameter", + "name": "configuration", + "type": { + "name": "Configuration", + "type": "reference", + "target": "225" + } + }, + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 392, + "kind": 32768, + "kindString": "Parameter", + "name": "client", + "type": { + "name": "ApifyClientAsync", + "type": "reference" + } + } + ], + "type": { + "name": "None", + "type": "literal", + "value": null + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Initialize the charging manager - this is called by the `Actor` class and shouldn't be invoked manually." + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 393, + "module": "_charging", + "name": "__aenter__", + "parsedDocstring": { + "text": "Initialize the charging manager - this is called by the `Actor` class and shouldn't be invoked manually." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_charging.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 125 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Initialize the charging manager - this is called by the `Actor` class and shouldn't be invoked manually." + } + ] + }, + "flags": {}, + "id": 394, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "__aenter__", + "parameters": [], + "type": { + "name": "None", + "type": "literal", + "value": null + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 395, + "module": "_charging", + "name": "__aexit__", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_charging.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 168 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "flags": {}, + "id": 396, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "__aexit__", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 397, + "kind": 32768, + "kindString": "Parameter", + "name": "exc_type", + "type": { + "name": "type[BaseException] | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "type", + "typeArguments": [ + { + "type": "reference", + "name": "BaseException" + } + ] + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 398, + "kind": 32768, + "kindString": "Parameter", + "name": "exc_value", + "type": { + "name": "BaseException | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "BaseException" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 399, + "kind": 32768, + "kindString": "Parameter", + "name": "exc_traceback", + "type": { + "name": "TracebackType | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "TracebackType" + }, + { + "type": "literal", + "value": null + } + ] + } + } + ], + "type": { + "name": "None", + "type": "literal", + "value": null + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "decorations": [ + { + "name": "ensure_context" + } + ], + "flags": {}, + "groups": [], + "id": 400, + "module": "_charging", + "name": "charge", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_charging.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 180 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "flags": {}, + "id": 401, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "charge", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 402, + "kind": 32768, + "kindString": "Parameter", + "name": "event_name", + "type": { + "name": "str", + "type": "reference" + } + }, + { + "defaultValue": "1", + "flags": { + "isOptional": true, + "keyword-only": false + }, + "id": 403, + "kind": 32768, + "kindString": "Parameter", + "name": "count", + "type": { + "name": "int", + "type": "reference" + } + } + ], + "type": { + "name": "ChargeResult", + "type": "reference", + "target": "379" + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "decorations": [ + { + "name": "ensure_context" + } + ], + "flags": {}, + "groups": [], + "id": 404, + "module": "_charging", + "name": "calculate_total_charged_amount", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_charging.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 270 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "flags": {}, + "id": 405, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "calculate_total_charged_amount", + "parameters": [], + "type": { + "name": "Decimal", + "type": "reference" + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "decorations": [ + { + "name": "ensure_context" + } + ], + "flags": {}, + "groups": [], + "id": 406, + "module": "_charging", + "name": "calculate_max_event_charge_count_within_limit", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_charging.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 277 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "flags": {}, + "id": 407, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "calculate_max_event_charge_count_within_limit", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 408, + "kind": 32768, + "kindString": "Parameter", + "name": "event_name", + "type": { + "name": "str", + "type": "reference" + } + } + ], + "type": { + "name": "int | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "int" + }, + { + "type": "literal", + "value": null + } + ] + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "decorations": [ + { + "name": "ensure_context" + } + ], + "flags": {}, + "groups": [], + "id": 409, + "module": "_charging", + "name": "get_pricing_info", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_charging.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 294 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "flags": {}, + "id": 410, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "get_pricing_info", + "parameters": [], + "type": { + "name": "ActorPricingInfo", + "type": "reference", + "target": "383" + } + } + ] + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 412, + "module": "_charging", + "name": "charge_count", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_charging.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 309 + } + ], + "type": { + "name": "int", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 413, + "module": "_charging", + "name": "total_charged_amount", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_charging.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 310 + } + ], + "type": { + "name": "Decimal", + "type": "reference" + } + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "decorations": [ + { + "name": "dataclass" + } + ], + "flags": {}, + "groups": [ + { + "children": [ + 412, + 413 + ], + "title": "Properties" + } + ], + "id": 411, + "module": "_charging", + "name": "ChargingStateItem", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_charging.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 308 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 415, + "module": "_charging", + "name": "price", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_charging.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 315 + } + ], + "type": { + "name": "Decimal", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 416, + "module": "_charging", + "name": "title", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_charging.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 316 + } + ], + "type": { + "name": "str", + "type": "reference" + } + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "decorations": [ + { + "name": "dataclass" + } + ], + "flags": {}, + "groups": [ + { + "children": [ + 415, + 416 + ], + "title": "Properties" + } + ], + "id": 414, + "module": "_charging", + "name": "PricingInfoItem", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_charging.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 314 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 418, + "module": "_platform_event_manager", + "name": "name", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_platform_event_manager.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 36 + } + ], + "type": { + "name": "Literal", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "Event.PERSIST_STATE" + } + ] + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 419, + "module": "_platform_event_manager", + "name": "data", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_platform_event_manager.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 37 + } + ], + "type": { + "name": "EventPersistStateData", + "type": "reference" + } + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "decorations": [ + { + "args": "('Data structures')", + "name": "docs_group" + } + ], + "flags": {}, + "groups": [ + { + "children": [ + 419, + 418 + ], + "title": "Properties" + } + ], + "id": 417, + "module": "_platform_event_manager", + "name": "PersistStateEvent", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_platform_event_manager.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 35 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 421, + "module": "_platform_event_manager", + "name": "mem_avg_bytes", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_platform_event_manager.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 42 + } + ], + "type": { + "name": "float", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 422, + "module": "_platform_event_manager", + "name": "mem_current_bytes", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_platform_event_manager.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 43 + } + ], + "type": { + "name": "float", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 423, + "module": "_platform_event_manager", + "name": "mem_max_bytes", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_platform_event_manager.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 44 + } + ], + "type": { + "name": "float", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 424, + "module": "_platform_event_manager", + "name": "cpu_avg_usage", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_platform_event_manager.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 45 + } + ], + "type": { + "name": "float", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 425, + "module": "_platform_event_manager", + "name": "cpu_max_usage", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_platform_event_manager.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 46 + } + ], + "type": { + "name": "float", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 426, + "module": "_platform_event_manager", + "name": "cpu_current_usage", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_platform_event_manager.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 47 + } + ], + "type": { + "name": "float", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 427, + "module": "_platform_event_manager", + "name": "is_cpu_overloaded", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_platform_event_manager.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 48 + } + ], + "type": { + "name": "bool", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 428, + "module": "_platform_event_manager", + "name": "created_at", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_platform_event_manager.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 49 + } + ], + "type": { + "name": "datetime", + "type": "reference" + } + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 429, + "module": "_platform_event_manager", + "name": "to_crawlee_format", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_platform_event_manager.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 51 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "flags": {}, + "id": 430, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "to_crawlee_format", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 431, + "kind": 32768, + "kindString": "Parameter", + "name": "dedicated_cpus", + "type": { + "name": "float", + "type": "reference" + } + } + ], + "type": { + "name": "EventSystemInfoData", + "type": "reference" + } + } + ] + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "decorations": [ + { + "args": "('Data structures')", + "name": "docs_group" + } + ], + "flags": {}, + "groups": [ + { + "children": [ + 429 + ], + "title": "Methods" + }, + { + "children": [ + 424, + 426, + 425, + 428, + 427, + 421, + 422, + 423 + ], + "title": "Properties" + } + ], + "id": 420, + "module": "_platform_event_manager", + "name": "SystemInfoEventData", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_platform_event_manager.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 41 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 433, + "module": "_platform_event_manager", + "name": "name", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_platform_event_manager.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 69 + } + ], + "type": { + "name": "Literal", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "Event.SYSTEM_INFO" + } + ] + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 434, + "module": "_platform_event_manager", + "name": "data", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_platform_event_manager.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 70 + } + ], + "type": { + "name": "SystemInfoEventData", + "type": "reference", + "target": "420" + } + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "decorations": [ + { + "args": "('Data structures')", + "name": "docs_group" + } + ], + "flags": {}, + "groups": [ + { + "children": [ + 434, + 433 + ], + "title": "Properties" + } + ], + "id": 432, + "module": "_platform_event_manager", + "name": "SystemInfoEvent", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_platform_event_manager.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 68 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 436, + "module": "_platform_event_manager", + "name": "name", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_platform_event_manager.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 75 + } + ], + "type": { + "name": "Literal", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "Event.MIGRATING" + } + ] + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 437, + "module": "_platform_event_manager", + "name": "data", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_platform_event_manager.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 76 + } + ], + "type": { + "name": "EventMigratingData", + "type": "reference" + } + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "decorations": [ + { + "args": "('Data structures')", + "name": "docs_group" + } + ], + "flags": {}, + "groups": [ + { + "children": [ + 437, + 436 + ], + "title": "Properties" + } + ], + "id": 435, + "module": "_platform_event_manager", + "name": "MigratingEvent", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_platform_event_manager.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 74 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 439, + "module": "_platform_event_manager", + "name": "name", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_platform_event_manager.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 81 + } + ], + "type": { + "name": "Literal", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "Event.ABORTING" + } + ] + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 440, + "module": "_platform_event_manager", + "name": "data", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_platform_event_manager.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 82 + } + ], + "type": { + "name": "EventAbortingData", + "type": "reference" + } + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "decorations": [ + { + "args": "('Data structures')", + "name": "docs_group" + } + ], + "flags": {}, + "groups": [ + { + "children": [ + 440, + 439 + ], + "title": "Properties" + } + ], + "id": 438, + "module": "_platform_event_manager", + "name": "AbortingEvent", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_platform_event_manager.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 80 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 442, + "module": "_platform_event_manager", + "name": "name", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_platform_event_manager.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 87 + } + ], + "type": { + "name": "Literal", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "Event.EXIT" + } + ] + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 443, + "module": "_platform_event_manager", + "name": "data", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_platform_event_manager.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 88 + } + ], + "type": { + "name": "EventExitData", + "type": "reference" + } + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "decorations": [ + { + "args": "('Data structures')", + "name": "docs_group" + } + ], + "flags": {}, + "groups": [ + { + "children": [ + 443, + 442 + ], + "title": "Properties" + } + ], + "id": 441, + "module": "_platform_event_manager", + "name": "ExitEvent", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_platform_event_manager.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 86 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 445, + "module": "_platform_event_manager", + "name": "name", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_platform_event_manager.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 93 + } + ], + "type": { + "name": "Literal", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "Event.SESSION_RETIRED" + }, + { + "type": "reference", + "name": "Event.BROWSER_LAUNCHED" + }, + { + "type": "reference", + "name": "Event.BROWSER_RETIRED" + }, + { + "type": "reference", + "name": "Event.BROWSER_CLOSED" + }, + { + "type": "reference", + "name": "Event.PAGE_CREATED" + }, + { + "type": "reference", + "name": "Event.PAGE_CLOSED" + } + ] + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 446, + "module": "_platform_event_manager", + "name": "data", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_platform_event_manager.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 101 + } + ], + "type": { + "name": "Any", + "type": "reference" + } + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "decorations": [ + { + "args": "('Data structures')", + "name": "docs_group" + } + ], + "flags": {}, + "groups": [ + { + "children": [ + 446, + 445 + ], + "title": "Properties" + } + ], + "id": 444, + "module": "_platform_event_manager", + "name": "EventWithoutData", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_platform_event_manager.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 92 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 448, + "module": "_platform_event_manager", + "name": "name", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_platform_event_manager.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 106 + } + ], + "type": { + "name": "Literal", + "type": "reference", + "typeArguments": [ + { + "type": "literal", + "value": "cpuInfo" + } + ] + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 449, + "module": "_platform_event_manager", + "name": "data", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_platform_event_manager.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 107 + } + ], + "type": { + "name": "dict", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "reference", + "name": "Any" + } + ] + } + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "decorations": [ + { + "args": "('Data structures')", + "name": "docs_group" + } + ], + "flags": {}, + "groups": [ + { + "children": [ + 449, + 448 + ], + "title": "Properties" + } + ], + "id": 447, + "module": "_platform_event_manager", + "name": "DeprecatedEvent", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_platform_event_manager.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 105 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 451, + "module": "_platform_event_manager", + "name": "name", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_platform_event_manager.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 112 + } + ], + "type": { + "name": "str", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 452, + "module": "_platform_event_manager", + "name": "data", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_platform_event_manager.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 113 + } + ], + "type": { + "name": "dict", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "reference", + "name": "Any" + } + ] + } + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "decorations": [ + { + "args": "('Data structures')", + "name": "docs_group" + } + ], + "flags": {}, + "groups": [ + { + "children": [ + 452, + 451 + ], + "title": "Properties" + } + ], + "id": 450, + "module": "_platform_event_manager", + "name": "UnknownEvent", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_platform_event_manager.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 111 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 453, + "module": "_platform_event_manager", + "name": "EventMessage", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_platform_event_manager.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 116 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 454, + "module": "_platform_event_manager", + "name": "event_data_adapter", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_platform_event_manager.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 126 + } + ], + "type": { + "name": "TypeAdapter", + "type": "reference", + "typeArguments": [ + { + "type": "union", + "types": [ + { + "type": "union", + "types": [ + { + "type": "reference", + "name": "EventMessage", + "target": "453" + }, + { + "type": "reference", + "name": "DeprecatedEvent", + "target": "447" + } + ] + }, + { + "type": "reference", + "name": "UnknownEvent", + "target": "450" + } + ] + } + ] + } + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Create an instance of the EventManager.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 456, + "module": "_platform_event_manager", + "name": "__init__", + "parsedDocstring": { + "text": "Create an instance of the EventManager.\n", + "args": { + "config": "The Actor configuration to be used in this event manager.", + "kwargs": "Event manager options - forwarded to the base class" + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_platform_event_manager.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 151 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Create an instance of the EventManager.\n" + } + ] + }, + "flags": {}, + "id": 457, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "__init__", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The Actor configuration to be used in this event manager." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 458, + "kind": 32768, + "kindString": "Parameter", + "name": "config", + "type": { + "name": "Configuration", + "type": "reference", + "target": "225" + } + } + ], + "type": { + "name": "None", + "type": "literal", + "value": null + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "decorations": [ + { + "name": "override" + } + ], + "flags": {}, + "groups": [], + "id": 460, + "module": "_platform_event_manager", + "name": "__aenter__", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_platform_event_manager.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 165 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "flags": {}, + "id": 461, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "__aenter__", + "parameters": [], + "type": { + "name": "Self", + "type": "reference" + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "decorations": [ + { + "name": "override" + } + ], + "flags": {}, + "groups": [], + "id": 462, + "module": "_platform_event_manager", + "name": "__aexit__", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_platform_event_manager.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 183 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "flags": {}, + "id": 463, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "__aexit__", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 464, + "kind": 32768, + "kindString": "Parameter", + "name": "exc_type", + "type": { + "name": "type[BaseException] | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "type", + "typeArguments": [ + { + "type": "reference", + "name": "BaseException" + } + ] + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 465, + "kind": 32768, + "kindString": "Parameter", + "name": "exc_value", + "type": { + "name": "BaseException | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "BaseException" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 466, + "kind": 32768, + "kindString": "Parameter", + "name": "exc_traceback", + "type": { + "name": "TracebackType | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "TracebackType" + }, + { + "type": "literal", + "value": null + } + ] + } + } + ], + "type": { + "name": "None", + "type": "literal", + "value": null + } + } + ] + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "A class for managing Actor events.\n\nYou shouldn't use this class directly,\nbut instead use it via the `Actor.on()` and `Actor.off()` methods." + } + ] + }, + "decorations": [ + { + "args": "('Classes')", + "name": "docs_group" + } + ], + "flags": {}, + "groups": [ + { + "children": [ + 460, + 462, + 456 + ], + "title": "Methods" + } + ], + "id": 455, + "module": "_platform_event_manager", + "name": "PlatformEventManager", + "parsedDocstring": { + "text": "A class for managing Actor events.\n\nYou shouldn't use this class directly,\nbut instead use it via the `Actor.on()` and `Actor.off()` methods." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_platform_event_manager.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 139 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 467, + "module": "_utils", + "name": "get_system_info", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_utils.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 9 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "flags": {}, + "id": 468, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "get_system_info", + "parameters": [], + "type": { + "name": "dict", + "type": "reference" + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 469, + "module": "_utils", + "name": "is_running_in_ipython", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_utils.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 26 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "flags": {}, + "id": 470, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "is_running_in_ipython", + "parameters": [], + "type": { + "name": "bool", + "type": "reference" + } + } + ] + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 471, + "module": "_utils", + "name": "GroupName", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_utils.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 30 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Mark a symbol for rendering and grouping in documentation.\n\nThis decorator is used solely for documentation purposes and does not modify the behavior\nof the decorated callable.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 472, + "module": "_utils", + "name": "docs_group", + "parsedDocstring": { + "text": "Mark a symbol for rendering and grouping in documentation.\n\nThis decorator is used solely for documentation purposes and does not modify the behavior\nof the decorated callable.\n", + "args": { + "group_name": "The documentation group to which the symbol belongs.\n" + }, + "returns": "The original callable without modification." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_utils.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 33 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "The original callable without modification." + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Mark a symbol for rendering and grouping in documentation.\n\nThis decorator is used solely for documentation purposes and does not modify the behavior\nof the decorated callable.\n" + } + ] + }, + "flags": {}, + "id": 473, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "docs_group", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The documentation group to which the symbol belongs.\n" + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 474, + "kind": 32768, + "kindString": "Parameter", + "name": "group_name", + "type": { + "name": "GroupName", + "type": "reference", + "target": "471" + } + } + ], + "type": { + "name": "Callable", + "type": "reference" + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Rename a symbol for documentation rendering.\n\nThis decorator modifies only the displayed name of the symbol in the generated documentation\nand does not affect its runtime behavior.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 475, + "module": "_utils", + "name": "docs_name", + "parsedDocstring": { + "text": "Rename a symbol for documentation rendering.\n\nThis decorator modifies only the displayed name of the symbol in the generated documentation\nand does not affect its runtime behavior.\n", + "args": { + "symbol_name": "The name to be used in the documentation.\n" + }, + "returns": "The original callable without modification." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_utils.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 52 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "The original callable without modification." + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Rename a symbol for documentation rendering.\n\nThis decorator modifies only the displayed name of the symbol in the generated documentation\nand does not affect its runtime behavior.\n" + } + ] + }, + "flags": {}, + "id": 476, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "docs_name", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The name to be used in the documentation.\n" + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 477, + "kind": 32768, + "kindString": "Parameter", + "name": "symbol_name", + "type": { + "name": "str", + "type": "reference" + } + } + ], + "type": { + "name": "Callable", + "type": "reference" + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Generate a basic authentication header for the given username and password." + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 478, + "module": "scrapy.utils", + "name": "get_basic_auth_header", + "parsedDocstring": { + "text": "Generate a basic authentication header for the given username and password." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/scrapy/utils.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 14 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Generate a basic authentication header for the given username and password." + } + ] + }, + "flags": {}, + "id": 479, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "get_basic_auth_header", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 480, + "kind": 32768, + "kindString": "Parameter", + "name": "username", + "type": { + "name": "str", + "type": "reference" + } + }, + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 481, + "kind": 32768, + "kindString": "Parameter", + "name": "password", + "type": { + "name": "str", + "type": "reference" + } + }, + { + "defaultValue": "'latin-1'", + "flags": { + "isOptional": true, + "keyword-only": false + }, + "id": 482, + "kind": 32768, + "kindString": "Parameter", + "name": "auth_encoding", + "type": { + "name": "str", + "type": "reference" + } + } + ], + "type": { + "name": "bytes", + "type": "reference" + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Integrates Apify configuration into a Scrapy project settings.\n\nNote: The function directly modifies the passed `settings` object and also returns it.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 483, + "module": "scrapy.utils", + "name": "apply_apify_settings", + "parsedDocstring": { + "text": "Integrates Apify configuration into a Scrapy project settings.\n\nNote: The function directly modifies the passed `settings` object and also returns it.\n", + "args": { + "settings": "Scrapy project settings to be modified.", + "proxy_config": "Proxy configuration to be stored in the settings.\n" + }, + "returns": "Scrapy project settings with custom configurations." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/scrapy/utils.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 21 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "Scrapy project settings with custom configurations." + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Integrates Apify configuration into a Scrapy project settings.\n\nNote: The function directly modifies the passed `settings` object and also returns it.\n" + } + ] + }, + "flags": {}, + "id": 484, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "apply_apify_settings", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Scrapy project settings to be modified." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 485, + "kind": 32768, + "kindString": "Parameter", + "name": "settings", + "type": { + "name": "Settings | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "Settings" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Proxy configuration to be stored in the settings.\n" + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 486, + "kind": 32768, + "kindString": "Parameter", + "name": "proxy_config", + "type": { + "name": "dict | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "dict" + }, + { + "type": "literal", + "value": null + } + ] + } + } + ], + "type": { + "name": "Settings", + "type": "reference" + } + } + ] + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 487, + "module": "scrapy._async_thread", + "name": "logger", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/scrapy/_async_thread.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 13 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 489, + "module": "scrapy._async_thread", + "name": "__init__", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/scrapy/_async_thread.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 23 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "flags": {}, + "id": 490, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "__init__", + "parameters": [], + "type": { + "name": "None", + "type": "literal", + "value": null + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Run a coroutine on an event loop running in a separate thread.\n\nThis method schedules the coroutine to run on the event loop and blocks until the coroutine completes\nor the specified timeout is reached.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 491, + "module": "scrapy._async_thread", + "name": "run_coro", + "parsedDocstring": { + "text": "Run a coroutine on an event loop running in a separate thread.\n\nThis method schedules the coroutine to run on the event loop and blocks until the coroutine completes\nor the specified timeout is reached.\n", + "args": { + "coro": "The coroutine to run.", + "timeout": "The maximum number of seconds to wait for the coroutine to finish.\n" + }, + "returns": "The result returned by the coroutine." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/scrapy/_async_thread.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 33 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "The result returned by the coroutine." + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Run a coroutine on an event loop running in a separate thread.\n\nThis method schedules the coroutine to run on the event loop and blocks until the coroutine completes\nor the specified timeout is reached.\n" + } + ] + }, + "flags": {}, + "id": 492, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "run_coro", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The coroutine to run." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 493, + "kind": 32768, + "kindString": "Parameter", + "name": "coro", + "type": { + "name": "Coroutine", + "type": "reference" + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The maximum number of seconds to wait for the coroutine to finish.\n" + } + ] + }, + "defaultValue": "timedelta(seconds=60)", + "flags": { + "isOptional": true, + "keyword-only": false + }, + "id": 494, + "kind": 32768, + "kindString": "Parameter", + "name": "timeout", + "type": { + "name": "timedelta", + "type": "reference" + } + } + ], + "type": { + "name": "Any", + "type": "reference" + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Close the event loop and its thread gracefully.\n\nThis method cancels all pending tasks, stops the event loop, and waits for the thread to exit.\nIf the thread does not exit within the given timeout, a forced shutdown is attempted.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 495, + "module": "scrapy._async_thread", + "name": "close", + "parsedDocstring": { + "text": "Close the event loop and its thread gracefully.\n\nThis method cancels all pending tasks, stops the event loop, and waits for the thread to exit.\nIf the thread does not exit within the given timeout, a forced shutdown is attempted.\n", + "args": { + "timeout": "The maximum number of seconds to wait for the event loop thread to exit." + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/scrapy/_async_thread.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 70 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Close the event loop and its thread gracefully.\n\nThis method cancels all pending tasks, stops the event loop, and waits for the thread to exit.\nIf the thread does not exit within the given timeout, a forced shutdown is attempted.\n" + } + ] + }, + "flags": {}, + "id": 496, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "close", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The maximum number of seconds to wait for the event loop thread to exit." + } + ] + }, + "defaultValue": "timedelta(seconds=60)", + "flags": { + "isOptional": true, + "keyword-only": false + }, + "id": 497, + "kind": 32768, + "kindString": "Parameter", + "name": "timeout", + "type": { + "name": "timedelta", + "type": "reference" + } + } + ], + "type": { + "name": "None", + "type": "literal", + "value": null + } + } + ] + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Class for running an asyncio event loop in a separate thread.\n\nThis allows running asynchronous coroutines from synchronous code by executingthem on an event loop\nthat runs in its own dedicated thread." + } + ] + }, + "flags": {}, + "groups": [ + { + "children": [ + 489, + 495, + 491 + ], + "title": "Methods" + } + ], + "id": 488, + "module": "scrapy._async_thread", + "name": "AsyncThread", + "parsedDocstring": { + "text": "Class for running an asyncio event loop in a separate thread.\n\nThis allows running asynchronous coroutines from synchronous code by executingthem on an event loop\nthat runs in its own dedicated thread." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/scrapy/_async_thread.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 16 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 498, + "module": "scrapy.scheduler", + "name": "logger", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/scrapy/scheduler.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 21 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 500, + "module": "scrapy.scheduler", + "name": "__init__", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/scrapy/scheduler.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 30 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "flags": {}, + "id": 501, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "__init__", + "parameters": [], + "type": { + "name": "None", + "type": "literal", + "value": null + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Open the scheduler.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 502, + "module": "scrapy.scheduler", + "name": "open", + "parsedDocstring": { + "text": "Open the scheduler.\n", + "args": { + "spider": "The spider that the scheduler is associated with." + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/scrapy/scheduler.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 43 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Open the scheduler.\n" + } + ] + }, + "flags": {}, + "id": 503, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "open", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The spider that the scheduler is associated with." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 504, + "kind": 32768, + "kindString": "Parameter", + "name": "spider", + "type": { + "name": "Spider", + "type": "reference" + } + } + ], + "type": { + "name": "Deferred[None] | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "Deferred", + "typeArguments": [ + { + "type": "literal", + "value": null + } + ] + }, + { + "type": "literal", + "value": null + } + ] + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Close the scheduler.\n\nShut down the event loop and its thread gracefully.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 505, + "module": "scrapy.scheduler", + "name": "close", + "parsedDocstring": { + "text": "Close the scheduler.\n\nShut down the event loop and its thread gracefully.\n", + "args": { + "reason": "The reason for closing the spider." + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/scrapy/scheduler.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 66 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Close the scheduler.\n\nShut down the event loop and its thread gracefully.\n" + } + ] + }, + "flags": {}, + "id": 506, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "close", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The reason for closing the spider." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 507, + "kind": 32768, + "kindString": "Parameter", + "name": "reason", + "type": { + "name": "str", + "type": "reference" + } + } + ], + "type": { + "name": "None", + "type": "literal", + "value": null + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Check if the scheduler has any pending requests.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 508, + "module": "scrapy.scheduler", + "name": "has_pending_requests", + "parsedDocstring": { + "text": "Check if the scheduler has any pending requests.\n", + "returns": "True if the scheduler has any pending requests, False otherwise." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/scrapy/scheduler.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 87 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "True if the scheduler has any pending requests, False otherwise." + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Check if the scheduler has any pending requests.\n" + } + ] + }, + "flags": {}, + "id": 509, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "has_pending_requests", + "parameters": [], + "type": { + "name": "bool", + "type": "reference" + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Add a request to the scheduler.\n\nThis could be called from either from a spider or a downloader middleware (e.g. redirect, retry, ...).\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 510, + "module": "scrapy.scheduler", + "name": "enqueue_request", + "parsedDocstring": { + "text": "Add a request to the scheduler.\n\nThis could be called from either from a spider or a downloader middleware (e.g. redirect, retry, ...).\n", + "args": { + "request": "The request to add to the scheduler.\n" + }, + "returns": "True if the request was successfully enqueued, False otherwise." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/scrapy/scheduler.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 104 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "True if the request was successfully enqueued, False otherwise." + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Add a request to the scheduler.\n\nThis could be called from either from a spider or a downloader middleware (e.g. redirect, retry, ...).\n" + } + ] + }, + "flags": {}, + "id": 511, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "enqueue_request", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The request to add to the scheduler.\n" + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 512, + "kind": 32768, + "kindString": "Parameter", + "name": "request", + "type": { + "name": "Request", + "type": "reference" + } + } + ], + "type": { + "name": "bool", + "type": "reference" + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Fetch the next request from the scheduler.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 513, + "module": "scrapy.scheduler", + "name": "next_request", + "parsedDocstring": { + "text": "Fetch the next request from the scheduler.\n", + "returns": "The next request, or None if there are no more requests." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/scrapy/scheduler.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 138 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "The next request, or None if there are no more requests." + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Fetch the next request from the scheduler.\n" + } + ] + }, + "flags": {}, + "id": 514, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "next_request", + "parameters": [], + "type": { + "name": "Request | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "Request" + }, + { + "type": "literal", + "value": null + } + ] + } + } + ] + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "A Scrapy scheduler that uses the Apify `RequestQueue` to manage requests.\n\nThis scheduler requires the asyncio Twisted reactor to be installed." + } + ] + }, + "flags": {}, + "groups": [ + { + "children": [ + 500, + 505, + 510, + 508, + 513, + 502 + ], + "title": "Methods" + } + ], + "id": 499, + "module": "scrapy.scheduler", + "name": "ApifyScheduler", + "parsedDocstring": { + "text": "A Scrapy scheduler that uses the Apify `RequestQueue` to manage requests.\n\nThis scheduler requires the asyncio Twisted reactor to be installed." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/scrapy/scheduler.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 24 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Create a new instance.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 516, + "module": "scrapy.middlewares.apify_proxy", + "name": "__init__", + "parsedDocstring": { + "text": "Create a new instance.\n", + "args": { + "proxy_settings": "Dictionary containing proxy settings, provided by the Actor input.", + "auth_encoding": "Encoding for basic authentication (default is 'latin-1')." + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/scrapy/middlewares/apify_proxy.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 28 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Create a new instance.\n" + } + ] + }, + "flags": {}, + "id": 517, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "__init__", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Dictionary containing proxy settings, provided by the Actor input." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 518, + "kind": 32768, + "kindString": "Parameter", + "name": "proxy_settings", + "type": { + "name": "dict", + "type": "reference" + } + } + ], + "type": { + "name": "None", + "type": "literal", + "value": null + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Create an instance of ApifyHttpProxyMiddleware from a Scrapy Crawler.\n\n\nReturns: Instance of the class." + } + ] + }, + "decorations": [ + { + "name": "classmethod" + } + ], + "flags": {}, + "groups": [], + "id": 519, + "module": "scrapy.middlewares.apify_proxy", + "name": "from_crawler", + "parsedDocstring": { + "text": "Create an instance of ApifyHttpProxyMiddleware from a Scrapy Crawler.\n\n\nReturns: Instance of the class.", + "args": { + "crawler": "Scrapy Crawler object.\n" + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/scrapy/middlewares/apify_proxy.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 39 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Create an instance of ApifyHttpProxyMiddleware from a Scrapy Crawler.\n\n\nReturns: Instance of the class." + } + ] + }, + "flags": {}, + "id": 520, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "from_crawler", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Scrapy Crawler object.\n" + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 521, + "kind": 32768, + "kindString": "Parameter", + "name": "crawler", + "type": { + "name": "Crawler", + "type": "reference" + } + } + ], + "type": { + "name": "ApifyHttpProxyMiddleware", + "type": "reference", + "target": "515" + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Process a Scrapy request by assigning a new proxy.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 522, + "module": "scrapy.middlewares.apify_proxy", + "name": "process_request", + "parsedDocstring": { + "text": "Process a Scrapy request by assigning a new proxy.\n", + "args": { + "request": "Scrapy Request object.", + "spider": "Scrapy Spider object.\n" + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/scrapy/middlewares/apify_proxy.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 67 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Process a Scrapy request by assigning a new proxy.\n" + } + ] + }, + "flags": {}, + "id": 523, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "process_request", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Scrapy Request object." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 524, + "kind": 32768, + "kindString": "Parameter", + "name": "request", + "type": { + "name": "Request", + "type": "reference" + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Scrapy Spider object.\n" + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 525, + "kind": 32768, + "kindString": "Parameter", + "name": "spider", + "type": { + "name": "Spider", + "type": "reference" + } + } + ], + "type": { + "name": "None", + "type": "literal", + "value": null + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Process an exception that occurs during request processing.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 526, + "module": "scrapy.middlewares.apify_proxy", + "name": "process_exception", + "parsedDocstring": { + "text": "Process an exception that occurs during request processing.\n", + "args": { + "request": "Scrapy Request object.", + "exception": "Exception object.", + "spider": "Scrapy Spider object.\n" + }, + "returns": "Returning None, meaning Scrapy will continue processing this exception, executing any other\nprocess_exception() methods of installed middleware, until no middleware is left and the default\nexception handling kicks in." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/scrapy/middlewares/apify_proxy.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 89 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "Returning None, meaning Scrapy will continue processing this exception, executing any other\nprocess_exception() methods of installed middleware, until no middleware is left and the default\nexception handling kicks in." + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Process an exception that occurs during request processing.\n" + } + ] + }, + "flags": {}, + "id": 527, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "process_exception", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Scrapy Request object." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 528, + "kind": 32768, + "kindString": "Parameter", + "name": "request", + "type": { + "name": "Request", + "type": "reference" + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Exception object." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 529, + "kind": 32768, + "kindString": "Parameter", + "name": "exception", + "type": { + "name": "Exception", + "type": "reference" + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Scrapy Spider object.\n" + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 530, + "kind": 32768, + "kindString": "Parameter", + "name": "spider", + "type": { + "name": "Spider", + "type": "reference" + } + } + ], + "type": { + "name": "None", + "type": "literal", + "value": null + } + } + ] + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Apify HTTP proxy middleware for Scrapy.\n\nThis middleware enhances request processing by adding a 'proxy' field to the request's meta and an authentication\nheader. It draws inspiration from the `HttpProxyMiddleware` included by default in Scrapy projects. The proxy URL\nis sourced from the settings under the `APIFY_PROXY_SETTINGS` key. The value of this key, a dictionary, should be\nprovided by the Actor input. An example of the proxy settings:\n\nproxy_settings = {'useApifyProxy': true, 'apifyProxyGroups': []}" + } + ] + }, + "flags": {}, + "groups": [ + { + "children": [ + 516, + 519, + 526, + 522 + ], + "title": "Methods" + } + ], + "id": 515, + "module": "scrapy.middlewares.apify_proxy", + "name": "ApifyHttpProxyMiddleware", + "parsedDocstring": { + "text": "Apify HTTP proxy middleware for Scrapy.\n\nThis middleware enhances request processing by adding a 'proxy' field to the request's meta and an authentication\nheader. It draws inspiration from the `HttpProxyMiddleware` included by default in Scrapy projects. The proxy URL\nis sourced from the settings under the `APIFY_PROXY_SETTINGS` key. The value of this key, a dictionary, should be\nprovided by the Actor input. An example of the proxy settings:\n\nproxy_settings = {'useApifyProxy': true, 'apifyProxyGroups': []}" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/scrapy/middlewares/apify_proxy.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 17 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 531, + "module": "scrapy.requests", + "name": "logger", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/scrapy/requests.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 16 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Convert a Scrapy request to an Apify request.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 532, + "module": "scrapy.requests", + "name": "to_apify_request", + "parsedDocstring": { + "text": "Convert a Scrapy request to an Apify request.\n", + "args": { + "scrapy_request": "The Scrapy request to be converted.", + "spider": "The Scrapy spider that the request is associated with.\n" + }, + "returns": "The converted Apify request if the conversion was successful, otherwise None." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/scrapy/requests.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 19 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "The converted Apify request if the conversion was successful, otherwise None." + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Convert a Scrapy request to an Apify request.\n" + } + ] + }, + "flags": {}, + "id": 533, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "to_apify_request", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The Scrapy request to be converted." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 534, + "kind": 32768, + "kindString": "Parameter", + "name": "scrapy_request", + "type": { + "name": "ScrapyRequest", + "type": "reference" + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The Scrapy spider that the request is associated with.\n" + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 535, + "kind": 32768, + "kindString": "Parameter", + "name": "spider", + "type": { + "name": "Spider", + "type": "reference" + } + } + ], + "type": { + "name": "ApifyRequest | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "ApifyRequest" + }, + { + "type": "literal", + "value": null + } + ] + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Convert an Apify request to a Scrapy request.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 536, + "module": "scrapy.requests", + "name": "to_scrapy_request", + "parsedDocstring": { + "text": "Convert an Apify request to a Scrapy request.\n", + "args": { + "apify_request": "The Apify request to be converted.", + "spider": "The Scrapy spider that the request is associated with.\n" + }, + "returns": "The converted Scrapy request." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/scrapy/requests.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 82 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "The converted Scrapy request." + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Convert an Apify request to a Scrapy request.\n" + } + ] + }, + "flags": {}, + "id": 537, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "to_scrapy_request", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The Apify request to be converted." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 538, + "kind": 32768, + "kindString": "Parameter", + "name": "apify_request", + "type": { + "name": "ApifyRequest", + "type": "reference" + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The Scrapy spider that the request is associated with.\n" + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 539, + "kind": 32768, + "kindString": "Parameter", + "name": "spider", + "type": { + "name": "Spider", + "type": "reference" + } + } + ], + "type": { + "name": "ScrapyRequest", + "type": "reference" + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Start Twisted's reactor and execute the provided Actor coroutine.\n\nThis function initiates the Twisted reactor and runs the given asyncio coroutine (typically the\nActor's main) by converting it to a Deferred. This bridges the asyncio and Twisted event loops,\nenabling the Apify and Scrapy integration to work together." + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 540, + "module": "scrapy._actor_runner", + "name": "run_scrapy_actor", + "parsedDocstring": { + "text": "Start Twisted's reactor and execute the provided Actor coroutine.\n\nThis function initiates the Twisted reactor and runs the given asyncio coroutine (typically the\nActor's main) by converting it to a Deferred. This bridges the asyncio and Twisted event loops,\nenabling the Apify and Scrapy integration to work together." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/scrapy/_actor_runner.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 19 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Start Twisted's reactor and execute the provided Actor coroutine.\n\nThis function initiates the Twisted reactor and runs the given asyncio coroutine (typically the\nActor's main) by converting it to a Deferred. This bridges the asyncio and Twisted event loops,\nenabling the Apify and Scrapy integration to work together." + } + ] + }, + "flags": {}, + "id": 541, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "run_scrapy_actor", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 542, + "kind": 32768, + "kindString": "Parameter", + "name": "coro", + "type": { + "name": "Coroutine", + "type": "reference" + } + } + ], + "type": { + "name": "None", + "type": "literal", + "value": null + } + } + ] + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 543, + "module": "scrapy.extensions._httpcache", + "name": "logger", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/scrapy/extensions/_httpcache.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 26 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 545, + "module": "scrapy.extensions._httpcache", + "name": "__init__", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/scrapy/extensions/_httpcache.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 38 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "flags": {}, + "id": 546, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "__init__", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 547, + "kind": 32768, + "kindString": "Parameter", + "name": "settings", + "type": { + "name": "BaseSettings", + "type": "reference" + } + } + ], + "type": { + "name": "None", + "type": "literal", + "value": null + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Open the cache storage for a spider." + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 548, + "module": "scrapy.extensions._httpcache", + "name": "open_spider", + "parsedDocstring": { + "text": "Open the cache storage for a spider." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/scrapy/extensions/_httpcache.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 46 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Open the cache storage for a spider." + } + ] + }, + "flags": {}, + "id": 549, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "open_spider", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 550, + "kind": 32768, + "kindString": "Parameter", + "name": "spider", + "type": { + "name": "Spider", + "type": "reference" + } + } + ], + "type": { + "name": "None", + "type": "literal", + "value": null + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Close the cache storage for a spider." + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 551, + "module": "scrapy.extensions._httpcache", + "name": "close_spider", + "parsedDocstring": { + "text": "Close the cache storage for a spider." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/scrapy/extensions/_httpcache.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 65 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Close the cache storage for a spider." + } + ] + }, + "flags": {}, + "id": 552, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "close_spider", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 553, + "kind": 32768, + "kindString": "Parameter", + "name": "_", + "type": { + "name": "Spider", + "type": "reference" + } + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": false + }, + "id": 554, + "kind": 32768, + "kindString": "Parameter", + "name": "current_time", + "type": { + "name": "int | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "int" + }, + { + "type": "literal", + "value": null + } + ] + } + } + ], + "type": { + "name": "None", + "type": "literal", + "value": null + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Retrieve a response from the cache storage." + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 555, + "module": "scrapy.extensions._httpcache", + "name": "retrieve_response", + "parsedDocstring": { + "text": "Retrieve a response from the cache storage." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/scrapy/extensions/_httpcache.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 108 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Retrieve a response from the cache storage." + } + ] + }, + "flags": {}, + "id": 556, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "retrieve_response", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 557, + "kind": 32768, + "kindString": "Parameter", + "name": "_", + "type": { + "name": "Spider", + "type": "reference" + } + }, + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 558, + "kind": 32768, + "kindString": "Parameter", + "name": "request", + "type": { + "name": "Request", + "type": "reference" + } + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": false + }, + "id": 559, + "kind": 32768, + "kindString": "Parameter", + "name": "current_time", + "type": { + "name": "int | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "int" + }, + { + "type": "literal", + "value": null + } + ] + } + } + ], + "type": { + "name": "Response | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "Response" + }, + { + "type": "literal", + "value": null + } + ] + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Store a response in the cache storage." + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 560, + "module": "scrapy.extensions._httpcache", + "name": "store_response", + "parsedDocstring": { + "text": "Store a response in the cache storage." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/scrapy/extensions/_httpcache.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 140 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Store a response in the cache storage." + } + ] + }, + "flags": {}, + "id": 561, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "store_response", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 562, + "kind": 32768, + "kindString": "Parameter", + "name": "_", + "type": { + "name": "Spider", + "type": "reference" + } + }, + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 563, + "kind": 32768, + "kindString": "Parameter", + "name": "request", + "type": { + "name": "Request", + "type": "reference" + } + }, + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 564, + "kind": 32768, + "kindString": "Parameter", + "name": "response", + "type": { + "name": "Response", + "type": "reference" + } + } + ], + "type": { + "name": "None", + "type": "literal", + "value": null + } + } + ] + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "A Scrapy cache storage that uses the Apify `KeyValueStore` to store responses.\n\nIt can be set as a storage for Scrapy's built-in `HttpCacheMiddleware`, which caches\nresponses to requests. See HTTPCache middleware settings (prefixed with `HTTPCACHE_`)\nin the Scrapy documentation for more information. Requires the asyncio Twisted reactor\nto be installed." + } + ] + }, + "flags": {}, + "groups": [ + { + "children": [ + 545, + 551, + 548, + 555, + 560 + ], + "title": "Methods" + } + ], + "id": 544, + "module": "scrapy.extensions._httpcache", + "name": "ApifyCacheStorage", + "parsedDocstring": { + "text": "A Scrapy cache storage that uses the Apify `KeyValueStore` to store responses.\n\nIt can be set as a storage for Scrapy's built-in `HttpCacheMiddleware`, which caches\nresponses to requests. See HTTPCache middleware settings (prefixed with `HTTPCACHE_`)\nin the Scrapy documentation for more information. Requires the asyncio Twisted reactor\nto be installed." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/scrapy/extensions/_httpcache.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 29 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Dump a dictionary to a gzip-compressed byte stream." + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 565, + "module": "scrapy.extensions._httpcache", + "name": "to_gzip", + "parsedDocstring": { + "text": "Dump a dictionary to a gzip-compressed byte stream." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/scrapy/extensions/_httpcache.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 160 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Dump a dictionary to a gzip-compressed byte stream." + } + ] + }, + "flags": {}, + "id": 566, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "to_gzip", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 567, + "kind": 32768, + "kindString": "Parameter", + "name": "data", + "type": { + "name": "dict", + "type": "reference" + } + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": false + }, + "id": 568, + "kind": 32768, + "kindString": "Parameter", + "name": "mtime", + "type": { + "name": "int | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "int" + }, + { + "type": "literal", + "value": null + } + ] + } + } + ], + "type": { + "name": "bytes", + "type": "reference" + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Load a dictionary from a gzip-compressed byte stream." + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 569, + "module": "scrapy.extensions._httpcache", + "name": "from_gzip", + "parsedDocstring": { + "text": "Load a dictionary from a gzip-compressed byte stream." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/scrapy/extensions/_httpcache.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 168 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Load a dictionary from a gzip-compressed byte stream." + } + ] + }, + "flags": {}, + "id": 570, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "from_gzip", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 571, + "kind": 32768, + "kindString": "Parameter", + "name": "gzip_bytes", + "type": { + "name": "bytes", + "type": "reference" + } + } + ], + "type": { + "name": "dict", + "type": "reference" + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Read the modification time from a gzip-compressed byte stream without decompressing the data." + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 572, + "module": "scrapy.extensions._httpcache", + "name": "read_gzip_time", + "parsedDocstring": { + "text": "Read the modification time from a gzip-compressed byte stream without decompressing the data." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/scrapy/extensions/_httpcache.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 175 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Read the modification time from a gzip-compressed byte stream without decompressing the data." + } + ] + }, + "flags": {}, + "id": 573, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "read_gzip_time", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 574, + "kind": 32768, + "kindString": "Parameter", + "name": "gzip_bytes", + "type": { + "name": "bytes", + "type": "reference" + } + } + ], + "type": { + "name": "int", + "type": "reference" + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Get the key value store name for a spider.\n\nThe key value store name is derived from the spider name by replacing all special characters\nwith hyphens and trimming leading and trailing hyphens. The resulting name is prefixed with\n'httpcache-' and truncated to the maximum length.\n\nThe documentation\n[about storages](https://docs.apify.com/platform/storage/usage#named-and-unnamed-storages)\nmentions that names can be up to 63 characters long, so the default max length is set to 60.\n\nSuch naming isn't unique per spider, but should be sufficiently unique for most use cases.\nThe name of the key value store should indicate to which spider it belongs, e.g. in\nthe listing in the Apify's console.\n\n\nReturns: Key value store name.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 575, + "module": "scrapy.extensions._httpcache", + "name": "get_kvs_name", + "parsedDocstring": { + "text": "Get the key value store name for a spider.\n\nThe key value store name is derived from the spider name by replacing all special characters\nwith hyphens and trimming leading and trailing hyphens. The resulting name is prefixed with\n'httpcache-' and truncated to the maximum length.\n\nThe documentation\n[about storages](https://docs.apify.com/platform/storage/usage#named-and-unnamed-storages)\nmentions that names can be up to 63 characters long, so the default max length is set to 60.\n\nSuch naming isn't unique per spider, but should be sufficiently unique for most use cases.\nThe name of the key value store should indicate to which spider it belongs, e.g. in\nthe listing in the Apify's console.\n\n\nReturns: Key value store name.\n", + "args": { + "spider_name": "Value of the Spider instance's name attribute.", + "max_length": "Maximum length of the key value store name.\n" + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/scrapy/extensions/_httpcache.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 183 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Get the key value store name for a spider.\n\nThe key value store name is derived from the spider name by replacing all special characters\nwith hyphens and trimming leading and trailing hyphens. The resulting name is prefixed with\n'httpcache-' and truncated to the maximum length.\n\nThe documentation\n[about storages](https://docs.apify.com/platform/storage/usage#named-and-unnamed-storages)\nmentions that names can be up to 63 characters long, so the default max length is set to 60.\n\nSuch naming isn't unique per spider, but should be sufficiently unique for most use cases.\nThe name of the key value store should indicate to which spider it belongs, e.g. in\nthe listing in the Apify's console.\n\n\nReturns: Key value store name.\n" + } + ] + }, + "flags": {}, + "id": 576, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "get_kvs_name", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Value of the Spider instance's name attribute." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 577, + "kind": 32768, + "kindString": "Parameter", + "name": "spider_name", + "type": { + "name": "str", + "type": "reference" + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Maximum length of the key value store name.\n" + } + ] + }, + "defaultValue": "60", + "flags": { + "isOptional": true, + "keyword-only": false + }, + "id": 578, + "kind": 32768, + "kindString": "Parameter", + "name": "max_length", + "type": { + "name": "int", + "type": "reference" + } + } + ], + "type": { + "name": "str", + "type": "reference" + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Configure logging for Apify Actors and adjust Scrapy's logging settings." + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 579, + "module": "scrapy._logging_config", + "name": "initialize_logging", + "parsedDocstring": { + "text": "Configure logging for Apify Actors and adjust Scrapy's logging settings." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/scrapy/_logging_config.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 26 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Configure logging for Apify Actors and adjust Scrapy's logging settings." + } + ] + }, + "flags": {}, + "id": 580, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "initialize_logging", + "parameters": [], + "type": { + "name": "None", + "type": "literal", + "value": null + } + } + ] + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 581, + "module": "scrapy.pipelines.actor_dataset_push", + "name": "logger", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/scrapy/pipelines/actor_dataset_push.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 13 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Pushes the provided Scrapy item to the Actor's default dataset." + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 583, + "module": "scrapy.pipelines.actor_dataset_push", + "name": "process_item", + "parsedDocstring": { + "text": "Pushes the provided Scrapy item to the Actor's default dataset." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/scrapy/pipelines/actor_dataset_push.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 22 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Pushes the provided Scrapy item to the Actor's default dataset." + } + ] + }, + "flags": {}, + "id": 584, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "process_item", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 585, + "kind": 32768, + "kindString": "Parameter", + "name": "item", + "type": { + "name": "Item", + "type": "reference" + } + }, + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 586, + "kind": 32768, + "kindString": "Parameter", + "name": "spider", + "type": { + "name": "Spider", + "type": "reference" + } + } + ], + "type": { + "name": "Item", + "type": "reference" + } + } + ] + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "A Scrapy pipeline for pushing items to an Actor's default dataset.\n\nThis pipeline is designed to be enabled only when the Scrapy project is run as an Actor." + } + ] + }, + "flags": {}, + "groups": [ + { + "children": [ + 583 + ], + "title": "Methods" + } + ], + "id": 582, + "module": "scrapy.pipelines.actor_dataset_push", + "name": "ActorDatasetPushPipeline", + "parsedDocstring": { + "text": "A Scrapy pipeline for pushing items to an Actor's default dataset.\n\nThis pipeline is designed to be enabled only when the Scrapy project is run as an Actor." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/scrapy/pipelines/actor_dataset_push.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 16 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 588, + "module": "apify_storage_client._key_value_store_collection_client", + "name": "__init__", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/apify_storage_client/_key_value_store_collection_client.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 17 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "flags": {}, + "id": 589, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "__init__", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 590, + "kind": 32768, + "kindString": "Parameter", + "name": "apify_dataset_collection_client", + "type": { + "name": "KeyValueStoreCollectionClientAsync", + "type": "reference" + } + } + ], + "type": { + "name": "None", + "type": "literal", + "value": null + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "decorations": [ + { + "name": "override" + } + ], + "flags": {}, + "groups": [], + "id": 591, + "module": "apify_storage_client._key_value_store_collection_client", + "name": "get_or_create", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/apify_storage_client/_key_value_store_collection_client.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 21 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "flags": {}, + "id": 592, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "get_or_create", + "parameters": [ + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 593, + "kind": 32768, + "kindString": "Parameter", + "name": "id", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 594, + "kind": 32768, + "kindString": "Parameter", + "name": "name", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 595, + "kind": 32768, + "kindString": "Parameter", + "name": "schema", + "type": { + "name": "dict | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "dict" + }, + { + "type": "literal", + "value": null + } + ] + } + } + ], + "type": { + "name": "KeyValueStoreMetadata", + "type": "reference" + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "decorations": [ + { + "name": "override" + } + ], + "flags": {}, + "groups": [], + "id": 596, + "module": "apify_storage_client._key_value_store_collection_client", + "name": "list", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/apify_storage_client/_key_value_store_collection_client.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 36 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "flags": {}, + "id": 597, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "list", + "parameters": [ + { + "defaultValue": "False", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 598, + "kind": 32768, + "kindString": "Parameter", + "name": "unnamed", + "type": { + "name": "bool", + "type": "reference" + } + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 599, + "kind": 32768, + "kindString": "Parameter", + "name": "limit", + "type": { + "name": "int | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "int" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 600, + "kind": 32768, + "kindString": "Parameter", + "name": "offset", + "type": { + "name": "int | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "int" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "defaultValue": "False", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 601, + "kind": 32768, + "kindString": "Parameter", + "name": "desc", + "type": { + "name": "bool", + "type": "reference" + } + } + ], + "type": { + "name": "KeyValueStoreListPage", + "type": "reference" + } + } + ] + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Key-value store collection resource client implementation based on the Apify platform storage." + } + ] + }, + "flags": {}, + "groups": [ + { + "children": [ + 588, + 591, + 596 + ], + "title": "Methods" + } + ], + "id": 587, + "module": "apify_storage_client._key_value_store_collection_client", + "name": "KeyValueStoreCollectionClient", + "parsedDocstring": { + "text": "Key-value store collection resource client implementation based on the Apify platform storage." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/apify_storage_client/_key_value_store_collection_client.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 14 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 603, + "module": "apify_storage_client._apify_storage_client", + "name": "__init__", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/apify_storage_client/_apify_storage_client.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 27 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "flags": {}, + "id": 604, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "__init__", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 605, + "kind": 32768, + "kindString": "Parameter", + "name": "configuration", + "type": { + "name": "Configuration", + "type": "reference", + "target": "225" + } + } + ], + "type": { + "name": "None", + "type": "literal", + "value": null + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "decorations": [ + { + "name": "classmethod" + } + ], + "flags": {}, + "groups": [], + "id": 606, + "module": "apify_storage_client._apify_storage_client", + "name": "from_config", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/apify_storage_client/_apify_storage_client.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 39 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "flags": {}, + "id": 607, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "from_config", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 608, + "kind": 32768, + "kindString": "Parameter", + "name": "config", + "type": { + "name": "Configuration", + "type": "reference", + "target": "225" + } + } + ], + "type": { + "name": "ApifyStorageClient", + "type": "reference", + "target": "602" + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "decorations": [ + { + "name": "override" + } + ], + "flags": {}, + "groups": [], + "id": 609, + "module": "apify_storage_client._apify_storage_client", + "name": "dataset", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/apify_storage_client/_apify_storage_client.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 43 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "flags": {}, + "id": 610, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "dataset", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 611, + "kind": 32768, + "kindString": "Parameter", + "name": "id", + "type": { + "name": "str", + "type": "reference" + } + } + ], + "type": { + "name": "DatasetClient", + "type": "reference", + "target": "742" + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "decorations": [ + { + "name": "override" + } + ], + "flags": {}, + "groups": [], + "id": 612, + "module": "apify_storage_client._apify_storage_client", + "name": "datasets", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/apify_storage_client/_apify_storage_client.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 47 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "flags": {}, + "id": 613, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "datasets", + "parameters": [], + "type": { + "name": "DatasetCollectionClient", + "type": "reference", + "target": "628" + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "decorations": [ + { + "name": "override" + } + ], + "flags": {}, + "groups": [], + "id": 614, + "module": "apify_storage_client._apify_storage_client", + "name": "key_value_store", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/apify_storage_client/_apify_storage_client.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 51 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "flags": {}, + "id": 615, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "key_value_store", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 616, + "kind": 32768, + "kindString": "Parameter", + "name": "id", + "type": { + "name": "str", + "type": "reference" + } + } + ], + "type": { + "name": "KeyValueStoreClient", + "type": "reference", + "target": "706" + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "decorations": [ + { + "name": "override" + } + ], + "flags": {}, + "groups": [], + "id": 617, + "module": "apify_storage_client._apify_storage_client", + "name": "key_value_stores", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/apify_storage_client/_apify_storage_client.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 55 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "flags": {}, + "id": 618, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "key_value_stores", + "parameters": [], + "type": { + "name": "KeyValueStoreCollectionClient", + "type": "reference", + "target": "587" + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "decorations": [ + { + "name": "override" + } + ], + "flags": {}, + "groups": [], + "id": 619, + "module": "apify_storage_client._apify_storage_client", + "name": "request_queue", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/apify_storage_client/_apify_storage_client.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 59 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "flags": {}, + "id": 620, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "request_queue", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 621, + "kind": 32768, + "kindString": "Parameter", + "name": "id", + "type": { + "name": "str", + "type": "reference" + } + } + ], + "type": { + "name": "RequestQueueClient", + "type": "reference", + "target": "643" + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "decorations": [ + { + "name": "override" + } + ], + "flags": {}, + "groups": [], + "id": 622, + "module": "apify_storage_client._apify_storage_client", + "name": "request_queues", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/apify_storage_client/_apify_storage_client.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 63 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "flags": {}, + "id": 623, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "request_queues", + "parameters": [], + "type": { + "name": "RequestQueueCollectionClient", + "type": "reference", + "target": "691" + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "decorations": [ + { + "name": "override" + } + ], + "flags": {}, + "groups": [], + "id": 624, + "module": "apify_storage_client._apify_storage_client", + "name": "purge_on_start", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/apify_storage_client/_apify_storage_client.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 67 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "flags": {}, + "id": 625, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "purge_on_start", + "parameters": [], + "type": { + "name": "None", + "type": "literal", + "value": null + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "decorations": [ + { + "name": "override" + } + ], + "flags": {}, + "groups": [], + "id": 626, + "module": "apify_storage_client._apify_storage_client", + "name": "get_rate_limit_errors", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/apify_storage_client/_apify_storage_client.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 71 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "flags": {}, + "id": 627, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "get_rate_limit_errors", + "parameters": [], + "type": { + "name": "dict", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "int" + }, + { + "type": "reference", + "name": "int" + } + ] + } + } + ] + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "A storage client implementation based on the Apify platform storage." + } + ] + }, + "decorations": [ + { + "args": "('Classes')", + "name": "docs_group" + } + ], + "flags": {}, + "groups": [ + { + "children": [ + 603, + 609, + 612, + 606, + 626, + 614, + 617, + 624, + 619, + 622 + ], + "title": "Methods" + } + ], + "id": 602, + "module": "apify_storage_client._apify_storage_client", + "name": "ApifyStorageClient", + "parsedDocstring": { + "text": "A storage client implementation based on the Apify platform storage." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/apify_storage_client/_apify_storage_client.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 24 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 629, + "module": "apify_storage_client._dataset_collection_client", + "name": "__init__", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/apify_storage_client/_dataset_collection_client.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 17 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "flags": {}, + "id": 630, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "__init__", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 631, + "kind": 32768, + "kindString": "Parameter", + "name": "apify_dataset_collection_client", + "type": { + "name": "DatasetCollectionClientAsync", + "type": "reference" + } + } + ], + "type": { + "name": "None", + "type": "literal", + "value": null + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "decorations": [ + { + "name": "override" + } + ], + "flags": {}, + "groups": [], + "id": 632, + "module": "apify_storage_client._dataset_collection_client", + "name": "get_or_create", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/apify_storage_client/_dataset_collection_client.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 21 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "flags": {}, + "id": 633, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "get_or_create", + "parameters": [ + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 634, + "kind": 32768, + "kindString": "Parameter", + "name": "id", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 635, + "kind": 32768, + "kindString": "Parameter", + "name": "name", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 636, + "kind": 32768, + "kindString": "Parameter", + "name": "schema", + "type": { + "name": "dict | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "dict" + }, + { + "type": "literal", + "value": null + } + ] + } + } + ], + "type": { + "name": "DatasetMetadata", + "type": "reference" + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "decorations": [ + { + "name": "override" + } + ], + "flags": {}, + "groups": [], + "id": 637, + "module": "apify_storage_client._dataset_collection_client", + "name": "list", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/apify_storage_client/_dataset_collection_client.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 36 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "flags": {}, + "id": 638, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "list", + "parameters": [ + { + "defaultValue": "False", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 639, + "kind": 32768, + "kindString": "Parameter", + "name": "unnamed", + "type": { + "name": "bool", + "type": "reference" + } + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 640, + "kind": 32768, + "kindString": "Parameter", + "name": "limit", + "type": { + "name": "int | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "int" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 641, + "kind": 32768, + "kindString": "Parameter", + "name": "offset", + "type": { + "name": "int | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "int" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "defaultValue": "False", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 642, + "kind": 32768, + "kindString": "Parameter", + "name": "desc", + "type": { + "name": "bool", + "type": "reference" + } + } + ], + "type": { + "name": "DatasetListPage", + "type": "reference" + } + } + ] + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Dataset collection resource client implementation based on the Apify platform storage." + } + ] + }, + "flags": {}, + "groups": [ + { + "children": [ + 629, + 632, + 637 + ], + "title": "Methods" + } + ], + "id": 628, + "module": "apify_storage_client._dataset_collection_client", + "name": "DatasetCollectionClient", + "parsedDocstring": { + "text": "Dataset collection resource client implementation based on the Apify platform storage." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/apify_storage_client/_dataset_collection_client.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 14 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 644, + "module": "apify_storage_client._request_queue_client", + "name": "__init__", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/apify_storage_client/_request_queue_client.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 27 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "flags": {}, + "id": 645, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "__init__", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 646, + "kind": 32768, + "kindString": "Parameter", + "name": "apify_request_queue_client", + "type": { + "name": "RequestQueueClientAsync", + "type": "reference" + } + } + ], + "type": { + "name": "None", + "type": "literal", + "value": null + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "decorations": [ + { + "name": "override" + } + ], + "flags": {}, + "groups": [], + "id": 647, + "module": "apify_storage_client._request_queue_client", + "name": "get", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/apify_storage_client/_request_queue_client.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 31 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "flags": {}, + "id": 648, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "get", + "parameters": [], + "type": { + "name": "RequestQueueMetadata | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "RequestQueueMetadata" + }, + { + "type": "literal", + "value": null + } + ] + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "decorations": [ + { + "name": "override" + } + ], + "flags": {}, + "groups": [], + "id": 649, + "module": "apify_storage_client._request_queue_client", + "name": "update", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/apify_storage_client/_request_queue_client.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 36 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "flags": {}, + "id": 650, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "update", + "parameters": [ + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 651, + "kind": 32768, + "kindString": "Parameter", + "name": "name", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + } + ], + "type": { + "name": "RequestQueueMetadata", + "type": "reference" + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "decorations": [ + { + "name": "override" + } + ], + "flags": {}, + "groups": [], + "id": 652, + "module": "apify_storage_client._request_queue_client", + "name": "delete", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/apify_storage_client/_request_queue_client.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 49 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "flags": {}, + "id": 653, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "delete", + "parameters": [], + "type": { + "name": "None", + "type": "literal", + "value": null + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "decorations": [ + { + "name": "override" + } + ], + "flags": {}, + "groups": [], + "id": 654, + "module": "apify_storage_client._request_queue_client", + "name": "list_head", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/apify_storage_client/_request_queue_client.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 53 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "flags": {}, + "id": 655, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "list_head", + "parameters": [ + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 656, + "kind": 32768, + "kindString": "Parameter", + "name": "limit", + "type": { + "name": "int | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "int" + }, + { + "type": "literal", + "value": null + } + ] + } + } + ], + "type": { + "name": "RequestQueueHead", + "type": "reference" + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "decorations": [ + { + "name": "override" + } + ], + "flags": {}, + "groups": [], + "id": 657, + "module": "apify_storage_client._request_queue_client", + "name": "list_and_lock_head", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/apify_storage_client/_request_queue_client.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 61 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "flags": {}, + "id": 658, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "list_and_lock_head", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 659, + "kind": 32768, + "kindString": "Parameter", + "name": "lock_secs", + "type": { + "name": "int", + "type": "reference" + } + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 660, + "kind": 32768, + "kindString": "Parameter", + "name": "limit", + "type": { + "name": "int | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "int" + }, + { + "type": "literal", + "value": null + } + ] + } + } + ], + "type": { + "name": "RequestQueueHeadWithLocks", + "type": "reference" + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "decorations": [ + { + "name": "override" + } + ], + "flags": {}, + "groups": [], + "id": 661, + "module": "apify_storage_client._request_queue_client", + "name": "add_request", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/apify_storage_client/_request_queue_client.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 70 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "flags": {}, + "id": 662, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "add_request", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 663, + "kind": 32768, + "kindString": "Parameter", + "name": "request", + "type": { + "name": "Request", + "type": "reference" + } + }, + { + "defaultValue": "False", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 664, + "kind": 32768, + "kindString": "Parameter", + "name": "forefront", + "type": { + "name": "bool", + "type": "reference" + } + } + ], + "type": { + "name": "ProcessedRequest", + "type": "reference" + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "decorations": [ + { + "name": "override" + } + ], + "flags": {}, + "groups": [], + "id": 665, + "module": "apify_storage_client._request_queue_client", + "name": "get_request", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/apify_storage_client/_request_queue_client.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 90 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "flags": {}, + "id": 666, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "get_request", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 667, + "kind": 32768, + "kindString": "Parameter", + "name": "request_id", + "type": { + "name": "str", + "type": "reference" + } + } + ], + "type": { + "name": "Request | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "Request" + }, + { + "type": "literal", + "value": null + } + ] + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "decorations": [ + { + "name": "override" + } + ], + "flags": {}, + "groups": [], + "id": 668, + "module": "apify_storage_client._request_queue_client", + "name": "update_request", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/apify_storage_client/_request_queue_client.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 95 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "flags": {}, + "id": 669, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "update_request", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 670, + "kind": 32768, + "kindString": "Parameter", + "name": "request", + "type": { + "name": "Request", + "type": "reference" + } + }, + { + "defaultValue": "False", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 671, + "kind": 32768, + "kindString": "Parameter", + "name": "forefront", + "type": { + "name": "bool", + "type": "reference" + } + } + ], + "type": { + "name": "ProcessedRequest", + "type": "reference" + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "decorations": [ + { + "name": "override" + } + ], + "flags": {}, + "groups": [], + "id": 672, + "module": "apify_storage_client._request_queue_client", + "name": "delete_request", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/apify_storage_client/_request_queue_client.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 112 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "flags": {}, + "id": 673, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "delete_request", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 674, + "kind": 32768, + "kindString": "Parameter", + "name": "request_id", + "type": { + "name": "str", + "type": "reference" + } + } + ], + "type": { + "name": "None", + "type": "literal", + "value": null + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "decorations": [ + { + "name": "override" + } + ], + "flags": {}, + "groups": [], + "id": 675, + "module": "apify_storage_client._request_queue_client", + "name": "prolong_request_lock", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/apify_storage_client/_request_queue_client.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 116 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "flags": {}, + "id": 676, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "prolong_request_lock", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 677, + "kind": 32768, + "kindString": "Parameter", + "name": "request_id", + "type": { + "name": "str", + "type": "reference" + } + }, + { + "defaultValue": "False", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 678, + "kind": 32768, + "kindString": "Parameter", + "name": "forefront", + "type": { + "name": "bool", + "type": "reference" + } + }, + { + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 679, + "kind": 32768, + "kindString": "Parameter", + "name": "lock_secs", + "type": { + "name": "int", + "type": "reference" + } + } + ], + "type": { + "name": "ProlongRequestLockResponse", + "type": "reference" + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "decorations": [ + { + "name": "override" + } + ], + "flags": {}, + "groups": [], + "id": 680, + "module": "apify_storage_client._request_queue_client", + "name": "delete_request_lock", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/apify_storage_client/_request_queue_client.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 132 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "flags": {}, + "id": 681, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "delete_request_lock", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 682, + "kind": 32768, + "kindString": "Parameter", + "name": "request_id", + "type": { + "name": "str", + "type": "reference" + } + }, + { + "defaultValue": "False", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 683, + "kind": 32768, + "kindString": "Parameter", + "name": "forefront", + "type": { + "name": "bool", + "type": "reference" + } + } + ], + "type": { + "name": "None", + "type": "literal", + "value": null + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "decorations": [ + { + "name": "override" + } + ], + "flags": {}, + "groups": [], + "id": 684, + "module": "apify_storage_client._request_queue_client", + "name": "batch_add_requests", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/apify_storage_client/_request_queue_client.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 144 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "flags": {}, + "id": 685, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "batch_add_requests", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 686, + "kind": 32768, + "kindString": "Parameter", + "name": "requests", + "type": { + "name": "Sequence", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "Request" + } + ] + } + }, + { + "defaultValue": "False", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 687, + "kind": 32768, + "kindString": "Parameter", + "name": "forefront", + "type": { + "name": "bool", + "type": "reference" + } + } + ], + "type": { + "name": "BatchRequestsOperationResponse", + "type": "reference" + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "decorations": [ + { + "name": "override" + } + ], + "flags": {}, + "groups": [], + "id": 688, + "module": "apify_storage_client._request_queue_client", + "name": "batch_delete_requests", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/apify_storage_client/_request_queue_client.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 166 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "flags": {}, + "id": 689, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "batch_delete_requests", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 690, + "kind": 32768, + "kindString": "Parameter", + "name": "requests", + "type": { + "name": "list", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "Request" + } + ], + "target": "596" + } + } + ], + "type": { + "name": "BatchRequestsOperationResponse", + "type": "reference" + } + } + ] + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Request queue resource client implementation based on the Apify platform storage." + } + ] + }, + "flags": {}, + "groups": [ + { + "children": [ + 644, + 661, + 684, + 688, + 652, + 672, + 680, + 647, + 665, + 657, + 654, + 675, + 649, + 668 + ], + "title": "Methods" + } + ], + "id": 643, + "module": "apify_storage_client._request_queue_client", + "name": "RequestQueueClient", + "parsedDocstring": { + "text": "Request queue resource client implementation based on the Apify platform storage." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/apify_storage_client/_request_queue_client.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 24 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 692, + "module": "apify_storage_client._request_queue_collection_client", + "name": "__init__", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/apify_storage_client/_request_queue_collection_client.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 17 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "flags": {}, + "id": 693, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "__init__", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 694, + "kind": 32768, + "kindString": "Parameter", + "name": "apify_request_queue_collection_client", + "type": { + "name": "RequestQueueCollectionClientAsync", + "type": "reference" + } + } + ], + "type": { + "name": "None", + "type": "literal", + "value": null + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "decorations": [ + { + "name": "override" + } + ], + "flags": {}, + "groups": [], + "id": 695, + "module": "apify_storage_client._request_queue_collection_client", + "name": "get_or_create", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/apify_storage_client/_request_queue_collection_client.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 21 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "flags": {}, + "id": 696, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "get_or_create", + "parameters": [ + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 697, + "kind": 32768, + "kindString": "Parameter", + "name": "id", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 698, + "kind": 32768, + "kindString": "Parameter", + "name": "name", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 699, + "kind": 32768, + "kindString": "Parameter", + "name": "schema", + "type": { + "name": "dict | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "dict" + }, + { + "type": "literal", + "value": null + } + ] + } + } + ], + "type": { + "name": "RequestQueueMetadata", + "type": "reference" + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "decorations": [ + { + "name": "override" + } + ], + "flags": {}, + "groups": [], + "id": 700, + "module": "apify_storage_client._request_queue_collection_client", + "name": "list", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/apify_storage_client/_request_queue_collection_client.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 36 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "flags": {}, + "id": 701, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "list", + "parameters": [ + { + "defaultValue": "False", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 702, + "kind": 32768, + "kindString": "Parameter", + "name": "unnamed", + "type": { + "name": "bool", + "type": "reference" + } + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 703, + "kind": 32768, + "kindString": "Parameter", + "name": "limit", + "type": { + "name": "int | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "int" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 704, + "kind": 32768, + "kindString": "Parameter", + "name": "offset", + "type": { + "name": "int | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "int" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "defaultValue": "False", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 705, + "kind": 32768, + "kindString": "Parameter", + "name": "desc", + "type": { + "name": "bool", + "type": "reference" + } + } + ], + "type": { + "name": "RequestQueueListPage", + "type": "reference" + } + } + ] + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Request queue collection resource client implementation based on the Apify platform storage." + } + ] + }, + "flags": {}, + "groups": [ + { + "children": [ + 692, + 695, + 700 + ], + "title": "Methods" + } + ], + "id": 691, + "module": "apify_storage_client._request_queue_collection_client", + "name": "RequestQueueCollectionClient", + "parsedDocstring": { + "text": "Request queue collection resource client implementation based on the Apify platform storage." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/apify_storage_client/_request_queue_collection_client.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 14 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 707, + "module": "apify_storage_client._key_value_store_client", + "name": "__init__", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/apify_storage_client/_key_value_store_client.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 26 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "flags": {}, + "id": 708, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "__init__", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 709, + "kind": 32768, + "kindString": "Parameter", + "name": "apify_key_value_store_client", + "type": { + "name": "KeyValueStoreClientAsync", + "type": "reference" + } + }, + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 710, + "kind": 32768, + "kindString": "Parameter", + "name": "api_public_base_url", + "type": { + "name": "str", + "type": "reference" + } + } + ], + "type": { + "name": "None", + "type": "literal", + "value": null + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "decorations": [ + { + "name": "override" + } + ], + "flags": {}, + "groups": [], + "id": 711, + "module": "apify_storage_client._key_value_store_client", + "name": "get", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/apify_storage_client/_key_value_store_client.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 31 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "flags": {}, + "id": 712, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "get", + "parameters": [], + "type": { + "name": "KeyValueStoreMetadata | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "KeyValueStoreMetadata" + }, + { + "type": "literal", + "value": null + } + ] + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "decorations": [ + { + "name": "override" + } + ], + "flags": {}, + "groups": [], + "id": 713, + "module": "apify_storage_client._key_value_store_client", + "name": "update", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/apify_storage_client/_key_value_store_client.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 36 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "flags": {}, + "id": 714, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "update", + "parameters": [ + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 715, + "kind": 32768, + "kindString": "Parameter", + "name": "name", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + } + ], + "type": { + "name": "KeyValueStoreMetadata", + "type": "reference" + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "decorations": [ + { + "name": "override" + } + ], + "flags": {}, + "groups": [], + "id": 716, + "module": "apify_storage_client._key_value_store_client", + "name": "delete", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/apify_storage_client/_key_value_store_client.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 44 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "flags": {}, + "id": 717, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "delete", + "parameters": [], + "type": { + "name": "None", + "type": "literal", + "value": null + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "decorations": [ + { + "name": "override" + } + ], + "flags": {}, + "groups": [], + "id": 718, + "module": "apify_storage_client._key_value_store_client", + "name": "list_keys", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/apify_storage_client/_key_value_store_client.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 48 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "flags": {}, + "id": 719, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "list_keys", + "parameters": [ + { + "defaultValue": "1000", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 720, + "kind": 32768, + "kindString": "Parameter", + "name": "limit", + "type": { + "name": "int", + "type": "reference" + } + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 721, + "kind": 32768, + "kindString": "Parameter", + "name": "exclusive_start_key", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + } + ], + "type": { + "name": "KeyValueStoreListKeysPage", + "type": "reference" + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "decorations": [ + { + "name": "override" + } + ], + "flags": {}, + "groups": [], + "id": 722, + "module": "apify_storage_client._key_value_store_client", + "name": "get_record", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/apify_storage_client/_key_value_store_client.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 57 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "flags": {}, + "id": 723, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "get_record", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 724, + "kind": 32768, + "kindString": "Parameter", + "name": "key", + "type": { + "name": "str", + "type": "reference" + } + } + ], + "type": { + "name": "KeyValueStoreRecord | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "KeyValueStoreRecord" + }, + { + "type": "literal", + "value": null + } + ] + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "decorations": [ + { + "name": "override" + } + ], + "flags": {}, + "groups": [], + "id": 725, + "module": "apify_storage_client._key_value_store_client", + "name": "get_record_as_bytes", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/apify_storage_client/_key_value_store_client.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 62 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "flags": {}, + "id": 726, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "get_record_as_bytes", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 727, + "kind": 32768, + "kindString": "Parameter", + "name": "key", + "type": { + "name": "str", + "type": "reference" + } + } + ], + "type": { + "name": "KeyValueStoreRecord | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "KeyValueStoreRecord" + }, + { + "type": "literal", + "value": null + } + ] + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "decorations": [ + { + "name": "override" + } + ], + "flags": {}, + "groups": [], + "id": 728, + "module": "apify_storage_client._key_value_store_client", + "name": "stream_record", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/apify_storage_client/_key_value_store_client.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 67 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "flags": {}, + "id": 729, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "stream_record", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 730, + "kind": 32768, + "kindString": "Parameter", + "name": "key", + "type": { + "name": "str", + "type": "reference" + } + } + ], + "type": { + "name": "AbstractAsyncContextManager", + "type": "reference", + "typeArguments": [ + { + "type": "union", + "types": [ + { + "type": "reference", + "name": "KeyValueStoreRecord", + "typeArguments": [ + { + "type": "reference", + "name": "Response" + } + ] + }, + { + "type": "literal", + "value": null + } + ] + } + ] + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "decorations": [ + { + "name": "override" + } + ], + "flags": {}, + "groups": [], + "id": 731, + "module": "apify_storage_client._key_value_store_client", + "name": "set_record", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/apify_storage_client/_key_value_store_client.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 76 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "flags": {}, + "id": 732, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "set_record", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 733, + "kind": 32768, + "kindString": "Parameter", + "name": "key", + "type": { + "name": "str", + "type": "reference" + } + }, + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 734, + "kind": 32768, + "kindString": "Parameter", + "name": "value", + "type": { + "name": "Any", + "type": "reference" + } + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": false + }, + "id": 735, + "kind": 32768, + "kindString": "Parameter", + "name": "content_type", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + } + ], + "type": { + "name": "None", + "type": "literal", + "value": null + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "decorations": [ + { + "name": "override" + } + ], + "flags": {}, + "groups": [], + "id": 736, + "module": "apify_storage_client._key_value_store_client", + "name": "delete_record", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/apify_storage_client/_key_value_store_client.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 84 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "flags": {}, + "id": 737, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "delete_record", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 738, + "kind": 32768, + "kindString": "Parameter", + "name": "key", + "type": { + "name": "str", + "type": "reference" + } + } + ], + "type": { + "name": "None", + "type": "literal", + "value": null + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Get a URL for the given key that may be used to publicly access the value in the remote key-value store.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 739, + "module": "apify_storage_client._key_value_store_client", + "name": "get_public_url", + "parsedDocstring": { + "text": "Get a URL for the given key that may be used to publicly access the value in the remote key-value store.\n", + "args": { + "key": "The key for which the URL should be generated." + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/apify_storage_client/_key_value_store_client.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 89 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Get a URL for the given key that may be used to publicly access the value in the remote key-value store.\n" + } + ] + }, + "flags": {}, + "id": 740, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "get_public_url", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The key for which the URL should be generated." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 741, + "kind": 32768, + "kindString": "Parameter", + "name": "key", + "type": { + "name": "str", + "type": "reference" + } + } + ], + "type": { + "name": "str", + "type": "reference" + } + } + ] + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Key-value store resource client implementation based on the Apify platform storage." + } + ] + }, + "flags": {}, + "groups": [ + { + "children": [ + 707, + 716, + 736, + 711, + 739, + 722, + 725, + 718, + 731, + 728, + 713 + ], + "title": "Methods" + } + ], + "id": 706, + "module": "apify_storage_client._key_value_store_client", + "name": "KeyValueStoreClient", + "parsedDocstring": { + "text": "Key-value store resource client implementation based on the Apify platform storage." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/apify_storage_client/_key_value_store_client.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 23 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 743, + "module": "apify_storage_client._dataset_client", + "name": "__init__", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/apify_storage_client/_dataset_client.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 23 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "flags": {}, + "id": 744, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "__init__", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 745, + "kind": 32768, + "kindString": "Parameter", + "name": "apify_dataset_client", + "type": { + "name": "DatasetClientAsync", + "type": "reference" + } + } + ], + "type": { + "name": "None", + "type": "literal", + "value": null + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "decorations": [ + { + "name": "override" + } + ], + "flags": {}, + "groups": [], + "id": 746, + "module": "apify_storage_client._dataset_client", + "name": "get", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/apify_storage_client/_dataset_client.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 27 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "flags": {}, + "id": 747, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "get", + "parameters": [], + "type": { + "name": "DatasetMetadata | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "DatasetMetadata" + }, + { + "type": "literal", + "value": null + } + ] + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "decorations": [ + { + "name": "override" + } + ], + "flags": {}, + "groups": [], + "id": 748, + "module": "apify_storage_client._dataset_client", + "name": "update", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/apify_storage_client/_dataset_client.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 32 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "flags": {}, + "id": 749, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "update", + "parameters": [ + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 750, + "kind": 32768, + "kindString": "Parameter", + "name": "name", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + } + ], + "type": { + "name": "DatasetMetadata", + "type": "reference" + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "decorations": [ + { + "name": "override" + } + ], + "flags": {}, + "groups": [], + "id": 751, + "module": "apify_storage_client._dataset_client", + "name": "delete", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/apify_storage_client/_dataset_client.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 44 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "flags": {}, + "id": 752, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "delete", + "parameters": [], + "type": { + "name": "None", + "type": "literal", + "value": null + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "decorations": [ + { + "name": "override" + } + ], + "flags": {}, + "groups": [], + "id": 753, + "module": "apify_storage_client._dataset_client", + "name": "list_items", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/apify_storage_client/_dataset_client.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 48 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "flags": {}, + "id": 754, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "list_items", + "parameters": [ + { + "defaultValue": "0", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 755, + "kind": 32768, + "kindString": "Parameter", + "name": "offset", + "type": { + "name": "int | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "int" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "defaultValue": "BaseDatasetClient._LIST_ITEMS_LIMIT", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 756, + "kind": 32768, + "kindString": "Parameter", + "name": "limit", + "type": { + "name": "int | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "int" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "defaultValue": "False", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 757, + "kind": 32768, + "kindString": "Parameter", + "name": "clean", + "type": { + "name": "bool", + "type": "reference" + } + }, + { + "defaultValue": "False", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 758, + "kind": 32768, + "kindString": "Parameter", + "name": "desc", + "type": { + "name": "bool", + "type": "reference" + } + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 759, + "kind": 32768, + "kindString": "Parameter", + "name": "fields", + "type": { + "name": "list[str] | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "list", + "typeArguments": [ + { + "type": "reference", + "name": "str" + } + ], + "target": "596" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 760, + "kind": 32768, + "kindString": "Parameter", + "name": "omit", + "type": { + "name": "list[str] | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "list", + "typeArguments": [ + { + "type": "reference", + "name": "str" + } + ], + "target": "596" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 761, + "kind": 32768, + "kindString": "Parameter", + "name": "unwind", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "defaultValue": "False", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 762, + "kind": 32768, + "kindString": "Parameter", + "name": "skip_empty", + "type": { + "name": "bool", + "type": "reference" + } + }, + { + "defaultValue": "False", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 763, + "kind": 32768, + "kindString": "Parameter", + "name": "skip_hidden", + "type": { + "name": "bool", + "type": "reference" + } + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 764, + "kind": 32768, + "kindString": "Parameter", + "name": "flatten", + "type": { + "name": "list[str] | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "list", + "typeArguments": [ + { + "type": "reference", + "name": "str" + } + ], + "target": "596" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 765, + "kind": 32768, + "kindString": "Parameter", + "name": "view", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + } + ], + "type": { + "name": "DatasetItemsListPage", + "type": "reference" + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "decorations": [ + { + "name": "override" + } + ], + "flags": {}, + "groups": [], + "id": 766, + "module": "apify_storage_client._dataset_client", + "name": "iterate_items", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/apify_storage_client/_dataset_client.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 82 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "flags": {}, + "id": 767, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "iterate_items", + "parameters": [ + { + "defaultValue": "0", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 768, + "kind": 32768, + "kindString": "Parameter", + "name": "offset", + "type": { + "name": "int", + "type": "reference" + } + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 769, + "kind": 32768, + "kindString": "Parameter", + "name": "limit", + "type": { + "name": "int | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "int" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "defaultValue": "False", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 770, + "kind": 32768, + "kindString": "Parameter", + "name": "clean", + "type": { + "name": "bool", + "type": "reference" + } + }, + { + "defaultValue": "False", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 771, + "kind": 32768, + "kindString": "Parameter", + "name": "desc", + "type": { + "name": "bool", + "type": "reference" + } + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 772, + "kind": 32768, + "kindString": "Parameter", + "name": "fields", + "type": { + "name": "list[str] | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "list", + "typeArguments": [ + { + "type": "reference", + "name": "str" + } + ], + "target": "596" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 773, + "kind": 32768, + "kindString": "Parameter", + "name": "omit", + "type": { + "name": "list[str] | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "list", + "typeArguments": [ + { + "type": "reference", + "name": "str" + } + ], + "target": "596" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 774, + "kind": 32768, + "kindString": "Parameter", + "name": "unwind", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "defaultValue": "False", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 775, + "kind": 32768, + "kindString": "Parameter", + "name": "skip_empty", + "type": { + "name": "bool", + "type": "reference" + } + }, + { + "defaultValue": "False", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 776, + "kind": 32768, + "kindString": "Parameter", + "name": "skip_hidden", + "type": { + "name": "bool", + "type": "reference" + } + } + ], + "type": { + "name": "AsyncIterator", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "dict" + } + ] + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "decorations": [ + { + "name": "override" + } + ], + "flags": {}, + "groups": [], + "id": 777, + "module": "apify_storage_client._dataset_client", + "name": "get_items_as_bytes", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/apify_storage_client/_dataset_client.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 109 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "flags": {}, + "id": 778, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "get_items_as_bytes", + "parameters": [ + { + "defaultValue": "'json'", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 779, + "kind": 32768, + "kindString": "Parameter", + "name": "item_format", + "type": { + "name": "str", + "type": "reference" + } + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 780, + "kind": 32768, + "kindString": "Parameter", + "name": "offset", + "type": { + "name": "int | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "int" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 781, + "kind": 32768, + "kindString": "Parameter", + "name": "limit", + "type": { + "name": "int | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "int" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "defaultValue": "False", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 782, + "kind": 32768, + "kindString": "Parameter", + "name": "desc", + "type": { + "name": "bool", + "type": "reference" + } + }, + { + "defaultValue": "False", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 783, + "kind": 32768, + "kindString": "Parameter", + "name": "clean", + "type": { + "name": "bool", + "type": "reference" + } + }, + { + "defaultValue": "False", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 784, + "kind": 32768, + "kindString": "Parameter", + "name": "bom", + "type": { + "name": "bool", + "type": "reference" + } + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 785, + "kind": 32768, + "kindString": "Parameter", + "name": "delimiter", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 786, + "kind": 32768, + "kindString": "Parameter", + "name": "fields", + "type": { + "name": "list[str] | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "list", + "typeArguments": [ + { + "type": "reference", + "name": "str" + } + ], + "target": "596" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 787, + "kind": 32768, + "kindString": "Parameter", + "name": "omit", + "type": { + "name": "list[str] | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "list", + "typeArguments": [ + { + "type": "reference", + "name": "str" + } + ], + "target": "596" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 788, + "kind": 32768, + "kindString": "Parameter", + "name": "unwind", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "defaultValue": "False", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 789, + "kind": 32768, + "kindString": "Parameter", + "name": "skip_empty", + "type": { + "name": "bool", + "type": "reference" + } + }, + { + "defaultValue": "False", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 790, + "kind": 32768, + "kindString": "Parameter", + "name": "skip_header_row", + "type": { + "name": "bool", + "type": "reference" + } + }, + { + "defaultValue": "False", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 791, + "kind": 32768, + "kindString": "Parameter", + "name": "skip_hidden", + "type": { + "name": "bool", + "type": "reference" + } + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 792, + "kind": 32768, + "kindString": "Parameter", + "name": "xml_root", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 793, + "kind": 32768, + "kindString": "Parameter", + "name": "xml_row", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 794, + "kind": 32768, + "kindString": "Parameter", + "name": "flatten", + "type": { + "name": "list[str] | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "list", + "typeArguments": [ + { + "type": "reference", + "name": "str" + } + ], + "target": "596" + }, + { + "type": "literal", + "value": null + } + ] + } + } + ], + "type": { + "name": "bytes", + "type": "reference" + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "decorations": [ + { + "name": "override" + } + ], + "flags": {}, + "groups": [], + "id": 795, + "module": "apify_storage_client._dataset_client", + "name": "stream_items", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/apify_storage_client/_dataset_client.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 149 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "flags": {}, + "id": 796, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "stream_items", + "parameters": [ + { + "defaultValue": "'json'", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 797, + "kind": 32768, + "kindString": "Parameter", + "name": "item_format", + "type": { + "name": "str", + "type": "reference" + } + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 798, + "kind": 32768, + "kindString": "Parameter", + "name": "offset", + "type": { + "name": "int | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "int" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 799, + "kind": 32768, + "kindString": "Parameter", + "name": "limit", + "type": { + "name": "int | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "int" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "defaultValue": "False", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 800, + "kind": 32768, + "kindString": "Parameter", + "name": "desc", + "type": { + "name": "bool", + "type": "reference" + } + }, + { + "defaultValue": "False", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 801, + "kind": 32768, + "kindString": "Parameter", + "name": "clean", + "type": { + "name": "bool", + "type": "reference" + } + }, + { + "defaultValue": "False", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 802, + "kind": 32768, + "kindString": "Parameter", + "name": "bom", + "type": { + "name": "bool", + "type": "reference" + } + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 803, + "kind": 32768, + "kindString": "Parameter", + "name": "delimiter", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 804, + "kind": 32768, + "kindString": "Parameter", + "name": "fields", + "type": { + "name": "list[str] | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "list", + "typeArguments": [ + { + "type": "reference", + "name": "str" + } + ], + "target": "596" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 805, + "kind": 32768, + "kindString": "Parameter", + "name": "omit", + "type": { + "name": "list[str] | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "list", + "typeArguments": [ + { + "type": "reference", + "name": "str" + } + ], + "target": "596" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 806, + "kind": 32768, + "kindString": "Parameter", + "name": "unwind", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "defaultValue": "False", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 807, + "kind": 32768, + "kindString": "Parameter", + "name": "skip_empty", + "type": { + "name": "bool", + "type": "reference" + } + }, + { + "defaultValue": "False", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 808, + "kind": 32768, + "kindString": "Parameter", + "name": "skip_header_row", + "type": { + "name": "bool", + "type": "reference" + } + }, + { + "defaultValue": "False", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 809, + "kind": 32768, + "kindString": "Parameter", + "name": "skip_hidden", + "type": { + "name": "bool", + "type": "reference" + } + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 810, + "kind": 32768, + "kindString": "Parameter", + "name": "xml_root", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 811, + "kind": 32768, + "kindString": "Parameter", + "name": "xml_row", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + } + ], + "type": { + "name": "AbstractAsyncContextManager", + "type": "reference", + "typeArguments": [ + { + "type": "union", + "types": [ + { + "type": "reference", + "name": "Response" + }, + { + "type": "literal", + "value": null + } + ] + } + ] + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "decorations": [ + { + "name": "override" + } + ], + "flags": {}, + "groups": [], + "id": 812, + "module": "apify_storage_client._dataset_client", + "name": "push_items", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/apify_storage_client/_dataset_client.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 187 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "flags": {}, + "id": 813, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "push_items", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 814, + "kind": 32768, + "kindString": "Parameter", + "name": "items", + "type": { + "name": "JsonSerializable", + "type": "reference" + } + } + ], + "type": { + "name": "None", + "type": "literal", + "value": null + } + } + ] + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Dataset resource client implementation based on the Apify platform storage." + } + ] + }, + "flags": {}, + "groups": [ + { + "children": [ + 743, + 751, + 746, + 777, + 766, + 753, + 812, + 795, + 748 + ], + "title": "Methods" + } + ], + "id": 742, + "module": "apify_storage_client._dataset_client", + "name": "DatasetClient", + "parsedDocstring": { + "text": "Dataset resource client implementation based on the Apify platform storage." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/apify_storage_client/_dataset_client.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 20 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 815, + "module": "storages._request_list", + "name": "URL_NO_COMMAS_REGEX", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storages/_request_list.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 18 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 817, + "module": "storages._request_list", + "name": "method", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storages/_request_list.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 24 + } + ], + "type": { + "name": "HttpMethod", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 818, + "module": "storages._request_list", + "name": "payload", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storages/_request_list.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 25 + } + ], + "type": { + "name": "str", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 819, + "module": "storages._request_list", + "name": "headers", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storages/_request_list.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 26 + } + ], + "type": { + "name": "dict", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "reference", + "name": "str" + } + ] + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 820, + "module": "storages._request_list", + "name": "user_data", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storages/_request_list.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 27 + } + ], + "type": { + "name": "dict", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "reference", + "name": "str" + } + ] + } + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [ + { + "children": [ + 819, + 817, + 818, + 820 + ], + "title": "Properties" + } + ], + "id": 816, + "module": "storages._request_list", + "name": "_RequestDetails", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storages/_request_list.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 23 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "extendedBy": [ + { + "name": "_RequestsFromUrlInput", + "target": "821", + "type": "reference" + }, + { + "name": "_SimpleUrlInput", + "target": "823", + "type": "reference" + } + ] + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 822, + "module": "storages._request_list", + "name": "requests_from_url", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storages/_request_list.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 31 + } + ], + "type": { + "name": "str", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 832, + "module": "storages._request_list", + "name": "method", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storages/_request_list.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 24 + } + ], + "type": { + "name": "HttpMethod", + "type": "reference" + }, + "inheritedFrom": { + "name": "_RequestDetails.method", + "target": 817, + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 833, + "module": "storages._request_list", + "name": "payload", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storages/_request_list.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 25 + } + ], + "type": { + "name": "str", + "type": "reference" + }, + "inheritedFrom": { + "name": "_RequestDetails.payload", + "target": 818, + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 834, + "module": "storages._request_list", + "name": "headers", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storages/_request_list.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 26 + } + ], + "type": { + "name": "dict", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "reference", + "name": "str" + } + ] + }, + "inheritedFrom": { + "name": "_RequestDetails.headers", + "target": 819, + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 835, + "module": "storages._request_list", + "name": "user_data", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storages/_request_list.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 27 + } + ], + "type": { + "name": "dict", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "reference", + "name": "str" + } + ] + }, + "inheritedFrom": { + "name": "_RequestDetails.user_data", + "target": 820, + "type": "reference" + } + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [ + { + "children": [ + 834, + 832, + 833, + 822, + 835 + ], + "title": "Properties" + } + ], + "id": 821, + "module": "storages._request_list", + "name": "_RequestsFromUrlInput", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storages/_request_list.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 30 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "extendedTypes": [ + { + "name": "_RequestDetails", + "target": "816", + "type": "reference" + } + ] + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 824, + "module": "storages._request_list", + "name": "url", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storages/_request_list.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 35 + } + ], + "type": { + "name": "str", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 836, + "module": "storages._request_list", + "name": "method", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storages/_request_list.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 24 + } + ], + "type": { + "name": "HttpMethod", + "type": "reference" + }, + "inheritedFrom": { + "name": "_RequestDetails.method", + "target": 817, + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 837, + "module": "storages._request_list", + "name": "payload", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storages/_request_list.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 25 + } + ], + "type": { + "name": "str", + "type": "reference" + }, + "inheritedFrom": { + "name": "_RequestDetails.payload", + "target": 818, + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 838, + "module": "storages._request_list", + "name": "headers", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storages/_request_list.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 26 + } + ], + "type": { + "name": "dict", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "reference", + "name": "str" + } + ] + }, + "inheritedFrom": { + "name": "_RequestDetails.headers", + "target": 819, + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 839, + "module": "storages._request_list", + "name": "user_data", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storages/_request_list.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 27 + } + ], + "type": { + "name": "dict", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "reference", + "name": "str" + } + ] + }, + "inheritedFrom": { + "name": "_RequestDetails.user_data", + "target": 820, + "type": "reference" + } + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [ + { + "children": [ + 838, + 836, + 837, + 824, + 839 + ], + "title": "Properties" + } + ], + "id": 823, + "module": "storages._request_list", + "name": "_SimpleUrlInput", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storages/_request_list.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 34 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "extendedTypes": [ + { + "name": "_RequestDetails", + "target": "816", + "type": "reference" + } + ] + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 825, + "module": "storages._request_list", + "name": "url_input_adapter", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storages/_request_list.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 38 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Initialize a new instance from request list source input.\n\n\n### Usage\n\n```python\nexample_input = [\n # Gather urls from response body.\n {'requestsFromUrl': 'https://crawlee.dev/file.txt', 'method': 'GET'},\n # Directly include this url.\n {'url': 'https://crawlee.dev', 'method': 'GET'}\n]\nrequest_list = await RequestList.open(request_list_sources_input=example_input)\n```" + } + ] + }, + "decorations": [ + { + "name": "staticmethod" + } + ], + "flags": {}, + "groups": [], + "id": 827, + "module": "storages._request_list", + "name": "open", + "parsedDocstring": { + "text": "Initialize a new instance from request list source input.\n\n\n### Usage\n\n```python\nexample_input = [\n # Gather urls from response body.\n {'requestsFromUrl': 'https://crawlee.dev/file.txt', 'method': 'GET'},\n # Directly include this url.\n {'url': 'https://crawlee.dev', 'method': 'GET'}\n]\nrequest_list = await RequestList.open(request_list_sources_input=example_input)\n```", + "args": { + "name": "Name of the returned RequestList.", + "request_list_sources_input": "List of dicts with either url key or requestsFromUrl key.", + "http_client": "Client that will be used to send get request to urls defined by value of requestsFromUrl keys.\n" + }, + "returns": "RequestList created from request_list_sources_input." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storages/_request_list.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 49 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "RequestList created from request_list_sources_input." + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Initialize a new instance from request list source input.\n\n\n### Usage\n\n```python\nexample_input = [\n # Gather urls from response body.\n {'requestsFromUrl': 'https://crawlee.dev/file.txt', 'method': 'GET'},\n # Directly include this url.\n {'url': 'https://crawlee.dev', 'method': 'GET'}\n]\nrequest_list = await RequestList.open(request_list_sources_input=example_input)\n```" + } + ] + }, + "flags": {}, + "id": 828, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "open", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Name of the returned RequestList." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": false + }, + "id": 829, + "kind": 32768, + "kindString": "Parameter", + "name": "name", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "List of dicts with either url key or requestsFromUrl key." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": false + }, + "id": 830, + "kind": 32768, + "kindString": "Parameter", + "name": "request_list_sources_input", + "type": { + "name": "list[dict[str, Any]] | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "list", + "typeArguments": [ + { + "type": "reference", + "name": "dict", + "typeArguments": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "reference", + "name": "Any" + } + ] + } + ], + "target": "596" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Client that will be used to send get request to urls defined by value of requestsFromUrl keys.\n" + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": false + }, + "id": 831, + "kind": 32768, + "kindString": "Parameter", + "name": "http_client", + "type": { + "name": "HttpClient | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "HttpClient" + }, + { + "type": "literal", + "value": null + } + ] + } + } + ], + "type": { + "name": "RequestList", + "type": "reference", + "target": "826" + } + } + ] + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Extends crawlee RequestList.\n\nMethod open is used to create RequestList from actor's requestListSources input." + } + ] + }, + "decorations": [ + { + "args": "('Classes')", + "name": "docs_group" + } + ], + "flags": {}, + "groups": [ + { + "children": [ + 827 + ], + "title": "Methods" + } + ], + "id": 826, + "module": "storages._request_list", + "name": "RequestList", + "parsedDocstring": { + "text": "Extends crawlee RequestList.\n\nMethod open is used to create RequestList from actor's requestListSources input." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storages/_request_list.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 42 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Get the storage ID." + } + ] + }, + "decorations": [ + { + "name": "property" + }, + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 3170, + "module": "storages._base", + "name": "id", + "parsedDocstring": { + "text": "Get the storage ID." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_base.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 20, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storages/_base.py#L20" + } + ], + "type": { + "name": "str", + "type": "reference", + "target": 0, + "ref": {} + }, + "permalink": "/python/api/class/Storage#id", + "parentId": 2330 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Get the storage name." + } + ] + }, + "decorations": [ + { + "name": "property" + }, + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 3171, + "module": "storages._base", + "name": "name", + "parsedDocstring": { + "text": "Get the storage name." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_base.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 25, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storages/_base.py#L25" + } + ], + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "permalink": "/python/api/class/Storage#name", + "parentId": 2330 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Get the storage metadata." + } + ] + }, + "decorations": [ + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 3172, + "module": "storages._base", + "name": "get_metadata", + "parsedDocstring": { + "text": "Get the storage metadata." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_base.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 29, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storages/_base.py#L29" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Get the storage metadata." + } + ] + }, + "flags": {}, + "id": 3173, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "get_metadata", + "parameters": [], + "type": { + "name": "DatasetMetadata | KeyValueStoreMetadata | RequestQueueMetadata", + "type": "union", + "types": [ + { + "type": "union", + "types": [ + { + "type": "reference", + "name": "DatasetMetadata", + "target": "1314", + "ref": { + "id": 2153, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 42 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/DatasetMetadata" + } + }, + { + "type": "reference", + "name": "KeyValueStoreMetadata", + "target": "1317", + "ref": { + "id": 2156, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 52 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/KeyValueStoreMetadata" + } + } + ] + }, + { + "type": "reference", + "name": "RequestQueueMetadata", + "target": "1319", + "ref": { + "id": 2158, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 59 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/RequestQueueMetadata" + } + } + ] + }, + "overwrites": { + "name": "Storage.get_metadata", + "target": 2333, + "type": "reference", + "ref": { + "id": 3172, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_base.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 29 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/Storage#get_metadata" + } + }, + "parentId": 2333 + } + ], + "permalink": "/python/api/class/Storage#get_metadata", + "parentId": 2330 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Open a storage, either restore existing or create a new one.\n" + } + ] + }, + "decorations": [ + { + "name": "classmethod" + }, + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 3174, + "module": "storages._base", + "name": "open", + "parsedDocstring": { + "text": "Open a storage, either restore existing or create a new one.\n", + "args": { + "id": "The storage ID.", + "name": "The storage name (global scope, persists across runs). Name can only contain letters \"a\" through \"z\",\nthe digits \"0\" through \"9\", and the hyphen (\"-\") but only in the middle of the string\n(e.g. \"my-value-1\").", + "alias": "The storage alias (run scope, creates unnamed storage).", + "configuration": "Configuration object used during the storage creation or restoration process.", + "storage_client": "Underlying storage client to use. If not provided, the default global storage client\nfrom the service locator will be used." + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_base.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 34, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storages/_base.py#L34" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Open a storage, either restore existing or create a new one.\n" + } + ] + }, + "flags": {}, + "id": 3175, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "open", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The storage ID." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 3176, + "kind": 32768, + "kindString": "Parameter", + "name": "id", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2336 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The storage name (global scope, persists across runs). Name can only contain letters \"a\" through \"z\",\nthe digits \"0\" through \"9\", and the hyphen (\"-\") but only in the middle of the string\n(e.g. \"my-value-1\")." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 3177, + "kind": 32768, + "kindString": "Parameter", + "name": "name", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2336 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The storage alias (run scope, creates unnamed storage)." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 3178, + "kind": 32768, + "kindString": "Parameter", + "name": "alias", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2336 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Configuration object used during the storage creation or restoration process." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 3179, + "kind": 32768, + "kindString": "Parameter", + "name": "configuration", + "type": { + "name": "Configuration | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "Configuration", + "target": "2", + "ref": { + "id": 841, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/configuration.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 20 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/Configuration" + } + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2336 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Underlying storage client to use. If not provided, the default global storage client\nfrom the service locator will be used." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 3180, + "kind": 32768, + "kindString": "Parameter", + "name": "storage_client", + "type": { + "name": "StorageClient | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "StorageClient", + "target": "1282", + "ref": { + "id": 2121, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 19 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/StorageClient" + } + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2336 + } + ], + "type": { + "name": "Storage", + "type": "reference", + "target": "2330", + "ref": { + "id": 3169, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_base.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 15 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/Storage" + } + }, + "overwrites": { + "name": "Storage.open", + "target": 2335, + "type": "reference", + "ref": { + "id": 3174, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_base.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 34 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/Storage#open" + } + }, + "parentId": 2335 + } + ], + "permalink": "/python/api/class/Storage#open", + "parentId": 2330 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Drop the storage, removing it from the underlying storage client and clearing the cache." + } + ] + }, + "decorations": [ + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 3181, + "module": "storages._base", + "name": "drop", + "parsedDocstring": { + "text": "Drop the storage, removing it from the underlying storage client and clearing the cache." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_base.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 57, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storages/_base.py#L57" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Drop the storage, removing it from the underlying storage client and clearing the cache." + } + ] + }, + "flags": {}, + "id": 3182, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "drop", + "parameters": [], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "overwrites": { + "name": "Storage.drop", + "target": 2342, + "type": "reference", + "ref": { + "id": 3181, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_base.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 57 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/Storage#drop" + } + }, + "parentId": 2342 + } + ], + "permalink": "/python/api/class/Storage#drop", + "parentId": 2330 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Purge the storage, removing all items from the underlying storage client.\n\nThis method does not remove the storage itself, e.g. don't remove the metadata,\nbut clears all items within it." + } + ] + }, + "decorations": [ + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 3183, + "module": "storages._base", + "name": "purge", + "parsedDocstring": { + "text": "Purge the storage, removing all items from the underlying storage client.\n\nThis method does not remove the storage itself, e.g. don't remove the metadata,\nbut clears all items within it." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_base.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 61, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storages/_base.py#L61" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Purge the storage, removing all items from the underlying storage client.\n\nThis method does not remove the storage itself, e.g. don't remove the metadata,\nbut clears all items within it." + } + ] + }, + "flags": {}, + "id": 3184, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "purge", + "parameters": [], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "overwrites": { + "name": "Storage.purge", + "target": 2344, + "type": "reference", + "ref": { + "id": 3183, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_base.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 61 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/Storage#purge" + } + }, + "parentId": 2344 + } + ], + "permalink": "/python/api/class/Storage#purge", + "parentId": 2330 + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Base class for storages." + } + ] + }, + "decorations": [ + { + "args": "('Storages')", + "name": "docs_group" + } + ], + "flags": {}, + "groups": [ + { + "children": [ + 3181, + 3172, + 3174, + 3183 + ], + "title": "Methods" + }, + { + "children": [ + 3170, + 3171 + ], + "title": "Properties" + } + ], + "id": 3169, + "module": "storages._base", + "name": "Storage", + "parsedDocstring": { + "text": "Base class for storages." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_base.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 15, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storages/_base.py#L15" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "extendedBy": [ + { + "name": "KeyValueStore", + "target": "2269", + "type": "reference", + "ref": { + "id": 3108, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_key_value_store.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 40 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/KeyValueStore" + } + }, + { + "name": "RequestQueue", + "target": "2348", + "type": "reference", + "ref": { + "id": 3187, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_request_queue.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 33 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/RequestQueue" + } + }, + { + "name": "Dataset", + "target": "2402", + "type": "reference", + "ref": { + "id": 3241, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_dataset.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 33 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/Dataset" + } + } + ], + "permalink": "/python/api/class/Storage", + "categories": [] + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Initialize a new instance.\n\nPreferably use the `Dataset.open` constructor to create a new instance.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 5644, + "module": "storages._dataset", + "name": "__init__", + "parsedDocstring": { + "text": "Initialize a new instance.\n\nPreferably use the `Dataset.open` constructor to create a new instance.\n", + "args": { + "client": "An instance of a storage client.", + "id": "The unique identifier of the storage.", + "name": "The name of the storage, if available." + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_dataset.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 69, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storages/_dataset.py#L69" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Initialize a new instance.\n\nPreferably use the `Dataset.open` constructor to create a new instance.\n" + } + ] + }, + "flags": {}, + "id": 5645, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "__init__", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "An instance of a storage client." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 5646, + "kind": 32768, + "kindString": "Parameter", + "name": "client", + "type": { + "name": "DatasetClient", + "type": "reference", + "target": "1220", + "ref": { + "id": 4461, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_dataset_client.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 13 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/DatasetClient" + } + }, + "parentId": 2404 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The unique identifier of the storage." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 5647, + "kind": 32768, + "kindString": "Parameter", + "name": "id", + "type": { + "name": "str", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2404 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The name of the storage, if available." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 5648, + "kind": 32768, + "kindString": "Parameter", + "name": "name", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2404 + } + ], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "parentId": 2403 + } + ], + "permalink": "/python/api/class/Dataset#__init__", + "parentId": 2402 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Get the storage ID." + } + ] + }, + "decorations": [ + { + "name": "property" + }, + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 5649, + "module": "storages._base", + "name": "id", + "parsedDocstring": { + "text": "Get the storage ID." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_dataset.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 87, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storages/_dataset.py#L87" + } + ], + "type": { + "name": "str", + "type": "reference", + "target": 0, + "ref": {} + }, + "overwrites": { + "name": "Storage.id", + "target": 2331, + "type": "reference", + "ref": { + "id": 5572, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_base.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 20 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api/class/Storage#id" + } + }, + "permalink": "/python/api/class/Dataset#id", + "parentId": 2402 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Get the storage name." + } + ] + }, + "decorations": [ + { + "name": "property" + }, + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 5650, + "module": "storages._base", + "name": "name", + "parsedDocstring": { + "text": "Get the storage name." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_dataset.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 92, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storages/_dataset.py#L92" + } + ], + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "overwrites": { + "name": "Storage.name", + "target": 2332, + "type": "reference", + "ref": { + "id": 5573, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_base.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 25 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api/class/Storage#name" + } + }, + "permalink": "/python/api/class/Dataset#name", + "parentId": 2402 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Get the storage metadata." + } + ] + }, + "decorations": [ + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 5651, + "module": "storages._base", + "name": "get_metadata", + "parsedDocstring": { + "text": "Get the storage metadata." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_dataset.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 96, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storages/_dataset.py#L96" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Get the storage metadata." + } + ] + }, + "flags": {}, + "id": 5575, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "get_metadata", + "parameters": [], + "type": { + "name": "DatasetMetadata | KeyValueStoreMetadata | RequestQueueMetadata", + "type": "union", + "types": [ + { + "type": "union", + "types": [ + { + "type": "reference", + "name": "DatasetMetadata", + "target": "1314", + "ref": { + "id": 4555, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 42 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/DatasetMetadata" + } + }, + { + "type": "reference", + "name": "KeyValueStoreMetadata", + "target": "1317", + "ref": { + "id": 4558, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 52 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/KeyValueStoreMetadata" + } + } + ] + }, + { + "type": "reference", + "name": "RequestQueueMetadata", + "target": "1319", + "ref": { + "id": 4560, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 59 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/RequestQueueMetadata" + } + } + ] + }, + "overwrites": { + "name": "Storage.get_metadata", + "target": 2333, + "type": "reference", + "ref": { + "id": 5574, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_base.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 29 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/Storage#get_metadata" + } + }, + "parentId": 2410 + } + ], + "overwrites": { + "name": "Storage.get_metadata", + "target": 2333, + "type": "reference", + "ref": { + "id": 5574, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_base.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 29 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/Storage#get_metadata" + } + }, + "permalink": "/python/api/class/Dataset#get_metadata", + "parentId": 2402 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Open a storage, either restore existing or create a new one.\n" + } + ] + }, + "decorations": [ + { + "name": "classmethod" + }, + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 5653, + "module": "storages._base", + "name": "open", + "parsedDocstring": { + "text": "Open a storage, either restore existing or create a new one.\n", + "args": { + "id": "The storage ID.", + "name": "The storage name (global scope, persists across runs). Name can only contain letters \"a\" through \"z\",\nthe digits \"0\" through \"9\", and the hyphen (\"-\") but only in the middle of the string\n(e.g. \"my-value-1\").", + "alias": "The storage alias (run scope, creates unnamed storage).", + "configuration": "Configuration object used during the storage creation or restoration process.", + "storage_client": "Underlying storage client to use. If not provided, the default global storage client\nfrom the service locator will be used." + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_dataset.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 101, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storages/_dataset.py#L101" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Open a storage, either restore existing or create a new one.\n" + } + ] + }, + "flags": {}, + "id": 5577, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "open", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The storage ID." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 5578, + "kind": 32768, + "kindString": "Parameter", + "name": "id", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2336 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The storage name (global scope, persists across runs). Name can only contain letters \"a\" through \"z\",\nthe digits \"0\" through \"9\", and the hyphen (\"-\") but only in the middle of the string\n(e.g. \"my-value-1\")." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 5579, + "kind": 32768, + "kindString": "Parameter", + "name": "name", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2336 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The storage alias (run scope, creates unnamed storage)." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 5580, + "kind": 32768, + "kindString": "Parameter", + "name": "alias", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2336 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Configuration object used during the storage creation or restoration process." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 5581, + "kind": 32768, + "kindString": "Parameter", + "name": "configuration", + "type": { + "name": "Configuration | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "Configuration", + "target": "2", + "ref": { + "id": 3243, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/configuration.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 20 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/Configuration" + } + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2336 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Underlying storage client to use. If not provided, the default global storage client\nfrom the service locator will be used." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 5582, + "kind": 32768, + "kindString": "Parameter", + "name": "storage_client", + "type": { + "name": "StorageClient | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "StorageClient", + "target": "1282", + "ref": { + "id": 4523, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 19 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/StorageClient" + } + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2336 + } + ], + "type": { + "name": "Storage", + "type": "reference", + "target": "2330", + "ref": { + "id": 5571, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_base.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 15 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/Storage" + } + }, + "overwrites": { + "name": "Storage.open", + "target": 2335, + "type": "reference", + "ref": { + "id": 5576, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_base.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 34 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/Storage#open" + } + }, + "parentId": 2412 + } + ], + "overwrites": { + "name": "Storage.open", + "target": 2335, + "type": "reference", + "ref": { + "id": 5576, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_base.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 34 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/Storage#open" + } + }, + "permalink": "/python/api/class/Dataset#open", + "parentId": 2402 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Drop the storage, removing it from the underlying storage client and clearing the cache." + } + ] + }, + "decorations": [ + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 5660, + "module": "storages._base", + "name": "drop", + "parsedDocstring": { + "text": "Drop the storage, removing it from the underlying storage client and clearing the cache." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_dataset.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 128, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storages/_dataset.py#L128" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Drop the storage, removing it from the underlying storage client and clearing the cache." + } + ] + }, + "flags": {}, + "id": 5584, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "drop", + "parameters": [], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "overwrites": { + "name": "Storage.drop", + "target": 2342, + "type": "reference", + "ref": { + "id": 5583, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_base.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 57 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/Storage#drop" + } + }, + "parentId": 2419 + } + ], + "overwrites": { + "name": "Storage.drop", + "target": 2342, + "type": "reference", + "ref": { + "id": 5583, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_base.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 57 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/Storage#drop" + } + }, + "permalink": "/python/api/class/Dataset#drop", + "parentId": 2402 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Purge the storage, removing all items from the underlying storage client.\n\nThis method does not remove the storage itself, e.g. don't remove the metadata,\nbut clears all items within it." + } + ] + }, + "decorations": [ + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 5662, + "module": "storages._base", + "name": "purge", + "parsedDocstring": { + "text": "Purge the storage, removing all items from the underlying storage client.\n\nThis method does not remove the storage itself, e.g. don't remove the metadata,\nbut clears all items within it." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_dataset.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 134, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storages/_dataset.py#L134" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Purge the storage, removing all items from the underlying storage client.\n\nThis method does not remove the storage itself, e.g. don't remove the metadata,\nbut clears all items within it." + } + ] + }, + "flags": {}, + "id": 5586, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "purge", + "parameters": [], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "overwrites": { + "name": "Storage.purge", + "target": 2344, + "type": "reference", + "ref": { + "id": 5585, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_base.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 61 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/Storage#purge" + } + }, + "parentId": 2421 + } + ], + "overwrites": { + "name": "Storage.purge", + "target": 2344, + "type": "reference", + "ref": { + "id": 5585, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_base.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 61 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/Storage#purge" + } + }, + "permalink": "/python/api/class/Dataset#purge", + "parentId": 2402 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Store an object or an array of objects to the dataset.\n\nThe size of the data is limited by the receiving API and therefore `push_data()` will only\nallow objects whose JSON representation is smaller than 9MB. When an array is passed,\nnone of the included objects may be larger than 9MB, but the array itself may be of any size.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 5664, + "module": "storages._dataset", + "name": "push_data", + "parsedDocstring": { + "text": "Store an object or an array of objects to the dataset.\n\nThe size of the data is limited by the receiving API and therefore `push_data()` will only\nallow objects whose JSON representation is smaller than 9MB. When an array is passed,\nnone of the included objects may be larger than 9MB, but the array itself may be of any size.\n", + "args": { + "data": "A JSON serializable data structure to be stored in the dataset. The JSON representation\nof each item must be smaller than 9MB." + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_dataset.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 137, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storages/_dataset.py#L137" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Store an object or an array of objects to the dataset.\n\nThe size of the data is limited by the receiving API and therefore `push_data()` will only\nallow objects whose JSON representation is smaller than 9MB. When an array is passed,\nnone of the included objects may be larger than 9MB, but the array itself may be of any size.\n" + } + ] + }, + "flags": {}, + "id": 5665, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "push_data", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "A JSON serializable data structure to be stored in the dataset. The JSON representation\nof each item must be smaller than 9MB." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 5666, + "kind": 32768, + "kindString": "Parameter", + "name": "data", + "type": { + "name": "list[dict[str, Any]] | dict[str, Any]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "list", + "typeArguments": [ + { + "type": "reference", + "name": "dict", + "typeArguments": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "reference", + "name": "Any", + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + }, + { + "type": "reference", + "name": "dict", + "typeArguments": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "reference", + "name": "Any", + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + } + ] + }, + "parentId": 2424 + } + ], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "parentId": 2423 + } + ], + "permalink": "/python/api/class/Dataset#push_data", + "parentId": 2402 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Retrieve a paginated list of items from a dataset based on various filtering parameters.\n\nThis method provides the flexibility to filter, sort, and modify the appearance of dataset items\nwhen listed. Each parameter modifies the result set according to its purpose. The method also\nsupports pagination through 'offset' and 'limit' parameters.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 5667, + "module": "storages._dataset", + "name": "get_data", + "parsedDocstring": { + "text": "Retrieve a paginated list of items from a dataset based on various filtering parameters.\n\nThis method provides the flexibility to filter, sort, and modify the appearance of dataset items\nwhen listed. Each parameter modifies the result set according to its purpose. The method also\nsupports pagination through 'offset' and 'limit' parameters.\n", + "args": { + "offset": "Skips the specified number of items at the start.", + "limit": "The maximum number of items to retrieve. Unlimited if None.", + "clean": "Return only non-empty items and excludes hidden fields. Shortcut for skip_hidden and skip_empty.", + "desc": "Set to True to sort results in descending order.", + "fields": "Fields to include in each item. Sorts fields as specified if provided.", + "omit": "Fields to exclude from each item.", + "unwind": "Unwinds items by a specified array field, turning each element into a separate item.", + "skip_empty": "Excludes empty items from the results if True.", + "skip_hidden": "Excludes fields starting with '#' if True.", + "flatten": "Fields to be flattened in returned items.", + "view": "Specifies the dataset view to be used.\n" + }, + "returns": "An object with filtered, sorted, and paginated dataset items plus pagination details." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_dataset.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 150, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storages/_dataset.py#L150" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "An object with filtered, sorted, and paginated dataset items plus pagination details." + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Retrieve a paginated list of items from a dataset based on various filtering parameters.\n\nThis method provides the flexibility to filter, sort, and modify the appearance of dataset items\nwhen listed. Each parameter modifies the result set according to its purpose. The method also\nsupports pagination through 'offset' and 'limit' parameters.\n" + } + ] + }, + "flags": {}, + "id": 5668, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "get_data", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Skips the specified number of items at the start." + } + ] + }, + "defaultValue": "0", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 5669, + "kind": 32768, + "kindString": "Parameter", + "name": "offset", + "type": { + "name": "int", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2427 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The maximum number of items to retrieve. Unlimited if None." + } + ] + }, + "defaultValue": "999_999_999_999", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 5670, + "kind": 32768, + "kindString": "Parameter", + "name": "limit", + "type": { + "name": "int | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "int", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2427 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Return only non-empty items and excludes hidden fields. Shortcut for skip_hidden and skip_empty." + } + ] + }, + "defaultValue": "False", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 5671, + "kind": 32768, + "kindString": "Parameter", + "name": "clean", + "type": { + "name": "bool", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2427 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Set to True to sort results in descending order." + } + ] + }, + "defaultValue": "False", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 5672, + "kind": 32768, + "kindString": "Parameter", + "name": "desc", + "type": { + "name": "bool", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2427 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Fields to include in each item. Sorts fields as specified if provided." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 5673, + "kind": 32768, + "kindString": "Parameter", + "name": "fields", + "type": { + "name": "list[str] | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "list", + "typeArguments": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2427 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Fields to exclude from each item." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 5674, + "kind": 32768, + "kindString": "Parameter", + "name": "omit", + "type": { + "name": "list[str] | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "list", + "typeArguments": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2427 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Unwinds items by a specified array field, turning each element into a separate item." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 5675, + "kind": 32768, + "kindString": "Parameter", + "name": "unwind", + "type": { + "name": "list[str] | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "list", + "typeArguments": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2427 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Excludes empty items from the results if True." + } + ] + }, + "defaultValue": "False", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 5676, + "kind": 32768, + "kindString": "Parameter", + "name": "skip_empty", + "type": { + "name": "bool", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2427 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Excludes fields starting with '#' if True." + } + ] + }, + "defaultValue": "False", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 5677, + "kind": 32768, + "kindString": "Parameter", + "name": "skip_hidden", + "type": { + "name": "bool", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2427 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Fields to be flattened in returned items." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 5678, + "kind": 32768, + "kindString": "Parameter", + "name": "flatten", + "type": { + "name": "list[str] | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "list", + "typeArguments": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2427 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Specifies the dataset view to be used.\n" + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 5679, + "kind": 32768, + "kindString": "Parameter", + "name": "view", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2427 + } + ], + "type": { + "name": "DatasetItemsListPage", + "type": "reference", + "target": "1333", + "ref": { + "id": 4574, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 110 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/DatasetItemsListPage" + } + }, + "parentId": 2426 + } + ], + "permalink": "/python/api/class/Dataset#get_data", + "parentId": 2402 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Iterate over items in the dataset according to specified filters and sorting.\n\nThis method allows for asynchronously iterating through dataset items while applying various filters such as\nskipping empty items, hiding specific fields, and sorting. It supports pagination via `offset` and `limit`\nparameters, and can modify the appearance of dataset items using `fields`, `omit`, `unwind`, `skip_empty`, and\n`skip_hidden` parameters.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 5680, + "module": "storages._dataset", + "name": "iterate_items", + "parsedDocstring": { + "text": "Iterate over items in the dataset according to specified filters and sorting.\n\nThis method allows for asynchronously iterating through dataset items while applying various filters such as\nskipping empty items, hiding specific fields, and sorting. It supports pagination via `offset` and `limit`\nparameters, and can modify the appearance of dataset items using `fields`, `omit`, `unwind`, `skip_empty`, and\n`skip_hidden` parameters.\n", + "args": { + "offset": "Skips the specified number of items at the start.", + "limit": "The maximum number of items to retrieve. Unlimited if None.", + "clean": "Return only non-empty items and excludes hidden fields. Shortcut for skip_hidden and skip_empty.", + "desc": "Set to True to sort results in descending order.", + "fields": "Fields to include in each item. Sorts fields as specified if provided.", + "omit": "Fields to exclude from each item.", + "unwind": "Unwinds items by a specified array field, turning each element into a separate item.", + "skip_empty": "Excludes empty items from the results if True.", + "skip_hidden": "Excludes fields starting with '#' if True.\n" + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_dataset.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 201, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storages/_dataset.py#L201" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Iterate over items in the dataset according to specified filters and sorting.\n\nThis method allows for asynchronously iterating through dataset items while applying various filters such as\nskipping empty items, hiding specific fields, and sorting. It supports pagination via `offset` and `limit`\nparameters, and can modify the appearance of dataset items using `fields`, `omit`, `unwind`, `skip_empty`, and\n`skip_hidden` parameters.\n" + } + ] + }, + "flags": {}, + "id": 5681, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "iterate_items", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Skips the specified number of items at the start." + } + ] + }, + "defaultValue": "0", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 5682, + "kind": 32768, + "kindString": "Parameter", + "name": "offset", + "type": { + "name": "int", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2440 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The maximum number of items to retrieve. Unlimited if None." + } + ] + }, + "defaultValue": "999_999_999_999", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 5683, + "kind": 32768, + "kindString": "Parameter", + "name": "limit", + "type": { + "name": "int | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "int", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2440 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Return only non-empty items and excludes hidden fields. Shortcut for skip_hidden and skip_empty." + } + ] + }, + "defaultValue": "False", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 5684, + "kind": 32768, + "kindString": "Parameter", + "name": "clean", + "type": { + "name": "bool", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2440 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Set to True to sort results in descending order." + } + ] + }, + "defaultValue": "False", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 5685, + "kind": 32768, + "kindString": "Parameter", + "name": "desc", + "type": { + "name": "bool", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2440 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Fields to include in each item. Sorts fields as specified if provided." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 5686, + "kind": 32768, + "kindString": "Parameter", + "name": "fields", + "type": { + "name": "list[str] | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "list", + "typeArguments": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2440 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Fields to exclude from each item." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 5687, + "kind": 32768, + "kindString": "Parameter", + "name": "omit", + "type": { + "name": "list[str] | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "list", + "typeArguments": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2440 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Unwinds items by a specified array field, turning each element into a separate item." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 5688, + "kind": 32768, + "kindString": "Parameter", + "name": "unwind", + "type": { + "name": "list[str] | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "list", + "typeArguments": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2440 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Excludes empty items from the results if True." + } + ] + }, + "defaultValue": "False", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 5689, + "kind": 32768, + "kindString": "Parameter", + "name": "skip_empty", + "type": { + "name": "bool", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2440 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Excludes fields starting with '#' if True.\n" + } + ] + }, + "defaultValue": "False", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 5690, + "kind": 32768, + "kindString": "Parameter", + "name": "skip_hidden", + "type": { + "name": "bool", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2440 + } + ], + "type": { + "name": "AsyncIterator", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "dict", + "typeArguments": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "reference", + "name": "Any", + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + }, + "parentId": 2439 + } + ], + "permalink": "/python/api/class/Dataset#iterate_items", + "parentId": 2402 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Retrieve a list of all items from the dataset according to specified filters and sorting.\n\nThis method collects all dataset items into a list while applying various filters such as\nskipping empty items, hiding specific fields, and sorting. It supports pagination via `offset` and `limit`\nparameters, and can modify the appearance of dataset items using `fields`, `omit`, `unwind`, `skip_empty`, and\n`skip_hidden` parameters.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 5691, + "module": "storages._dataset", + "name": "list_items", + "parsedDocstring": { + "text": "Retrieve a list of all items from the dataset according to specified filters and sorting.\n\nThis method collects all dataset items into a list while applying various filters such as\nskipping empty items, hiding specific fields, and sorting. It supports pagination via `offset` and `limit`\nparameters, and can modify the appearance of dataset items using `fields`, `omit`, `unwind`, `skip_empty`, and\n`skip_hidden` parameters.\n", + "args": { + "offset": "Skips the specified number of items at the start.", + "limit": "The maximum number of items to retrieve. Unlimited if None.", + "clean": "Return only non-empty items and excludes hidden fields. Shortcut for skip_hidden and skip_empty.", + "desc": "Set to True to sort results in descending order.", + "fields": "Fields to include in each item. Sorts fields as specified if provided.", + "omit": "Fields to exclude from each item.", + "unwind": "Unwinds items by a specified array field, turning each element into a separate item.", + "skip_empty": "Excludes empty items from the results if True.", + "skip_hidden": "Excludes fields starting with '#' if True.\n" + }, + "returns": "A list of dictionary objects, each representing a dataset item after applying\nthe specified filters and transformations." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_dataset.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 249, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storages/_dataset.py#L249" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "A list of dictionary objects, each representing a dataset item after applying\nthe specified filters and transformations." + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Retrieve a list of all items from the dataset according to specified filters and sorting.\n\nThis method collects all dataset items into a list while applying various filters such as\nskipping empty items, hiding specific fields, and sorting. It supports pagination via `offset` and `limit`\nparameters, and can modify the appearance of dataset items using `fields`, `omit`, `unwind`, `skip_empty`, and\n`skip_hidden` parameters.\n" + } + ] + }, + "flags": {}, + "id": 5692, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "list_items", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Skips the specified number of items at the start." + } + ] + }, + "defaultValue": "0", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 5693, + "kind": 32768, + "kindString": "Parameter", + "name": "offset", + "type": { + "name": "int", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2451 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The maximum number of items to retrieve. Unlimited if None." + } + ] + }, + "defaultValue": "999_999_999_999", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 5694, + "kind": 32768, + "kindString": "Parameter", + "name": "limit", + "type": { + "name": "int | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "int", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2451 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Return only non-empty items and excludes hidden fields. Shortcut for skip_hidden and skip_empty." + } + ] + }, + "defaultValue": "False", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 5695, + "kind": 32768, + "kindString": "Parameter", + "name": "clean", + "type": { + "name": "bool", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2451 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Set to True to sort results in descending order." + } + ] + }, + "defaultValue": "False", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 5696, + "kind": 32768, + "kindString": "Parameter", + "name": "desc", + "type": { + "name": "bool", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2451 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Fields to include in each item. Sorts fields as specified if provided." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 5697, + "kind": 32768, + "kindString": "Parameter", + "name": "fields", + "type": { + "name": "list[str] | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "list", + "typeArguments": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2451 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Fields to exclude from each item." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 5698, + "kind": 32768, + "kindString": "Parameter", + "name": "omit", + "type": { + "name": "list[str] | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "list", + "typeArguments": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2451 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Unwinds items by a specified array field, turning each element into a separate item." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 5699, + "kind": 32768, + "kindString": "Parameter", + "name": "unwind", + "type": { + "name": "list[str] | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "list", + "typeArguments": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2451 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Excludes empty items from the results if True." + } + ] + }, + "defaultValue": "False", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 5700, + "kind": 32768, + "kindString": "Parameter", + "name": "skip_empty", + "type": { + "name": "bool", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2451 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Excludes fields starting with '#' if True.\n" + } + ] + }, + "defaultValue": "False", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 5701, + "kind": 32768, + "kindString": "Parameter", + "name": "skip_hidden", + "type": { + "name": "bool", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2451 + } + ], + "type": { + "name": "list", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "dict", + "typeArguments": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "reference", + "name": "Any", + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + }, + "parentId": 2450 + } + ], + "permalink": "/python/api/class/Dataset#list_items", + "parentId": 2402 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Export the entire dataset into a specified file stored under a key in a key-value store.\n\nThis method consolidates all entries from a specified dataset into one file, which is then saved under a\ngiven key in a key-value store. The format of the exported file is determined by the `content_type` parameter.\nEither the dataset's ID or name should be specified, and similarly, either the target key-value store's ID or\nname should be used.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 5702, + "module": "storages._dataset", + "name": "export_to", + "parsedDocstring": { + "text": "Export the entire dataset into a specified file stored under a key in a key-value store.\n\nThis method consolidates all entries from a specified dataset into one file, which is then saved under a\ngiven key in a key-value store. The format of the exported file is determined by the `content_type` parameter.\nEither the dataset's ID or name should be specified, and similarly, either the target key-value store's ID or\nname should be used.\n", + "args": { + "key": "The key under which to save the data in the key-value store.", + "content_type": "The format in which to export the data.", + "to_kvs_id": "ID of the key-value store to save the exported file.\nSpecify only one of ID or name.", + "to_kvs_name": "Name of the key-value store to save the exported file.\nSpecify only one of ID or name.", + "to_kvs_storage_client": "Storage client to use for the key-value store.", + "to_kvs_configuration": "Configuration for the key-value store.", + "kwargs": "Additional parameters for the export operation, specific to the chosen content type." + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_dataset.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 323, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storages/_dataset.py#L323" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Export the entire dataset into a specified file stored under a key in a key-value store.\n\nThis method consolidates all entries from a specified dataset into one file, which is then saved under a\ngiven key in a key-value store. The format of the exported file is determined by the `content_type` parameter.\nEither the dataset's ID or name should be specified, and similarly, either the target key-value store's ID or\nname should be used.\n" + } + ] + }, + "flags": {}, + "id": 5703, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "export_to", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The key under which to save the data in the key-value store." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 5704, + "kind": 32768, + "kindString": "Parameter", + "name": "key", + "type": { + "name": "str", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2462 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The format in which to export the data." + } + ] + }, + "defaultValue": "'json'", + "flags": { + "isOptional": true, + "keyword-only": false + }, + "id": 5705, + "kind": 32768, + "kindString": "Parameter", + "name": "content_type", + "type": { + "name": "Literal", + "type": "reference", + "typeArguments": [ + { + "type": "literal", + "value": "json" + }, + { + "type": "literal", + "value": "csv" + } + ], + "target": 0, + "ref": {} + }, + "parentId": 2462 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "ID of the key-value store to save the exported file.\nSpecify only one of ID or name." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": false + }, + "id": 5706, + "kind": 32768, + "kindString": "Parameter", + "name": "to_kvs_id", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2462 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Name of the key-value store to save the exported file.\nSpecify only one of ID or name." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": false + }, + "id": 5707, + "kind": 32768, + "kindString": "Parameter", + "name": "to_kvs_name", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2462 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Storage client to use for the key-value store." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": false + }, + "id": 5708, + "kind": 32768, + "kindString": "Parameter", + "name": "to_kvs_storage_client", + "type": { + "name": "StorageClient | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "StorageClient", + "target": "1282", + "ref": { + "id": 4523, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 19 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/StorageClient" + } + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2462 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Configuration for the key-value store." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": false + }, + "id": 5709, + "kind": 32768, + "kindString": "Parameter", + "name": "to_kvs_configuration", + "type": { + "name": "Configuration | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "Configuration", + "target": "2", + "ref": { + "id": 3243, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/configuration.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 20 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/Configuration" + } + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2462 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Additional parameters for the export operation, specific to the chosen content type." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 5710, + "kind": 32768, + "kindString": "Parameter", + "name": "kwargs", + "type": { + "name": "Any", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2462 + } + ], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "parentId": 2461 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Export the entire dataset into a specified file stored under a key in a key-value store.\n\nThis method consolidates all entries from a specified dataset into one file, which is then saved under a\ngiven key in a key-value store. The format of the exported file is determined by the `content_type` parameter.\nEither the dataset's ID or name should be specified, and similarly, either the target key-value store's ID or\nname should be used.\n" + } + ] + }, + "flags": {}, + "id": 5711, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "export_to", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The key under which to save the data in the key-value store." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 5712, + "kind": 32768, + "kindString": "Parameter", + "name": "key", + "type": { + "name": "str", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2470 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The format in which to export the data." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 5713, + "kind": 32768, + "kindString": "Parameter", + "name": "content_type", + "type": { + "name": "Literal", + "type": "reference", + "typeArguments": [ + { + "type": "literal", + "value": "json" + } + ], + "target": 0, + "ref": {} + }, + "parentId": 2470 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "ID of the key-value store to save the exported file.\nSpecify only one of ID or name." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": false + }, + "id": 5714, + "kind": 32768, + "kindString": "Parameter", + "name": "to_kvs_id", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2470 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Name of the key-value store to save the exported file.\nSpecify only one of ID or name." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": false + }, + "id": 5715, + "kind": 32768, + "kindString": "Parameter", + "name": "to_kvs_name", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2470 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Storage client to use for the key-value store." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": false + }, + "id": 5716, + "kind": 32768, + "kindString": "Parameter", + "name": "to_kvs_storage_client", + "type": { + "name": "StorageClient | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "StorageClient", + "target": "1282", + "ref": { + "id": 4523, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 19 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/StorageClient" + } + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2470 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Configuration for the key-value store." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": false + }, + "id": 5717, + "kind": 32768, + "kindString": "Parameter", + "name": "to_kvs_configuration", + "type": { + "name": "Configuration | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "Configuration", + "target": "2", + "ref": { + "id": 3243, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/configuration.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 20 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/Configuration" + } + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2470 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "If True (default: False), dict keys that are not of a basic type (str, int, float, bool, None) will be skipped\ninstead of raising a `TypeError`." + } + ] + }, + "flags": { + "keyword-only": true, + "optional": true + }, + "groups": [], + "id": 3570, + "module": "_types", + "name": "skipkeys", + "parsedDocstring": { + "text": "If True (default: False), dict keys that are not of a basic type (str, int, float, bool, None) will be skipped\ninstead of raising a `TypeError`." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_types.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 759, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/_types.py#L759" + } + ], + "type": { + "name": "NotRequired", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "bool", + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + }, + "parentId": 2470 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Determines if non-ASCII characters should be escaped in the output JSON string." + } + ] + }, + "flags": { + "keyword-only": true, + "optional": true + }, + "groups": [], + "id": 3571, + "module": "_types", + "name": "ensure_ascii", + "parsedDocstring": { + "text": "Determines if non-ASCII characters should be escaped in the output JSON string." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_types.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 763, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/_types.py#L763" + } + ], + "type": { + "name": "NotRequired", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "bool", + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + }, + "parentId": 2470 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "If False (default: True), skips the circular reference check for container types. A circular reference will\nresult in a `RecursionError` or worse if unchecked." + } + ] + }, + "flags": { + "keyword-only": true, + "optional": true + }, + "groups": [], + "id": 3572, + "module": "_types", + "name": "check_circular", + "parsedDocstring": { + "text": "If False (default: True), skips the circular reference check for container types. A circular reference will\nresult in a `RecursionError` or worse if unchecked." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_types.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 766, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/_types.py#L766" + } + ], + "type": { + "name": "NotRequired", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "bool", + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + }, + "parentId": 2470 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "If False (default: True), raises a ValueError for out-of-range float values (nan, inf, -inf) to strictly comply\nwith the JSON specification. If True, uses their JavaScript equivalents (NaN, Infinity, -Infinity)." + } + ] + }, + "flags": { + "keyword-only": true, + "optional": true + }, + "groups": [], + "id": 3573, + "module": "_types", + "name": "allow_nan", + "parsedDocstring": { + "text": "If False (default: True), raises a ValueError for out-of-range float values (nan, inf, -inf) to strictly comply\nwith the JSON specification. If True, uses their JavaScript equivalents (NaN, Infinity, -Infinity)." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_types.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 770, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/_types.py#L770" + } + ], + "type": { + "name": "NotRequired", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "bool", + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + }, + "parentId": 2470 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Allows specifying a custom JSON encoder." + } + ] + }, + "flags": { + "keyword-only": true, + "optional": true + }, + "groups": [], + "id": 3574, + "module": "_types", + "name": "cls", + "parsedDocstring": { + "text": "Allows specifying a custom JSON encoder." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_types.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 774, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/_types.py#L774" + } + ], + "type": { + "name": "NotRequired", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "type", + "typeArguments": [ + { + "type": "reference", + "name": "json.JSONEncoder", + "target": 0, + "ref": {} + } + ], + "target": "3422", + "ref": { + "id": 6663, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_utils/sitemap.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 62 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api/class/SitemapSource#type" + } + } + ], + "target": 0, + "ref": {} + }, + "parentId": 2470 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Specifies the number of spaces for indentation in the pretty-printed JSON output." + } + ] + }, + "flags": { + "keyword-only": true, + "optional": true + }, + "groups": [], + "id": 3575, + "module": "_types", + "name": "indent", + "parsedDocstring": { + "text": "Specifies the number of spaces for indentation in the pretty-printed JSON output." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_types.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 777, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/_types.py#L777" + } + ], + "type": { + "name": "NotRequired", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "int", + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + }, + "parentId": 2470 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "A tuple of (item_separator, key_separator). The default is (', ', ': ') if indent is None and (',', ': ')\notherwise." + } + ] + }, + "flags": { + "keyword-only": true, + "optional": true + }, + "groups": [], + "id": 3576, + "module": "_types", + "name": "separators", + "parsedDocstring": { + "text": "A tuple of (item_separator, key_separator). The default is (', ', ': ') if indent is None and (',', ': ')\notherwise." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_types.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 780, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/_types.py#L780" + } + ], + "type": { + "name": "NotRequired", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "tuple", + "typeArguments": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + }, + "parentId": 2470 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "A function called for objects that can't be serialized otherwise. It should return a JSON-encodable version\nof the object or raise a `TypeError`." + } + ] + }, + "flags": { + "keyword-only": true, + "optional": true + }, + "groups": [], + "id": 3577, + "module": "_types", + "name": "default", + "parsedDocstring": { + "text": "A function called for objects that can't be serialized otherwise. It should return a JSON-encodable version\nof the object or raise a `TypeError`." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_types.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 784, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/_types.py#L784" + } + ], + "type": { + "name": "NotRequired", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "Callable", + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + }, + "parentId": 2470 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Specifies whether the output JSON object should have keys sorted alphabetically." + } + ] + }, + "flags": { + "keyword-only": true, + "optional": true + }, + "groups": [], + "id": 3578, + "module": "_types", + "name": "sort_keys", + "parsedDocstring": { + "text": "Specifies whether the output JSON object should have keys sorted alphabetically." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_types.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 788, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/_types.py#L788" + } + ], + "type": { + "name": "NotRequired", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "bool", + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + }, + "parentId": 2470 + } + ], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "parentId": 2461 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Export the entire dataset into a specified file stored under a key in a key-value store.\n\nThis method consolidates all entries from a specified dataset into one file, which is then saved under a\ngiven key in a key-value store. The format of the exported file is determined by the `content_type` parameter.\nEither the dataset's ID or name should be specified, and similarly, either the target key-value store's ID or\nname should be used.\n" + } + ] + }, + "flags": {}, + "id": 5719, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "export_to", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The key under which to save the data in the key-value store." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 5720, + "kind": 32768, + "kindString": "Parameter", + "name": "key", + "type": { + "name": "str", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2478 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The format in which to export the data." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 5721, + "kind": 32768, + "kindString": "Parameter", + "name": "content_type", + "type": { + "name": "Literal", + "type": "reference", + "typeArguments": [ + { + "type": "literal", + "value": "csv" + } + ], + "target": 0, + "ref": {} + }, + "parentId": 2478 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "ID of the key-value store to save the exported file.\nSpecify only one of ID or name." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": false + }, + "id": 5722, + "kind": 32768, + "kindString": "Parameter", + "name": "to_kvs_id", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2478 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Name of the key-value store to save the exported file.\nSpecify only one of ID or name." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": false + }, + "id": 5723, + "kind": 32768, + "kindString": "Parameter", + "name": "to_kvs_name", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2478 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Storage client to use for the key-value store." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": false + }, + "id": 5724, + "kind": 32768, + "kindString": "Parameter", + "name": "to_kvs_storage_client", + "type": { + "name": "StorageClient | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "StorageClient", + "target": "1282", + "ref": { + "id": 4523, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 19 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/StorageClient" + } + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2478 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Configuration for the key-value store." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": false + }, + "id": 5725, + "kind": 32768, + "kindString": "Parameter", + "name": "to_kvs_configuration", + "type": { + "name": "Configuration | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "Configuration", + "target": "2", + "ref": { + "id": 3243, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/configuration.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 20 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/Configuration" + } + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2478 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Specifies a dialect to be used in CSV parsing and writing." + } + ] + }, + "flags": { + "keyword-only": true, + "optional": true + }, + "groups": [], + "id": 3580, + "module": "_types", + "name": "dialect", + "parsedDocstring": { + "text": "Specifies a dialect to be used in CSV parsing and writing." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_types.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 795, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/_types.py#L795" + } + ], + "type": { + "name": "NotRequired", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + }, + "parentId": 2478 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "A one-character string used to separate fields. Defaults to ','." + } + ] + }, + "flags": { + "keyword-only": true, + "optional": true + }, + "groups": [], + "id": 3581, + "module": "_types", + "name": "delimiter", + "parsedDocstring": { + "text": "A one-character string used to separate fields. Defaults to ','." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_types.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 798, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/_types.py#L798" + } + ], + "type": { + "name": "NotRequired", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + }, + "parentId": 2478 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Controls how instances of `quotechar` inside a field should be quoted. When True, the character is doubled;\nwhen False, the `escapechar` is used as a prefix. Defaults to True." + } + ] + }, + "flags": { + "keyword-only": true, + "optional": true + }, + "groups": [], + "id": 3582, + "module": "_types", + "name": "doublequote", + "parsedDocstring": { + "text": "Controls how instances of `quotechar` inside a field should be quoted. When True, the character is doubled;\nwhen False, the `escapechar` is used as a prefix. Defaults to True." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_types.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 801, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/_types.py#L801" + } + ], + "type": { + "name": "NotRequired", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "bool", + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + }, + "parentId": 2478 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "A one-character string used to escape the delimiter if `quoting` is set to `QUOTE_NONE` and the `quotechar`\nif `doublequote` is False. Defaults to None, disabling escaping." + } + ] + }, + "flags": { + "keyword-only": true, + "optional": true + }, + "groups": [], + "id": 3583, + "module": "_types", + "name": "escapechar", + "parsedDocstring": { + "text": "A one-character string used to escape the delimiter if `quoting` is set to `QUOTE_NONE` and the `quotechar`\nif `doublequote` is False. Defaults to None, disabling escaping." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_types.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 805, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/_types.py#L805" + } + ], + "type": { + "name": "NotRequired", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + }, + "parentId": 2478 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "The string used to terminate lines produced by the writer. Defaults to '\\r\\n'." + } + ] + }, + "flags": { + "keyword-only": true, + "optional": true + }, + "groups": [], + "id": 3584, + "module": "_types", + "name": "lineterminator", + "parsedDocstring": { + "text": "The string used to terminate lines produced by the writer. Defaults to '\\r\\n'." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_types.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 809, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/_types.py#L809" + } + ], + "type": { + "name": "NotRequired", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + }, + "parentId": 2478 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "A one-character string used to quote fields containing special characters, like the delimiter or quotechar,\nor fields containing new-line characters. Defaults to '\"'." + } + ] + }, + "flags": { + "keyword-only": true, + "optional": true + }, + "groups": [], + "id": 3585, + "module": "_types", + "name": "quotechar", + "parsedDocstring": { + "text": "A one-character string used to quote fields containing special characters, like the delimiter or quotechar,\nor fields containing new-line characters. Defaults to '\"'." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_types.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 812, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/_types.py#L812" + } + ], + "type": { + "name": "NotRequired", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + }, + "parentId": 2478 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Controls when quotes should be generated by the writer and recognized by the reader. Can take any of\nthe `QUOTE_*` constants, with a default of `QUOTE_MINIMAL`." + } + ] + }, + "flags": { + "keyword-only": true, + "optional": true + }, + "groups": [], + "id": 3586, + "module": "_types", + "name": "quoting", + "parsedDocstring": { + "text": "Controls when quotes should be generated by the writer and recognized by the reader. Can take any of\nthe `QUOTE_*` constants, with a default of `QUOTE_MINIMAL`." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_types.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 816, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/_types.py#L816" + } + ], + "type": { + "name": "NotRequired", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "Literal", + "typeArguments": [ + { + "type": "literal", + "value": 0 + }, + { + "type": "literal", + "value": 1 + }, + { + "type": "literal", + "value": 2 + }, + { + "type": "literal", + "value": 3 + } + ], + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + }, + "parentId": 2478 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "When True, spaces immediately following the delimiter are ignored. Defaults to False." + } + ] + }, + "flags": { + "keyword-only": true, + "optional": true + }, + "groups": [], + "id": 3587, + "module": "_types", + "name": "skipinitialspace", + "parsedDocstring": { + "text": "When True, spaces immediately following the delimiter are ignored. Defaults to False." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_types.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 820, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/_types.py#L820" + } + ], + "type": { + "name": "NotRequired", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "bool", + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + }, + "parentId": 2478 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "When True, raises an exception on bad CSV input. Defaults to False." + } + ] + }, + "flags": { + "keyword-only": true, + "optional": true + }, + "groups": [], + "id": 3588, + "module": "_types", + "name": "strict", + "parsedDocstring": { + "text": "When True, raises an exception on bad CSV input. Defaults to False." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_types.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 823, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/_types.py#L823" + } + ], + "type": { + "name": "NotRequired", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "bool", + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + }, + "parentId": 2478 + } + ], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "parentId": 2461 + } + ], + "permalink": "/python/api/class/Dataset#export_to", + "parentId": 2402 + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Dataset is a storage for managing structured tabular data.\n\nThe dataset class provides a high-level interface for storing and retrieving structured data\nwith consistent schema, similar to database tables or spreadsheets. It abstracts the underlying\nstorage implementation details, offering a consistent API regardless of where the data is\nphysically stored.\n\nDataset operates in an append-only mode, allowing new records to be added but not modified\nor deleted after creation. This makes it particularly suitable for storing crawling results\nand other data that should be immutable once collected.\n\nThe class provides methods for adding data, retrieving data with various filtering options,\nand exporting data to different formats. You can create a dataset using the `open` class method,\nspecifying either a name or ID. The underlying storage implementation is determined by\nthe configured storage client.\n\n### Usage\n\n```python\nfrom crawlee.storages import Dataset\n\n# Open a dataset\ndataset = await Dataset.open(name='my-dataset')\n\n# Add data\nawait dataset.push_data({'title': 'Example Product', 'price': 99.99})\n\n# Retrieve filtered data\nresults = await dataset.get_data(limit=10, desc=True)\n\n# Export data\nawait dataset.export_to('results.json', content_type='json')\n```" + } + ] + }, + "decorations": [ + { + "args": "('Storages')", + "name": "docs_group" + } + ], + "flags": {}, + "groups": [ + { + "children": [ + 5644, + 5660, + 5702, + 5667, + 5651, + 5680, + 5691, + 5653, + 5662, + 5664 + ], + "title": "Methods" + }, + { + "children": [ + 5649, + 5650 + ], + "title": "Properties" + } + ], + "id": 5643, + "module": "storages._dataset", + "name": "Dataset", + "parsedDocstring": { + "text": "Dataset is a storage for managing structured tabular data.\n\nThe dataset class provides a high-level interface for storing and retrieving structured data\nwith consistent schema, similar to database tables or spreadsheets. It abstracts the underlying\nstorage implementation details, offering a consistent API regardless of where the data is\nphysically stored.\n\nDataset operates in an append-only mode, allowing new records to be added but not modified\nor deleted after creation. This makes it particularly suitable for storing crawling results\nand other data that should be immutable once collected.\n\nThe class provides methods for adding data, retrieving data with various filtering options,\nand exporting data to different formats. You can create a dataset using the `open` class method,\nspecifying either a name or ID. The underlying storage implementation is determined by\nthe configured storage client.\n\n### Usage\n\n```python\nfrom crawlee.storages import Dataset\n\n# Open a dataset\ndataset = await Dataset.open(name='my-dataset')\n\n# Add data\nawait dataset.push_data({'title': 'Example Product', 'price': 99.99})\n\n# Retrieve filtered data\nresults = await dataset.get_data(limit=10, desc=True)\n\n# Export data\nawait dataset.export_to('results.json', content_type='json')\n```" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_dataset.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 33, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storages/_dataset.py#L33" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "extendedTypes": [ + { + "name": "Storage", + "target": "2330", + "type": "reference", + "ref": { + "id": 5571, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_base.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 15 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/Storage" + } + } + ], + "permalink": "/python/api/class/Dataset", + "categories": [] + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Initialize a new instance.\n\nPreferably use the `KeyValueStore.open` constructor to create a new instance.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 8933, + "module": "storages._key_value_store", + "name": "__init__", + "parsedDocstring": { + "text": "Initialize a new instance.\n\nPreferably use the `KeyValueStore.open` constructor to create a new instance.\n", + "args": { + "client": "An instance of a storage client.", + "id": "The unique identifier of the storage.", + "name": "The name of the storage, if available." + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_key_value_store.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 78, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storages/_key_value_store.py#L78" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Initialize a new instance.\n\nPreferably use the `KeyValueStore.open` constructor to create a new instance.\n" + } + ] + }, + "flags": {}, + "id": 8934, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "__init__", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "An instance of a storage client." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 8935, + "kind": 32768, + "kindString": "Parameter", + "name": "client", + "type": { + "name": "KeyValueStoreClient", + "type": "reference", + "target": "1254", + "ref": { + "id": 7917, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_key_value_store_client.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 14 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/KeyValueStoreClient" + } + }, + "parentId": 2271 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The unique identifier of the storage." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 8936, + "kind": 32768, + "kindString": "Parameter", + "name": "id", + "type": { + "name": "str", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2271 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The name of the storage, if available." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 8937, + "kind": 32768, + "kindString": "Parameter", + "name": "name", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2271 + } + ], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "parentId": 2270 + } + ], + "permalink": "/python/api/class/KeyValueStore#__init__", + "parentId": 2269 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Get the storage ID." + } + ] + }, + "decorations": [ + { + "name": "property" + }, + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 8938, + "module": "storages._base", + "name": "id", + "parsedDocstring": { + "text": "Get the storage ID." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_key_value_store.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 99, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storages/_key_value_store.py#L99" + } + ], + "type": { + "name": "str", + "type": "reference", + "target": 0, + "ref": {} + }, + "overwrites": { + "name": "Storage.id", + "target": 2331, + "type": "reference", + "ref": { + "id": 8994, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_base.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 20 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api/class/Storage#id" + } + }, + "permalink": "/python/api/class/KeyValueStore#id", + "parentId": 2269 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Get the storage name." + } + ] + }, + "decorations": [ + { + "name": "property" + }, + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 8939, + "module": "storages._base", + "name": "name", + "parsedDocstring": { + "text": "Get the storage name." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_key_value_store.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 104, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storages/_key_value_store.py#L104" + } + ], + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "overwrites": { + "name": "Storage.name", + "target": 2332, + "type": "reference", + "ref": { + "id": 8995, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_base.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 25 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api/class/Storage#name" + } + }, + "permalink": "/python/api/class/KeyValueStore#name", + "parentId": 2269 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Get the storage metadata." + } + ] + }, + "decorations": [ + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 8940, + "module": "storages._base", + "name": "get_metadata", + "parsedDocstring": { + "text": "Get the storage metadata." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_key_value_store.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 108, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storages/_key_value_store.py#L108" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Get the storage metadata." + } + ] + }, + "flags": {}, + "id": 8997, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "get_metadata", + "parameters": [], + "type": { + "name": "DatasetMetadata | KeyValueStoreMetadata | RequestQueueMetadata", + "type": "union", + "types": [ + { + "type": "union", + "types": [ + { + "type": "reference", + "name": "DatasetMetadata", + "target": "1314", + "ref": { + "id": 7977, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 42 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/DatasetMetadata" + } + }, + { + "type": "reference", + "name": "KeyValueStoreMetadata", + "target": "1317", + "ref": { + "id": 7980, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 52 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/KeyValueStoreMetadata" + } + } + ] + }, + { + "type": "reference", + "name": "RequestQueueMetadata", + "target": "1319", + "ref": { + "id": 7982, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 59 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/RequestQueueMetadata" + } + } + ] + }, + "overwrites": { + "name": "Storage.get_metadata", + "target": 2333, + "type": "reference", + "ref": { + "id": 8996, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_base.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 29 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/Storage#get_metadata" + } + }, + "parentId": 2277 + } + ], + "overwrites": { + "name": "Storage.get_metadata", + "target": 2333, + "type": "reference", + "ref": { + "id": 8996, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_base.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 29 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/Storage#get_metadata" + } + }, + "permalink": "/python/api/class/KeyValueStore#get_metadata", + "parentId": 2269 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Open a storage, either restore existing or create a new one.\n" + } + ] + }, + "decorations": [ + { + "name": "classmethod" + }, + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 8942, + "module": "storages._base", + "name": "open", + "parsedDocstring": { + "text": "Open a storage, either restore existing or create a new one.\n", + "args": { + "id": "The storage ID.", + "name": "The storage name (global scope, persists across runs). Name can only contain letters \"a\" through \"z\",\nthe digits \"0\" through \"9\", and the hyphen (\"-\") but only in the middle of the string\n(e.g. \"my-value-1\").", + "alias": "The storage alias (run scope, creates unnamed storage).", + "configuration": "Configuration object used during the storage creation or restoration process.", + "storage_client": "Underlying storage client to use. If not provided, the default global storage client\nfrom the service locator will be used." + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_key_value_store.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 113, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storages/_key_value_store.py#L113" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Open a storage, either restore existing or create a new one.\n" + } + ] + }, + "flags": {}, + "id": 8999, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "open", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The storage ID." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 9000, + "kind": 32768, + "kindString": "Parameter", + "name": "id", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2336 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The storage name (global scope, persists across runs). Name can only contain letters \"a\" through \"z\",\nthe digits \"0\" through \"9\", and the hyphen (\"-\") but only in the middle of the string\n(e.g. \"my-value-1\")." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 9001, + "kind": 32768, + "kindString": "Parameter", + "name": "name", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2336 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The storage alias (run scope, creates unnamed storage)." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 9002, + "kind": 32768, + "kindString": "Parameter", + "name": "alias", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2336 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Configuration object used during the storage creation or restoration process." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 9003, + "kind": 32768, + "kindString": "Parameter", + "name": "configuration", + "type": { + "name": "Configuration | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "Configuration", + "target": "2", + "ref": { + "id": 6665, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/configuration.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 20 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/Configuration" + } + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2336 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Underlying storage client to use. If not provided, the default global storage client\nfrom the service locator will be used." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 9004, + "kind": 32768, + "kindString": "Parameter", + "name": "storage_client", + "type": { + "name": "StorageClient | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "StorageClient", + "target": "1282", + "ref": { + "id": 7945, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 19 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/StorageClient" + } + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2336 + } + ], + "type": { + "name": "Storage", + "type": "reference", + "target": "2330", + "ref": { + "id": 8993, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_base.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 15 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/Storage" + } + }, + "overwrites": { + "name": "Storage.open", + "target": 2335, + "type": "reference", + "ref": { + "id": 8998, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_base.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 34 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/Storage#open" + } + }, + "parentId": 2279 + } + ], + "overwrites": { + "name": "Storage.open", + "target": 2335, + "type": "reference", + "ref": { + "id": 8998, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_base.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 34 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/Storage#open" + } + }, + "permalink": "/python/api/class/KeyValueStore#open", + "parentId": 2269 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Drop the storage, removing it from the underlying storage client and clearing the cache." + } + ] + }, + "decorations": [ + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 8949, + "module": "storages._base", + "name": "drop", + "parsedDocstring": { + "text": "Drop the storage, removing it from the underlying storage client and clearing the cache." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_key_value_store.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 140, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storages/_key_value_store.py#L140" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Drop the storage, removing it from the underlying storage client and clearing the cache." + } + ] + }, + "flags": {}, + "id": 9006, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "drop", + "parameters": [], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "overwrites": { + "name": "Storage.drop", + "target": 2342, + "type": "reference", + "ref": { + "id": 9005, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_base.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 57 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/Storage#drop" + } + }, + "parentId": 2286 + } + ], + "overwrites": { + "name": "Storage.drop", + "target": 2342, + "type": "reference", + "ref": { + "id": 9005, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_base.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 57 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/Storage#drop" + } + }, + "permalink": "/python/api/class/KeyValueStore#drop", + "parentId": 2269 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Purge the storage, removing all items from the underlying storage client.\n\nThis method does not remove the storage itself, e.g. don't remove the metadata,\nbut clears all items within it." + } + ] + }, + "decorations": [ + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 8951, + "module": "storages._base", + "name": "purge", + "parsedDocstring": { + "text": "Purge the storage, removing all items from the underlying storage client.\n\nThis method does not remove the storage itself, e.g. don't remove the metadata,\nbut clears all items within it." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_key_value_store.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 148, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storages/_key_value_store.py#L148" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Purge the storage, removing all items from the underlying storage client.\n\nThis method does not remove the storage itself, e.g. don't remove the metadata,\nbut clears all items within it." + } + ] + }, + "flags": {}, + "id": 9008, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "purge", + "parameters": [], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "overwrites": { + "name": "Storage.purge", + "target": 2344, + "type": "reference", + "ref": { + "id": 9007, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_base.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 61 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/Storage#purge" + } + }, + "parentId": 2288 + } + ], + "overwrites": { + "name": "Storage.purge", + "target": 2344, + "type": "reference", + "ref": { + "id": 9007, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_base.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 61 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/Storage#purge" + } + }, + "permalink": "/python/api/class/KeyValueStore#purge", + "parentId": 2269 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Get a value from the KVS.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 8953, + "module": "storages._key_value_store", + "name": "get_value", + "parsedDocstring": { + "text": "Get a value from the KVS.\n", + "args": { + "key": "Key of the record to retrieve.", + "default_value": "Default value returned in case the record does not exist.\n" + }, + "returns": "The value associated with the given key. `default_value` is used in case the record does not exist." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_key_value_store.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 160, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storages/_key_value_store.py#L160" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "The value associated with the given key. `default_value` is used in case the record does not exist." + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Get a value from the KVS.\n" + } + ] + }, + "flags": {}, + "id": 8954, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "get_value", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Key of the record to retrieve." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 8955, + "kind": 32768, + "kindString": "Parameter", + "name": "key", + "type": { + "name": "str", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2291 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Default value returned in case the record does not exist.\n" + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": false + }, + "id": 8956, + "kind": 32768, + "kindString": "Parameter", + "name": "default_value", + "type": { + "name": "T | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "T", + "target": "83", + "ref": { + "id": 6746, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_types.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 36 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api#T" + } + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2291 + } + ], + "type": { + "name": "T | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "T", + "target": "83", + "ref": { + "id": 6746, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_types.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 36 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api#T" + } + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2290 + }, + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "The value associated with the given key. `default_value` is used in case the record does not exist." + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Get a value from the KVS.\n" + } + ] + }, + "flags": {}, + "id": 8985, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "get_value", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Key of the record to retrieve." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 8986, + "kind": 32768, + "kindString": "Parameter", + "name": "key", + "type": { + "name": "str", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2322 + } + ], + "type": { + "name": "Any", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2290 + }, + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "The value associated with the given key. `default_value` is used in case the record does not exist." + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Get a value from the KVS.\n" + } + ] + }, + "flags": {}, + "id": 8987, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "get_value", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Key of the record to retrieve." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 8988, + "kind": 32768, + "kindString": "Parameter", + "name": "key", + "type": { + "name": "str", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2324 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Default value returned in case the record does not exist.\n" + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 8989, + "kind": 32768, + "kindString": "Parameter", + "name": "default_value", + "type": { + "name": "T", + "type": "reference", + "target": "83", + "ref": { + "id": 6746, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_types.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 36 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api#T" + } + }, + "parentId": 2324 + } + ], + "type": { + "name": "T", + "type": "reference", + "target": "83", + "ref": { + "id": 6746, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_types.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 36 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api#T" + } + }, + "parentId": 2290 + }, + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "The value associated with the given key. `default_value` is used in case the record does not exist." + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Get a value from the KVS.\n" + } + ] + }, + "flags": {}, + "id": 8990, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "get_value", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Key of the record to retrieve." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 8991, + "kind": 32768, + "kindString": "Parameter", + "name": "key", + "type": { + "name": "str", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2327 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Default value returned in case the record does not exist.\n" + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": false + }, + "id": 8992, + "kind": 32768, + "kindString": "Parameter", + "name": "default_value", + "type": { + "name": "T | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "T", + "target": "83", + "ref": { + "id": 6746, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_types.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 36 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api#T" + } + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2327 + } + ], + "type": { + "name": "T | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "T", + "target": "83", + "ref": { + "id": 6746, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_types.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 36 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api#T" + } + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2290 + } + ], + "permalink": "/python/api/class/KeyValueStore#get_value", + "parentId": 2269 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Set a value in the KVS.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 8957, + "module": "storages._key_value_store", + "name": "set_value", + "parsedDocstring": { + "text": "Set a value in the KVS.\n", + "args": { + "key": "Key of the record to set.", + "value": "Value to set.", + "content_type": "The MIME content type string." + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_key_value_store.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 173, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storages/_key_value_store.py#L173" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Set a value in the KVS.\n" + } + ] + }, + "flags": {}, + "id": 8958, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "set_value", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Key of the record to set." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 8959, + "kind": 32768, + "kindString": "Parameter", + "name": "key", + "type": { + "name": "str", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2295 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Value to set." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 8960, + "kind": 32768, + "kindString": "Parameter", + "name": "value", + "type": { + "name": "Any", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2295 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The MIME content type string." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": false + }, + "id": 8961, + "kind": 32768, + "kindString": "Parameter", + "name": "content_type", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2295 + } + ], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "parentId": 2294 + } + ], + "permalink": "/python/api/class/KeyValueStore#set_value", + "parentId": 2269 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Delete a value from the KVS.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 8962, + "module": "storages._key_value_store", + "name": "delete_value", + "parsedDocstring": { + "text": "Delete a value from the KVS.\n", + "args": { + "key": "Key of the record to delete." + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_key_value_store.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 188, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storages/_key_value_store.py#L188" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Delete a value from the KVS.\n" + } + ] + }, + "flags": {}, + "id": 8963, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "delete_value", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Key of the record to delete." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 8964, + "kind": 32768, + "kindString": "Parameter", + "name": "key", + "type": { + "name": "str", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2300 + } + ], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "parentId": 2299 + } + ], + "permalink": "/python/api/class/KeyValueStore#delete_value", + "parentId": 2269 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Iterate over the existing keys in the KVS.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 8965, + "module": "storages._key_value_store", + "name": "iterate_keys", + "parsedDocstring": { + "text": "Iterate over the existing keys in the KVS.\n", + "args": { + "exclusive_start_key": "Key to start the iteration from.", + "limit": "Maximum number of keys to return. None means no limit.\n" + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_key_value_store.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 196, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storages/_key_value_store.py#L196" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Iterate over the existing keys in the KVS.\n" + } + ] + }, + "flags": {}, + "id": 8966, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "iterate_keys", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Key to start the iteration from." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": false + }, + "id": 8967, + "kind": 32768, + "kindString": "Parameter", + "name": "exclusive_start_key", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2303 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Maximum number of keys to return. None means no limit.\n" + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": false + }, + "id": 8968, + "kind": 32768, + "kindString": "Parameter", + "name": "limit", + "type": { + "name": "int | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "int", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2303 + } + ], + "type": { + "name": "AsyncIterator", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "KeyValueStoreRecordMetadata", + "target": "1325", + "ref": { + "id": 7988, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 78 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/KeyValueStoreRecordMetadata" + } + } + ], + "target": 0, + "ref": {} + }, + "parentId": 2302 + } + ], + "permalink": "/python/api/class/KeyValueStore#iterate_keys", + "parentId": 2269 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "List all the existing keys in the KVS.\n\nIt uses client's `iterate_keys` method to get the keys.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 8969, + "module": "storages._key_value_store", + "name": "list_keys", + "parsedDocstring": { + "text": "List all the existing keys in the KVS.\n\nIt uses client's `iterate_keys` method to get the keys.\n", + "args": { + "exclusive_start_key": "Key to start the iteration from.", + "limit": "Maximum number of keys to return.\n" + }, + "returns": "A list of keys in the KVS." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_key_value_store.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 216, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storages/_key_value_store.py#L216" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "A list of keys in the KVS." + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "List all the existing keys in the KVS.\n\nIt uses client's `iterate_keys` method to get the keys.\n" + } + ] + }, + "flags": {}, + "id": 8970, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "list_keys", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Key to start the iteration from." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": false + }, + "id": 8971, + "kind": 32768, + "kindString": "Parameter", + "name": "exclusive_start_key", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2307 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Maximum number of keys to return.\n" + } + ] + }, + "defaultValue": "1000", + "flags": { + "isOptional": true, + "keyword-only": false + }, + "id": 8972, + "kind": 32768, + "kindString": "Parameter", + "name": "limit", + "type": { + "name": "int", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2307 + } + ], + "type": { + "name": "list", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "KeyValueStoreRecordMetadata", + "target": "1325", + "ref": { + "id": 7988, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 78 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/KeyValueStoreRecordMetadata" + } + } + ], + "target": 0, + "ref": {} + }, + "parentId": 2306 + } + ], + "permalink": "/python/api/class/KeyValueStore#list_keys", + "parentId": 2269 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Check if a record with the given key exists in the key-value store.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 8973, + "module": "storages._key_value_store", + "name": "record_exists", + "parsedDocstring": { + "text": "Check if a record with the given key exists in the key-value store.\n", + "args": { + "key": "Key of the record to check for existence.\n" + }, + "returns": "True if a record with the given key exists, False otherwise." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_key_value_store.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 240, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storages/_key_value_store.py#L240" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "True if a record with the given key exists, False otherwise." + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Check if a record with the given key exists in the key-value store.\n" + } + ] + }, + "flags": {}, + "id": 8974, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "record_exists", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Key of the record to check for existence.\n" + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 8975, + "kind": 32768, + "kindString": "Parameter", + "name": "key", + "type": { + "name": "str", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2311 + } + ], + "type": { + "name": "bool", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2310 + } + ], + "permalink": "/python/api/class/KeyValueStore#record_exists", + "parentId": 2269 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Get the public URL for the given key.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 8976, + "module": "storages._key_value_store", + "name": "get_public_url", + "parsedDocstring": { + "text": "Get the public URL for the given key.\n", + "args": { + "key": "Key of the record for which URL is required.\n" + }, + "returns": "The public URL for the given key." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_key_value_store.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 251, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storages/_key_value_store.py#L251" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "The public URL for the given key." + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Get the public URL for the given key.\n" + } + ] + }, + "flags": {}, + "id": 8977, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "get_public_url", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Key of the record for which URL is required.\n" + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 8978, + "kind": 32768, + "kindString": "Parameter", + "name": "key", + "type": { + "name": "str", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2314 + } + ], + "type": { + "name": "str", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2313 + } + ], + "permalink": "/python/api/class/KeyValueStore#get_public_url", + "parentId": 2269 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Get a value from KVS that will be automatically saved on changes.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 8979, + "module": "storages._key_value_store", + "name": "get_auto_saved_value", + "parsedDocstring": { + "text": "Get a value from KVS that will be automatically saved on changes.\n", + "args": { + "key": "Key of the record, to store the value.", + "default_value": "Value to be used if the record does not exist yet. Should be a dictionary.\n" + }, + "returns": "Return the value of the key." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_key_value_store.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 262, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storages/_key_value_store.py#L262" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "Return the value of the key." + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Get a value from KVS that will be automatically saved on changes.\n" + } + ] + }, + "flags": {}, + "id": 8980, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "get_auto_saved_value", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Key of the record, to store the value." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 8981, + "kind": 32768, + "kindString": "Parameter", + "name": "key", + "type": { + "name": "str", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2317 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Value to be used if the record does not exist yet. Should be a dictionary.\n" + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": false + }, + "id": 8982, + "kind": 32768, + "kindString": "Parameter", + "name": "default_value", + "type": { + "name": "dict[str, JsonSerializable] | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "dict", + "typeArguments": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "reference", + "name": "JsonSerializable", + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2317 + } + ], + "type": { + "name": "dict", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "reference", + "name": "JsonSerializable", + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + }, + "parentId": 2316 + } + ], + "permalink": "/python/api/class/KeyValueStore#get_auto_saved_value", + "parentId": 2269 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Force autosaved values to be saved without waiting for an event in Event Manager." + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 8983, + "module": "storages._key_value_store", + "name": "persist_autosaved_values", + "parsedDocstring": { + "text": "Force autosaved values to be saved without waiting for an event in Event Manager." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_key_value_store.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 299, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storages/_key_value_store.py#L299" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Force autosaved values to be saved without waiting for an event in Event Manager." + } + ] + }, + "flags": {}, + "id": 8984, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "persist_autosaved_values", + "parameters": [], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "parentId": 2320 + } + ], + "permalink": "/python/api/class/KeyValueStore#persist_autosaved_values", + "parentId": 2269 + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Key-value store is a storage for reading and writing data records with unique key identifiers.\n\nThe key-value store class acts as a high-level interface for storing, retrieving, and managing data records\nidentified by unique string keys. It abstracts away the underlying storage implementation details,\nallowing you to work with the same API regardless of whether data is stored in memory, on disk,\nor in the cloud.\n\nEach data record is associated with a specific MIME content type, allowing storage of various\ndata formats such as JSON, text, images, HTML snapshots or any binary data. This class is\ncommonly used to store inputs, outputs, and other artifacts of crawler operations.\n\nYou can instantiate a key-value store using the `open` class method, which will create a store\nwith the specified name or id. The underlying storage implementation is determined by the configured\nstorage client.\n\n### Usage\n\n```python\nfrom crawlee.storages import KeyValueStore\n\n# Open a named key-value store\nkvs = await KeyValueStore.open(name='my-store')\n\n# Store and retrieve data\nawait kvs.set_value('product-1234.json', [{'name': 'Smartphone', 'price': 799.99}])\nproduct = await kvs.get_value('product-1234')\n```" + } + ] + }, + "decorations": [ + { + "args": "('Storages')", + "name": "docs_group" + } + ], + "flags": {}, + "groups": [ + { + "children": [ + 8933, + 8962, + 8949, + 8979, + 8940, + 8976, + 8953, + 8965, + 8969, + 8942, + 8983, + 8951, + 8973, + 8957 + ], + "title": "Methods" + }, + { + "children": [ + 8938, + 8939 + ], + "title": "Properties" + } + ], + "id": 8932, + "module": "storages._key_value_store", + "name": "KeyValueStore", + "parsedDocstring": { + "text": "Key-value store is a storage for reading and writing data records with unique key identifiers.\n\nThe key-value store class acts as a high-level interface for storing, retrieving, and managing data records\nidentified by unique string keys. It abstracts away the underlying storage implementation details,\nallowing you to work with the same API regardless of whether data is stored in memory, on disk,\nor in the cloud.\n\nEach data record is associated with a specific MIME content type, allowing storage of various\ndata formats such as JSON, text, images, HTML snapshots or any binary data. This class is\ncommonly used to store inputs, outputs, and other artifacts of crawler operations.\n\nYou can instantiate a key-value store using the `open` class method, which will create a store\nwith the specified name or id. The underlying storage implementation is determined by the configured\nstorage client.\n\n### Usage\n\n```python\nfrom crawlee.storages import KeyValueStore\n\n# Open a named key-value store\nkvs = await KeyValueStore.open(name='my-store')\n\n# Store and retrieve data\nawait kvs.set_value('product-1234.json', [{'name': 'Smartphone', 'price': 799.99}])\nproduct = await kvs.get_value('product-1234')\n```" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_key_value_store.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 40, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storages/_key_value_store.py#L40" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "extendedTypes": [ + { + "name": "Storage", + "target": "2330", + "type": "reference", + "ref": { + "id": 8993, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_base.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 15 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/Storage" + } + } + ], + "permalink": "/python/api/class/KeyValueStore", + "categories": [] + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Initialize a new instance.\n\nPreferably use the `RequestQueue.open` constructor to create a new instance.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 11357, + "module": "storages._request_queue", + "name": "__init__", + "parsedDocstring": { + "text": "Initialize a new instance.\n\nPreferably use the `RequestQueue.open` constructor to create a new instance.\n", + "args": { + "client": "An instance of a storage client.", + "id": "The unique identifier of the storage.", + "name": "The name of the storage, if available." + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_request_queue.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 74, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storages/_request_queue.py#L74" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Initialize a new instance.\n\nPreferably use the `RequestQueue.open` constructor to create a new instance.\n" + } + ] + }, + "flags": {}, + "id": 11358, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "__init__", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "An instance of a storage client." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 11359, + "kind": 32768, + "kindString": "Parameter", + "name": "client", + "type": { + "name": "RequestQueueClient", + "type": "reference", + "target": "1195", + "ref": { + "id": 10203, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_request_queue_client.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 13 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/RequestQueueClient" + } + }, + "parentId": 2350 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The unique identifier of the storage." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 11360, + "kind": 32768, + "kindString": "Parameter", + "name": "id", + "type": { + "name": "str", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2350 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The name of the storage, if available." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 11361, + "kind": 32768, + "kindString": "Parameter", + "name": "name", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2350 + } + ], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "parentId": 2349 + } + ], + "permalink": "/python/api/class/RequestQueue#__init__", + "parentId": 2348 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Get the storage ID." + } + ] + }, + "decorations": [ + { + "name": "property" + }, + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 11362, + "module": "storages._base", + "name": "id", + "parsedDocstring": { + "text": "Get the storage ID." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_request_queue.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 95, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storages/_request_queue.py#L95" + } + ], + "type": { + "name": "str", + "type": "reference", + "target": 0, + "ref": {} + }, + "overwrites": { + "name": "Storage.id", + "target": 2331, + "type": "reference", + "ref": { + "id": 11339, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_base.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 20 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api/class/Storage#id" + } + }, + "permalink": "/python/api/class/RequestQueue#id", + "parentId": 2348 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Get the storage name." + } + ] + }, + "decorations": [ + { + "name": "property" + }, + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 11363, + "module": "storages._base", + "name": "name", + "parsedDocstring": { + "text": "Get the storage name." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_request_queue.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 100, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storages/_request_queue.py#L100" + } + ], + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "overwrites": { + "name": "Storage.name", + "target": 2332, + "type": "reference", + "ref": { + "id": 11340, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_base.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 25 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api/class/Storage#name" + } + }, + "permalink": "/python/api/class/RequestQueue#name", + "parentId": 2348 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Get the storage metadata." + } + ] + }, + "decorations": [ + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 11364, + "module": "storages._base", + "name": "get_metadata", + "parsedDocstring": { + "text": "Get the storage metadata." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_request_queue.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 104, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storages/_request_queue.py#L104" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Get the storage metadata." + } + ] + }, + "flags": {}, + "id": 11342, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "get_metadata", + "parameters": [], + "type": { + "name": "DatasetMetadata | KeyValueStoreMetadata | RequestQueueMetadata", + "type": "union", + "types": [ + { + "type": "union", + "types": [ + { + "type": "reference", + "name": "DatasetMetadata", + "target": "1314", + "ref": { + "id": 10322, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 42 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/DatasetMetadata" + } + }, + { + "type": "reference", + "name": "KeyValueStoreMetadata", + "target": "1317", + "ref": { + "id": 10325, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 52 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/KeyValueStoreMetadata" + } + } + ] + }, + { + "type": "reference", + "name": "RequestQueueMetadata", + "target": "1319", + "ref": { + "id": 10327, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 59 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/RequestQueueMetadata" + } + } + ] + }, + "overwrites": { + "name": "Storage.get_metadata", + "target": 2333, + "type": "reference", + "ref": { + "id": 11341, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_base.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 29 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/Storage#get_metadata" + } + }, + "parentId": 2356 + } + ], + "overwrites": { + "name": "Storage.get_metadata", + "target": 2333, + "type": "reference", + "ref": { + "id": 11341, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_base.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 29 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/Storage#get_metadata" + } + }, + "permalink": "/python/api/class/RequestQueue#get_metadata", + "parentId": 2348 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Get the number of requests in the loader that have been handled." + } + ] + }, + "decorations": [ + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 11366, + "module": "request_loaders._request_loader", + "name": "get_handled_count", + "parsedDocstring": { + "text": "Get the number of requests in the loader that have been handled." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_request_queue.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 108, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storages/_request_queue.py#L108" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Get the number of requests in the loader that have been handled." + } + ] + }, + "flags": {}, + "id": 11944, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "get_handled_count", + "parameters": [], + "type": { + "name": "int", + "type": "reference", + "target": 0, + "ref": {} + }, + "overwrites": { + "name": "RequestManager.get_handled_count", + "target": 4143, + "type": "reference", + "ref": { + "id": 13151, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 29 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestManager#get_handled_count" + } + }, + "parentId": 2358 + } + ], + "overwrites": { + "name": "RequestManager.get_handled_count", + "target": 4143, + "type": "reference", + "ref": { + "id": 13151, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 29 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestManager#get_handled_count" + } + }, + "permalink": "/python/api/class/RequestQueue#get_handled_count", + "parentId": 2348 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Get an offline approximation of the total number of requests in the loader (i.e. pending + handled)." + } + ] + }, + "decorations": [ + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 11368, + "module": "request_loaders._request_loader", + "name": "get_total_count", + "parsedDocstring": { + "text": "Get an offline approximation of the total number of requests in the loader (i.e. pending + handled)." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_request_queue.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 113, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storages/_request_queue.py#L113" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Get an offline approximation of the total number of requests in the loader (i.e. pending + handled)." + } + ] + }, + "flags": {}, + "id": 11946, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "get_total_count", + "parameters": [], + "type": { + "name": "int", + "type": "reference", + "target": 0, + "ref": {} + }, + "overwrites": { + "name": "RequestManager.get_total_count", + "target": 4144, + "type": "reference", + "ref": { + "id": 13152, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 33 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestManager#get_total_count" + } + }, + "parentId": 2360 + } + ], + "overwrites": { + "name": "RequestManager.get_total_count", + "target": 4144, + "type": "reference", + "ref": { + "id": 13152, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 33 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestManager#get_total_count" + } + }, + "permalink": "/python/api/class/RequestQueue#get_total_count", + "parentId": 2348 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Open a storage, either restore existing or create a new one.\n" + } + ] + }, + "decorations": [ + { + "name": "classmethod" + }, + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 11370, + "module": "storages._base", + "name": "open", + "parsedDocstring": { + "text": "Open a storage, either restore existing or create a new one.\n", + "args": { + "id": "The storage ID.", + "name": "The storage name (global scope, persists across runs). Name can only contain letters \"a\" through \"z\",\nthe digits \"0\" through \"9\", and the hyphen (\"-\") but only in the middle of the string\n(e.g. \"my-value-1\").", + "alias": "The storage alias (run scope, creates unnamed storage).", + "configuration": "Configuration object used during the storage creation or restoration process.", + "storage_client": "Underlying storage client to use. If not provided, the default global storage client\nfrom the service locator will be used." + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_request_queue.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 119, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storages/_request_queue.py#L119" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Open a storage, either restore existing or create a new one.\n" + } + ] + }, + "flags": {}, + "id": 11344, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "open", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The storage ID." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 11345, + "kind": 32768, + "kindString": "Parameter", + "name": "id", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2336 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The storage name (global scope, persists across runs). Name can only contain letters \"a\" through \"z\",\nthe digits \"0\" through \"9\", and the hyphen (\"-\") but only in the middle of the string\n(e.g. \"my-value-1\")." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 11346, + "kind": 32768, + "kindString": "Parameter", + "name": "name", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2336 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The storage alias (run scope, creates unnamed storage)." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 11347, + "kind": 32768, + "kindString": "Parameter", + "name": "alias", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2336 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Configuration object used during the storage creation or restoration process." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 11348, + "kind": 32768, + "kindString": "Parameter", + "name": "configuration", + "type": { + "name": "Configuration | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "Configuration", + "target": "2", + "ref": { + "id": 9010, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/configuration.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 20 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/Configuration" + } + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2336 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Underlying storage client to use. If not provided, the default global storage client\nfrom the service locator will be used." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 11349, + "kind": 32768, + "kindString": "Parameter", + "name": "storage_client", + "type": { + "name": "StorageClient | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "StorageClient", + "target": "1282", + "ref": { + "id": 10290, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 19 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/StorageClient" + } + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2336 + } + ], + "type": { + "name": "Storage", + "type": "reference", + "target": "2330", + "ref": { + "id": 11338, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_base.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 15 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/Storage" + } + }, + "overwrites": { + "name": "Storage.open", + "target": 2335, + "type": "reference", + "ref": { + "id": 11343, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_base.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 34 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/Storage#open" + } + }, + "parentId": 2362 + } + ], + "overwrites": { + "name": "Storage.open", + "target": 2335, + "type": "reference", + "ref": { + "id": 11343, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_base.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 34 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/Storage#open" + } + }, + "permalink": "/python/api/class/RequestQueue#open", + "parentId": 2348 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Remove persistent state either from the Apify Cloud storage or from the local database." + } + ] + }, + "decorations": [ + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 11377, + "module": "request_loaders._request_manager", + "name": "drop", + "parsedDocstring": { + "text": "Remove persistent state either from the Apify Cloud storage or from the local database." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_request_queue.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 144, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storages/_request_queue.py#L144" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Remove persistent state either from the Apify Cloud storage or from the local database." + } + ] + }, + "flags": {}, + "id": 12011, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "drop", + "parameters": [], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "overwrites": { + "name": "Storage.drop", + "target": 2342, + "type": "reference", + "ref": { + "id": 11350, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_base.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 57 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/Storage#drop" + } + }, + "parentId": 2369 + } + ], + "overwrites": { + "name": "Storage.drop", + "target": 2342, + "type": "reference", + "ref": { + "id": 11350, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_base.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 57 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/Storage#drop" + } + }, + "permalink": "/python/api/class/RequestQueue#drop", + "parentId": 2348 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Remove all queued requests from this manager while keeping it usable.\n\nDefault implementation is a no-op. Subclasses with persistent state should override to clear it." + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 11379, + "module": "request_loaders._request_manager", + "name": "purge", + "parsedDocstring": { + "text": "Remove all queued requests from this manager while keeping it usable.\n\nDefault implementation is a no-op. Subclasses with persistent state should override to clear it." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_request_queue.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 152, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storages/_request_queue.py#L152" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Remove all queued requests from this manager while keeping it usable.\n\nDefault implementation is a no-op. Subclasses with persistent state should override to clear it." + } + ] + }, + "flags": {}, + "id": 12013, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "purge", + "parameters": [], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "overwrites": { + "name": "Storage.purge", + "target": 2344, + "type": "reference", + "ref": { + "id": 11352, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_base.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 61 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/Storage#purge" + } + }, + "parentId": 2371 + } + ], + "overwrites": { + "name": "Storage.purge", + "target": 2344, + "type": "reference", + "ref": { + "id": 11352, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_base.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 61 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/Storage#purge" + } + }, + "permalink": "/python/api/class/RequestQueue#purge", + "parentId": 2348 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Add a single request to the manager and store it in underlying resource client.\n" + } + ] + }, + "decorations": [ + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 11381, + "module": "request_loaders._request_manager", + "name": "add_request", + "parsedDocstring": { + "text": "Add a single request to the manager and store it in underlying resource client.\n", + "args": { + "request": "The request object (or its string representation) to be added to the manager.", + "forefront": "Determines whether the request should be added to the beginning (if True) or the end (if False)\nof the manager.\n" + }, + "returns": "Information about the request addition to the manager or None if the request was not added." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_request_queue.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 177, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storages/_request_queue.py#L177" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "Information about the request addition to the manager or None if the request was not added." + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Add a single request to the manager and store it in underlying resource client.\n" + } + ] + }, + "flags": {}, + "id": 12015, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "add_request", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The request object (or its string representation) to be added to the manager." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 12016, + "kind": 32768, + "kindString": "Parameter", + "name": "request", + "type": { + "name": "str | Request", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "reference", + "name": "Request", + "target": "409", + "ref": { + "id": 9417, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_request.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 145 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/Request" + } + } + ] + }, + "parentId": 3007 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Determines whether the request should be added to the beginning (if True) or the end (if False)\nof the manager.\n" + } + ] + }, + "defaultValue": "False", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 12017, + "kind": 32768, + "kindString": "Parameter", + "name": "forefront", + "type": { + "name": "bool", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 3007 + } + ], + "type": { + "name": "ProcessedRequest | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "ProcessedRequest", + "target": "1340", + "ref": { + "id": 10348, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 140 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/ProcessedRequest" + } + }, + { + "type": "literal", + "value": {} + } + ] + }, + "overwrites": { + "name": "RequestManager.add_request", + "target": 3006, + "type": "reference", + "ref": { + "id": 12014, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_manager.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 32 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestManager#add_request" + } + }, + "parentId": 2373 + } + ], + "overwrites": { + "name": "RequestManager.add_request", + "target": 3006, + "type": "reference", + "ref": { + "id": 12014, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_manager.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 32 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestManager#add_request" + } + }, + "permalink": "/python/api/class/RequestQueue#add_request", + "parentId": 2348 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Add requests to the manager in batches.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 11385, + "module": "request_loaders._request_manager", + "name": "add_requests", + "parsedDocstring": { + "text": "Add requests to the manager in batches.\n", + "args": { + "requests": "Requests to enqueue.", + "forefront": "If True, add requests to the beginning of the queue.", + "batch_size": "The number of requests to add in one batch.", + "wait_time_between_batches": "Time to wait between adding batches.", + "wait_for_all_requests_to_be_added": "If True, wait for all requests to be added before returning.", + "wait_for_all_requests_to_be_added_timeout": "Timeout for waiting for all requests to be added." + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_request_queue.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 201, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storages/_request_queue.py#L201" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Add requests to the manager in batches.\n" + } + ] + }, + "flags": {}, + "id": 12019, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "add_requests", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Requests to enqueue." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 12020, + "kind": 32768, + "kindString": "Parameter", + "name": "requests", + "type": { + "name": "Sequence", + "type": "reference", + "typeArguments": [ + { + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "reference", + "name": "Request", + "target": "409", + "ref": { + "id": 9417, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_request.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 145 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/Request" + } + } + ] + } + ], + "target": 0, + "ref": {} + }, + "parentId": 3011 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "If True, add requests to the beginning of the queue." + } + ] + }, + "defaultValue": "False", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 12021, + "kind": 32768, + "kindString": "Parameter", + "name": "forefront", + "type": { + "name": "bool", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 3011 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The number of requests to add in one batch." + } + ] + }, + "defaultValue": "1000", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 12022, + "kind": 32768, + "kindString": "Parameter", + "name": "batch_size", + "type": { + "name": "int", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 3011 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Time to wait between adding batches." + } + ] + }, + "defaultValue": "timedelta(seconds=1)", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 12023, + "kind": 32768, + "kindString": "Parameter", + "name": "wait_time_between_batches", + "type": { + "name": "timedelta", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 3011 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "If True, wait for all requests to be added before returning." + } + ] + }, + "defaultValue": "False", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 12024, + "kind": 32768, + "kindString": "Parameter", + "name": "wait_for_all_requests_to_be_added", + "type": { + "name": "bool", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 3011 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Timeout for waiting for all requests to be added." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 12025, + "kind": 32768, + "kindString": "Parameter", + "name": "wait_for_all_requests_to_be_added_timeout", + "type": { + "name": "timedelta | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "timedelta", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 3011 + } + ], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "overwrites": { + "name": "RequestManager.add_requests", + "target": 3010, + "type": "reference", + "ref": { + "id": 12018, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_manager.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 49 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestManager#add_requests" + } + }, + "parentId": 2377 + } + ], + "overwrites": { + "name": "RequestManager.add_requests", + "target": 3010, + "type": "reference", + "ref": { + "id": 12018, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_manager.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 49 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestManager#add_requests" + } + }, + "permalink": "/python/api/class/RequestQueue#add_requests", + "parentId": 2348 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Return the next request in the queue to be processed.\n\nOnce you successfully finish processing of the request, you need to call `RequestQueue.mark_request_as_handled`\nto mark the request as handled in the queue. If there was some error in processing the request, call\n`RequestQueue.reclaim_request` instead, so that the queue will give the request to some other consumer\nin another call to the `fetch_next_request` method.\n\nNote that the `None` return value does not mean the queue processing finished, it means there are currently\nno pending requests. To check whether all requests in queue were finished, use `RequestQueue.is_finished`\ninstead.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 11393, + "module": "storages._request_queue", + "name": "fetch_next_request", + "parsedDocstring": { + "text": "Return the next request in the queue to be processed.\n\nOnce you successfully finish processing of the request, you need to call `RequestQueue.mark_request_as_handled`\nto mark the request as handled in the queue. If there was some error in processing the request, call\n`RequestQueue.reclaim_request` instead, so that the queue will give the request to some other consumer\nin another call to the `fetch_next_request` method.\n\nNote that the `None` return value does not mean the queue processing finished, it means there are currently\nno pending requests. To check whether all requests in queue were finished, use `RequestQueue.is_finished`\ninstead.\n", + "returns": "The next request to process, or `None` if there are no more pending requests." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_request_queue.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 251, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storages/_request_queue.py#L251" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "The next request to process, or `None` if there are no more pending requests." + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Return the next request in the queue to be processed.\n\nOnce you successfully finish processing of the request, you need to call `RequestQueue.mark_request_as_handled`\nto mark the request as handled in the queue. If there was some error in processing the request, call\n`RequestQueue.reclaim_request` instead, so that the queue will give the request to some other consumer\nin another call to the `fetch_next_request` method.\n\nNote that the `None` return value does not mean the queue processing finished, it means there are currently\nno pending requests. To check whether all requests in queue were finished, use `RequestQueue.is_finished`\ninstead.\n" + } + ] + }, + "flags": {}, + "id": 11394, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "fetch_next_request", + "parameters": [], + "type": { + "name": "Request | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "Request", + "target": "409", + "ref": { + "id": 9417, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_request.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 145 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/Request" + } + }, + { + "type": "literal", + "value": {} + } + ] + }, + "overwrites": { + "name": "RequestManager.fetch_next_request", + "target": 4147, + "type": "reference", + "ref": { + "id": 13155, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 45 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestManager#fetch_next_request" + } + }, + "parentId": 2385 + } + ], + "overwrites": { + "name": "RequestManager.fetch_next_request", + "target": 4147, + "type": "reference", + "ref": { + "id": 13155, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 45 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestManager#fetch_next_request" + } + }, + "permalink": "/python/api/class/RequestQueue#fetch_next_request", + "parentId": 2348 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Retrieve a specific request from the queue by its ID.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 11395, + "module": "storages._request_queue", + "name": "get_request", + "parsedDocstring": { + "text": "Retrieve a specific request from the queue by its ID.\n", + "args": { + "unique_key": "Unique key of the request to retrieve.\n" + }, + "returns": "The request with the specified ID, or `None` if no such request exists." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_request_queue.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 268, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storages/_request_queue.py#L268" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "The request with the specified ID, or `None` if no such request exists." + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Retrieve a specific request from the queue by its ID.\n" + } + ] + }, + "flags": {}, + "id": 11396, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "get_request", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Unique key of the request to retrieve.\n" + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 11397, + "kind": 32768, + "kindString": "Parameter", + "name": "unique_key", + "type": { + "name": "str", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2388 + } + ], + "type": { + "name": "Request | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "Request", + "target": "409", + "ref": { + "id": 9417, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_request.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 145 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/Request" + } + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2387 + } + ], + "permalink": "/python/api/class/RequestQueue#get_request", + "parentId": 2348 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Mark a request as handled after successful processing.\n\nThis method should be called after a request has been successfully processed.\nOnce marked as handled, the request will be removed from the queue and will\nnot be returned in subsequent calls to `fetch_next_request` method.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 11398, + "module": "storages._request_queue", + "name": "mark_request_as_handled", + "parsedDocstring": { + "text": "Mark a request as handled after successful processing.\n\nThis method should be called after a request has been successfully processed.\nOnce marked as handled, the request will be removed from the queue and will\nnot be returned in subsequent calls to `fetch_next_request` method.\n", + "args": { + "request": "The request to mark as handled.\n" + }, + "returns": "Information about the queue operation." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_request_queue.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 279, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storages/_request_queue.py#L279" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "Information about the queue operation." + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Mark a request as handled after successful processing.\n\nThis method should be called after a request has been successfully processed.\nOnce marked as handled, the request will be removed from the queue and will\nnot be returned in subsequent calls to `fetch_next_request` method.\n" + } + ] + }, + "flags": {}, + "id": 11399, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "mark_request_as_handled", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The request to mark as handled.\n" + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 11400, + "kind": 32768, + "kindString": "Parameter", + "name": "request", + "type": { + "name": "Request", + "type": "reference", + "target": "409", + "ref": { + "id": 9417, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_request.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 145 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/Request" + } + }, + "parentId": 2391 + } + ], + "type": { + "name": "ProcessedRequest | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "ProcessedRequest", + "target": "1340", + "ref": { + "id": 10348, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 140 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/ProcessedRequest" + } + }, + { + "type": "literal", + "value": {} + } + ] + }, + "overwrites": { + "name": "RequestManager.mark_request_as_handled", + "target": 4148, + "type": "reference", + "ref": { + "id": 13156, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 53 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestManager#mark_request_as_handled" + } + }, + "parentId": 2390 + } + ], + "overwrites": { + "name": "RequestManager.mark_request_as_handled", + "target": 4148, + "type": "reference", + "ref": { + "id": 13156, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 53 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestManager#mark_request_as_handled" + } + }, + "permalink": "/python/api/class/RequestQueue#mark_request_as_handled", + "parentId": 2348 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Reclaim a failed request back to the queue for later processing.\n\nIf a request fails during processing, this method can be used to return it to the queue.\nThe request will be returned for processing again in a subsequent call\nto `RequestQueue.fetch_next_request`.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 11401, + "module": "storages._request_queue", + "name": "reclaim_request", + "parsedDocstring": { + "text": "Reclaim a failed request back to the queue for later processing.\n\nIf a request fails during processing, this method can be used to return it to the queue.\nThe request will be returned for processing again in a subsequent call\nto `RequestQueue.fetch_next_request`.\n", + "args": { + "request": "The request to return to the queue.", + "forefront": "If true, the request will be added to the beginning of the queue.\nOtherwise, it will be added to the end.\n" + }, + "returns": "Information about the queue operation." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_request_queue.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 294, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storages/_request_queue.py#L294" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "Information about the queue operation." + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Reclaim a failed request back to the queue for later processing.\n\nIf a request fails during processing, this method can be used to return it to the queue.\nThe request will be returned for processing again in a subsequent call\nto `RequestQueue.fetch_next_request`.\n" + } + ] + }, + "flags": {}, + "id": 11402, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "reclaim_request", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The request to return to the queue." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 11403, + "kind": 32768, + "kindString": "Parameter", + "name": "request", + "type": { + "name": "Request", + "type": "reference", + "target": "409", + "ref": { + "id": 9417, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_request.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 145 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/Request" + } + }, + "parentId": 2394 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "If true, the request will be added to the beginning of the queue.\nOtherwise, it will be added to the end.\n" + } + ] + }, + "defaultValue": "False", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 11404, + "kind": 32768, + "kindString": "Parameter", + "name": "forefront", + "type": { + "name": "bool", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2394 + } + ], + "type": { + "name": "ProcessedRequest | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "ProcessedRequest", + "target": "1340", + "ref": { + "id": 10348, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 140 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/ProcessedRequest" + } + }, + { + "type": "literal", + "value": {} + } + ] + }, + "overwrites": { + "name": "RequestManager.reclaim_request", + "target": 3018, + "type": "reference", + "ref": { + "id": 12026, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_manager.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 77 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestManager#reclaim_request" + } + }, + "parentId": 2393 + } + ], + "overwrites": { + "name": "RequestManager.reclaim_request", + "target": 3018, + "type": "reference", + "ref": { + "id": 12026, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_manager.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 77 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestManager#reclaim_request" + } + }, + "permalink": "/python/api/class/RequestQueue#reclaim_request", + "parentId": 2348 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Check if the request queue is empty.\n\nAn empty queue means that there are no requests currently in the queue, either pending or being processed.\nHowever, this does not necessarily mean that the crawling operation is finished, as there still might be\ntasks that could add additional requests to the queue.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 11405, + "module": "storages._request_queue", + "name": "is_empty", + "parsedDocstring": { + "text": "Check if the request queue is empty.\n\nAn empty queue means that there are no requests currently in the queue, either pending or being processed.\nHowever, this does not necessarily mean that the crawling operation is finished, as there still might be\ntasks that could add additional requests to the queue.\n", + "returns": "True if the request queue is empty, False otherwise." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_request_queue.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 316, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storages/_request_queue.py#L316" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "True if the request queue is empty, False otherwise." + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Check if the request queue is empty.\n\nAn empty queue means that there are no requests currently in the queue, either pending or being processed.\nHowever, this does not necessarily mean that the crawling operation is finished, as there still might be\ntasks that could add additional requests to the queue.\n" + } + ] + }, + "flags": {}, + "id": 11406, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "is_empty", + "parameters": [], + "type": { + "name": "bool", + "type": "reference", + "target": 0, + "ref": {} + }, + "overwrites": { + "name": "RequestManager.is_empty", + "target": 4145, + "type": "reference", + "ref": { + "id": 13153, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 37 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestManager#is_empty" + } + }, + "parentId": 2397 + } + ], + "overwrites": { + "name": "RequestManager.is_empty", + "target": 4145, + "type": "reference", + "ref": { + "id": 13153, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 37 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestManager#is_empty" + } + }, + "permalink": "/python/api/class/RequestQueue#is_empty", + "parentId": 2348 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Check if the request queue is finished.\n\nA finished queue means that all requests in the queue have been processed (the queue is empty) and there\nare no more tasks that could add additional requests to the queue. This is the definitive way to check\nif a crawling operation is complete.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 11407, + "module": "storages._request_queue", + "name": "is_finished", + "parsedDocstring": { + "text": "Check if the request queue is finished.\n\nA finished queue means that all requests in the queue have been processed (the queue is empty) and there\nare no more tasks that could add additional requests to the queue. This is the definitive way to check\nif a crawling operation is complete.\n", + "returns": "True if the request queue is finished (empty and no pending add operations), False otherwise." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_request_queue.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 328, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storages/_request_queue.py#L328" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "True if the request queue is finished (empty and no pending add operations), False otherwise." + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Check if the request queue is finished.\n\nA finished queue means that all requests in the queue have been processed (the queue is empty) and there\nare no more tasks that could add additional requests to the queue. This is the definitive way to check\nif a crawling operation is complete.\n" + } + ] + }, + "flags": {}, + "id": 11408, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "is_finished", + "parameters": [], + "type": { + "name": "bool", + "type": "reference", + "target": 0, + "ref": {} + }, + "overwrites": { + "name": "RequestManager.is_finished", + "target": 4146, + "type": "reference", + "ref": { + "id": 13154, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 41 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestManager#is_finished" + } + }, + "parentId": 2399 + } + ], + "overwrites": { + "name": "RequestManager.is_finished", + "target": 4146, + "type": "reference", + "ref": { + "id": 13154, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 41 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestManager#is_finished" + } + }, + "permalink": "/python/api/class/RequestQueue#is_finished", + "parentId": 2348 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Combine the loader with a request manager to support adding and reclaiming requests.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 13158, + "module": "request_loaders._request_loader", + "name": "to_tandem", + "parsedDocstring": { + "text": "Combine the loader with a request manager to support adding and reclaiming requests.\n", + "args": { + "request_manager": "Request manager to combine the loader with.\nIf None is given, the default request queue is used." + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 56, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/request_loaders/_request_loader.py#L56" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Combine the loader with a request manager to support adding and reclaiming requests.\n" + } + ] + }, + "flags": {}, + "id": 11957, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "to_tandem", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Request manager to combine the loader with.\nIf None is given, the default request queue is used." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": false + }, + "id": 11958, + "kind": 32768, + "kindString": "Parameter", + "name": "request_manager", + "type": { + "name": "RequestManager | None", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2949 + } + ], + "type": { + "name": "RequestManagerTandem", + "type": "reference", + "target": "2803", + "ref": { + "id": 11811, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_manager_tandem.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 24 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/RequestManagerTandem" + } + }, + "inheritedFrom": { + "name": "RequestLoader.to_tandem", + "target": 2948, + "type": "reference", + "ref": { + "id": 11956, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 56 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestLoader#to_tandem" + } + }, + "parentId": 4150 + } + ], + "inheritedFrom": { + "name": "RequestLoader.to_tandem", + "target": 2948, + "type": "reference", + "ref": { + "id": 11956, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 56 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestLoader#to_tandem" + } + }, + "permalink": "/python/api/class/RequestQueue#to_tandem", + "parentId": 2348 + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Request queue is a storage for managing HTTP requests.\n\nThe request queue class serves as a high-level interface for organizing and managing HTTP requests\nduring web crawling. It provides methods for adding, retrieving, and manipulating requests throughout\nthe crawling lifecycle, abstracting away the underlying storage implementation details.\n\nRequest queue maintains the state of each URL to be crawled, tracking whether it has been processed,\nis currently being handled, or is waiting in the queue. Each URL in the queue is uniquely identified\nby a `unique_key` property, which prevents duplicate processing unless explicitly configured otherwise.\n\nThe class supports both breadth-first and depth-first crawling strategies through its `forefront` parameter\nwhen adding requests. It also provides mechanisms for error handling and request reclamation when\nprocessing fails.\n\nYou can open a request queue using the `open` class method, specifying either a name or ID to identify\nthe queue. The underlying storage implementation is determined by the configured storage client.\n\n### Usage\n\n```python\nfrom crawlee.storages import RequestQueue\n\n# Open a request queue\nrq = await RequestQueue.open(name='my-queue')\n\n# Add a request\nawait rq.add_request('https://example.com')\n\n# Process requests\nrequest = await rq.fetch_next_request()\nif request:\n try:\n # Process the request\n # ...\n await rq.mark_request_as_handled(request)\n except Exception:\n await rq.reclaim_request(request)\n```" + } + ] + }, + "decorations": [ + { + "args": "('Storages')", + "name": "docs_group" + } + ], + "flags": {}, + "groups": [ + { + "children": [ + 11357, + 11381, + 11385, + 11377, + 11393, + 11366, + 11364, + 11395, + 11368, + 11405, + 11407, + 11398, + 11370, + 11379, + 11401, + 13158 + ], + "title": "Methods" + }, + { + "children": [ + 11362, + 11363 + ], + "title": "Properties" + } + ], + "id": 11356, + "module": "storages._request_queue", + "name": "RequestQueue", + "parsedDocstring": { + "text": "Request queue is a storage for managing HTTP requests.\n\nThe request queue class serves as a high-level interface for organizing and managing HTTP requests\nduring web crawling. It provides methods for adding, retrieving, and manipulating requests throughout\nthe crawling lifecycle, abstracting away the underlying storage implementation details.\n\nRequest queue maintains the state of each URL to be crawled, tracking whether it has been processed,\nis currently being handled, or is waiting in the queue. Each URL in the queue is uniquely identified\nby a `unique_key` property, which prevents duplicate processing unless explicitly configured otherwise.\n\nThe class supports both breadth-first and depth-first crawling strategies through its `forefront` parameter\nwhen adding requests. It also provides mechanisms for error handling and request reclamation when\nprocessing fails.\n\nYou can open a request queue using the `open` class method, specifying either a name or ID to identify\nthe queue. The underlying storage implementation is determined by the configured storage client.\n\n### Usage\n\n```python\nfrom crawlee.storages import RequestQueue\n\n# Open a request queue\nrq = await RequestQueue.open(name='my-queue')\n\n# Add a request\nawait rq.add_request('https://example.com')\n\n# Process requests\nrequest = await rq.fetch_next_request()\nif request:\n try:\n # Process the request\n # ...\n await rq.mark_request_as_handled(request)\n except Exception:\n await rq.reclaim_request(request)\n```" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_request_queue.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 33, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storages/_request_queue.py#L33" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "extendedTypes": [ + { + "name": "RequestManager", + "target": "3001", + "type": "reference", + "ref": { + "id": 12009, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_manager.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 18 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/RequestManager" + } + }, + { + "name": "Storage", + "target": "2330", + "type": "reference", + "ref": { + "id": 11338, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_base.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 15 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/Storage" + } + } + ], + "permalink": "/python/api/class/RequestQueue", + "categories": [] + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 14510, + "module": "storage_clients.models", + "name": "model_config", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 173, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/models.py#L173" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "permalink": "/python/api/class/AddRequestsResponse#model_config", + "parentId": 1351 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Successfully processed requests, including information about whether they were\nalready present in the queue and whether they had been handled previously." + } + ] + }, + "flags": {}, + "groups": [], + "id": 14511, + "module": "storage_clients.models", + "name": "processed_requests", + "parsedDocstring": { + "text": "Successfully processed requests, including information about whether they were\nalready present in the queue and whether they had been handled previously." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 175, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/models.py#L175" + } + ], + "type": { + "name": "list", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "ProcessedRequest", + "target": "1340", + "ref": { + "id": 14498, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 140 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/ProcessedRequest" + } + } + ], + "target": 0, + "ref": {} + }, + "permalink": "/python/api/class/AddRequestsResponse#processed_requests", + "parentId": 1351 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Requests that could not be processed, typically due to validation errors or other issues." + } + ] + }, + "flags": {}, + "groups": [], + "id": 14512, + "module": "storage_clients.models", + "name": "unprocessed_requests", + "parsedDocstring": { + "text": "Requests that could not be processed, typically due to validation errors or other issues." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 179, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/models.py#L179" + } + ], + "type": { + "name": "list", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "UnprocessedRequest", + "target": "1346", + "ref": { + "id": 14504, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 154 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/UnprocessedRequest" + } + } + ], + "target": 0, + "ref": {} + }, + "permalink": "/python/api/class/AddRequestsResponse#unprocessed_requests", + "parentId": 1351 + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Model for a response to add requests to a queue.\n\nContains detailed information about the processing results when adding multiple requests\nto a queue. This includes which requests were successfully processed and which ones\nencountered issues during processing." + } + ] + }, + "decorations": [ + { + "args": "('Storage data')", + "name": "docs_group" + } + ], + "flags": {}, + "groups": [ + { + "children": [ + 14510, + 14511, + 14512 + ], + "title": "Properties" + } + ], + "id": 14509, + "module": "storage_clients.models", + "name": "AddRequestsResponse", + "parsedDocstring": { + "text": "Model for a response to add requests to a queue.\n\nContains detailed information about the processing results when adding multiple requests\nto a queue. This includes which requests were successfully processed and which ones\nencountered issues during processing." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 165, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/models.py#L165" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "permalink": "/python/api/class/AddRequestsResponse", + "categories": [] + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 15846, + "module": "storage_clients.models", + "name": "model_config", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 113, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/models.py#L113" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "permalink": "/python/api/class/DatasetItemsListPage#model_config", + "parentId": 1333 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "The number of objects returned on this page." + } + ] + }, + "flags": {}, + "groups": [], + "id": 15847, + "module": "storage_clients.models", + "name": "count", + "parsedDocstring": { + "text": "The number of objects returned on this page." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 115, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/models.py#L115" + } + ], + "type": { + "name": "int", + "type": "reference", + "target": 0, + "ref": {} + }, + "permalink": "/python/api/class/DatasetItemsListPage#count", + "parentId": 1333 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "The starting position of the first object returned, as specified in the API call." + } + ] + }, + "flags": {}, + "groups": [], + "id": 15848, + "module": "storage_clients.models", + "name": "offset", + "parsedDocstring": { + "text": "The starting position of the first object returned, as specified in the API call." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 118, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/models.py#L118" + } + ], + "type": { + "name": "int", + "type": "reference", + "target": 0, + "ref": {} + }, + "permalink": "/python/api/class/DatasetItemsListPage#offset", + "parentId": 1333 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "The maximum number of objects to return, as specified in the API call." + } + ] + }, + "flags": {}, + "groups": [], + "id": 15849, + "module": "storage_clients.models", + "name": "limit", + "parsedDocstring": { + "text": "The maximum number of objects to return, as specified in the API call." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 121, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/models.py#L121" + } + ], + "type": { + "name": "int", + "type": "reference", + "target": 0, + "ref": {} + }, + "permalink": "/python/api/class/DatasetItemsListPage#limit", + "parentId": 1333 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "The total number of objects that match the criteria of the API call." + } + ] + }, + "flags": {}, + "groups": [], + "id": 15850, + "module": "storage_clients.models", + "name": "total", + "parsedDocstring": { + "text": "The total number of objects that match the criteria of the API call." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 124, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/models.py#L124" + } + ], + "type": { + "name": "int", + "type": "reference", + "target": 0, + "ref": {} + }, + "permalink": "/python/api/class/DatasetItemsListPage#total", + "parentId": 1333 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Indicates if the returned list is in descending order." + } + ] + }, + "flags": {}, + "groups": [], + "id": 15851, + "module": "storage_clients.models", + "name": "desc", + "parsedDocstring": { + "text": "Indicates if the returned list is in descending order." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 127, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/models.py#L127" + } + ], + "type": { + "name": "bool", + "type": "reference", + "target": 0, + "ref": {} + }, + "permalink": "/python/api/class/DatasetItemsListPage#desc", + "parentId": 1333 + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Model for a single page of dataset items returned from a collection list method." + } + ] + }, + "decorations": [ + { + "args": "('Storage data')", + "name": "docs_group" + } + ], + "flags": {}, + "groups": [ + { + "children": [ + 15847, + 15851, + 15849, + 15846, + 15848, + 15850 + ], + "title": "Properties" + } + ], + "id": 15845, + "module": "storage_clients.models", + "name": "DatasetItemsListPage", + "parsedDocstring": { + "text": "Model for a single page of dataset items returned from a collection list method." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 110, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/models.py#L110" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "permalink": "/python/api/class/DatasetItemsListPage", + "categories": [] + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 17166, + "module": "storage_clients.models", + "name": "model_config", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 45, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/models.py#L45" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "overwrites": { + "name": "StorageMetadata.model_config", + "target": 1308, + "type": "reference", + "ref": { + "id": 17159, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 23 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api/class/StorageMetadata#model_config" + } + }, + "permalink": "/python/api/class/DatasetMetadata#model_config", + "parentId": 1314 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "The number of items in the dataset." + } + ] + }, + "flags": {}, + "groups": [], + "id": 17167, + "module": "storage_clients.models", + "name": "item_count", + "parsedDocstring": { + "text": "The number of items in the dataset." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 47, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/models.py#L47" + } + ], + "type": { + "name": "int", + "type": "reference", + "target": 0, + "ref": {} + }, + "permalink": "/python/api/class/DatasetMetadata#item_count", + "parentId": 1314 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "The unique identifier of the storage." + } + ] + }, + "flags": {}, + "groups": [], + "id": 20060, + "module": "storage_clients.models", + "name": "id", + "parsedDocstring": { + "text": "The unique identifier of the storage." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 25, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/models.py#L25" + } + ], + "type": { + "name": "Annotated[str, Field(alias='id')]", + "type": "reference", + "target": 0, + "ref": {} + }, + "inheritedFrom": { + "name": "StorageMetadata.id", + "target": 1309, + "type": "reference", + "ref": { + "id": 17160, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 25 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api/class/StorageMetadata#id" + } + }, + "permalink": "/python/api/class/DatasetMetadata#id", + "parentId": 1314 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "The name of the storage." + } + ] + }, + "flags": {}, + "groups": [], + "id": 20061, + "module": "storage_clients.models", + "name": "name", + "parsedDocstring": { + "text": "The name of the storage." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 28, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/models.py#L28" + } + ], + "type": { + "name": "Annotated[str | None, Field(alias='name', default=None)]", + "type": "reference", + "target": 0, + "ref": {} + }, + "inheritedFrom": { + "name": "StorageMetadata.name", + "target": 1310, + "type": "reference", + "ref": { + "id": 17161, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 28 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api/class/StorageMetadata#name" + } + }, + "permalink": "/python/api/class/DatasetMetadata#name", + "parentId": 1314 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "The timestamp when the storage was last accessed." + } + ] + }, + "flags": {}, + "groups": [], + "id": 20062, + "module": "storage_clients.models", + "name": "accessed_at", + "parsedDocstring": { + "text": "The timestamp when the storage was last accessed." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 31, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/models.py#L31" + } + ], + "type": { + "name": "Annotated[datetime, Field(alias='accessedAt')]", + "type": "reference", + "target": 0, + "ref": {} + }, + "inheritedFrom": { + "name": "StorageMetadata.accessed_at", + "target": 1311, + "type": "reference", + "ref": { + "id": 17162, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 31 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api/class/StorageMetadata#accessed_at" + } + }, + "permalink": "/python/api/class/DatasetMetadata#accessed_at", + "parentId": 1314 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "The timestamp when the storage was created." + } + ] + }, + "flags": {}, + "groups": [], + "id": 20063, + "module": "storage_clients.models", + "name": "created_at", + "parsedDocstring": { + "text": "The timestamp when the storage was created." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 34, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/models.py#L34" + } + ], + "type": { + "name": "Annotated[datetime, Field(alias='createdAt')]", + "type": "reference", + "target": 0, + "ref": {} + }, + "inheritedFrom": { + "name": "StorageMetadata.created_at", + "target": 1312, + "type": "reference", + "ref": { + "id": 17163, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 34 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api/class/StorageMetadata#created_at" + } + }, + "permalink": "/python/api/class/DatasetMetadata#created_at", + "parentId": 1314 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "The timestamp when the storage was last modified." + } + ] + }, + "flags": {}, + "groups": [], + "id": 20064, + "module": "storage_clients.models", + "name": "modified_at", + "parsedDocstring": { + "text": "The timestamp when the storage was last modified." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 37, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/models.py#L37" + } + ], + "type": { + "name": "Annotated[datetime, Field(alias='modifiedAt')]", + "type": "reference", + "target": 0, + "ref": {} + }, + "inheritedFrom": { + "name": "StorageMetadata.modified_at", + "target": 1313, + "type": "reference", + "ref": { + "id": 17164, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 37 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api/class/StorageMetadata#modified_at" + } + }, + "permalink": "/python/api/class/DatasetMetadata#modified_at", + "parentId": 1314 + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Model for a dataset metadata." + } + ] + }, + "decorations": [ + { + "args": "('Storage data')", + "name": "docs_group" + } + ], + "flags": {}, + "groups": [ + { + "children": [ + 20062, + 20063, + 20060, + 17167, + 17166, + 20064, + 20061 + ], + "title": "Properties" + } + ], + "id": 17165, + "module": "storage_clients.models", + "name": "DatasetMetadata", + "parsedDocstring": { + "text": "Model for a dataset metadata." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 42, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/models.py#L42" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "extendedTypes": [ + { + "name": "StorageMetadata", + "target": "1307", + "type": "reference", + "ref": { + "id": 17158, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 17 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/StorageMetadata" + } + } + ], + "permalink": "/python/api/class/DatasetMetadata", + "categories": [] + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 21382, + "module": "storage_clients.models", + "name": "model_config", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 55, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/models.py#L55" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "overwrites": { + "name": "StorageMetadata.model_config", + "target": 1308, + "type": "reference", + "ref": { + "id": 21372, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 23 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api/class/StorageMetadata#model_config" + } + }, + "permalink": "/python/api/class/KeyValueStoreMetadata#model_config", + "parentId": 1317 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "The unique identifier of the storage." + } + ] + }, + "flags": {}, + "groups": [], + "id": 24278, + "module": "storage_clients.models", + "name": "id", + "parsedDocstring": { + "text": "The unique identifier of the storage." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 25, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/models.py#L25" + } + ], + "type": { + "name": "Annotated[str, Field(alias='id')]", + "type": "reference", + "target": 0, + "ref": {} + }, + "inheritedFrom": { + "name": "StorageMetadata.id", + "target": 1309, + "type": "reference", + "ref": { + "id": 21373, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 25 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api/class/StorageMetadata#id" + } + }, + "permalink": "/python/api/class/KeyValueStoreMetadata#id", + "parentId": 1317 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "The name of the storage." + } + ] + }, + "flags": {}, + "groups": [], + "id": 24279, + "module": "storage_clients.models", + "name": "name", + "parsedDocstring": { + "text": "The name of the storage." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 28, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/models.py#L28" + } + ], + "type": { + "name": "Annotated[str | None, Field(alias='name', default=None)]", + "type": "reference", + "target": 0, + "ref": {} + }, + "inheritedFrom": { + "name": "StorageMetadata.name", + "target": 1310, + "type": "reference", + "ref": { + "id": 21374, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 28 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api/class/StorageMetadata#name" + } + }, + "permalink": "/python/api/class/KeyValueStoreMetadata#name", + "parentId": 1317 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "The timestamp when the storage was last accessed." + } + ] + }, + "flags": {}, + "groups": [], + "id": 24280, + "module": "storage_clients.models", + "name": "accessed_at", + "parsedDocstring": { + "text": "The timestamp when the storage was last accessed." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 31, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/models.py#L31" + } + ], + "type": { + "name": "Annotated[datetime, Field(alias='accessedAt')]", + "type": "reference", + "target": 0, + "ref": {} + }, + "inheritedFrom": { + "name": "StorageMetadata.accessed_at", + "target": 1311, + "type": "reference", + "ref": { + "id": 21375, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 31 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api/class/StorageMetadata#accessed_at" + } + }, + "permalink": "/python/api/class/KeyValueStoreMetadata#accessed_at", + "parentId": 1317 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "The timestamp when the storage was created." + } + ] + }, + "flags": {}, + "groups": [], + "id": 24281, + "module": "storage_clients.models", + "name": "created_at", + "parsedDocstring": { + "text": "The timestamp when the storage was created." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 34, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/models.py#L34" + } + ], + "type": { + "name": "Annotated[datetime, Field(alias='createdAt')]", + "type": "reference", + "target": 0, + "ref": {} + }, + "inheritedFrom": { + "name": "StorageMetadata.created_at", + "target": 1312, + "type": "reference", + "ref": { + "id": 21376, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 34 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api/class/StorageMetadata#created_at" + } + }, + "permalink": "/python/api/class/KeyValueStoreMetadata#created_at", + "parentId": 1317 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "The timestamp when the storage was last modified." + } + ] + }, + "flags": {}, + "groups": [], + "id": 24282, + "module": "storage_clients.models", + "name": "modified_at", + "parsedDocstring": { + "text": "The timestamp when the storage was last modified." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 37, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/models.py#L37" + } + ], + "type": { + "name": "Annotated[datetime, Field(alias='modifiedAt')]", + "type": "reference", + "target": 0, + "ref": {} + }, + "inheritedFrom": { + "name": "StorageMetadata.modified_at", + "target": 1313, + "type": "reference", + "ref": { + "id": 21377, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 37 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api/class/StorageMetadata#modified_at" + } + }, + "permalink": "/python/api/class/KeyValueStoreMetadata#modified_at", + "parentId": 1317 + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Model for a key-value store metadata." + } + ] + }, + "decorations": [ + { + "args": "('Storage data')", + "name": "docs_group" + } + ], + "flags": {}, + "groups": [ + { + "children": [ + 24280, + 24281, + 24278, + 21382, + 24282, + 24279 + ], + "title": "Properties" + } + ], + "id": 21381, + "module": "storage_clients.models", + "name": "KeyValueStoreMetadata", + "parsedDocstring": { + "text": "Model for a key-value store metadata." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 52, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/models.py#L52" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "extendedTypes": [ + { + "name": "StorageMetadata", + "target": "1307", + "type": "reference", + "ref": { + "id": 21371, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 17 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/StorageMetadata" + } + } + ], + "permalink": "/python/api/class/KeyValueStoreMetadata", + "categories": [] + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 25613, + "module": "storage_clients.models", + "name": "model_config", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 103, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/models.py#L103" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "overwrites": { + "name": "KeyValueStoreRecordMetadata.model_config", + "target": 1326, + "type": "reference", + "ref": { + "id": 25608, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 81 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api/class/KeyValueStoreRecordMetadata#model_config" + } + }, + "permalink": "/python/api/class/KeyValueStoreRecord#model_config", + "parentId": 1330 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "The value of the record." + } + ] + }, + "flags": {}, + "groups": [], + "id": 25614, + "module": "storage_clients.models", + "name": "value", + "parsedDocstring": { + "text": "The value of the record." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 105, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/models.py#L105" + } + ], + "type": { + "name": "KvsValueType", + "type": "reference", + "target": "1306", + "ref": { + "id": 25588, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 13 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api#KvsValueType" + } + }, + "permalink": "/python/api/class/KeyValueStoreRecord#value", + "parentId": 1330 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "The key of the record.\n\nA unique identifier for the record in the key-value store." + } + ] + }, + "flags": {}, + "groups": [], + "id": 28488, + "module": "storage_clients.models", + "name": "key", + "parsedDocstring": { + "text": "The key of the record.\n\nA unique identifier for the record in the key-value store." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 83, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/models.py#L83" + } + ], + "type": { + "name": "Annotated[str, Field(alias='key')]", + "type": "reference", + "target": 0, + "ref": {} + }, + "inheritedFrom": { + "name": "KeyValueStoreRecordMetadata.key", + "target": 1327, + "type": "reference", + "ref": { + "id": 25609, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 83 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api/class/KeyValueStoreRecordMetadata#key" + } + }, + "permalink": "/python/api/class/KeyValueStoreRecord#key", + "parentId": 1330 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "The MIME type of the record.\n\nDescribe the format and type of data stored in the record, following the MIME specification." + } + ] + }, + "flags": {}, + "groups": [], + "id": 28489, + "module": "storage_clients.models", + "name": "content_type", + "parsedDocstring": { + "text": "The MIME type of the record.\n\nDescribe the format and type of data stored in the record, following the MIME specification." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 89, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/models.py#L89" + } + ], + "type": { + "name": "Annotated[str, Field(alias='contentType')]", + "type": "reference", + "target": 0, + "ref": {} + }, + "inheritedFrom": { + "name": "KeyValueStoreRecordMetadata.content_type", + "target": 1328, + "type": "reference", + "ref": { + "id": 25610, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 89 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api/class/KeyValueStoreRecordMetadata#content_type" + } + }, + "permalink": "/python/api/class/KeyValueStoreRecord#content_type", + "parentId": 1330 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "The size of the record in bytes." + } + ] + }, + "flags": {}, + "groups": [], + "id": 28490, + "module": "storage_clients.models", + "name": "size", + "parsedDocstring": { + "text": "The size of the record in bytes." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 95, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/models.py#L95" + } + ], + "type": { + "name": "Annotated[int | None, Field(alias='size', default=None)]", + "type": "reference", + "target": 0, + "ref": {} + }, + "inheritedFrom": { + "name": "KeyValueStoreRecordMetadata.size", + "target": 1329, + "type": "reference", + "ref": { + "id": 25611, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 95 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api/class/KeyValueStoreRecordMetadata#size" + } + }, + "permalink": "/python/api/class/KeyValueStoreRecord#size", + "parentId": 1330 + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Model for a key-value store record." + } + ] + }, + "decorations": [ + { + "args": "('Storage data')", + "name": "docs_group" + } + ], + "flags": {}, + "groups": [ + { + "children": [ + 28489, + 28488, + 25613, + 28490, + 25614 + ], + "title": "Properties" + } + ], + "id": 25612, + "module": "storage_clients.models", + "name": "KeyValueStoreRecord", + "parsedDocstring": { + "text": "Model for a key-value store record." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 100, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/models.py#L100" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "extendedTypes": [ + { + "name": "KeyValueStoreRecordMetadata", + "target": "1325", + "type": "reference", + "ref": { + "id": 25607, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 78 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/KeyValueStoreRecordMetadata" + } + } + ], + "permalink": "/python/api/class/KeyValueStoreRecord", + "categories": [] + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 29816, + "module": "storage_clients.models", + "name": "model_config", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 81, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/models.py#L81" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "permalink": "/python/api/class/KeyValueStoreRecordMetadata#model_config", + "parentId": 1325 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "The key of the record.\n\nA unique identifier for the record in the key-value store." + } + ] + }, + "flags": {}, + "groups": [], + "id": 29817, + "module": "storage_clients.models", + "name": "key", + "parsedDocstring": { + "text": "The key of the record.\n\nA unique identifier for the record in the key-value store." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 83, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/models.py#L83" + } + ], + "type": { + "name": "str", + "type": "reference", + "target": 0, + "ref": {} + }, + "permalink": "/python/api/class/KeyValueStoreRecordMetadata#key", + "parentId": 1325 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "The MIME type of the record.\n\nDescribe the format and type of data stored in the record, following the MIME specification." + } + ] + }, + "flags": {}, + "groups": [], + "id": 29818, + "module": "storage_clients.models", + "name": "content_type", + "parsedDocstring": { + "text": "The MIME type of the record.\n\nDescribe the format and type of data stored in the record, following the MIME specification." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 89, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/models.py#L89" + } + ], + "type": { + "name": "str", + "type": "reference", + "target": 0, + "ref": {} + }, + "permalink": "/python/api/class/KeyValueStoreRecordMetadata#content_type", + "parentId": 1325 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "The size of the record in bytes." + } + ] + }, + "flags": {}, + "groups": [], + "id": 29819, + "module": "storage_clients.models", + "name": "size", + "parsedDocstring": { + "text": "The size of the record in bytes." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 95, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/models.py#L95" + } + ], + "type": { + "name": "Annotated[int | None, Field(alias='size', default=None)]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "int", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "permalink": "/python/api/class/KeyValueStoreRecordMetadata#size", + "parentId": 1325 + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Model for a key-value store record metadata." + } + ] + }, + "decorations": [ + { + "args": "('Storage data')", + "name": "docs_group" + } + ], + "flags": {}, + "groups": [ + { + "children": [ + 29818, + 29817, + 29816, + 29819 + ], + "title": "Properties" + } + ], + "id": 29815, + "module": "storage_clients.models", + "name": "KeyValueStoreRecordMetadata", + "parsedDocstring": { + "text": "Model for a key-value store record metadata." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 78, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/models.py#L78" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "extendedBy": [ + { + "name": "KeyValueStoreRecord", + "target": "1330", + "type": "reference", + "ref": { + "id": 29820, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 100 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/KeyValueStoreRecord" + } + } + ], + "permalink": "/python/api/class/KeyValueStoreRecordMetadata", + "categories": [] + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 31161, + "module": "storage_clients.models", + "name": "model_config", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 143, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/models.py#L143" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "permalink": "/python/api/class/ProcessedRequest#model_config", + "parentId": 1340 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Internal representation of the request by the storage client. Only some clients use id." + } + ] + }, + "flags": {}, + "groups": [], + "id": 31162, + "module": "storage_clients.models", + "name": "id", + "parsedDocstring": { + "text": "Internal representation of the request by the storage client. Only some clients use id." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 145, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/models.py#L145" + } + ], + "type": { + "name": "Annotated[str | None, Field(alias='requestId', default=None)]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "permalink": "/python/api/class/ProcessedRequest#id", + "parentId": 1340 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 31163, + "module": "storage_clients.models", + "name": "unique_key", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 148, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/models.py#L148" + } + ], + "type": { + "name": "str", + "type": "reference", + "target": 0, + "ref": {} + }, + "permalink": "/python/api/class/ProcessedRequest#unique_key", + "parentId": 1340 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 31164, + "module": "storage_clients.models", + "name": "was_already_present", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 149, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/models.py#L149" + } + ], + "type": { + "name": "bool", + "type": "reference", + "target": 0, + "ref": {} + }, + "permalink": "/python/api/class/ProcessedRequest#was_already_present", + "parentId": 1340 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 31165, + "module": "storage_clients.models", + "name": "was_already_handled", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 150, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/models.py#L150" + } + ], + "type": { + "name": "bool", + "type": "reference", + "target": 0, + "ref": {} + }, + "permalink": "/python/api/class/ProcessedRequest#was_already_handled", + "parentId": 1340 + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Represents a processed request." + } + ] + }, + "decorations": [ + { + "args": "('Storage data')", + "name": "docs_group" + } + ], + "flags": {}, + "groups": [ + { + "children": [ + 31162, + 31161, + 31163, + 31165, + 31164 + ], + "title": "Properties" + } + ], + "id": 31160, + "module": "storage_clients.models", + "name": "ProcessedRequest", + "parsedDocstring": { + "text": "Represents a processed request." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 140, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/models.py#L140" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "permalink": "/python/api/class/ProcessedRequest", + "categories": [] + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 31575, + "module": "_request", + "name": "model_config", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_request.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 169, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/_request.py#L169" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "permalink": "/python/api/class/Request#model_config", + "parentId": 409 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "A unique key identifying the request. Two requests with the same `unique_key` are considered as pointing\nto the same URL.\n\nIf `unique_key` is not provided, then it is automatically generated by normalizing the URL.\nFor example, the URL of `HTTP://www.EXAMPLE.com/something/` will produce the `unique_key`\nof `http://www.example.com/something`.\n\nPass an arbitrary non-empty text value to the `unique_key` property to override the default behavior\nand specify which URLs shall be considered equal." + } + ] + }, + "flags": {}, + "groups": [], + "id": 31576, + "module": "_request", + "name": "unique_key", + "parsedDocstring": { + "text": "A unique key identifying the request. Two requests with the same `unique_key` are considered as pointing\nto the same URL.\n\nIf `unique_key` is not provided, then it is automatically generated by normalizing the URL.\nFor example, the URL of `HTTP://www.EXAMPLE.com/something/` will produce the `unique_key`\nof `http://www.example.com/something`.\n\nPass an arbitrary non-empty text value to the `unique_key` property to override the default behavior\nand specify which URLs shall be considered equal." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_request.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 171, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/_request.py#L171" + } + ], + "type": { + "name": "str", + "type": "reference", + "target": 0, + "ref": {} + }, + "permalink": "/python/api/class/Request#unique_key", + "parentId": 409 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "The URL of the web page to crawl. Must be a valid HTTP or HTTPS URL, and may include query parameters\nand fragments." + } + ] + }, + "flags": {}, + "groups": [], + "id": 31577, + "module": "_request", + "name": "url", + "parsedDocstring": { + "text": "The URL of the web page to crawl. Must be a valid HTTP or HTTPS URL, and may include query parameters\nand fragments." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_request.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 183, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/_request.py#L183" + } + ], + "type": { + "name": "str", + "type": "reference", + "target": 0, + "ref": {} + }, + "permalink": "/python/api/class/Request#url", + "parentId": 409 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "HTTP request method." + } + ] + }, + "flags": {}, + "groups": [], + "id": 31578, + "module": "_request", + "name": "method", + "parsedDocstring": { + "text": "HTTP request method." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_request.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 187, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/_request.py#L187" + } + ], + "type": { + "name": "HttpMethod", + "type": "reference", + "target": "84", + "ref": { + "id": 31249, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_types.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 38 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api#HttpMethod" + } + }, + "permalink": "/python/api/class/Request#method", + "parentId": 409 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "HTTP request payload." + } + ] + }, + "flags": {}, + "groups": [], + "id": 31579, + "module": "_request", + "name": "payload", + "parsedDocstring": { + "text": "HTTP request payload." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_request.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 190, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/_request.py#L190" + } + ], + "type": { + "name": "Annotated[ HttpPayload | None, BeforeValidator(lambda v: v.encode() if isinstance(v, str) else v), PlainSerializer(lambda v: v.decode() if isinstance(v, bytes) else v), Field(frozen=True), ]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "HttpPayload", + "target": "85", + "ref": { + "id": 31250, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_types.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 40 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api#HttpPayload" + } + }, + { + "type": "literal", + "value": {} + } + ] + }, + "permalink": "/python/api/class/Request#payload", + "parentId": 409 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Number of times the request has been retried." + } + ] + }, + "flags": {}, + "groups": [], + "id": 31580, + "module": "_request", + "name": "retry_count", + "parsedDocstring": { + "text": "Number of times the request has been retried." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_request.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 230, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/_request.py#L230" + } + ], + "type": { + "name": "int", + "type": "reference", + "target": 0, + "ref": {} + }, + "permalink": "/python/api/class/Request#retry_count", + "parentId": 409 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "If set to `True`, the request will not be retried in case of failure." + } + ] + }, + "flags": {}, + "groups": [], + "id": 31581, + "module": "_request", + "name": "no_retry", + "parsedDocstring": { + "text": "If set to `True`, the request will not be retried in case of failure." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_request.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 233, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/_request.py#L233" + } + ], + "type": { + "name": "bool", + "type": "reference", + "target": 0, + "ref": {} + }, + "permalink": "/python/api/class/Request#no_retry", + "parentId": 409 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "URL of the web page that was loaded. This can differ from the original URL in case of redirects." + } + ] + }, + "flags": {}, + "groups": [], + "id": 31582, + "module": "_request", + "name": "loaded_url", + "parsedDocstring": { + "text": "URL of the web page that was loaded. This can differ from the original URL in case of redirects." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_request.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 236, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/_request.py#L236" + } + ], + "type": { + "name": "Annotated[str | None, BeforeValidator(validate_http_url), Field(alias='loadedUrl')]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "permalink": "/python/api/class/Request#loaded_url", + "parentId": 409 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Timestamp when the request was handled." + } + ] + }, + "flags": {}, + "groups": [], + "id": 31583, + "module": "_request", + "name": "handled_at", + "parsedDocstring": { + "text": "Timestamp when the request was handled." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_request.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 239, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/_request.py#L239" + } + ], + "type": { + "name": "Annotated[datetime | None, Field(alias='handledAt')]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "datetime", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "permalink": "/python/api/class/Request#handled_at", + "parentId": 409 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Create a new `Request` instance from a URL.\n\nThis is recommended constructor for creating new `Request` instances. It generates a `Request` object from\na given URL with additional options to customize HTTP method, payload, unique key, and other request\nproperties. If no `unique_key` or `id` is provided, they are computed automatically based on the URL,\nmethod and payload. It depends on the `keep_url_fragment` and `use_extended_unique_key` flags.\n" + } + ] + }, + "decorations": [ + { + "name": "classmethod" + } + ], + "flags": {}, + "groups": [], + "id": 31584, + "module": "_request", + "name": "from_url", + "parsedDocstring": { + "text": "Create a new `Request` instance from a URL.\n\nThis is recommended constructor for creating new `Request` instances. It generates a `Request` object from\na given URL with additional options to customize HTTP method, payload, unique key, and other request\nproperties. If no `unique_key` or `id` is provided, they are computed automatically based on the URL,\nmethod and payload. It depends on the `keep_url_fragment` and `use_extended_unique_key` flags.\n", + "args": { + "url": "The URL of the request.", + "method": "The HTTP method of the request.", + "headers": "The HTTP headers of the request.", + "payload": "The data to be sent as the request body. Typically used with 'POST' or 'PUT' requests.", + "label": "A custom label to differentiate between request types. This is stored in `user_data`, and it is\nused for request routing (different requests go to different handlers).", + "session_id": "ID of a specific `Session` to which the request will be strictly bound.\nIf the session becomes unavailable when the request is processed, a `RequestCollisionError` will be\nraised.", + "unique_key": "A unique key identifying the request. If not provided, it is automatically computed based on\nthe URL and other parameters. Requests with the same `unique_key` are treated as identical.", + "keep_url_fragment": "Determines whether the URL fragment (e.g., ``section``) should be included in\nthe `unique_key` computation. This is only relevant when `unique_key` is not provided.", + "use_extended_unique_key": "Determines whether to include the HTTP method, ID Session and payload in the\n`unique_key` computation. This is only relevant when `unique_key` is not provided.", + "always_enqueue": "If set to `True`, the request will be enqueued even if it is already present in the queue.\nUsing this is not allowed when a custom `unique_key` is also provided and will result in a `ValueError`.", + "enqueue_strategy": "The strategy that will be used for enqueuing the request.", + "max_retries": "Maximum number of retries for this request. Allows to override the global `max_request_retries`\noption of `BasicCrawler`.", + "**kwargs": "Additional request properties." + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_request.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 243, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/_request.py#L243" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Create a new `Request` instance from a URL.\n\nThis is recommended constructor for creating new `Request` instances. It generates a `Request` object from\na given URL with additional options to customize HTTP method, payload, unique key, and other request\nproperties. If no `unique_key` or `id` is provided, they are computed automatically based on the URL,\nmethod and payload. It depends on the `keep_url_fragment` and `use_extended_unique_key` flags.\n" + } + ] + }, + "flags": {}, + "id": 31585, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "from_url", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The URL of the request." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 31586, + "kind": 32768, + "kindString": "Parameter", + "name": "url", + "type": { + "name": "str", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 420 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The HTTP method of the request." + } + ] + }, + "defaultValue": "'GET'", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 31587, + "kind": 32768, + "kindString": "Parameter", + "name": "method", + "type": { + "name": "HttpMethod", + "type": "reference", + "target": "84", + "ref": { + "id": 31249, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_types.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 38 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api#HttpMethod" + } + }, + "parentId": 420 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The HTTP headers of the request." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 31588, + "kind": 32768, + "kindString": "Parameter", + "name": "headers", + "type": { + "name": "HttpHeaders | dict[str, str] | None", + "type": "union", + "types": [ + { + "type": "union", + "types": [ + { + "type": "reference", + "name": "HttpHeaders", + "target": "91", + "ref": { + "id": 31256, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_types.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 63 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/HttpHeaders" + } + }, + { + "type": "reference", + "name": "dict", + "typeArguments": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + } + ] + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 420 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The data to be sent as the request body. Typically used with 'POST' or 'PUT' requests." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 31589, + "kind": 32768, + "kindString": "Parameter", + "name": "payload", + "type": { + "name": "HttpPayload | str | None", + "type": "union", + "types": [ + { + "type": "union", + "types": [ + { + "type": "reference", + "name": "HttpPayload", + "target": "85", + "ref": { + "id": 31250, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_types.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 40 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api#HttpPayload" + } + }, + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + } + ] + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 420 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "A custom label to differentiate between request types. This is stored in `user_data`, and it is\nused for request routing (different requests go to different handlers)." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 31590, + "kind": 32768, + "kindString": "Parameter", + "name": "label", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 420 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "ID of a specific `Session` to which the request will be strictly bound.\nIf the session becomes unavailable when the request is processed, a `RequestCollisionError` will be\nraised." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 31591, + "kind": 32768, + "kindString": "Parameter", + "name": "session_id", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 420 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "A unique key identifying the request. If not provided, it is automatically computed based on\nthe URL and other parameters. Requests with the same `unique_key` are treated as identical." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 31592, + "kind": 32768, + "kindString": "Parameter", + "name": "unique_key", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 420 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Determines whether the URL fragment (e.g., ``section``) should be included in\nthe `unique_key` computation. This is only relevant when `unique_key` is not provided." + } + ] + }, + "defaultValue": "False", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 31593, + "kind": 32768, + "kindString": "Parameter", + "name": "keep_url_fragment", + "type": { + "name": "bool", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 420 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Determines whether to include the HTTP method, ID Session and payload in the\n`unique_key` computation. This is only relevant when `unique_key` is not provided." + } + ] + }, + "defaultValue": "False", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 31594, + "kind": 32768, + "kindString": "Parameter", + "name": "use_extended_unique_key", + "type": { + "name": "bool", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 420 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "If set to `True`, the request will be enqueued even if it is already present in the queue.\nUsing this is not allowed when a custom `unique_key` is also provided and will result in a `ValueError`." + } + ] + }, + "defaultValue": "False", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 31595, + "kind": 32768, + "kindString": "Parameter", + "name": "always_enqueue", + "type": { + "name": "bool", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 420 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The strategy that will be used for enqueuing the request." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 31596, + "kind": 32768, + "kindString": "Parameter", + "name": "enqueue_strategy", + "type": { + "name": "EnqueueStrategy | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "EnqueueStrategy", + "target": "88", + "ref": { + "id": 31253, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_types.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 47 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api#EnqueueStrategy" + } + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 420 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Maximum number of retries for this request. Allows to override the global `max_request_retries`\noption of `BasicCrawler`." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 31597, + "kind": 32768, + "kindString": "Parameter", + "name": "max_retries", + "type": { + "name": "int | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "int", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 420 + }, + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 31598, + "kind": 32768, + "kindString": "Parameter", + "name": "kwargs", + "type": { + "name": "Any", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 420 + } + ], + "type": { + "name": "Self", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 419 + } + ], + "permalink": "/python/api/class/Request#from_url", + "parentId": 409 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Get the value of a specific query parameter from the URL." + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 31599, + "module": "_request", + "name": "get_query_param_from_url", + "parsedDocstring": { + "text": "Get the value of a specific query parameter from the URL." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_request.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 344, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/_request.py#L344" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Get the value of a specific query parameter from the URL." + } + ] + }, + "flags": {}, + "id": 31600, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "get_query_param_from_url", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 31601, + "kind": 32768, + "kindString": "Parameter", + "name": "param", + "type": { + "name": "str", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 435 + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 31602, + "kind": 32768, + "kindString": "Parameter", + "name": "default", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 435 + } + ], + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 434 + } + ], + "permalink": "/python/api/class/Request#get_query_param_from_url", + "parentId": 409 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "A string used to differentiate between arbitrary request types." + } + ] + }, + "decorations": [ + { + "name": "property" + } + ], + "flags": {}, + "groups": [], + "id": 31603, + "module": "_request", + "name": "label", + "parsedDocstring": { + "text": "A string used to differentiate between arbitrary request types." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_request.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 350, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/_request.py#L350" + } + ], + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "permalink": "/python/api/class/Request#label", + "parentId": 409 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "The ID of the bound session, if there is any." + } + ] + }, + "decorations": [ + { + "name": "property" + } + ], + "flags": {}, + "groups": [], + "id": 31604, + "module": "_request", + "name": "session_id", + "parsedDocstring": { + "text": "The ID of the bound session, if there is any." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_request.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 355, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/_request.py#L355" + } + ], + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "permalink": "/python/api/class/Request#session_id", + "parentId": 409 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Crawlee-specific configuration stored in the `user_data`." + } + ] + }, + "decorations": [ + { + "name": "property" + } + ], + "flags": {}, + "groups": [], + "id": 31605, + "module": "_request", + "name": "crawlee_data", + "parsedDocstring": { + "text": "Crawlee-specific configuration stored in the `user_data`." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_request.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 360, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/_request.py#L360" + } + ], + "type": { + "name": "CrawleeRequestData", + "type": "reference", + "target": "358", + "ref": { + "id": 31523, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_request.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 34 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/CrawleeRequestData" + } + }, + "permalink": "/python/api/class/Request#crawlee_data", + "parentId": 409 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "The depth of the request in the crawl tree." + } + ] + }, + "decorations": [ + { + "name": "property" + } + ], + "flags": {}, + "groups": [], + "id": 31606, + "module": "_request", + "name": "crawl_depth", + "parsedDocstring": { + "text": "The depth of the request in the crawl tree." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_request.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 369, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/_request.py#L369" + } + ], + "type": { + "name": "int", + "type": "reference", + "target": 0, + "ref": {} + }, + "permalink": "/python/api/class/Request#crawl_depth", + "parentId": 409 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "decorations": [ + { + "args": ".setter", + "name": "crawl_depth" + } + ], + "flags": {}, + "groups": [], + "id": 31607, + "module": "_request", + "name": "crawl_depth", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_request.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 374, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/_request.py#L374" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "flags": {}, + "id": 31608, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "crawl_depth", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 31609, + "kind": 32768, + "kindString": "Parameter", + "name": "new_value", + "type": { + "name": "int", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 443 + } + ], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "parentId": 442 + } + ], + "permalink": "/python/api/class/Request#crawl_depth", + "parentId": 409 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Crawlee-specific request handling state." + } + ] + }, + "decorations": [ + { + "name": "property" + } + ], + "flags": {}, + "groups": [], + "id": 31610, + "module": "_request", + "name": "state", + "parsedDocstring": { + "text": "Crawlee-specific request handling state." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_request.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 378, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/_request.py#L378" + } + ], + "type": { + "name": "RequestState", + "type": "reference", + "target": "349", + "ref": { + "id": 31514, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_request.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 21 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/RequestState" + } + }, + "permalink": "/python/api/class/Request#state", + "parentId": 409 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "decorations": [ + { + "args": ".setter", + "name": "state" + } + ], + "flags": {}, + "groups": [], + "id": 31611, + "module": "_request", + "name": "state", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_request.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 383, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/_request.py#L383" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "flags": {}, + "id": 31612, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "state", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 31613, + "kind": 32768, + "kindString": "Parameter", + "name": "new_state", + "type": { + "name": "RequestState", + "type": "reference", + "target": "349", + "ref": { + "id": 31514, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_request.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 21 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/RequestState" + } + }, + "parentId": 447 + } + ], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "parentId": 446 + } + ], + "permalink": "/python/api/class/Request#state", + "parentId": 409 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Crawlee-specific limit on the number of retries of the request." + } + ] + }, + "decorations": [ + { + "name": "property" + } + ], + "flags": {}, + "groups": [], + "id": 31614, + "module": "_request", + "name": "max_retries", + "parsedDocstring": { + "text": "Crawlee-specific limit on the number of retries of the request." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_request.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 387, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/_request.py#L387" + } + ], + "type": { + "name": "int | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "int", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "permalink": "/python/api/class/Request#max_retries", + "parentId": 409 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Crawlee-specific number of finished session rotations for the request." + } + ] + }, + "decorations": [ + { + "name": "property" + } + ], + "flags": {}, + "groups": [], + "id": 31615, + "module": "_request", + "name": "session_rotation_count", + "parsedDocstring": { + "text": "Crawlee-specific number of finished session rotations for the request." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_request.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 392, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/_request.py#L392" + } + ], + "type": { + "name": "int | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "int", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "permalink": "/python/api/class/Request#session_rotation_count", + "parentId": 409 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "decorations": [ + { + "args": ".setter", + "name": "session_rotation_count" + } + ], + "flags": {}, + "groups": [], + "id": 31616, + "module": "_request", + "name": "session_rotation_count", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_request.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 397, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/_request.py#L397" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "flags": {}, + "id": 31617, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "session_rotation_count", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 31618, + "kind": 32768, + "kindString": "Parameter", + "name": "new_session_rotation_count", + "type": { + "name": "int", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 452 + } + ], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "parentId": 451 + } + ], + "permalink": "/python/api/class/Request#session_rotation_count", + "parentId": 409 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "The strategy that was used for enqueuing the request." + } + ] + }, + "decorations": [ + { + "name": "property" + } + ], + "flags": {}, + "groups": [], + "id": 31619, + "module": "_request", + "name": "enqueue_strategy", + "parsedDocstring": { + "text": "The strategy that was used for enqueuing the request." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_request.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 401, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/_request.py#L401" + } + ], + "type": { + "name": "EnqueueStrategy", + "type": "reference", + "target": "88", + "ref": { + "id": 31253, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_types.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 47 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api#EnqueueStrategy" + } + }, + "permalink": "/python/api/class/Request#enqueue_strategy", + "parentId": 409 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "decorations": [ + { + "args": ".setter", + "name": "enqueue_strategy" + } + ], + "flags": {}, + "groups": [], + "id": 31620, + "module": "_request", + "name": "enqueue_strategy", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_request.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 406, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/_request.py#L406" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "flags": {}, + "id": 31621, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "enqueue_strategy", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 31622, + "kind": 32768, + "kindString": "Parameter", + "name": "new_enqueue_strategy", + "type": { + "name": "EnqueueStrategy", + "type": "reference", + "target": "88", + "ref": { + "id": 31253, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_types.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 47 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api#EnqueueStrategy" + } + }, + "parentId": 456 + } + ], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "parentId": 455 + } + ], + "permalink": "/python/api/class/Request#enqueue_strategy", + "parentId": 409 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "The last proxy tier used to process the request." + } + ] + }, + "decorations": [ + { + "name": "property" + } + ], + "flags": {}, + "groups": [], + "id": 31623, + "module": "_request", + "name": "last_proxy_tier", + "parsedDocstring": { + "text": "The last proxy tier used to process the request." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_request.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 410, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/_request.py#L410" + } + ], + "type": { + "name": "int | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "int", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "permalink": "/python/api/class/Request#last_proxy_tier", + "parentId": 409 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "decorations": [ + { + "args": ".setter", + "name": "last_proxy_tier" + } + ], + "flags": {}, + "groups": [], + "id": 31624, + "module": "_request", + "name": "last_proxy_tier", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_request.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 415, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/_request.py#L415" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "flags": {}, + "id": 31625, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "last_proxy_tier", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 31626, + "kind": 32768, + "kindString": "Parameter", + "name": "new_value", + "type": { + "name": "int", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 460 + } + ], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "parentId": 459 + } + ], + "permalink": "/python/api/class/Request#last_proxy_tier", + "parentId": 409 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Indicate whether the request should be enqueued at the front of the queue." + } + ] + }, + "decorations": [ + { + "name": "property" + } + ], + "flags": {}, + "groups": [], + "id": 31627, + "module": "_request", + "name": "forefront", + "parsedDocstring": { + "text": "Indicate whether the request should be enqueued at the front of the queue." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_request.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 419, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/_request.py#L419" + } + ], + "type": { + "name": "bool", + "type": "reference", + "target": 0, + "ref": {} + }, + "permalink": "/python/api/class/Request#forefront", + "parentId": 409 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "decorations": [ + { + "args": ".setter", + "name": "forefront" + } + ], + "flags": {}, + "groups": [], + "id": 31628, + "module": "_request", + "name": "forefront", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_request.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 424, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/_request.py#L424" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "flags": {}, + "id": 31629, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "forefront", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 31630, + "kind": 32768, + "kindString": "Parameter", + "name": "new_value", + "type": { + "name": "bool", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 464 + } + ], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "parentId": 463 + } + ], + "permalink": "/python/api/class/Request#forefront", + "parentId": 409 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Indicates whether the request was handled." + } + ] + }, + "decorations": [ + { + "name": "property" + } + ], + "flags": {}, + "groups": [], + "id": 31631, + "module": "_request", + "name": "was_already_handled", + "parsedDocstring": { + "text": "Indicates whether the request was handled." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_request.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 428, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/_request.py#L428" + } + ], + "type": { + "name": "bool", + "type": "reference", + "target": 0, + "ref": {} + }, + "permalink": "/python/api/class/Request#was_already_handled", + "parentId": 409 + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Represents a request in the Crawlee framework, containing the necessary information for crawling operations.\n\nThe `Request` class is one of the core components in Crawlee, utilized by various components such as request\nproviders, HTTP clients, crawlers, and more. It encapsulates the essential data for executing web requests,\nincluding the URL, HTTP method, headers, payload, and user data. The user data allows custom information\nto be stored and persisted throughout the request lifecycle, including its retries.\n\nKey functionalities include managing the request's identifier (`id`), unique key (`unique_key`) that is used\nfor request deduplication, controlling retries, handling state management, and enabling configuration for session\nrotation and proxy handling.\n\nThe recommended way to create a new instance is by using the `Request.from_url` constructor, which automatically\ngenerates a unique key and identifier based on the URL and request parameters.\n\n### Usage\n\n```python\nfrom crawlee import Request\n\nrequest = Request.from_url('https://crawlee.dev')\n```" + } + ] + }, + "decorations": [ + { + "args": "('Storage data')", + "name": "docs_group" + } + ], + "flags": {}, + "groups": [ + { + "children": [ + 31607, + 31620, + 31628, + 31584, + 31599, + 31624, + 31616, + 31611 + ], + "title": "Methods" + }, + { + "children": [ + 31606, + 31605, + 31619, + 31627, + 31583, + 31603, + 31623, + 31582, + 31614, + 31578, + 31575, + 31581, + 31579, + 31580, + 31604, + 31615, + 31610, + 31576, + 31577, + 31631 + ], + "title": "Properties" + } + ], + "id": 31574, + "module": "_request", + "name": "Request", + "parsedDocstring": { + "text": "Represents a request in the Crawlee framework, containing the necessary information for crawling operations.\n\nThe `Request` class is one of the core components in Crawlee, utilized by various components such as request\nproviders, HTTP clients, crawlers, and more. It encapsulates the essential data for executing web requests,\nincluding the URL, HTTP method, headers, payload, and user data. The user data allows custom information\nto be stored and persisted throughout the request lifecycle, including its retries.\n\nKey functionalities include managing the request's identifier (`id`), unique key (`unique_key`) that is used\nfor request deduplication, controlling retries, handling state management, and enabling configuration for session\nrotation and proxy handling.\n\nThe recommended way to create a new instance is by using the `Request.from_url` constructor, which automatically\ngenerates a unique key and identifier based on the URL and request parameters.\n\n### Usage\n\n```python\nfrom crawlee import Request\n\nrequest = Request.from_url('https://crawlee.dev')\n```" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_request.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 145, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/_request.py#L145" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "extendedBy": [ + { + "name": "RequestWithLock", + "target": "467", + "type": "reference", + "ref": { + "id": 31632, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_request.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 433 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/RequestWithLock" + } + } + ], + "permalink": "/python/api/class/Request", + "categories": [] + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 32952, + "module": "storage_clients.models", + "name": "model_config", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 62, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/models.py#L62" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "overwrites": { + "name": "StorageMetadata.model_config", + "target": 1308, + "type": "reference", + "ref": { + "id": 32940, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 23 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api/class/StorageMetadata#model_config" + } + }, + "permalink": "/python/api/class/RequestQueueMetadata#model_config", + "parentId": 1319 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Indicates whether the queue has been accessed by multiple clients (consumers)." + } + ] + }, + "flags": {}, + "groups": [], + "id": 32953, + "module": "storage_clients.models", + "name": "had_multiple_clients", + "parsedDocstring": { + "text": "Indicates whether the queue has been accessed by multiple clients (consumers)." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 64, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/models.py#L64" + } + ], + "type": { + "name": "bool", + "type": "reference", + "target": 0, + "ref": {} + }, + "permalink": "/python/api/class/RequestQueueMetadata#had_multiple_clients", + "parentId": 1319 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "The number of requests that have been handled from the queue." + } + ] + }, + "flags": {}, + "groups": [], + "id": 32954, + "module": "storage_clients.models", + "name": "handled_request_count", + "parsedDocstring": { + "text": "The number of requests that have been handled from the queue." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 67, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/models.py#L67" + } + ], + "type": { + "name": "int", + "type": "reference", + "target": 0, + "ref": {} + }, + "permalink": "/python/api/class/RequestQueueMetadata#handled_request_count", + "parentId": 1319 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "The number of requests that are still pending in the queue." + } + ] + }, + "flags": {}, + "groups": [], + "id": 32955, + "module": "storage_clients.models", + "name": "pending_request_count", + "parsedDocstring": { + "text": "The number of requests that are still pending in the queue." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 70, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/models.py#L70" + } + ], + "type": { + "name": "int", + "type": "reference", + "target": 0, + "ref": {} + }, + "permalink": "/python/api/class/RequestQueueMetadata#pending_request_count", + "parentId": 1319 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "The total number of requests that have been added to the queue." + } + ] + }, + "flags": {}, + "groups": [], + "id": 32956, + "module": "storage_clients.models", + "name": "total_request_count", + "parsedDocstring": { + "text": "The total number of requests that have been added to the queue." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 73, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/models.py#L73" + } + ], + "type": { + "name": "int", + "type": "reference", + "target": 0, + "ref": {} + }, + "permalink": "/python/api/class/RequestQueueMetadata#total_request_count", + "parentId": 1319 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "The unique identifier of the storage." + } + ] + }, + "flags": {}, + "groups": [], + "id": 35851, + "module": "storage_clients.models", + "name": "id", + "parsedDocstring": { + "text": "The unique identifier of the storage." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 25, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/models.py#L25" + } + ], + "type": { + "name": "Annotated[str, Field(alias='id')]", + "type": "reference", + "target": 0, + "ref": {} + }, + "inheritedFrom": { + "name": "StorageMetadata.id", + "target": 1309, + "type": "reference", + "ref": { + "id": 32941, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 25 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api/class/StorageMetadata#id" + } + }, + "permalink": "/python/api/class/RequestQueueMetadata#id", + "parentId": 1319 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "The name of the storage." + } + ] + }, + "flags": {}, + "groups": [], + "id": 35852, + "module": "storage_clients.models", + "name": "name", + "parsedDocstring": { + "text": "The name of the storage." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 28, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/models.py#L28" + } + ], + "type": { + "name": "Annotated[str | None, Field(alias='name', default=None)]", + "type": "reference", + "target": 0, + "ref": {} + }, + "inheritedFrom": { + "name": "StorageMetadata.name", + "target": 1310, + "type": "reference", + "ref": { + "id": 32942, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 28 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api/class/StorageMetadata#name" + } + }, + "permalink": "/python/api/class/RequestQueueMetadata#name", + "parentId": 1319 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "The timestamp when the storage was last accessed." + } + ] + }, + "flags": {}, + "groups": [], + "id": 35853, + "module": "storage_clients.models", + "name": "accessed_at", + "parsedDocstring": { + "text": "The timestamp when the storage was last accessed." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 31, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/models.py#L31" + } + ], + "type": { + "name": "Annotated[datetime, Field(alias='accessedAt')]", + "type": "reference", + "target": 0, + "ref": {} + }, + "inheritedFrom": { + "name": "StorageMetadata.accessed_at", + "target": 1311, + "type": "reference", + "ref": { + "id": 32943, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 31 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api/class/StorageMetadata#accessed_at" + } + }, + "permalink": "/python/api/class/RequestQueueMetadata#accessed_at", + "parentId": 1319 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "The timestamp when the storage was created." + } + ] + }, + "flags": {}, + "groups": [], + "id": 35854, + "module": "storage_clients.models", + "name": "created_at", + "parsedDocstring": { + "text": "The timestamp when the storage was created." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 34, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/models.py#L34" + } + ], + "type": { + "name": "Annotated[datetime, Field(alias='createdAt')]", + "type": "reference", + "target": 0, + "ref": {} + }, + "inheritedFrom": { + "name": "StorageMetadata.created_at", + "target": 1312, + "type": "reference", + "ref": { + "id": 32944, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 34 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api/class/StorageMetadata#created_at" + } + }, + "permalink": "/python/api/class/RequestQueueMetadata#created_at", + "parentId": 1319 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "The timestamp when the storage was last modified." + } + ] + }, + "flags": {}, + "groups": [], + "id": 35855, + "module": "storage_clients.models", + "name": "modified_at", + "parsedDocstring": { + "text": "The timestamp when the storage was last modified." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 37, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/models.py#L37" + } + ], + "type": { + "name": "Annotated[datetime, Field(alias='modifiedAt')]", + "type": "reference", + "target": 0, + "ref": {} + }, + "inheritedFrom": { + "name": "StorageMetadata.modified_at", + "target": 1313, + "type": "reference", + "ref": { + "id": 32945, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 37 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api/class/StorageMetadata#modified_at" + } + }, + "permalink": "/python/api/class/RequestQueueMetadata#modified_at", + "parentId": 1319 + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Model for a request queue metadata." + } + ] + }, + "decorations": [ + { + "args": "('Storage data')", + "name": "docs_group" + } + ], + "flags": {}, + "groups": [ + { + "children": [ + 35853, + 35854, + 32953, + 32954, + 35851, + 32952, + 35855, + 35852, + 32955, + 32956 + ], + "title": "Properties" + } + ], + "id": 32951, + "module": "storage_clients.models", + "name": "RequestQueueMetadata", + "parsedDocstring": { + "text": "Model for a request queue metadata." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 59, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/models.py#L59" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "extendedTypes": [ + { + "name": "StorageMetadata", + "target": "1307", + "type": "reference", + "ref": { + "id": 32939, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 17 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/StorageMetadata" + } + } + ], + "permalink": "/python/api/class/RequestQueueMetadata", + "categories": [] + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 37163, + "module": "storage_clients.models", + "name": "model_config", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 23, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/models.py#L23" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "permalink": "/python/api/class/StorageMetadata#model_config", + "parentId": 1307 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "The unique identifier of the storage." + } + ] + }, + "flags": {}, + "groups": [], + "id": 37164, + "module": "storage_clients.models", + "name": "id", + "parsedDocstring": { + "text": "The unique identifier of the storage." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 25, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/models.py#L25" + } + ], + "type": { + "name": "str", + "type": "reference", + "target": 0, + "ref": {} + }, + "permalink": "/python/api/class/StorageMetadata#id", + "parentId": 1307 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "The name of the storage." + } + ] + }, + "flags": {}, + "groups": [], + "id": 37165, + "module": "storage_clients.models", + "name": "name", + "parsedDocstring": { + "text": "The name of the storage." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 28, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/models.py#L28" + } + ], + "type": { + "name": "Annotated[str | None, Field(alias='name', default=None)]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "permalink": "/python/api/class/StorageMetadata#name", + "parentId": 1307 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "The timestamp when the storage was last accessed." + } + ] + }, + "flags": {}, + "groups": [], + "id": 37166, + "module": "storage_clients.models", + "name": "accessed_at", + "parsedDocstring": { + "text": "The timestamp when the storage was last accessed." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 31, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/models.py#L31" + } + ], + "type": { + "name": "datetime", + "type": "reference", + "target": 0, + "ref": {} + }, + "permalink": "/python/api/class/StorageMetadata#accessed_at", + "parentId": 1307 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "The timestamp when the storage was created." + } + ] + }, + "flags": {}, + "groups": [], + "id": 37167, + "module": "storage_clients.models", + "name": "created_at", + "parsedDocstring": { + "text": "The timestamp when the storage was created." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 34, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/models.py#L34" + } + ], + "type": { + "name": "datetime", + "type": "reference", + "target": 0, + "ref": {} + }, + "permalink": "/python/api/class/StorageMetadata#created_at", + "parentId": 1307 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "The timestamp when the storage was last modified." + } + ] + }, + "flags": {}, + "groups": [], + "id": 37168, + "module": "storage_clients.models", + "name": "modified_at", + "parsedDocstring": { + "text": "The timestamp when the storage was last modified." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 37, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/models.py#L37" + } + ], + "type": { + "name": "datetime", + "type": "reference", + "target": 0, + "ref": {} + }, + "permalink": "/python/api/class/StorageMetadata#modified_at", + "parentId": 1307 + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Represents the base model for storage metadata.\n\nIt contains common fields shared across all specific storage types." + } + ] + }, + "decorations": [ + { + "args": "('Storage data')", + "name": "docs_group" + } + ], + "flags": {}, + "groups": [ + { + "children": [ + 37166, + 37167, + 37164, + 37163, + 37168, + 37165 + ], + "title": "Properties" + } + ], + "id": 37162, + "module": "storage_clients.models", + "name": "StorageMetadata", + "parsedDocstring": { + "text": "Represents the base model for storage metadata.\n\nIt contains common fields shared across all specific storage types." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 17, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/models.py#L17" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "extendedBy": [ + { + "name": "DatasetMetadata", + "target": "1314", + "type": "reference", + "ref": { + "id": 37169, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 42 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/DatasetMetadata" + } + }, + { + "name": "KeyValueStoreMetadata", + "target": "1317", + "type": "reference", + "ref": { + "id": 37172, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 52 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/KeyValueStoreMetadata" + } + }, + { + "name": "RequestQueueMetadata", + "target": "1319", + "type": "reference", + "ref": { + "id": 37174, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 59 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/RequestQueueMetadata" + } + } + ], + "permalink": "/python/api/class/StorageMetadata", + "categories": [] + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Initialize a new instance.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 39330, + "module": "events._event_manager", + "name": "__init__", + "parsedDocstring": { + "text": "Initialize a new instance.\n", + "args": { + "persist_state_interval": "Interval between emitted `PersistState` events to maintain state persistence.", + "close_timeout": "Optional timeout for canceling pending event listeners if they exceed this duration." + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_event_manager.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 63, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/events/_event_manager.py#L63" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Initialize a new instance.\n" + } + ] + }, + "flags": {}, + "id": 39331, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "__init__", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Interval between emitted `PersistState` events to maintain state persistence." + } + ] + }, + "defaultValue": "timedelta(minutes=1)", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 39332, + "kind": 32768, + "kindString": "Parameter", + "name": "persist_state_interval", + "type": { + "name": "timedelta", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2157 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Optional timeout for canceling pending event listeners if they exceed this duration." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 39333, + "kind": 32768, + "kindString": "Parameter", + "name": "close_timeout", + "type": { + "name": "timedelta | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "timedelta", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2157 + } + ], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "parentId": 2156 + } + ], + "permalink": "/python/api/class/EventManager#__init__", + "parentId": 2155 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Indicate whether the context is active." + } + ] + }, + "decorations": [ + { + "name": "property" + } + ], + "flags": {}, + "groups": [], + "id": 39334, + "module": "events._event_manager", + "name": "active", + "parsedDocstring": { + "text": "Indicate whether the context is active." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_event_manager.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 100, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/events/_event_manager.py#L100" + } + ], + "type": { + "name": "bool", + "type": "reference", + "target": 0, + "ref": {} + }, + "permalink": "/python/api/class/EventManager#active", + "parentId": 2155 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Initialize the event manager upon entering the async context." + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 39335, + "module": "events._event_manager", + "name": "__aenter__", + "parsedDocstring": { + "text": "Initialize the event manager upon entering the async context." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_event_manager.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 104, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/events/_event_manager.py#L104" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Initialize the event manager upon entering the async context." + } + ] + }, + "flags": {}, + "id": 39336, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "__aenter__", + "parameters": [], + "type": { + "name": "EventManager", + "type": "reference", + "target": "2155", + "ref": { + "id": 39329, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_event_manager.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 55 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/EventManager" + } + }, + "parentId": 2161 + } + ], + "permalink": "/python/api/class/EventManager#__aenter__", + "parentId": 2155 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Close the local event manager upon exiting the async context.\n\nThis will stop listening for the events, and it will wait for all the event listeners to finish.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 39337, + "module": "events._event_manager", + "name": "__aexit__", + "parsedDocstring": { + "text": "Close the local event manager upon exiting the async context.\n\nThis will stop listening for the events, and it will wait for all the event listeners to finish.\n" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_event_manager.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 113, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/events/_event_manager.py#L113" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Close the local event manager upon exiting the async context.\n\nThis will stop listening for the events, and it will wait for all the event listeners to finish.\n" + } + ] + }, + "flags": {}, + "id": 39338, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "__aexit__", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 39339, + "kind": 32768, + "kindString": "Parameter", + "name": "exc_type", + "type": { + "name": "type[BaseException] | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "type", + "typeArguments": [ + { + "type": "reference", + "name": "BaseException", + "target": 0, + "ref": {} + } + ], + "target": "3422", + "ref": { + "id": 40596, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_utils/sitemap.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 62 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api/class/SitemapSource#type" + } + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2164 + }, + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 39340, + "kind": 32768, + "kindString": "Parameter", + "name": "exc_value", + "type": { + "name": "BaseException | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "BaseException", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2164 + }, + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 39341, + "kind": 32768, + "kindString": "Parameter", + "name": "exc_traceback", + "type": { + "name": "TracebackType | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "TracebackType", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2164 + } + ], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "parentId": 2163 + } + ], + "permalink": "/python/api/class/EventManager#__aexit__", + "parentId": 2155 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Register an event listener for a specific event.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 39342, + "module": "events._event_manager", + "name": "on", + "parsedDocstring": { + "text": "Register an event listener for a specific event.\n", + "args": { + "event": "The event for which to listen to.", + "listener": "The function (sync or async) which is to be called when the event is emitted." + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_event_manager.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 160, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/events/_event_manager.py#L160" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Register an event listener for a specific event.\n" + } + ] + }, + "flags": {}, + "id": 39343, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "on", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The event for which to listen to." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 39344, + "kind": 32768, + "kindString": "Parameter", + "name": "event", + "type": { + "name": "Event", + "type": "reference", + "target": "2116", + "ref": { + "id": 39290, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 15 + } + ], + "kind": 8, + "permalink": "https://crawlee.dev/python/api/enum/Event" + } + }, + "parentId": 2169 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The function (sync or async) which is to be called when the event is emitted." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 39345, + "kind": 32768, + "kindString": "Parameter", + "name": "listener", + "type": { + "name": "EventListener", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "Any", + "target": 0, + "ref": {} + } + ], + "target": "2150", + "ref": { + "id": 39324, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 112 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api#EventListener" + } + }, + "parentId": 2169 + } + ], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "parentId": 2168 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Register an event listener for a specific event.\n" + } + ] + }, + "flags": {}, + "id": 39357, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "on", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The event for which to listen to." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 39358, + "kind": 32768, + "kindString": "Parameter", + "name": "event", + "type": { + "name": "Literal", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "Event.PERSIST_STATE", + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + }, + "parentId": 2183 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The function (sync or async) which is to be called when the event is emitted." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 39359, + "kind": 32768, + "kindString": "Parameter", + "name": "listener", + "type": { + "name": "EventListener", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "EventPersistStateData", + "target": "2129", + "ref": { + "id": 39303, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 40 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/EventPersistStateData" + } + } + ], + "target": "2150", + "ref": { + "id": 39324, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 112 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api#EventListener" + } + }, + "parentId": 2183 + } + ], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "parentId": 2168 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Register an event listener for a specific event.\n" + } + ] + }, + "flags": {}, + "id": 39360, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "on", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The event for which to listen to." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 39361, + "kind": 32768, + "kindString": "Parameter", + "name": "event", + "type": { + "name": "Literal", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "Event.SYSTEM_INFO", + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + }, + "parentId": 2186 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The function (sync or async) which is to be called when the event is emitted." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 39362, + "kind": 32768, + "kindString": "Parameter", + "name": "listener", + "type": { + "name": "EventListener", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "EventSystemInfoData", + "target": "2132", + "ref": { + "id": 39306, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 49 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/EventSystemInfoData" + } + } + ], + "target": "2150", + "ref": { + "id": 39324, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 112 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api#EventListener" + } + }, + "parentId": 2186 + } + ], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "parentId": 2168 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Register an event listener for a specific event.\n" + } + ] + }, + "flags": {}, + "id": 39363, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "on", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The event for which to listen to." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 39364, + "kind": 32768, + "kindString": "Parameter", + "name": "event", + "type": { + "name": "Literal", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "Event.MIGRATING", + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + }, + "parentId": 2189 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The function (sync or async) which is to be called when the event is emitted." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 39365, + "kind": 32768, + "kindString": "Parameter", + "name": "listener", + "type": { + "name": "EventListener", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "EventMigratingData", + "target": "2136", + "ref": { + "id": 39310, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 62 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/EventMigratingData" + } + } + ], + "target": "2150", + "ref": { + "id": 39324, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 112 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api#EventListener" + } + }, + "parentId": 2189 + } + ], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "parentId": 2168 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Register an event listener for a specific event.\n" + } + ] + }, + "flags": {}, + "id": 39366, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "on", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The event for which to listen to." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 39367, + "kind": 32768, + "kindString": "Parameter", + "name": "event", + "type": { + "name": "Literal", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "Event.ABORTING", + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + }, + "parentId": 2192 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The function (sync or async) which is to be called when the event is emitted." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 39368, + "kind": 32768, + "kindString": "Parameter", + "name": "listener", + "type": { + "name": "EventListener", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "EventAbortingData", + "target": "2139", + "ref": { + "id": 39313, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 73 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/EventAbortingData" + } + } + ], + "target": "2150", + "ref": { + "id": 39324, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 112 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api#EventListener" + } + }, + "parentId": 2192 + } + ], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "parentId": 2168 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Register an event listener for a specific event.\n" + } + ] + }, + "flags": {}, + "id": 39369, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "on", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The event for which to listen to." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 39370, + "kind": 32768, + "kindString": "Parameter", + "name": "event", + "type": { + "name": "Literal", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "Event.EXIT", + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + }, + "parentId": 2195 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The function (sync or async) which is to be called when the event is emitted." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 39371, + "kind": 32768, + "kindString": "Parameter", + "name": "listener", + "type": { + "name": "EventListener", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "EventExitData", + "target": "2141", + "ref": { + "id": 39315, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 80 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/EventExitData" + } + } + ], + "target": "2150", + "ref": { + "id": 39324, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 112 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api#EventListener" + } + }, + "parentId": 2195 + } + ], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "parentId": 2168 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Register an event listener for a specific event.\n" + } + ] + }, + "flags": {}, + "id": 39372, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "on", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The event for which to listen to." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 39373, + "kind": 32768, + "kindString": "Parameter", + "name": "event", + "type": { + "name": "Literal", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "Event.CRAWLER_STATUS", + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + }, + "parentId": 2198 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The function (sync or async) which is to be called when the event is emitted." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 39374, + "kind": 32768, + "kindString": "Parameter", + "name": "listener", + "type": { + "name": "EventListener", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "EventCrawlerStatusData", + "target": "2143", + "ref": { + "id": 39317, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 87 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/EventCrawlerStatusData" + } + } + ], + "target": "2150", + "ref": { + "id": 39324, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 112 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api#EventListener" + } + }, + "parentId": 2198 + } + ], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "parentId": 2168 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Register an event listener for a specific event.\n" + } + ] + }, + "flags": {}, + "id": 39375, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "on", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The event for which to listen to." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 39376, + "kind": 32768, + "kindString": "Parameter", + "name": "event", + "type": { + "name": "Event", + "type": "reference", + "target": "2116", + "ref": { + "id": 39290, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 15 + } + ], + "kind": 8, + "permalink": "https://crawlee.dev/python/api/enum/Event" + } + }, + "parentId": 2201 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The function (sync or async) which is to be called when the event is emitted." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 39377, + "kind": 32768, + "kindString": "Parameter", + "name": "listener", + "type": { + "name": "EventListener", + "type": "reference", + "typeArguments": [ + { + "type": "literal", + "value": {} + } + ], + "target": "2150", + "ref": { + "id": 39324, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 112 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api#EventListener" + } + }, + "parentId": 2201 + } + ], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "parentId": 2168 + } + ], + "permalink": "/python/api/class/EventManager#on", + "parentId": 2155 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Remove a specific listener or all listeners for an event.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 39346, + "module": "events._event_manager", + "name": "off", + "parsedDocstring": { + "text": "Remove a specific listener or all listeners for an event.\n", + "args": { + "event": "The Actor event for which to remove listeners.", + "listener": "The listener which is supposed to be removed. If not passed, all listeners of this event\nare removed." + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_event_manager.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 210, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/events/_event_manager.py#L210" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Remove a specific listener or all listeners for an event.\n" + } + ] + }, + "flags": {}, + "id": 39347, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "off", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The Actor event for which to remove listeners." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 39348, + "kind": 32768, + "kindString": "Parameter", + "name": "event", + "type": { + "name": "Event", + "type": "reference", + "target": "2116", + "ref": { + "id": 39290, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 15 + } + ], + "kind": 8, + "permalink": "https://crawlee.dev/python/api/enum/Event" + } + }, + "parentId": 2173 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The listener which is supposed to be removed. If not passed, all listeners of this event\nare removed." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 39349, + "kind": 32768, + "kindString": "Parameter", + "name": "listener", + "type": { + "name": "EventListener[Any] | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "EventListener", + "typeArguments": [ + { + "type": "reference", + "name": "Any", + "target": 0, + "ref": {} + } + ], + "target": "2150", + "ref": { + "id": 39324, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 112 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api#EventListener" + } + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2173 + } + ], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "parentId": 2172 + } + ], + "permalink": "/python/api/class/EventManager#off", + "parentId": 2155 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Emit an event with the associated data to all registered listeners.\n" + } + ] + }, + "decorations": [ + { + "name": "ensure_context" + } + ], + "flags": {}, + "groups": [], + "id": 39350, + "module": "events._event_manager", + "name": "emit", + "parsedDocstring": { + "text": "Emit an event with the associated data to all registered listeners.\n", + "args": { + "event": "The event which will be emitted.", + "event_data": "The data which will be passed to the event listeners." + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_event_manager.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 242, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/events/_event_manager.py#L242" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Emit an event with the associated data to all registered listeners.\n" + } + ] + }, + "flags": {}, + "id": 39351, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "emit", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The event which will be emitted." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 39352, + "kind": 32768, + "kindString": "Parameter", + "name": "event", + "type": { + "name": "Event", + "type": "reference", + "target": "2116", + "ref": { + "id": 39290, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 15 + } + ], + "kind": 8, + "permalink": "https://crawlee.dev/python/api/enum/Event" + } + }, + "parentId": 2177 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The data which will be passed to the event listeners." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 39353, + "kind": 32768, + "kindString": "Parameter", + "name": "event_data", + "type": { + "name": "EventData", + "type": "reference", + "target": "2147", + "ref": { + "id": 39321, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 99 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api#EventData" + } + }, + "parentId": 2177 + } + ], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "parentId": 2176 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Emit an event with the associated data to all registered listeners.\n" + } + ] + }, + "flags": {}, + "id": 39378, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "emit", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The event which will be emitted." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 39379, + "kind": 32768, + "kindString": "Parameter", + "name": "event", + "type": { + "name": "Literal", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "Event.PERSIST_STATE", + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + }, + "parentId": 2204 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The data which will be passed to the event listeners." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 39380, + "kind": 32768, + "kindString": "Parameter", + "name": "event_data", + "type": { + "name": "EventPersistStateData", + "type": "reference", + "target": "2129", + "ref": { + "id": 39303, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 40 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/EventPersistStateData" + } + }, + "parentId": 2204 + } + ], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "parentId": 2176 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Emit an event with the associated data to all registered listeners.\n" + } + ] + }, + "flags": {}, + "id": 39381, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "emit", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The event which will be emitted." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 39382, + "kind": 32768, + "kindString": "Parameter", + "name": "event", + "type": { + "name": "Literal", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "Event.SYSTEM_INFO", + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + }, + "parentId": 2207 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The data which will be passed to the event listeners." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 39383, + "kind": 32768, + "kindString": "Parameter", + "name": "event_data", + "type": { + "name": "EventSystemInfoData", + "type": "reference", + "target": "2132", + "ref": { + "id": 39306, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 49 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/EventSystemInfoData" + } + }, + "parentId": 2207 + } + ], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "parentId": 2176 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Emit an event with the associated data to all registered listeners.\n" + } + ] + }, + "flags": {}, + "id": 39384, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "emit", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The event which will be emitted." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 39385, + "kind": 32768, + "kindString": "Parameter", + "name": "event", + "type": { + "name": "Literal", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "Event.MIGRATING", + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + }, + "parentId": 2210 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The data which will be passed to the event listeners." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 39386, + "kind": 32768, + "kindString": "Parameter", + "name": "event_data", + "type": { + "name": "EventMigratingData", + "type": "reference", + "target": "2136", + "ref": { + "id": 39310, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 62 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/EventMigratingData" + } + }, + "parentId": 2210 + } + ], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "parentId": 2176 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Emit an event with the associated data to all registered listeners.\n" + } + ] + }, + "flags": {}, + "id": 39387, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "emit", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The event which will be emitted." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 39388, + "kind": 32768, + "kindString": "Parameter", + "name": "event", + "type": { + "name": "Literal", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "Event.ABORTING", + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + }, + "parentId": 2213 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The data which will be passed to the event listeners." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 39389, + "kind": 32768, + "kindString": "Parameter", + "name": "event_data", + "type": { + "name": "EventAbortingData", + "type": "reference", + "target": "2139", + "ref": { + "id": 39313, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 73 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/EventAbortingData" + } + }, + "parentId": 2213 + } + ], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "parentId": 2176 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Emit an event with the associated data to all registered listeners.\n" + } + ] + }, + "flags": {}, + "id": 39390, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "emit", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The event which will be emitted." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 39391, + "kind": 32768, + "kindString": "Parameter", + "name": "event", + "type": { + "name": "Literal", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "Event.EXIT", + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + }, + "parentId": 2216 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The data which will be passed to the event listeners." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 39392, + "kind": 32768, + "kindString": "Parameter", + "name": "event_data", + "type": { + "name": "EventExitData", + "type": "reference", + "target": "2141", + "ref": { + "id": 39315, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 80 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/EventExitData" + } + }, + "parentId": 2216 + } + ], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "parentId": 2176 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Emit an event with the associated data to all registered listeners.\n" + } + ] + }, + "flags": {}, + "id": 39393, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "emit", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The event which will be emitted." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 39394, + "kind": 32768, + "kindString": "Parameter", + "name": "event", + "type": { + "name": "Literal", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "Event.CRAWLER_STATUS", + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + }, + "parentId": 2219 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The data which will be passed to the event listeners." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 39395, + "kind": 32768, + "kindString": "Parameter", + "name": "event_data", + "type": { + "name": "EventCrawlerStatusData", + "type": "reference", + "target": "2143", + "ref": { + "id": 39317, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 87 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/EventCrawlerStatusData" + } + }, + "parentId": 2219 + } + ], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "parentId": 2176 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Emit an event with the associated data to all registered listeners.\n" + } + ] + }, + "flags": {}, + "id": 39396, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "emit", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The event which will be emitted." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 39397, + "kind": 32768, + "kindString": "Parameter", + "name": "event", + "type": { + "name": "Event", + "type": "reference", + "target": "2116", + "ref": { + "id": 39290, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 15 + } + ], + "kind": 8, + "permalink": "https://crawlee.dev/python/api/enum/Event" + } + }, + "parentId": 2222 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The data which will be passed to the event listeners." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 39398, + "kind": 32768, + "kindString": "Parameter", + "name": "event_data", + "type": { + "name": "Any", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2222 + } + ], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "parentId": 2176 + } + ], + "permalink": "/python/api/class/EventManager#emit", + "parentId": 2155 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Wait for all currently executing event listeners to complete.\n" + } + ] + }, + "decorations": [ + { + "name": "ensure_context" + } + ], + "flags": {}, + "groups": [], + "id": 39354, + "module": "events._event_manager", + "name": "wait_for_all_listeners_to_complete", + "parsedDocstring": { + "text": "Wait for all currently executing event listeners to complete.\n", + "args": { + "timeout": "The maximum time to wait for the event listeners to finish. If they do not complete within\nthe specified timeout, they will be canceled." + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_event_manager.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 252, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/events/_event_manager.py#L252" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Wait for all currently executing event listeners to complete.\n" + } + ] + }, + "flags": {}, + "id": 39355, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "wait_for_all_listeners_to_complete", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The maximum time to wait for the event listeners to finish. If they do not complete within\nthe specified timeout, they will be canceled." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 39356, + "kind": 32768, + "kindString": "Parameter", + "name": "timeout", + "type": { + "name": "timedelta | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "timedelta", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2181 + } + ], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "parentId": 2180 + } + ], + "permalink": "/python/api/class/EventManager#wait_for_all_listeners_to_complete", + "parentId": 2155 + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Manage events and their listeners, enabling registration, emission, and execution control.\n\nIt allows for registering event listeners, emitting events, and ensuring all listeners complete their execution.\nBuilt on top of `pyee.asyncio.AsyncIOEventEmitter`. It implements additional features such as waiting for all\nlisteners to complete and emitting `PersistState` events at regular intervals." + } + ] + }, + "decorations": [ + { + "args": "('Event managers')", + "name": "docs_group" + } + ], + "flags": {}, + "groups": [ + { + "children": [ + 39335, + 39337, + 39330, + 39350, + 39346, + 39342, + 39354 + ], + "title": "Methods" + }, + { + "children": [ + 39334 + ], + "title": "Properties" + } + ], + "id": 39329, + "module": "events._event_manager", + "name": "EventManager", + "parsedDocstring": { + "text": "Manage events and their listeners, enabling registration, emission, and execution control.\n\nIt allows for registering event listeners, emitting events, and ensuring all listeners complete their execution.\nBuilt on top of `pyee.asyncio.AsyncIOEventEmitter`. It implements additional features such as waiting for all\nlisteners to complete and emitting `PersistState` events at regular intervals." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_event_manager.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 55, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/events/_event_manager.py#L55" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "extendedBy": [ + { + "name": "LocalEventManager", + "target": "2226", + "type": "reference", + "ref": { + "id": 39400, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_local_event_manager.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 26 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/LocalEventManager" + } + } + ], + "permalink": "/python/api/class/EventManager", + "categories": [] + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Initialize a new instance.\n\nIn most cases, you should use the `from_config` constructor to create a new instance based on\nthe provided configuration.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 42823, + "module": "events._local_event_manager", + "name": "__init__", + "parsedDocstring": { + "text": "Initialize a new instance.\n\nIn most cases, you should use the `from_config` constructor to create a new instance based on\nthe provided configuration.\n", + "args": { + "system_info_interval": "Interval at which `SystemInfo` events are emitted.", + "event_manager_options": "Additional options for the parent class." + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_local_event_manager.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 34, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/events/_local_event_manager.py#L34" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Initialize a new instance.\n\nIn most cases, you should use the `from_config` constructor to create a new instance based on\nthe provided configuration.\n" + } + ] + }, + "flags": {}, + "id": 42824, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "__init__", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Interval at which `SystemInfo` events are emitted." + } + ] + }, + "defaultValue": "timedelta(seconds=1)", + "flags": { + "isOptional": true, + "keyword-only": false + }, + "id": 42825, + "kind": 32768, + "kindString": "Parameter", + "name": "system_info_interval", + "type": { + "name": "timedelta", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2228 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Interval between emitted `PersistState` events to maintain state persistence." + } + ] + }, + "flags": { + "keyword-only": true, + "optional": true + }, + "groups": [], + "id": 42749, + "module": "events._event_manager", + "name": "persist_state_interval", + "parsedDocstring": { + "text": "Interval between emitted `PersistState` events to maintain state persistence." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_event_manager.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 47, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/events/_event_manager.py#L47" + } + ], + "type": { + "name": "NotRequired", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "timedelta", + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + }, + "parentId": 2228 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Optional timeout for canceling pending event listeners if they exceed this duration." + } + ] + }, + "flags": { + "keyword-only": true, + "optional": true + }, + "groups": [], + "id": 42750, + "module": "events._event_manager", + "name": "close_timeout", + "parsedDocstring": { + "text": "Optional timeout for canceling pending event listeners if they exceed this duration." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_event_manager.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 50, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/events/_event_manager.py#L50" + } + ], + "type": { + "name": "NotRequired", + "type": "reference", + "typeArguments": [ + { + "type": "union", + "types": [ + { + "type": "reference", + "name": "timedelta", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + } + ], + "target": 0, + "ref": {} + }, + "parentId": 2228 + } + ], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "overwrites": { + "name": "EventManager.__init__", + "target": 2156, + "type": "reference", + "ref": { + "id": 42752, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_event_manager.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 63 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/EventManager#__init__" + } + }, + "parentId": 2227 + } + ], + "overwrites": { + "name": "EventManager.__init__", + "target": 2156, + "type": "reference", + "ref": { + "id": 42752, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_event_manager.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 63 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/EventManager#__init__" + } + }, + "permalink": "/python/api/class/LocalEventManager#__init__", + "parentId": 2226 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Initialize a new instance based on the provided `Configuration`.\n" + } + ] + }, + "decorations": [ + { + "name": "classmethod" + } + ], + "flags": {}, + "groups": [], + "id": 42827, + "module": "events._local_event_manager", + "name": "from_config", + "parsedDocstring": { + "text": "Initialize a new instance based on the provided `Configuration`.\n", + "args": { + "config": "The `Configuration` instance. Uses the global (default) one if not provided." + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_local_event_manager.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 59, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/events/_local_event_manager.py#L59" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Initialize a new instance based on the provided `Configuration`.\n" + } + ] + }, + "flags": {}, + "id": 42828, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "from_config", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The `Configuration` instance. Uses the global (default) one if not provided." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": false + }, + "id": 42829, + "kind": 32768, + "kindString": "Parameter", + "name": "config", + "type": { + "name": "Configuration | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "Configuration", + "target": "2", + "ref": { + "id": 40598, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/configuration.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 20 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/Configuration" + } + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2232 + } + ], + "type": { + "name": "LocalEventManager", + "type": "reference", + "target": "2226", + "ref": { + "id": 42822, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_local_event_manager.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 26 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/LocalEventManager" + } + }, + "parentId": 2231 + } + ], + "permalink": "/python/api/class/LocalEventManager#from_config", + "parentId": 2226 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Initialize the local event manager upon entering the async context.\n\nIt starts emitting system info events at regular intervals." + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 42830, + "module": "events._local_event_manager", + "name": "__aenter__", + "parsedDocstring": { + "text": "Initialize the local event manager upon entering the async context.\n\nIt starts emitting system info events at regular intervals." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_local_event_manager.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 72, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/events/_local_event_manager.py#L72" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Initialize the local event manager upon entering the async context.\n\nIt starts emitting system info events at regular intervals." + } + ] + }, + "flags": {}, + "id": 42831, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "__aenter__", + "parameters": [], + "type": { + "name": "LocalEventManager", + "type": "reference", + "target": "2226", + "ref": { + "id": 42822, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_local_event_manager.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 26 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/LocalEventManager" + } + }, + "overwrites": { + "name": "EventManager.__aenter__", + "target": 2161, + "type": "reference", + "ref": { + "id": 42757, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_event_manager.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 104 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/EventManager#__aenter__" + } + }, + "parentId": 2234 + } + ], + "overwrites": { + "name": "EventManager.__aenter__", + "target": 2161, + "type": "reference", + "ref": { + "id": 42757, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_event_manager.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 104 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/EventManager#__aenter__" + } + }, + "permalink": "/python/api/class/LocalEventManager#__aenter__", + "parentId": 2226 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Close the local event manager upon exiting the async context.\n\nIt stops emitting system info events and closes the event manager." + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 42832, + "module": "events._local_event_manager", + "name": "__aexit__", + "parsedDocstring": { + "text": "Close the local event manager upon exiting the async context.\n\nIt stops emitting system info events and closes the event manager." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_local_event_manager.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 84, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/events/_local_event_manager.py#L84" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Close the local event manager upon exiting the async context.\n\nIt stops emitting system info events and closes the event manager." + } + ] + }, + "flags": {}, + "id": 42833, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "__aexit__", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 42834, + "kind": 32768, + "kindString": "Parameter", + "name": "exc_type", + "type": { + "name": "type[BaseException] | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "type", + "typeArguments": [ + { + "type": "reference", + "name": "BaseException", + "target": 0, + "ref": {} + } + ], + "target": "3422", + "ref": { + "id": 44018, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_utils/sitemap.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 62 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api/class/SitemapSource#type" + } + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2237 + }, + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 42835, + "kind": 32768, + "kindString": "Parameter", + "name": "exc_value", + "type": { + "name": "BaseException | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "BaseException", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2237 + }, + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 42836, + "kind": 32768, + "kindString": "Parameter", + "name": "exc_traceback", + "type": { + "name": "TracebackType | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "TracebackType", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2237 + } + ], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "overwrites": { + "name": "EventManager.__aexit__", + "target": 2163, + "type": "reference", + "ref": { + "id": 42759, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_event_manager.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 113 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/EventManager#__aexit__" + } + }, + "parentId": 2236 + } + ], + "overwrites": { + "name": "EventManager.__aexit__", + "target": 2163, + "type": "reference", + "ref": { + "id": 42759, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_event_manager.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 113 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/EventManager#__aexit__" + } + }, + "permalink": "/python/api/class/LocalEventManager#__aexit__", + "parentId": 2226 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Indicate whether the context is active." + } + ] + }, + "decorations": [ + { + "name": "property" + } + ], + "flags": {}, + "groups": [], + "id": 44750, + "module": "events._event_manager", + "name": "active", + "parsedDocstring": { + "text": "Indicate whether the context is active." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_event_manager.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 100, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/events/_event_manager.py#L100" + } + ], + "type": { + "name": "bool", + "type": "reference", + "target": 0, + "ref": {} + }, + "inheritedFrom": { + "name": "EventManager.active", + "target": 2160, + "type": "reference", + "ref": { + "id": 42756, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_event_manager.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 100 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api/class/EventManager#active" + } + }, + "permalink": "/python/api/class/LocalEventManager#active", + "parentId": 2226 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Register an event listener for a specific event.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 44751, + "module": "events._event_manager", + "name": "on", + "parsedDocstring": { + "text": "Register an event listener for a specific event.\n", + "args": { + "event": "The event for which to listen to.", + "listener": "The function (sync or async) which is to be called when the event is emitted." + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_event_manager.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 160, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/events/_event_manager.py#L160" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Register an event listener for a specific event.\n" + } + ] + }, + "flags": {}, + "id": 42765, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "on", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The event for which to listen to." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 42766, + "kind": 32768, + "kindString": "Parameter", + "name": "event", + "type": { + "name": "Event", + "type": "reference", + "target": "2116", + "ref": { + "id": 42712, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 15 + } + ], + "kind": 8, + "permalink": "https://crawlee.dev/python/api/enum/Event" + } + }, + "parentId": 2169 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The function (sync or async) which is to be called when the event is emitted." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 42767, + "kind": 32768, + "kindString": "Parameter", + "name": "listener", + "type": { + "name": "EventListener[Any]", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2169 + } + ], + "type": { + "name": "None", + "type": "reference", + "target": 0, + "ref": {} + }, + "inheritedFrom": { + "name": "EventManager.on", + "target": 2168, + "type": "reference", + "ref": { + "id": 42764, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_event_manager.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 160 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/EventManager#on" + } + }, + "parentId": 4155 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Register an event listener for a specific event.\n" + } + ] + }, + "flags": {}, + "id": 42779, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "on", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The event for which to listen to." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 42780, + "kind": 32768, + "kindString": "Parameter", + "name": "event", + "type": { + "name": "Literal[Event.PERSIST_STATE]", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2183 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The function (sync or async) which is to be called when the event is emitted." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 42781, + "kind": 32768, + "kindString": "Parameter", + "name": "listener", + "type": { + "name": "EventListener[EventPersistStateData]", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2183 + } + ], + "type": { + "name": "None", + "type": "reference", + "target": 0, + "ref": {} + }, + "inheritedFrom": { + "name": "EventManager.on", + "target": 2168, + "type": "reference", + "ref": { + "id": 42764, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_event_manager.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 160 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/EventManager#on" + } + }, + "parentId": 4155 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Register an event listener for a specific event.\n" + } + ] + }, + "flags": {}, + "id": 42782, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "on", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The event for which to listen to." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 42783, + "kind": 32768, + "kindString": "Parameter", + "name": "event", + "type": { + "name": "Literal[Event.SYSTEM_INFO]", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2186 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The function (sync or async) which is to be called when the event is emitted." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 42784, + "kind": 32768, + "kindString": "Parameter", + "name": "listener", + "type": { + "name": "EventListener[EventSystemInfoData]", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2186 + } + ], + "type": { + "name": "None", + "type": "reference", + "target": 0, + "ref": {} + }, + "inheritedFrom": { + "name": "EventManager.on", + "target": 2168, + "type": "reference", + "ref": { + "id": 42764, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_event_manager.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 160 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/EventManager#on" + } + }, + "parentId": 4155 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Register an event listener for a specific event.\n" + } + ] + }, + "flags": {}, + "id": 42785, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "on", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The event for which to listen to." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 42786, + "kind": 32768, + "kindString": "Parameter", + "name": "event", + "type": { + "name": "Literal[Event.MIGRATING]", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2189 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The function (sync or async) which is to be called when the event is emitted." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 42787, + "kind": 32768, + "kindString": "Parameter", + "name": "listener", + "type": { + "name": "EventListener[EventMigratingData]", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2189 + } + ], + "type": { + "name": "None", + "type": "reference", + "target": 0, + "ref": {} + }, + "inheritedFrom": { + "name": "EventManager.on", + "target": 2168, + "type": "reference", + "ref": { + "id": 42764, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_event_manager.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 160 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/EventManager#on" + } + }, + "parentId": 4155 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Register an event listener for a specific event.\n" + } + ] + }, + "flags": {}, + "id": 42788, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "on", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The event for which to listen to." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 42789, + "kind": 32768, + "kindString": "Parameter", + "name": "event", + "type": { + "name": "Literal[Event.ABORTING]", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2192 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The function (sync or async) which is to be called when the event is emitted." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 42790, + "kind": 32768, + "kindString": "Parameter", + "name": "listener", + "type": { + "name": "EventListener[EventAbortingData]", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2192 + } + ], + "type": { + "name": "None", + "type": "reference", + "target": 0, + "ref": {} + }, + "inheritedFrom": { + "name": "EventManager.on", + "target": 2168, + "type": "reference", + "ref": { + "id": 42764, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_event_manager.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 160 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/EventManager#on" + } + }, + "parentId": 4155 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Register an event listener for a specific event.\n" + } + ] + }, + "flags": {}, + "id": 42791, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "on", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The event for which to listen to." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 42792, + "kind": 32768, + "kindString": "Parameter", + "name": "event", + "type": { + "name": "Literal[Event.EXIT]", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2195 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The function (sync or async) which is to be called when the event is emitted." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 42793, + "kind": 32768, + "kindString": "Parameter", + "name": "listener", + "type": { + "name": "EventListener[EventExitData]", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2195 + } + ], + "type": { + "name": "None", + "type": "reference", + "target": 0, + "ref": {} + }, + "inheritedFrom": { + "name": "EventManager.on", + "target": 2168, + "type": "reference", + "ref": { + "id": 42764, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_event_manager.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 160 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/EventManager#on" + } + }, + "parentId": 4155 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Register an event listener for a specific event.\n" + } + ] + }, + "flags": {}, + "id": 42794, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "on", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The event for which to listen to." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 42795, + "kind": 32768, + "kindString": "Parameter", + "name": "event", + "type": { + "name": "Literal[Event.CRAWLER_STATUS]", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2198 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The function (sync or async) which is to be called when the event is emitted." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 42796, + "kind": 32768, + "kindString": "Parameter", + "name": "listener", + "type": { + "name": "EventListener[EventCrawlerStatusData]", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2198 + } + ], + "type": { + "name": "None", + "type": "reference", + "target": 0, + "ref": {} + }, + "inheritedFrom": { + "name": "EventManager.on", + "target": 2168, + "type": "reference", + "ref": { + "id": 42764, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_event_manager.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 160 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/EventManager#on" + } + }, + "parentId": 4155 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Register an event listener for a specific event.\n" + } + ] + }, + "flags": {}, + "id": 42797, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "on", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The event for which to listen to." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 42798, + "kind": 32768, + "kindString": "Parameter", + "name": "event", + "type": { + "name": "Event", + "type": "reference", + "target": "2116", + "ref": { + "id": 42712, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 15 + } + ], + "kind": 8, + "permalink": "https://crawlee.dev/python/api/enum/Event" + } + }, + "parentId": 2201 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The function (sync or async) which is to be called when the event is emitted." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 42799, + "kind": 32768, + "kindString": "Parameter", + "name": "listener", + "type": { + "name": "EventListener[None]", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2201 + } + ], + "type": { + "name": "None", + "type": "reference", + "target": 0, + "ref": {} + }, + "inheritedFrom": { + "name": "EventManager.on", + "target": 2168, + "type": "reference", + "ref": { + "id": 42764, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_event_manager.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 160 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/EventManager#on" + } + }, + "parentId": 4155 + } + ], + "inheritedFrom": { + "name": "EventManager.on", + "target": 2168, + "type": "reference", + "ref": { + "id": 42764, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_event_manager.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 160 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/EventManager#on" + } + }, + "permalink": "/python/api/class/LocalEventManager#on", + "parentId": 2226 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Remove a specific listener or all listeners for an event.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 44752, + "module": "events._event_manager", + "name": "off", + "parsedDocstring": { + "text": "Remove a specific listener or all listeners for an event.\n", + "args": { + "event": "The Actor event for which to remove listeners.", + "listener": "The listener which is supposed to be removed. If not passed, all listeners of this event\nare removed." + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_event_manager.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 210, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/events/_event_manager.py#L210" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Remove a specific listener or all listeners for an event.\n" + } + ] + }, + "flags": {}, + "id": 42769, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "off", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The Actor event for which to remove listeners." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 42770, + "kind": 32768, + "kindString": "Parameter", + "name": "event", + "type": { + "name": "Event", + "type": "reference", + "target": "2116", + "ref": { + "id": 42712, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 15 + } + ], + "kind": 8, + "permalink": "https://crawlee.dev/python/api/enum/Event" + } + }, + "parentId": 2173 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The listener which is supposed to be removed. If not passed, all listeners of this event\nare removed." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 42771, + "kind": 32768, + "kindString": "Parameter", + "name": "listener", + "type": { + "name": "EventListener[Any] | None", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2173 + } + ], + "type": { + "name": "None", + "type": "reference", + "target": 0, + "ref": {} + }, + "inheritedFrom": { + "name": "EventManager.off", + "target": 2172, + "type": "reference", + "ref": { + "id": 42768, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_event_manager.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 210 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/EventManager#off" + } + }, + "parentId": 4156 + } + ], + "inheritedFrom": { + "name": "EventManager.off", + "target": 2172, + "type": "reference", + "ref": { + "id": 42768, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_event_manager.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 210 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/EventManager#off" + } + }, + "permalink": "/python/api/class/LocalEventManager#off", + "parentId": 2226 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Emit an event with the associated data to all registered listeners.\n" + } + ] + }, + "decorations": [ + { + "name": "ensure_context" + } + ], + "flags": {}, + "groups": [], + "id": 44753, + "module": "events._event_manager", + "name": "emit", + "parsedDocstring": { + "text": "Emit an event with the associated data to all registered listeners.\n", + "args": { + "event": "The event which will be emitted.", + "event_data": "The data which will be passed to the event listeners." + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_event_manager.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 242, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/events/_event_manager.py#L242" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Emit an event with the associated data to all registered listeners.\n" + } + ] + }, + "flags": {}, + "id": 42773, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "emit", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The event which will be emitted." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 42774, + "kind": 32768, + "kindString": "Parameter", + "name": "event", + "type": { + "name": "Event", + "type": "reference", + "target": "2116", + "ref": { + "id": 42712, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 15 + } + ], + "kind": 8, + "permalink": "https://crawlee.dev/python/api/enum/Event" + } + }, + "parentId": 2177 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The data which will be passed to the event listeners." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 42775, + "kind": 32768, + "kindString": "Parameter", + "name": "event_data", + "type": { + "name": "EventData", + "type": "reference", + "target": "2147", + "ref": { + "id": 42743, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 99 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api#EventData" + } + }, + "parentId": 2177 + } + ], + "type": { + "name": "None", + "type": "reference", + "target": 0, + "ref": {} + }, + "inheritedFrom": { + "name": "EventManager.emit", + "target": 2176, + "type": "reference", + "ref": { + "id": 42772, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_event_manager.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 242 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/EventManager#emit" + } + }, + "parentId": 4157 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Emit an event with the associated data to all registered listeners.\n" + } + ] + }, + "flags": {}, + "id": 42800, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "emit", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The event which will be emitted." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 42801, + "kind": 32768, + "kindString": "Parameter", + "name": "event", + "type": { + "name": "Literal[Event.PERSIST_STATE]", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2204 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The data which will be passed to the event listeners." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 42802, + "kind": 32768, + "kindString": "Parameter", + "name": "event_data", + "type": { + "name": "EventPersistStateData", + "type": "reference", + "target": "2129", + "ref": { + "id": 42725, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 40 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/EventPersistStateData" + } + }, + "parentId": 2204 + } + ], + "type": { + "name": "None", + "type": "reference", + "target": 0, + "ref": {} + }, + "inheritedFrom": { + "name": "EventManager.emit", + "target": 2176, + "type": "reference", + "ref": { + "id": 42772, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_event_manager.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 242 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/EventManager#emit" + } + }, + "parentId": 4157 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Emit an event with the associated data to all registered listeners.\n" + } + ] + }, + "flags": {}, + "id": 42803, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "emit", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The event which will be emitted." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 42804, + "kind": 32768, + "kindString": "Parameter", + "name": "event", + "type": { + "name": "Literal[Event.SYSTEM_INFO]", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2207 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The data which will be passed to the event listeners." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 42805, + "kind": 32768, + "kindString": "Parameter", + "name": "event_data", + "type": { + "name": "EventSystemInfoData", + "type": "reference", + "target": "2132", + "ref": { + "id": 42728, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 49 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/EventSystemInfoData" + } + }, + "parentId": 2207 + } + ], + "type": { + "name": "None", + "type": "reference", + "target": 0, + "ref": {} + }, + "inheritedFrom": { + "name": "EventManager.emit", + "target": 2176, + "type": "reference", + "ref": { + "id": 42772, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_event_manager.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 242 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/EventManager#emit" + } + }, + "parentId": 4157 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Emit an event with the associated data to all registered listeners.\n" + } + ] + }, + "flags": {}, + "id": 42806, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "emit", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The event which will be emitted." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 42807, + "kind": 32768, + "kindString": "Parameter", + "name": "event", + "type": { + "name": "Literal[Event.MIGRATING]", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2210 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The data which will be passed to the event listeners." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 42808, + "kind": 32768, + "kindString": "Parameter", + "name": "event_data", + "type": { + "name": "EventMigratingData", + "type": "reference", + "target": "2136", + "ref": { + "id": 42732, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 62 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/EventMigratingData" + } + }, + "parentId": 2210 + } + ], + "type": { + "name": "None", + "type": "reference", + "target": 0, + "ref": {} + }, + "inheritedFrom": { + "name": "EventManager.emit", + "target": 2176, + "type": "reference", + "ref": { + "id": 42772, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_event_manager.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 242 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/EventManager#emit" + } + }, + "parentId": 4157 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Emit an event with the associated data to all registered listeners.\n" + } + ] + }, + "flags": {}, + "id": 42809, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "emit", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The event which will be emitted." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 42810, + "kind": 32768, + "kindString": "Parameter", + "name": "event", + "type": { + "name": "Literal[Event.ABORTING]", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2213 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The data which will be passed to the event listeners." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 42811, + "kind": 32768, + "kindString": "Parameter", + "name": "event_data", + "type": { + "name": "EventAbortingData", + "type": "reference", + "target": "2139", + "ref": { + "id": 42735, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 73 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/EventAbortingData" + } + }, + "parentId": 2213 + } + ], + "type": { + "name": "None", + "type": "reference", + "target": 0, + "ref": {} + }, + "inheritedFrom": { + "name": "EventManager.emit", + "target": 2176, + "type": "reference", + "ref": { + "id": 42772, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_event_manager.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 242 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/EventManager#emit" + } + }, + "parentId": 4157 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Emit an event with the associated data to all registered listeners.\n" + } + ] + }, + "flags": {}, + "id": 42812, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "emit", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The event which will be emitted." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 42813, + "kind": 32768, + "kindString": "Parameter", + "name": "event", + "type": { + "name": "Literal[Event.EXIT]", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2216 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The data which will be passed to the event listeners." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 42814, + "kind": 32768, + "kindString": "Parameter", + "name": "event_data", + "type": { + "name": "EventExitData", + "type": "reference", + "target": "2141", + "ref": { + "id": 42737, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 80 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/EventExitData" + } + }, + "parentId": 2216 + } + ], + "type": { + "name": "None", + "type": "reference", + "target": 0, + "ref": {} + }, + "inheritedFrom": { + "name": "EventManager.emit", + "target": 2176, + "type": "reference", + "ref": { + "id": 42772, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_event_manager.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 242 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/EventManager#emit" + } + }, + "parentId": 4157 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Emit an event with the associated data to all registered listeners.\n" + } + ] + }, + "flags": {}, + "id": 42815, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "emit", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The event which will be emitted." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 42816, + "kind": 32768, + "kindString": "Parameter", + "name": "event", + "type": { + "name": "Literal[Event.CRAWLER_STATUS]", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2219 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The data which will be passed to the event listeners." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 42817, + "kind": 32768, + "kindString": "Parameter", + "name": "event_data", + "type": { + "name": "EventCrawlerStatusData", + "type": "reference", + "target": "2143", + "ref": { + "id": 42739, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 87 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/EventCrawlerStatusData" + } + }, + "parentId": 2219 + } + ], + "type": { + "name": "None", + "type": "reference", + "target": 0, + "ref": {} + }, + "inheritedFrom": { + "name": "EventManager.emit", + "target": 2176, + "type": "reference", + "ref": { + "id": 42772, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_event_manager.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 242 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/EventManager#emit" + } + }, + "parentId": 4157 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Emit an event with the associated data to all registered listeners.\n" + } + ] + }, + "flags": {}, + "id": 42818, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "emit", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The event which will be emitted." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 42819, + "kind": 32768, + "kindString": "Parameter", + "name": "event", + "type": { + "name": "Event", + "type": "reference", + "target": "2116", + "ref": { + "id": 42712, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 15 + } + ], + "kind": 8, + "permalink": "https://crawlee.dev/python/api/enum/Event" + } + }, + "parentId": 2222 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The data which will be passed to the event listeners." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 42820, + "kind": 32768, + "kindString": "Parameter", + "name": "event_data", + "type": { + "name": "Any", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2222 + } + ], + "type": { + "name": "None", + "type": "reference", + "target": 0, + "ref": {} + }, + "inheritedFrom": { + "name": "EventManager.emit", + "target": 2176, + "type": "reference", + "ref": { + "id": 42772, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_event_manager.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 242 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/EventManager#emit" + } + }, + "parentId": 4157 + } + ], + "inheritedFrom": { + "name": "EventManager.emit", + "target": 2176, + "type": "reference", + "ref": { + "id": 42772, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_event_manager.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 242 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/EventManager#emit" + } + }, + "permalink": "/python/api/class/LocalEventManager#emit", + "parentId": 2226 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Wait for all currently executing event listeners to complete.\n" + } + ] + }, + "decorations": [ + { + "name": "ensure_context" + } + ], + "flags": {}, + "groups": [], + "id": 44754, + "module": "events._event_manager", + "name": "wait_for_all_listeners_to_complete", + "parsedDocstring": { + "text": "Wait for all currently executing event listeners to complete.\n", + "args": { + "timeout": "The maximum time to wait for the event listeners to finish. If they do not complete within\nthe specified timeout, they will be canceled." + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_event_manager.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 252, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/events/_event_manager.py#L252" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Wait for all currently executing event listeners to complete.\n" + } + ] + }, + "flags": {}, + "id": 42777, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "wait_for_all_listeners_to_complete", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The maximum time to wait for the event listeners to finish. If they do not complete within\nthe specified timeout, they will be canceled." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 42778, + "kind": 32768, + "kindString": "Parameter", + "name": "timeout", + "type": { + "name": "timedelta | None", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2181 + } + ], + "type": { + "name": "None", + "type": "reference", + "target": 0, + "ref": {} + }, + "inheritedFrom": { + "name": "EventManager.wait_for_all_listeners_to_complete", + "target": 2180, + "type": "reference", + "ref": { + "id": 42776, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_event_manager.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 252 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/EventManager#wait_for_all_listeners_to_complete" + } + }, + "parentId": 4158 + } + ], + "inheritedFrom": { + "name": "EventManager.wait_for_all_listeners_to_complete", + "target": 2180, + "type": "reference", + "ref": { + "id": 42776, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_event_manager.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 252 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/EventManager#wait_for_all_listeners_to_complete" + } + }, + "permalink": "/python/api/class/LocalEventManager#wait_for_all_listeners_to_complete", + "parentId": 2226 + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Event manager for local environments.\n\nIt extends the `EventManager` to emit `SystemInfo` events at regular intervals. The `LocalEventManager`\nis intended to be used in local environments, where the system metrics are required managing the `Snapshotter`\nand `AutoscaledPool`." + } + ] + }, + "decorations": [ + { + "args": "('Event managers')", + "name": "docs_group" + } + ], + "flags": {}, + "groups": [ + { + "children": [ + 42830, + 42832, + 42823, + 44753, + 42827, + 44752, + 44751, + 44754 + ], + "title": "Methods" + }, + { + "children": [ + 44750 + ], + "title": "Properties" + } + ], + "id": 42822, + "module": "events._local_event_manager", + "name": "LocalEventManager", + "parsedDocstring": { + "text": "Event manager for local environments.\n\nIt extends the `EventManager` to emit `SystemInfo` events at regular intervals. The `LocalEventManager`\nis intended to be used in local environments, where the system metrics are required managing the `Snapshotter`\nand `AutoscaledPool`." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_local_event_manager.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 26, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/events/_local_event_manager.py#L26" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "extendedTypes": [ + { + "name": "EventManager", + "target": "2155", + "type": "reference", + "ref": { + "id": 42751, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_event_manager.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 55 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/EventManager" + } + } + ], + "permalink": "/python/api/class/LocalEventManager", + "categories": [] + }, + { + "kind": 8, + "kindString": "Enumeration", + "children": [ + { + "kind": 16, + "kindString": "Enumeration Member", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 46871, + "module": "events._types", + "name": "PERSIST_STATE", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 19, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/events/_types.py#L19" + } + ], + "type": { + "type": "literal", + "value": "'persistState'" + }, + "permalink": "/python/api/enum/Event#PERSIST_STATE", + "parentId": 2116 + }, + { + "kind": 16, + "kindString": "Enumeration Member", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 46872, + "module": "events._types", + "name": "SYSTEM_INFO", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 20, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/events/_types.py#L20" + } + ], + "type": { + "type": "literal", + "value": "'systemInfo'" + }, + "permalink": "/python/api/enum/Event#SYSTEM_INFO", + "parentId": 2116 + }, + { + "kind": 16, + "kindString": "Enumeration Member", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 46873, + "module": "events._types", + "name": "MIGRATING", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 21, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/events/_types.py#L21" + } + ], + "type": { + "type": "literal", + "value": "'migrating'" + }, + "permalink": "/python/api/enum/Event#MIGRATING", + "parentId": 2116 + }, + { + "kind": 16, + "kindString": "Enumeration Member", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 46874, + "module": "events._types", + "name": "ABORTING", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 22, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/events/_types.py#L22" + } + ], + "type": { + "type": "literal", + "value": "'aborting'" + }, + "permalink": "/python/api/enum/Event#ABORTING", + "parentId": 2116 + }, + { + "kind": 16, + "kindString": "Enumeration Member", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 46875, + "module": "events._types", + "name": "EXIT", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 23, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/events/_types.py#L23" + } + ], + "type": { + "type": "literal", + "value": "'exit'" + }, + "permalink": "/python/api/enum/Event#EXIT", + "parentId": 2116 + }, + { + "kind": 16, + "kindString": "Enumeration Member", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 46876, + "module": "events._types", + "name": "SESSION_RETIRED", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 26, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/events/_types.py#L26" + } + ], + "type": { + "type": "literal", + "value": "'sessionRetired'" + }, + "permalink": "/python/api/enum/Event#SESSION_RETIRED", + "parentId": 2116 + }, + { + "kind": 16, + "kindString": "Enumeration Member", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 46877, + "module": "events._types", + "name": "BROWSER_LAUNCHED", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 29, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/events/_types.py#L29" + } + ], + "type": { + "type": "literal", + "value": "'browserLaunched'" + }, + "permalink": "/python/api/enum/Event#BROWSER_LAUNCHED", + "parentId": 2116 + }, + { + "kind": 16, + "kindString": "Enumeration Member", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 46878, + "module": "events._types", + "name": "BROWSER_RETIRED", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 30, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/events/_types.py#L30" + } + ], + "type": { + "type": "literal", + "value": "'browserRetired'" + }, + "permalink": "/python/api/enum/Event#BROWSER_RETIRED", + "parentId": 2116 + }, + { + "kind": 16, + "kindString": "Enumeration Member", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 46879, + "module": "events._types", + "name": "BROWSER_CLOSED", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 31, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/events/_types.py#L31" + } + ], + "type": { + "type": "literal", + "value": "'browserClosed'" + }, + "permalink": "/python/api/enum/Event#BROWSER_CLOSED", + "parentId": 2116 + }, + { + "kind": 16, + "kindString": "Enumeration Member", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 46880, + "module": "events._types", + "name": "PAGE_CREATED", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 32, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/events/_types.py#L32" + } + ], + "type": { + "type": "literal", + "value": "'pageCreated'" + }, + "permalink": "/python/api/enum/Event#PAGE_CREATED", + "parentId": 2116 + }, + { + "kind": 16, + "kindString": "Enumeration Member", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 46881, + "module": "events._types", + "name": "PAGE_CLOSED", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 33, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/events/_types.py#L33" + } + ], + "type": { + "type": "literal", + "value": "'pageClosed'" + }, + "permalink": "/python/api/enum/Event#PAGE_CLOSED", + "parentId": 2116 + }, + { + "kind": 16, + "kindString": "Enumeration Member", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 46882, + "module": "events._types", + "name": "CRAWLER_STATUS", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 36, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/events/_types.py#L36" + } + ], + "type": { + "type": "literal", + "value": "'crawlerStatus'" + }, + "permalink": "/python/api/enum/Event#CRAWLER_STATUS", + "parentId": 2116 + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Names of all possible events that can be emitted using an `EventManager`." + } + ] + }, + "decorations": [ + { + "args": "('Event data')", + "name": "docs_group" + } + ], + "flags": {}, + "groups": [ + { + "children": [ + 46874, + 46879, + 46877, + 46878, + 46882, + 46875, + 46873, + 46881, + 46880, + 46871, + 46876, + 46872 + ], + "title": "Enumeration members" + } + ], + "id": 46870, + "module": "events._types", + "name": "Event", + "parsedDocstring": { + "text": "Names of all possible events that can be emitted using an `EventManager`." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 15, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/events/_types.py#L15" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "permalink": "/python/api/enum/Event", + "categories": [] + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 49022, + "module": "events._types", + "name": "model_config", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 76, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/events/_types.py#L76" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "permalink": "/python/api/class/EventAbortingData#model_config", + "parentId": 2139 + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Data for the aborting event." + } + ] + }, + "decorations": [ + { + "args": "('Event data')", + "name": "docs_group" + } + ], + "flags": {}, + "groups": [ + { + "children": [ + 49022 + ], + "title": "Properties" + } + ], + "id": 49021, + "module": "events._types", + "name": "EventAbortingData", + "parsedDocstring": { + "text": "Data for the aborting event." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 73, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/events/_types.py#L73" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "permalink": "/python/api/class/EventAbortingData", + "categories": [] + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 51164, + "module": "events._types", + "name": "model_config", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 83, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/events/_types.py#L83" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "permalink": "/python/api/class/EventExitData#model_config", + "parentId": 2141 + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Data for the exit event." + } + ] + }, + "decorations": [ + { + "args": "('Event data')", + "name": "docs_group" + } + ], + "flags": {}, + "groups": [ + { + "children": [ + 51164 + ], + "title": "Properties" + } + ], + "id": 51163, + "module": "events._types", + "name": "EventExitData", + "parsedDocstring": { + "text": "Data for the exit event." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 80, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/events/_types.py#L80" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "permalink": "/python/api/class/EventExitData", + "categories": [] + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 53301, + "module": "events._types", + "name": "model_config", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 65, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/events/_types.py#L65" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "permalink": "/python/api/class/EventMigratingData#model_config", + "parentId": 2136 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 53302, + "module": "events._types", + "name": "time_remaining", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 69, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/events/_types.py#L69" + } + ], + "type": { + "name": "Annotated[timedelta_secs | None, Field(alias='timeRemainingSecs')]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "timedelta_secs", + "target": "3321", + "ref": { + "id": 54485, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_utils/models.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 68 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api#timedelta_secs" + } + }, + { + "type": "literal", + "value": {} + } + ] + }, + "permalink": "/python/api/class/EventMigratingData#time_remaining", + "parentId": 2136 + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Data for the migrating event." + } + ] + }, + "decorations": [ + { + "args": "('Event data')", + "name": "docs_group" + } + ], + "flags": {}, + "groups": [ + { + "children": [ + 53301, + 53302 + ], + "title": "Properties" + } + ], + "id": 53300, + "module": "events._types", + "name": "EventMigratingData", + "parsedDocstring": { + "text": "Data for the migrating event." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 62, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/events/_types.py#L62" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "permalink": "/python/api/class/EventMigratingData", + "categories": [] + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 56615, + "module": "events._types", + "name": "model_config", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 43, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/events/_types.py#L43" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "permalink": "/python/api/class/EventPersistStateData#model_config", + "parentId": 2129 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 56616, + "module": "events._types", + "name": "is_migrating", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 45, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/events/_types.py#L45" + } + ], + "type": { + "name": "bool", + "type": "reference", + "target": 0, + "ref": {} + }, + "permalink": "/python/api/class/EventPersistStateData#is_migrating", + "parentId": 2129 + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Data for the persist state event." + } + ] + }, + "decorations": [ + { + "args": "('Event data')", + "name": "docs_group" + } + ], + "flags": {}, + "groups": [ + { + "children": [ + 56616, + 56615 + ], + "title": "Properties" + } + ], + "id": 56614, + "module": "events._types", + "name": "EventPersistStateData", + "parsedDocstring": { + "text": "Data for the persist state event." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 40, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/events/_types.py#L40" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "permalink": "/python/api/class/EventPersistStateData", + "categories": [] + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 58749, + "module": "events._types", + "name": "model_config", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 52, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/events/_types.py#L52" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "permalink": "/python/api/class/EventSystemInfoData#model_config", + "parentId": 2132 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 58750, + "module": "events._types", + "name": "cpu_info", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 54, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/events/_types.py#L54" + } + ], + "type": { + "name": "CpuInfo", + "type": "reference", + "target": "3342", + "ref": { + "id": 59958, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_utils/system.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 36 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/CpuInfo" + } + }, + "permalink": "/python/api/class/EventSystemInfoData#cpu_info", + "parentId": 2132 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 58751, + "module": "events._types", + "name": "memory_info", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 55, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/events/_types.py#L55" + } + ], + "type": { + "name": "MemoryUsageInfo", + "type": "reference", + "target": "3345", + "ref": { + "id": 59961, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_utils/system.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 59 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/MemoryUsageInfo" + } + }, + "permalink": "/python/api/class/EventSystemInfoData#memory_info", + "parentId": 2132 + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Data for the system info event." + } + ] + }, + "decorations": [ + { + "args": "('Event data')", + "name": "docs_group" + } + ], + "flags": {}, + "groups": [ + { + "children": [ + 58750, + 58751, + 58749 + ], + "title": "Properties" + } + ], + "id": 58748, + "module": "events._types", + "name": "EventSystemInfoData", + "parsedDocstring": { + "text": "Data for the system info event." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 49, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/events/_types.py#L49" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "permalink": "/python/api/class/EventSystemInfoData", + "categories": [] + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Return a cache key that can differentiate between different storages of this and other clients.\n\nCan be based on configuration or on the client itself. By default, returns a module and name of the client\nclass." + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 61244, + "module": "storage_clients._base._storage_client", + "name": "get_storage_client_cache_key", + "parsedDocstring": { + "text": "Return a cache key that can differentiate between different storages of this and other clients.\n\nCan be based on configuration or on the client itself. By default, returns a module and name of the client\nclass." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 33, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/_base/_storage_client.py#L33" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Return a cache key that can differentiate between different storages of this and other clients.\n\nCan be based on configuration or on the client itself. By default, returns a module and name of the client\nclass." + } + ] + }, + "flags": {}, + "id": 61245, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "get_storage_client_cache_key", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 61246, + "kind": 32768, + "kindString": "Parameter", + "name": "configuration", + "type": { + "name": "Configuration", + "type": "reference", + "target": "2", + "ref": { + "id": 59963, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/configuration.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 20 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/Configuration" + } + }, + "parentId": 1284 + } + ], + "type": { + "name": "Hashable", + "type": "reference", + "target": 0, + "ref": {} + }, + "overwrites": { + "name": "StorageClient.get_storage_client_cache_key", + "target": 1283, + "type": "reference", + "ref": { + "id": 61244, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 33 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/StorageClient#get_storage_client_cache_key" + } + }, + "parentId": 1283 + } + ], + "permalink": "/python/api/class/StorageClient#get_storage_client_cache_key", + "parentId": 1282 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Create a dataset client." + } + ] + }, + "decorations": [ + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 61247, + "module": "storage_clients._base._storage_client", + "name": "create_dataset_client", + "parsedDocstring": { + "text": "Create a dataset client." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 42, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/_base/_storage_client.py#L42" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Create a dataset client." + } + ] + }, + "flags": {}, + "id": 61248, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "create_dataset_client", + "parameters": [ + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 61249, + "kind": 32768, + "kindString": "Parameter", + "name": "id", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 1287 + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 61250, + "kind": 32768, + "kindString": "Parameter", + "name": "name", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 1287 + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 61251, + "kind": 32768, + "kindString": "Parameter", + "name": "alias", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 1287 + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 61252, + "kind": 32768, + "kindString": "Parameter", + "name": "configuration", + "type": { + "name": "Configuration | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "Configuration", + "target": "2", + "ref": { + "id": 59963, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/configuration.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 20 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/Configuration" + } + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 1287 + } + ], + "type": { + "name": "DatasetClient", + "type": "reference", + "target": "1220", + "ref": { + "id": 61181, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_dataset_client.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 13 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/DatasetClient" + } + }, + "overwrites": { + "name": "StorageClient.create_dataset_client", + "target": 1286, + "type": "reference", + "ref": { + "id": 61247, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 42 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/StorageClient#create_dataset_client" + } + }, + "parentId": 1286 + } + ], + "permalink": "/python/api/class/StorageClient#create_dataset_client", + "parentId": 1282 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Create a key-value store client." + } + ] + }, + "decorations": [ + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 61253, + "module": "storage_clients._base._storage_client", + "name": "create_kvs_client", + "parsedDocstring": { + "text": "Create a key-value store client." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 53, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/_base/_storage_client.py#L53" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Create a key-value store client." + } + ] + }, + "flags": {}, + "id": 61254, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "create_kvs_client", + "parameters": [ + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 61255, + "kind": 32768, + "kindString": "Parameter", + "name": "id", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 1293 + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 61256, + "kind": 32768, + "kindString": "Parameter", + "name": "name", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 1293 + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 61257, + "kind": 32768, + "kindString": "Parameter", + "name": "alias", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 1293 + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 61258, + "kind": 32768, + "kindString": "Parameter", + "name": "configuration", + "type": { + "name": "Configuration | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "Configuration", + "target": "2", + "ref": { + "id": 59963, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/configuration.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 20 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/Configuration" + } + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 1293 + } + ], + "type": { + "name": "KeyValueStoreClient", + "type": "reference", + "target": "1254", + "ref": { + "id": 61215, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_key_value_store_client.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 14 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/KeyValueStoreClient" + } + }, + "overwrites": { + "name": "StorageClient.create_kvs_client", + "target": 1292, + "type": "reference", + "ref": { + "id": 61253, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 53 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/StorageClient#create_kvs_client" + } + }, + "parentId": 1292 + } + ], + "permalink": "/python/api/class/StorageClient#create_kvs_client", + "parentId": 1282 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Create a request queue client." + } + ] + }, + "decorations": [ + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 61259, + "module": "storage_clients._base._storage_client", + "name": "create_rq_client", + "parsedDocstring": { + "text": "Create a request queue client." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 64, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/_base/_storage_client.py#L64" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Create a request queue client." + } + ] + }, + "flags": {}, + "id": 61260, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "create_rq_client", + "parameters": [ + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 61261, + "kind": 32768, + "kindString": "Parameter", + "name": "id", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 1299 + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 61262, + "kind": 32768, + "kindString": "Parameter", + "name": "name", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 1299 + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 61263, + "kind": 32768, + "kindString": "Parameter", + "name": "alias", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 1299 + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 61264, + "kind": 32768, + "kindString": "Parameter", + "name": "configuration", + "type": { + "name": "Configuration | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "Configuration", + "target": "2", + "ref": { + "id": 59963, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/configuration.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 20 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/Configuration" + } + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 1299 + } + ], + "type": { + "name": "RequestQueueClient", + "type": "reference", + "target": "1195", + "ref": { + "id": 61156, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_request_queue_client.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 13 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/RequestQueueClient" + } + }, + "overwrites": { + "name": "StorageClient.create_rq_client", + "target": 1298, + "type": "reference", + "ref": { + "id": 61259, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 64 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/StorageClient#create_rq_client" + } + }, + "parentId": 1298 + } + ], + "permalink": "/python/api/class/StorageClient#create_rq_client", + "parentId": 1282 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Return statistics about rate limit errors encountered by the HTTP client in storage client." + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 61265, + "module": "storage_clients._base._storage_client", + "name": "get_rate_limit_errors", + "parsedDocstring": { + "text": "Return statistics about rate limit errors encountered by the HTTP client in storage client." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 74, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/_base/_storage_client.py#L74" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Return statistics about rate limit errors encountered by the HTTP client in storage client." + } + ] + }, + "flags": {}, + "id": 61266, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "get_rate_limit_errors", + "parameters": [], + "type": { + "name": "dict", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "int", + "target": 0, + "ref": {} + }, + { + "type": "reference", + "name": "int", + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + }, + "parentId": 1304 + } + ], + "permalink": "/python/api/class/StorageClient#get_rate_limit_errors", + "parentId": 1282 + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Base class for storage clients.\n\nThe `StorageClient` serves as an abstract base class that defines the interface for accessing Crawlee's\nstorage types: datasets, key-value stores, and request queues. It provides methods to open clients for\neach of these storage types and handles common functionality.\n\nStorage clients implementations can be provided for various backends (file system, memory, databases,\nvarious cloud providers, etc.) to support different use cases from development to production environments.\n\nEach storage client implementation is responsible for ensuring proper initialization, data persistence\n(where applicable), and consistent access patterns across all storage types it supports." + } + ] + }, + "decorations": [ + { + "args": "('Storage clients')", + "name": "docs_group" + } + ], + "flags": {}, + "groups": [ + { + "children": [ + 61247, + 61253, + 61259, + 61265, + 61244 + ], + "title": "Methods" + } + ], + "id": 61243, + "module": "storage_clients._base._storage_client", + "name": "StorageClient", + "parsedDocstring": { + "text": "Base class for storage clients.\n\nThe `StorageClient` serves as an abstract base class that defines the interface for accessing Crawlee's\nstorage types: datasets, key-value stores, and request queues. It provides methods to open clients for\neach of these storage types and handles common functionality.\n\nStorage clients implementations can be provided for various backends (file system, memory, databases,\nvarious cloud providers, etc.) to support different use cases from development to production environments.\n\nEach storage client implementation is responsible for ensuring proper initialization, data persistence\n(where applicable), and consistent access patterns across all storage types it supports." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 19, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/_base/_storage_client.py#L19" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "extendedBy": [ + { + "name": "MemoryStorageClient", + "target": "1468", + "type": "reference", + "ref": { + "id": 61429, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_memory/_storage_client.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 15 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/MemoryStorageClient" + } + }, + { + "name": "FileSystemStorageClient", + "target": "1624", + "type": "reference", + "ref": { + "id": 61585, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_file_system/_storage_client.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 20 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/FileSystemStorageClient" + } + }, + { + "name": "SqlStorageClient", + "target": "1893", + "type": "reference", + "ref": { + "id": 61854, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_sql/_storage_client.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 35 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/SqlStorageClient" + } + }, + { + "name": "RedisStorageClient", + "target": "2091", + "type": "reference", + "ref": { + "id": 62052, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_redis/_storage_client.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 19 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/RedisStorageClient" + } + } + ], + "permalink": "/python/api/class/StorageClient", + "categories": [] + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Create a dataset client." + } + ] + }, + "decorations": [ + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 63521, + "module": "storage_clients._base._storage_client", + "name": "create_dataset_client", + "parsedDocstring": { + "text": "Create a dataset client." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_memory/_storage_client.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 31, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/_memory/_storage_client.py#L31" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Create a dataset client." + } + ] + }, + "flags": {}, + "id": 63339, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "create_dataset_client", + "parameters": [ + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 63340, + "kind": 32768, + "kindString": "Parameter", + "name": "id", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 1287 + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 63341, + "kind": 32768, + "kindString": "Parameter", + "name": "name", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 1287 + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 63342, + "kind": 32768, + "kindString": "Parameter", + "name": "alias", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 1287 + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 63343, + "kind": 32768, + "kindString": "Parameter", + "name": "configuration", + "type": { + "name": "Configuration | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "Configuration", + "target": "2", + "ref": { + "id": 62054, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/configuration.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 20 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/Configuration" + } + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 1287 + } + ], + "type": { + "name": "DatasetClient", + "type": "reference", + "target": "1220", + "ref": { + "id": 63272, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_dataset_client.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 13 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/DatasetClient" + } + }, + "overwrites": { + "name": "StorageClient.create_dataset_client", + "target": 1286, + "type": "reference", + "ref": { + "id": 63338, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 42 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/StorageClient#create_dataset_client" + } + }, + "parentId": 1469 + } + ], + "overwrites": { + "name": "StorageClient.create_dataset_client", + "target": 1286, + "type": "reference", + "ref": { + "id": 63338, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 42 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/StorageClient#create_dataset_client" + } + }, + "permalink": "/python/api/class/MemoryStorageClient#create_dataset_client", + "parentId": 1468 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Create a key-value store client." + } + ] + }, + "decorations": [ + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 63527, + "module": "storage_clients._base._storage_client", + "name": "create_kvs_client", + "parsedDocstring": { + "text": "Create a key-value store client." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_memory/_storage_client.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 45, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/_memory/_storage_client.py#L45" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Create a key-value store client." + } + ] + }, + "flags": {}, + "id": 63345, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "create_kvs_client", + "parameters": [ + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 63346, + "kind": 32768, + "kindString": "Parameter", + "name": "id", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 1293 + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 63347, + "kind": 32768, + "kindString": "Parameter", + "name": "name", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 1293 + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 63348, + "kind": 32768, + "kindString": "Parameter", + "name": "alias", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 1293 + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 63349, + "kind": 32768, + "kindString": "Parameter", + "name": "configuration", + "type": { + "name": "Configuration | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "Configuration", + "target": "2", + "ref": { + "id": 62054, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/configuration.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 20 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/Configuration" + } + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 1293 + } + ], + "type": { + "name": "KeyValueStoreClient", + "type": "reference", + "target": "1254", + "ref": { + "id": 63306, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_key_value_store_client.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 14 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/KeyValueStoreClient" + } + }, + "overwrites": { + "name": "StorageClient.create_kvs_client", + "target": 1292, + "type": "reference", + "ref": { + "id": 63344, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 53 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/StorageClient#create_kvs_client" + } + }, + "parentId": 1475 + } + ], + "overwrites": { + "name": "StorageClient.create_kvs_client", + "target": 1292, + "type": "reference", + "ref": { + "id": 63344, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 53 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/StorageClient#create_kvs_client" + } + }, + "permalink": "/python/api/class/MemoryStorageClient#create_kvs_client", + "parentId": 1468 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Create a request queue client." + } + ] + }, + "decorations": [ + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 63533, + "module": "storage_clients._base._storage_client", + "name": "create_rq_client", + "parsedDocstring": { + "text": "Create a request queue client." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_memory/_storage_client.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 59, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/_memory/_storage_client.py#L59" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Create a request queue client." + } + ] + }, + "flags": {}, + "id": 63351, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "create_rq_client", + "parameters": [ + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 63352, + "kind": 32768, + "kindString": "Parameter", + "name": "id", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 1299 + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 63353, + "kind": 32768, + "kindString": "Parameter", + "name": "name", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 1299 + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 63354, + "kind": 32768, + "kindString": "Parameter", + "name": "alias", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 1299 + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 63355, + "kind": 32768, + "kindString": "Parameter", + "name": "configuration", + "type": { + "name": "Configuration | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "Configuration", + "target": "2", + "ref": { + "id": 62054, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/configuration.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 20 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/Configuration" + } + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 1299 + } + ], + "type": { + "name": "RequestQueueClient", + "type": "reference", + "target": "1195", + "ref": { + "id": 63247, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_request_queue_client.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 13 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/RequestQueueClient" + } + }, + "overwrites": { + "name": "StorageClient.create_rq_client", + "target": 1298, + "type": "reference", + "ref": { + "id": 63350, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 64 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/StorageClient#create_rq_client" + } + }, + "parentId": 1481 + } + ], + "overwrites": { + "name": "StorageClient.create_rq_client", + "target": 1298, + "type": "reference", + "ref": { + "id": 63350, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 64 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/StorageClient#create_rq_client" + } + }, + "permalink": "/python/api/class/MemoryStorageClient#create_rq_client", + "parentId": 1468 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Return a cache key that can differentiate between different storages of this and other clients.\n\nCan be based on configuration or on the client itself. By default, returns a module and name of the client\nclass." + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 66251, + "module": "storage_clients._base._storage_client", + "name": "get_storage_client_cache_key", + "parsedDocstring": { + "text": "Return a cache key that can differentiate between different storages of this and other clients.\n\nCan be based on configuration or on the client itself. By default, returns a module and name of the client\nclass." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 33, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/_base/_storage_client.py#L33" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Return a cache key that can differentiate between different storages of this and other clients.\n\nCan be based on configuration or on the client itself. By default, returns a module and name of the client\nclass." + } + ] + }, + "flags": {}, + "id": 63336, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "get_storage_client_cache_key", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 63337, + "kind": 32768, + "kindString": "Parameter", + "name": "configuration", + "type": { + "name": "Configuration", + "type": "reference", + "target": "2", + "ref": { + "id": 62054, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/configuration.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 20 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/Configuration" + } + }, + "parentId": 1284 + } + ], + "type": { + "name": "Hashable", + "type": "reference", + "target": 0, + "ref": {} + }, + "inheritedFrom": { + "name": "StorageClient.get_storage_client_cache_key", + "target": 1283, + "type": "reference", + "ref": { + "id": 63335, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 33 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/StorageClient#get_storage_client_cache_key" + } + }, + "parentId": 4199 + } + ], + "inheritedFrom": { + "name": "StorageClient.get_storage_client_cache_key", + "target": 1283, + "type": "reference", + "ref": { + "id": 63335, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 33 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/StorageClient#get_storage_client_cache_key" + } + }, + "permalink": "/python/api/class/MemoryStorageClient#get_storage_client_cache_key", + "parentId": 1468 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Return statistics about rate limit errors encountered by the HTTP client in storage client." + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 66252, + "module": "storage_clients._base._storage_client", + "name": "get_rate_limit_errors", + "parsedDocstring": { + "text": "Return statistics about rate limit errors encountered by the HTTP client in storage client." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 74, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/_base/_storage_client.py#L74" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Return statistics about rate limit errors encountered by the HTTP client in storage client." + } + ] + }, + "flags": {}, + "id": 63357, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "get_rate_limit_errors", + "parameters": [], + "type": { + "name": "dict[int, int]", + "type": "reference", + "target": 0, + "ref": {} + }, + "inheritedFrom": { + "name": "StorageClient.get_rate_limit_errors", + "target": 1304, + "type": "reference", + "ref": { + "id": 63356, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 74 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/StorageClient#get_rate_limit_errors" + } + }, + "parentId": 4200 + } + ], + "inheritedFrom": { + "name": "StorageClient.get_rate_limit_errors", + "target": 1304, + "type": "reference", + "ref": { + "id": 63356, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 74 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/StorageClient#get_rate_limit_errors" + } + }, + "permalink": "/python/api/class/MemoryStorageClient#get_rate_limit_errors", + "parentId": 1468 + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Memory implementation of the storage client.\n\nThis storage client provides access to datasets, key-value stores, and request queues that store all data\nin memory using Python data structures (lists and dictionaries). No data is persisted between process runs,\nmeaning all stored data is lost when the program terminates.\n\nThe memory implementation provides fast access to data but is limited by available memory and does not\nsupport data sharing across different processes. All storage operations happen entirely in memory with\nno disk operations.\n\nThe memory storage client is useful for testing and development environments, or short-lived crawler\noperations where persistence is not required." + } + ] + }, + "decorations": [ + { + "args": "('Storage clients')", + "name": "docs_group" + } + ], + "flags": {}, + "groups": [ + { + "children": [ + 63521, + 63527, + 63533, + 66252, + 66251 + ], + "title": "Methods" + } + ], + "id": 63520, + "module": "storage_clients._memory._storage_client", + "name": "MemoryStorageClient", + "parsedDocstring": { + "text": "Memory implementation of the storage client.\n\nThis storage client provides access to datasets, key-value stores, and request queues that store all data\nin memory using Python data structures (lists and dictionaries). No data is persisted between process runs,\nmeaning all stored data is lost when the program terminates.\n\nThe memory implementation provides fast access to data but is limited by available memory and does not\nsupport data sharing across different processes. All storage operations happen entirely in memory with\nno disk operations.\n\nThe memory storage client is useful for testing and development environments, or short-lived crawler\noperations where persistence is not required." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_memory/_storage_client.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 15, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/_memory/_storage_client.py#L15" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "extendedTypes": [ + { + "name": "StorageClient", + "target": "1282", + "type": "reference", + "ref": { + "id": 63334, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 19 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/StorageClient" + } + } + ], + "permalink": "/python/api/class/MemoryStorageClient", + "categories": [] + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Return a cache key that can differentiate between different storages of this and other clients.\n\nCan be based on configuration or on the client itself. By default, returns a module and name of the client\nclass." + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 67877, + "module": "storage_clients._base._storage_client", + "name": "get_storage_client_cache_key", + "parsedDocstring": { + "text": "Return a cache key that can differentiate between different storages of this and other clients.\n\nCan be based on configuration or on the client itself. By default, returns a module and name of the client\nclass." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_file_system/_storage_client.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 38, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/_file_system/_storage_client.py#L38" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Return a cache key that can differentiate between different storages of this and other clients.\n\nCan be based on configuration or on the client itself. By default, returns a module and name of the client\nclass." + } + ] + }, + "flags": {}, + "id": 67536, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "get_storage_client_cache_key", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 67537, + "kind": 32768, + "kindString": "Parameter", + "name": "configuration", + "type": { + "name": "Configuration", + "type": "reference", + "target": "2", + "ref": { + "id": 66254, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/configuration.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 20 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/Configuration" + } + }, + "parentId": 1284 + } + ], + "type": { + "name": "Hashable", + "type": "reference", + "target": 0, + "ref": {} + }, + "overwrites": { + "name": "StorageClient.get_storage_client_cache_key", + "target": 1283, + "type": "reference", + "ref": { + "id": 67535, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 33 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/StorageClient#get_storage_client_cache_key" + } + }, + "parentId": 1625 + } + ], + "overwrites": { + "name": "StorageClient.get_storage_client_cache_key", + "target": 1283, + "type": "reference", + "ref": { + "id": 67535, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 33 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/StorageClient#get_storage_client_cache_key" + } + }, + "permalink": "/python/api/class/FileSystemStorageClient#get_storage_client_cache_key", + "parentId": 1624 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Create a dataset client." + } + ] + }, + "decorations": [ + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 67880, + "module": "storage_clients._base._storage_client", + "name": "create_dataset_client", + "parsedDocstring": { + "text": "Create a dataset client." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_file_system/_storage_client.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 43, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/_file_system/_storage_client.py#L43" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Create a dataset client." + } + ] + }, + "flags": {}, + "id": 67539, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "create_dataset_client", + "parameters": [ + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 67540, + "kind": 32768, + "kindString": "Parameter", + "name": "id", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 1287 + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 67541, + "kind": 32768, + "kindString": "Parameter", + "name": "name", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 1287 + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 67542, + "kind": 32768, + "kindString": "Parameter", + "name": "alias", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 1287 + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 67543, + "kind": 32768, + "kindString": "Parameter", + "name": "configuration", + "type": { + "name": "Configuration | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "Configuration", + "target": "2", + "ref": { + "id": 66254, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/configuration.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 20 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/Configuration" + } + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 1287 + } + ], + "type": { + "name": "DatasetClient", + "type": "reference", + "target": "1220", + "ref": { + "id": 67472, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_dataset_client.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 13 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/DatasetClient" + } + }, + "overwrites": { + "name": "StorageClient.create_dataset_client", + "target": 1286, + "type": "reference", + "ref": { + "id": 67538, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 42 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/StorageClient#create_dataset_client" + } + }, + "parentId": 1628 + } + ], + "overwrites": { + "name": "StorageClient.create_dataset_client", + "target": 1286, + "type": "reference", + "ref": { + "id": 67538, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 42 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/StorageClient#create_dataset_client" + } + }, + "permalink": "/python/api/class/FileSystemStorageClient#create_dataset_client", + "parentId": 1624 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Create a key-value store client." + } + ] + }, + "decorations": [ + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 67886, + "module": "storage_clients._base._storage_client", + "name": "create_kvs_client", + "parsedDocstring": { + "text": "Create a key-value store client." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_file_system/_storage_client.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 57, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/_file_system/_storage_client.py#L57" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Create a key-value store client." + } + ] + }, + "flags": {}, + "id": 67545, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "create_kvs_client", + "parameters": [ + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 67546, + "kind": 32768, + "kindString": "Parameter", + "name": "id", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 1293 + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 67547, + "kind": 32768, + "kindString": "Parameter", + "name": "name", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 1293 + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 67548, + "kind": 32768, + "kindString": "Parameter", + "name": "alias", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 1293 + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 67549, + "kind": 32768, + "kindString": "Parameter", + "name": "configuration", + "type": { + "name": "Configuration | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "Configuration", + "target": "2", + "ref": { + "id": 66254, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/configuration.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 20 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/Configuration" + } + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 1293 + } + ], + "type": { + "name": "KeyValueStoreClient", + "type": "reference", + "target": "1254", + "ref": { + "id": 67506, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_key_value_store_client.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 14 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/KeyValueStoreClient" + } + }, + "overwrites": { + "name": "StorageClient.create_kvs_client", + "target": 1292, + "type": "reference", + "ref": { + "id": 67544, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 53 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/StorageClient#create_kvs_client" + } + }, + "parentId": 1634 + } + ], + "overwrites": { + "name": "StorageClient.create_kvs_client", + "target": 1292, + "type": "reference", + "ref": { + "id": 67544, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 53 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/StorageClient#create_kvs_client" + } + }, + "permalink": "/python/api/class/FileSystemStorageClient#create_kvs_client", + "parentId": 1624 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Create a request queue client." + } + ] + }, + "decorations": [ + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 67892, + "module": "storage_clients._base._storage_client", + "name": "create_rq_client", + "parsedDocstring": { + "text": "Create a request queue client." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_file_system/_storage_client.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 71, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/_file_system/_storage_client.py#L71" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Create a request queue client." + } + ] + }, + "flags": {}, + "id": 67551, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "create_rq_client", + "parameters": [ + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 67552, + "kind": 32768, + "kindString": "Parameter", + "name": "id", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 1299 + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 67553, + "kind": 32768, + "kindString": "Parameter", + "name": "name", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 1299 + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 67554, + "kind": 32768, + "kindString": "Parameter", + "name": "alias", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 1299 + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 67555, + "kind": 32768, + "kindString": "Parameter", + "name": "configuration", + "type": { + "name": "Configuration | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "Configuration", + "target": "2", + "ref": { + "id": 66254, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/configuration.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 20 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/Configuration" + } + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 1299 + } + ], + "type": { + "name": "RequestQueueClient", + "type": "reference", + "target": "1195", + "ref": { + "id": 67447, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_request_queue_client.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 13 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/RequestQueueClient" + } + }, + "overwrites": { + "name": "StorageClient.create_rq_client", + "target": 1298, + "type": "reference", + "ref": { + "id": 67550, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 64 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/StorageClient#create_rq_client" + } + }, + "parentId": 1640 + } + ], + "overwrites": { + "name": "StorageClient.create_rq_client", + "target": 1298, + "type": "reference", + "ref": { + "id": 67550, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 64 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/StorageClient#create_rq_client" + } + }, + "permalink": "/python/api/class/FileSystemStorageClient#create_rq_client", + "parentId": 1624 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Return statistics about rate limit errors encountered by the HTTP client in storage client." + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 70453, + "module": "storage_clients._base._storage_client", + "name": "get_rate_limit_errors", + "parsedDocstring": { + "text": "Return statistics about rate limit errors encountered by the HTTP client in storage client." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 74, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/_base/_storage_client.py#L74" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Return statistics about rate limit errors encountered by the HTTP client in storage client." + } + ] + }, + "flags": {}, + "id": 67557, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "get_rate_limit_errors", + "parameters": [], + "type": { + "name": "dict[int, int]", + "type": "reference", + "target": 0, + "ref": {} + }, + "inheritedFrom": { + "name": "StorageClient.get_rate_limit_errors", + "target": 1304, + "type": "reference", + "ref": { + "id": 67556, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 74 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/StorageClient#get_rate_limit_errors" + } + }, + "parentId": 4201 + } + ], + "inheritedFrom": { + "name": "StorageClient.get_rate_limit_errors", + "target": 1304, + "type": "reference", + "ref": { + "id": 67556, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 74 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/StorageClient#get_rate_limit_errors" + } + }, + "permalink": "/python/api/class/FileSystemStorageClient#get_rate_limit_errors", + "parentId": 1624 + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "File system implementation of the storage client.\n\nThis storage client provides access to datasets, key-value stores, and request queues that persist data\nto the local file system. Each storage type is implemented with its own specific file system client\nthat stores data in a structured directory hierarchy.\n\nData is stored in JSON format in predictable file paths, making it easy to inspect and manipulate\nthe stored data outside of the Crawlee application if needed.\n\nAll data persists between program runs but is limited to access from the local machine\nwhere the files are stored.\n\nWarning: This storage client is not safe for concurrent access from multiple crawler processes.\nUse it only when running a single crawler process at a time." + } + ] + }, + "decorations": [ + { + "args": "('Storage clients')", + "name": "docs_group" + } + ], + "flags": {}, + "groups": [ + { + "children": [ + 67880, + 67886, + 67892, + 70453, + 67877 + ], + "title": "Methods" + } + ], + "id": 67876, + "module": "storage_clients._file_system._storage_client", + "name": "FileSystemStorageClient", + "parsedDocstring": { + "text": "File system implementation of the storage client.\n\nThis storage client provides access to datasets, key-value stores, and request queues that persist data\nto the local file system. Each storage type is implemented with its own specific file system client\nthat stores data in a structured directory hierarchy.\n\nData is stored in JSON format in predictable file paths, making it easy to inspect and manipulate\nthe stored data outside of the Crawlee application if needed.\n\nAll data persists between program runs but is limited to access from the local machine\nwhere the files are stored.\n\nWarning: This storage client is not safe for concurrent access from multiple crawler processes.\nUse it only when running a single crawler process at a time." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_file_system/_storage_client.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 20, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/_file_system/_storage_client.py#L20" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "extendedTypes": [ + { + "name": "StorageClient", + "target": "1282", + "type": "reference", + "ref": { + "id": 67534, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 19 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/StorageClient" + } + } + ], + "permalink": "/python/api/class/FileSystemStorageClient", + "categories": [] + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Initialize the SQL storage client.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 72347, + "module": "storage_clients._sql._storage_client", + "name": "__init__", + "parsedDocstring": { + "text": "Initialize the SQL storage client.\n", + "args": { + "connection_string": "Database connection string (e.g., \"sqlite+aiosqlite:///crawlee.db\").\nIf not provided, defaults to SQLite database in the storage directory.", + "engine": "Pre-configured AsyncEngine instance. If provided, connection_string is ignored." + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_sql/_storage_client.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 57, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/_sql/_storage_client.py#L57" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Initialize the SQL storage client.\n" + } + ] + }, + "flags": {}, + "id": 72348, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "__init__", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Database connection string (e.g., \"sqlite+aiosqlite:///crawlee.db\").\nIf not provided, defaults to SQLite database in the storage directory." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 72349, + "kind": 32768, + "kindString": "Parameter", + "name": "connection_string", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 1895 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Pre-configured AsyncEngine instance. If provided, connection_string is ignored." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 72350, + "kind": 32768, + "kindString": "Parameter", + "name": "engine", + "type": { + "name": "AsyncEngine | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "AsyncEngine", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 1895 + } + ], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "parentId": 1894 + } + ], + "permalink": "/python/api/class/SqlStorageClient#__init__", + "parentId": 1893 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Async context manager entry." + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 72351, + "module": "storage_clients._sql._storage_client", + "name": "__aenter__", + "parsedDocstring": { + "text": "Async context manager entry." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_sql/_storage_client.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 88, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/_sql/_storage_client.py#L88" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Async context manager entry." + } + ] + }, + "flags": {}, + "id": 72352, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "__aenter__", + "parameters": [], + "type": { + "name": "SqlStorageClient", + "type": "reference", + "target": "1893", + "ref": { + "id": 72346, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_sql/_storage_client.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 35 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/SqlStorageClient" + } + }, + "parentId": 1898 + } + ], + "permalink": "/python/api/class/SqlStorageClient#__aenter__", + "parentId": 1893 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Async context manager exit." + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 72353, + "module": "storage_clients._sql._storage_client", + "name": "__aexit__", + "parsedDocstring": { + "text": "Async context manager exit." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_sql/_storage_client.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 92, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/_sql/_storage_client.py#L92" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Async context manager exit." + } + ] + }, + "flags": {}, + "id": 72354, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "__aexit__", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 72355, + "kind": 32768, + "kindString": "Parameter", + "name": "exc_type", + "type": { + "name": "type[BaseException] | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "type", + "typeArguments": [ + { + "type": "reference", + "name": "BaseException", + "target": 0, + "ref": {} + } + ], + "target": "3422", + "ref": { + "id": 73875, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_utils/sitemap.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 62 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api/class/SitemapSource#type" + } + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 1901 + }, + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 72356, + "kind": 32768, + "kindString": "Parameter", + "name": "exc_value", + "type": { + "name": "BaseException | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "BaseException", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 1901 + }, + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 72357, + "kind": 32768, + "kindString": "Parameter", + "name": "exc_traceback", + "type": { + "name": "TracebackType | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "TracebackType", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 1901 + } + ], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "parentId": 1900 + } + ], + "permalink": "/python/api/class/SqlStorageClient#__aexit__", + "parentId": 1893 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Get the SQLAlchemy AsyncEngine instance." + } + ] + }, + "decorations": [ + { + "name": "property" + } + ], + "flags": {}, + "groups": [], + "id": 72358, + "module": "storage_clients._sql._storage_client", + "name": "engine", + "parsedDocstring": { + "text": "Get the SQLAlchemy AsyncEngine instance." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_sql/_storage_client.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 102, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/_sql/_storage_client.py#L102" + } + ], + "type": { + "name": "AsyncEngine", + "type": "reference", + "target": 0, + "ref": {} + }, + "permalink": "/python/api/class/SqlStorageClient#engine", + "parentId": 1893 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Get the database dialect name." + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 72359, + "module": "storage_clients._sql._storage_client", + "name": "get_dialect_name", + "parsedDocstring": { + "text": "Get the database dialect name." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_sql/_storage_client.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 108, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/_sql/_storage_client.py#L108" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Get the database dialect name." + } + ] + }, + "flags": {}, + "id": 72360, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "get_dialect_name", + "parameters": [], + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 1906 + } + ], + "permalink": "/python/api/class/SqlStorageClient#get_dialect_name", + "parentId": 1893 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Initialize the database schema.\n\nThis method creates all necessary tables if they don't exist.\nShould be called before using the storage client." + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 72361, + "module": "storage_clients._sql._storage_client", + "name": "initialize", + "parsedDocstring": { + "text": "Initialize the database schema.\n\nThis method creates all necessary tables if they don't exist.\nShould be called before using the storage client." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_sql/_storage_client.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 112, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/_sql/_storage_client.py#L112" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Initialize the database schema.\n\nThis method creates all necessary tables if they don't exist.\nShould be called before using the storage client." + } + ] + }, + "flags": {}, + "id": 72362, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "initialize", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 72363, + "kind": 32768, + "kindString": "Parameter", + "name": "configuration", + "type": { + "name": "Configuration", + "type": "reference", + "target": "2", + "ref": { + "id": 70455, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/configuration.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 20 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/Configuration" + } + }, + "parentId": 1909 + } + ], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "parentId": 1908 + } + ], + "permalink": "/python/api/class/SqlStorageClient#initialize", + "parentId": 1893 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Close the database connection pool." + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 72364, + "module": "storage_clients._sql._storage_client", + "name": "close", + "parsedDocstring": { + "text": "Close the database connection pool." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_sql/_storage_client.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 157, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/_sql/_storage_client.py#L157" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Close the database connection pool." + } + ] + }, + "flags": {}, + "id": 72365, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "close", + "parameters": [], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "parentId": 1911 + } + ], + "permalink": "/python/api/class/SqlStorageClient#close", + "parentId": 1893 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Create a new database session.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 72366, + "module": "storage_clients._sql._storage_client", + "name": "create_session", + "parsedDocstring": { + "text": "Create a new database session.\n", + "returns": "A new AsyncSession instance." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_sql/_storage_client.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 167, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/_sql/_storage_client.py#L167" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "A new AsyncSession instance." + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Create a new database session.\n" + } + ] + }, + "flags": {}, + "id": 72367, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "create_session", + "parameters": [], + "type": { + "name": "AsyncSession", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 1913 + } + ], + "permalink": "/python/api/class/SqlStorageClient#create_session", + "parentId": 1893 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Create a dataset client." + } + ] + }, + "decorations": [ + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 72368, + "module": "storage_clients._base._storage_client", + "name": "create_dataset_client", + "parsedDocstring": { + "text": "Create a dataset client." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_sql/_storage_client.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 178, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/_sql/_storage_client.py#L178" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Create a dataset client." + } + ] + }, + "flags": {}, + "id": 71740, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "create_dataset_client", + "parameters": [ + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 71741, + "kind": 32768, + "kindString": "Parameter", + "name": "id", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 1287 + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 71742, + "kind": 32768, + "kindString": "Parameter", + "name": "name", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 1287 + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 71743, + "kind": 32768, + "kindString": "Parameter", + "name": "alias", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 1287 + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 71744, + "kind": 32768, + "kindString": "Parameter", + "name": "configuration", + "type": { + "name": "Configuration | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "Configuration", + "target": "2", + "ref": { + "id": 70455, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/configuration.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 20 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/Configuration" + } + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 1287 + } + ], + "type": { + "name": "DatasetClient", + "type": "reference", + "target": "1220", + "ref": { + "id": 71673, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_dataset_client.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 13 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/DatasetClient" + } + }, + "overwrites": { + "name": "StorageClient.create_dataset_client", + "target": 1286, + "type": "reference", + "ref": { + "id": 71739, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 42 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/StorageClient#create_dataset_client" + } + }, + "parentId": 1915 + } + ], + "overwrites": { + "name": "StorageClient.create_dataset_client", + "target": 1286, + "type": "reference", + "ref": { + "id": 71739, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 42 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/StorageClient#create_dataset_client" + } + }, + "permalink": "/python/api/class/SqlStorageClient#create_dataset_client", + "parentId": 1893 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Create a key-value store client." + } + ] + }, + "decorations": [ + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 72374, + "module": "storage_clients._base._storage_client", + "name": "create_kvs_client", + "parsedDocstring": { + "text": "Create a key-value store client." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_sql/_storage_client.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 200, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/_sql/_storage_client.py#L200" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Create a key-value store client." + } + ] + }, + "flags": {}, + "id": 71746, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "create_kvs_client", + "parameters": [ + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 71747, + "kind": 32768, + "kindString": "Parameter", + "name": "id", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 1293 + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 71748, + "kind": 32768, + "kindString": "Parameter", + "name": "name", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 1293 + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 71749, + "kind": 32768, + "kindString": "Parameter", + "name": "alias", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 1293 + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 71750, + "kind": 32768, + "kindString": "Parameter", + "name": "configuration", + "type": { + "name": "Configuration | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "Configuration", + "target": "2", + "ref": { + "id": 70455, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/configuration.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 20 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/Configuration" + } + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 1293 + } + ], + "type": { + "name": "KeyValueStoreClient", + "type": "reference", + "target": "1254", + "ref": { + "id": 71707, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_key_value_store_client.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 14 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/KeyValueStoreClient" + } + }, + "overwrites": { + "name": "StorageClient.create_kvs_client", + "target": 1292, + "type": "reference", + "ref": { + "id": 71745, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 53 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/StorageClient#create_kvs_client" + } + }, + "parentId": 1921 + } + ], + "overwrites": { + "name": "StorageClient.create_kvs_client", + "target": 1292, + "type": "reference", + "ref": { + "id": 71745, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 53 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/StorageClient#create_kvs_client" + } + }, + "permalink": "/python/api/class/SqlStorageClient#create_kvs_client", + "parentId": 1893 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Create a request queue client." + } + ] + }, + "decorations": [ + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 72380, + "module": "storage_clients._base._storage_client", + "name": "create_rq_client", + "parsedDocstring": { + "text": "Create a request queue client." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_sql/_storage_client.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 222, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/_sql/_storage_client.py#L222" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Create a request queue client." + } + ] + }, + "flags": {}, + "id": 71752, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "create_rq_client", + "parameters": [ + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 71753, + "kind": 32768, + "kindString": "Parameter", + "name": "id", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 1299 + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 71754, + "kind": 32768, + "kindString": "Parameter", + "name": "name", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 1299 + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 71755, + "kind": 32768, + "kindString": "Parameter", + "name": "alias", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 1299 + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 71756, + "kind": 32768, + "kindString": "Parameter", + "name": "configuration", + "type": { + "name": "Configuration | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "Configuration", + "target": "2", + "ref": { + "id": 70455, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/configuration.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 20 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/Configuration" + } + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 1299 + } + ], + "type": { + "name": "RequestQueueClient", + "type": "reference", + "target": "1195", + "ref": { + "id": 71648, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_request_queue_client.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 13 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/RequestQueueClient" + } + }, + "overwrites": { + "name": "StorageClient.create_rq_client", + "target": 1298, + "type": "reference", + "ref": { + "id": 71751, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 64 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/StorageClient#create_rq_client" + } + }, + "parentId": 1927 + } + ], + "overwrites": { + "name": "StorageClient.create_rq_client", + "target": 1298, + "type": "reference", + "ref": { + "id": 71751, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 64 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/StorageClient#create_rq_client" + } + }, + "permalink": "/python/api/class/SqlStorageClient#create_rq_client", + "parentId": 1893 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Return a cache key that can differentiate between different storages of this and other clients.\n\nCan be based on configuration or on the client itself. By default, returns a module and name of the client\nclass." + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 74655, + "module": "storage_clients._base._storage_client", + "name": "get_storage_client_cache_key", + "parsedDocstring": { + "text": "Return a cache key that can differentiate between different storages of this and other clients.\n\nCan be based on configuration or on the client itself. By default, returns a module and name of the client\nclass." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 33, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/_base/_storage_client.py#L33" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Return a cache key that can differentiate between different storages of this and other clients.\n\nCan be based on configuration or on the client itself. By default, returns a module and name of the client\nclass." + } + ] + }, + "flags": {}, + "id": 71737, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "get_storage_client_cache_key", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 71738, + "kind": 32768, + "kindString": "Parameter", + "name": "configuration", + "type": { + "name": "Configuration", + "type": "reference", + "target": "2", + "ref": { + "id": 70455, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/configuration.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 20 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/Configuration" + } + }, + "parentId": 1284 + } + ], + "type": { + "name": "Hashable", + "type": "reference", + "target": 0, + "ref": {} + }, + "inheritedFrom": { + "name": "StorageClient.get_storage_client_cache_key", + "target": 1283, + "type": "reference", + "ref": { + "id": 71736, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 33 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/StorageClient#get_storage_client_cache_key" + } + }, + "parentId": 4202 + } + ], + "inheritedFrom": { + "name": "StorageClient.get_storage_client_cache_key", + "target": 1283, + "type": "reference", + "ref": { + "id": 71736, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 33 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/StorageClient#get_storage_client_cache_key" + } + }, + "permalink": "/python/api/class/SqlStorageClient#get_storage_client_cache_key", + "parentId": 1893 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Return statistics about rate limit errors encountered by the HTTP client in storage client." + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 74656, + "module": "storage_clients._base._storage_client", + "name": "get_rate_limit_errors", + "parsedDocstring": { + "text": "Return statistics about rate limit errors encountered by the HTTP client in storage client." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 74, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/_base/_storage_client.py#L74" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Return statistics about rate limit errors encountered by the HTTP client in storage client." + } + ] + }, + "flags": {}, + "id": 71758, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "get_rate_limit_errors", + "parameters": [], + "type": { + "name": "dict[int, int]", + "type": "reference", + "target": 0, + "ref": {} + }, + "inheritedFrom": { + "name": "StorageClient.get_rate_limit_errors", + "target": 1304, + "type": "reference", + "ref": { + "id": 71757, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 74 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/StorageClient#get_rate_limit_errors" + } + }, + "parentId": 4203 + } + ], + "inheritedFrom": { + "name": "StorageClient.get_rate_limit_errors", + "target": 1304, + "type": "reference", + "ref": { + "id": 71757, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 74 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/StorageClient#get_rate_limit_errors" + } + }, + "permalink": "/python/api/class/SqlStorageClient#get_rate_limit_errors", + "parentId": 1893 + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "SQL implementation of the storage client.\n\nThis storage client provides access to datasets, key-value stores, and request queues that persist data\nto a SQL database using SQLAlchemy 2+. Each storage type uses two tables: one for metadata and one for\nrecords.\n\nThe client accepts either a database connection string or a pre-configured AsyncEngine. If neither is\nprovided, it creates a default SQLite database 'crawlee.db' in the storage directory.\n\nDatabase schema is automatically created during initialization. SQLite databases receive performance\noptimizations including WAL mode and increased cache size.\n\n\n:::warning Warning\nThis is an experimental feature. The behavior and interface may change in future versions.\n:::" + } + ] + }, + "decorations": [ + { + "args": "('Storage clients')", + "name": "docs_group" + } + ], + "flags": {}, + "groups": [ + { + "children": [ + 72351, + 72353, + 72347, + 72364, + 72368, + 72374, + 72380, + 72366, + 72359, + 74656, + 74655, + 72361 + ], + "title": "Methods" + }, + { + "children": [ + 72358 + ], + "title": "Properties" + } + ], + "id": 72346, + "module": "storage_clients._sql._storage_client", + "name": "SqlStorageClient", + "parsedDocstring": { + "text": "SQL implementation of the storage client.\n\nThis storage client provides access to datasets, key-value stores, and request queues that persist data\nto a SQL database using SQLAlchemy 2+. Each storage type uses two tables: one for metadata and one for\nrecords.\n\nThe client accepts either a database connection string or a pre-configured AsyncEngine. If neither is\nprovided, it creates a default SQLite database 'crawlee.db' in the storage directory.\n\nDatabase schema is automatically created during initialization. SQLite databases receive performance\noptimizations including WAL mode and increased cache size.\n\n\n:::warning Warning\nThis is an experimental feature. The behavior and interface may change in future versions.\n:::" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_sql/_storage_client.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 35, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/storage_clients/_sql/_storage_client.py#L35" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "extendedTypes": [ + { + "name": "StorageClient", + "target": "1282", + "type": "reference", + "ref": { + "id": 71735, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 19 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/StorageClient" + } + } + ], + "permalink": "/python/api/class/SqlStorageClient", + "categories": [] + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Get the number of requests in the loader that have been handled." + } + ] + }, + "decorations": [ + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 77591, + "module": "request_loaders._request_loader", + "name": "get_handled_count", + "parsedDocstring": { + "text": "Get the number of requests in the loader that have been handled." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 29, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/request_loaders/_request_loader.py#L29" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Get the number of requests in the loader that have been handled." + } + ] + }, + "flags": {}, + "id": 77592, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "get_handled_count", + "parameters": [], + "type": { + "name": "int", + "type": "reference", + "target": 0, + "ref": {} + }, + "overwrites": { + "name": "RequestLoader.get_handled_count", + "target": 2935, + "type": "reference", + "ref": { + "id": 77591, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 29 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestLoader#get_handled_count" + } + }, + "parentId": 2935 + } + ], + "permalink": "/python/api/class/RequestLoader#get_handled_count", + "parentId": 2934 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Get an offline approximation of the total number of requests in the loader (i.e. pending + handled)." + } + ] + }, + "decorations": [ + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 77593, + "module": "request_loaders._request_loader", + "name": "get_total_count", + "parsedDocstring": { + "text": "Get an offline approximation of the total number of requests in the loader (i.e. pending + handled)." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 33, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/request_loaders/_request_loader.py#L33" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Get an offline approximation of the total number of requests in the loader (i.e. pending + handled)." + } + ] + }, + "flags": {}, + "id": 77594, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "get_total_count", + "parameters": [], + "type": { + "name": "int", + "type": "reference", + "target": 0, + "ref": {} + }, + "overwrites": { + "name": "RequestLoader.get_total_count", + "target": 2937, + "type": "reference", + "ref": { + "id": 77593, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 33 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestLoader#get_total_count" + } + }, + "parentId": 2937 + } + ], + "permalink": "/python/api/class/RequestLoader#get_total_count", + "parentId": 2934 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Return True if there are no more requests in the loader (there might still be unfinished requests)." + } + ] + }, + "decorations": [ + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 77595, + "module": "request_loaders._request_loader", + "name": "is_empty", + "parsedDocstring": { + "text": "Return True if there are no more requests in the loader (there might still be unfinished requests)." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 37, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/request_loaders/_request_loader.py#L37" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Return True if there are no more requests in the loader (there might still be unfinished requests)." + } + ] + }, + "flags": {}, + "id": 77596, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "is_empty", + "parameters": [], + "type": { + "name": "bool", + "type": "reference", + "target": 0, + "ref": {} + }, + "overwrites": { + "name": "RequestLoader.is_empty", + "target": 2939, + "type": "reference", + "ref": { + "id": 77595, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 37 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestLoader#is_empty" + } + }, + "parentId": 2939 + } + ], + "permalink": "/python/api/class/RequestLoader#is_empty", + "parentId": 2934 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Return True if all requests have been handled." + } + ] + }, + "decorations": [ + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 77597, + "module": "request_loaders._request_loader", + "name": "is_finished", + "parsedDocstring": { + "text": "Return True if all requests have been handled." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 41, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/request_loaders/_request_loader.py#L41" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Return True if all requests have been handled." + } + ] + }, + "flags": {}, + "id": 77598, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "is_finished", + "parameters": [], + "type": { + "name": "bool", + "type": "reference", + "target": 0, + "ref": {} + }, + "overwrites": { + "name": "RequestLoader.is_finished", + "target": 2941, + "type": "reference", + "ref": { + "id": 77597, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 41 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestLoader#is_finished" + } + }, + "parentId": 2941 + } + ], + "permalink": "/python/api/class/RequestLoader#is_finished", + "parentId": 2934 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Return the next request to be processed, or `None` if there are no more pending requests.\n\nThe method should return `None` if and only if `is_finished` would return `True`. In other cases, the method\nshould wait until a request appears." + } + ] + }, + "decorations": [ + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 77599, + "module": "request_loaders._request_loader", + "name": "fetch_next_request", + "parsedDocstring": { + "text": "Return the next request to be processed, or `None` if there are no more pending requests.\n\nThe method should return `None` if and only if `is_finished` would return `True`. In other cases, the method\nshould wait until a request appears." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 45, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/request_loaders/_request_loader.py#L45" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Return the next request to be processed, or `None` if there are no more pending requests.\n\nThe method should return `None` if and only if `is_finished` would return `True`. In other cases, the method\nshould wait until a request appears." + } + ] + }, + "flags": {}, + "id": 77600, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "fetch_next_request", + "parameters": [], + "type": { + "name": "Request | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "Request", + "target": "409", + "ref": { + "id": 75065, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_request.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 145 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/Request" + } + }, + { + "type": "literal", + "value": {} + } + ] + }, + "overwrites": { + "name": "RequestLoader.fetch_next_request", + "target": 2943, + "type": "reference", + "ref": { + "id": 77599, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 45 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestLoader#fetch_next_request" + } + }, + "parentId": 2943 + } + ], + "permalink": "/python/api/class/RequestLoader#fetch_next_request", + "parentId": 2934 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Mark a request as handled after a successful processing (or after giving up retrying)." + } + ] + }, + "decorations": [ + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 77601, + "module": "request_loaders._request_loader", + "name": "mark_request_as_handled", + "parsedDocstring": { + "text": "Mark a request as handled after a successful processing (or after giving up retrying)." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 53, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/request_loaders/_request_loader.py#L53" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Mark a request as handled after a successful processing (or after giving up retrying)." + } + ] + }, + "flags": {}, + "id": 77602, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "mark_request_as_handled", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 77603, + "kind": 32768, + "kindString": "Parameter", + "name": "request", + "type": { + "name": "Request", + "type": "reference", + "target": "409", + "ref": { + "id": 75065, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_request.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 145 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/Request" + } + }, + "parentId": 2946 + } + ], + "type": { + "name": "ProcessedRequest | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "ProcessedRequest", + "target": "1340", + "ref": { + "id": 75996, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 140 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/ProcessedRequest" + } + }, + { + "type": "literal", + "value": {} + } + ] + }, + "overwrites": { + "name": "RequestLoader.mark_request_as_handled", + "target": 2945, + "type": "reference", + "ref": { + "id": 77601, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 53 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestLoader#mark_request_as_handled" + } + }, + "parentId": 2945 + } + ], + "permalink": "/python/api/class/RequestLoader#mark_request_as_handled", + "parentId": 2934 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Combine the loader with a request manager to support adding and reclaiming requests.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 77604, + "module": "request_loaders._request_loader", + "name": "to_tandem", + "parsedDocstring": { + "text": "Combine the loader with a request manager to support adding and reclaiming requests.\n", + "args": { + "request_manager": "Request manager to combine the loader with.\nIf None is given, the default request queue is used." + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 56, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/request_loaders/_request_loader.py#L56" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Combine the loader with a request manager to support adding and reclaiming requests.\n" + } + ] + }, + "flags": {}, + "id": 77605, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "to_tandem", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Request manager to combine the loader with.\nIf None is given, the default request queue is used." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": false + }, + "id": 77606, + "kind": 32768, + "kindString": "Parameter", + "name": "request_manager", + "type": { + "name": "RequestManager | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "RequestManager", + "target": "3001", + "ref": { + "id": 77657, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_manager.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 18 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/RequestManager" + } + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2949 + } + ], + "type": { + "name": "RequestManagerTandem", + "type": "reference", + "target": "2803", + "ref": { + "id": 77459, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_manager_tandem.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 24 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/RequestManagerTandem" + } + }, + "parentId": 2948 + } + ], + "permalink": "/python/api/class/RequestLoader#to_tandem", + "parentId": 2934 + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "An abstract class defining the interface for classes that provide access to a read-only stream of requests.\n\nRequest loaders are used to manage and provide access to a storage of crawling requests.\n\nKey responsibilities:\n- Fetching the next request to be processed.\n- Marking requests as successfully handled after processing.\n- Managing state information such as the total and handled request counts." + } + ] + }, + "decorations": [ + { + "args": "('Request loaders')", + "name": "docs_group" + } + ], + "flags": {}, + "groups": [ + { + "children": [ + 77599, + 77591, + 77593, + 77595, + 77597, + 77601, + 77604 + ], + "title": "Methods" + } + ], + "id": 77590, + "module": "request_loaders._request_loader", + "name": "RequestLoader", + "parsedDocstring": { + "text": "An abstract class defining the interface for classes that provide access to a read-only stream of requests.\n\nRequest loaders are used to manage and provide access to a storage of crawling requests.\n\nKey responsibilities:\n- Fetching the next request to be processed.\n- Marking requests as successfully handled after processing.\n- Managing state information such as the total and handled request counts." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 17, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/request_loaders/_request_loader.py#L17" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "extendedBy": [ + { + "name": "RequestList", + "target": "2913", + "type": "reference", + "ref": { + "id": 77569, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_list.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 32 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/RequestList" + } + }, + { + "name": "SitemapRequestLoader", + "target": "2963", + "type": "reference", + "ref": { + "id": 77619, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_sitemap_request_loader.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 94 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/SitemapRequestLoader" + } + }, + { + "name": "RequestManager", + "target": "3001", + "type": "reference", + "ref": { + "id": 77657, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_manager.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 18 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/RequestManager" + } + } + ], + "permalink": "/python/api/class/RequestLoader", + "categories": [] + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Remove persistent state either from the Apify Cloud storage or from the local database." + } + ] + }, + "decorations": [ + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 80659, + "module": "request_loaders._request_manager", + "name": "drop", + "parsedDocstring": { + "text": "Remove persistent state either from the Apify Cloud storage or from the local database." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_manager.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 22, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/request_loaders/_request_manager.py#L22" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Remove persistent state either from the Apify Cloud storage or from the local database." + } + ] + }, + "flags": {}, + "id": 80660, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "drop", + "parameters": [], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "overwrites": { + "name": "Storage.drop", + "target": 2342, + "type": "reference", + "ref": { + "id": 79999, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_base.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 57 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/Storage#drop" + } + }, + "parentId": 3002 + } + ], + "permalink": "/python/api/class/RequestManager#drop", + "parentId": 3001 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Remove all queued requests from this manager while keeping it usable.\n\nDefault implementation is a no-op. Subclasses with persistent state should override to clear it." + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 80661, + "module": "request_loaders._request_manager", + "name": "purge", + "parsedDocstring": { + "text": "Remove all queued requests from this manager while keeping it usable.\n\nDefault implementation is a no-op. Subclasses with persistent state should override to clear it." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_manager.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 25, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/request_loaders/_request_manager.py#L25" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Remove all queued requests from this manager while keeping it usable.\n\nDefault implementation is a no-op. Subclasses with persistent state should override to clear it." + } + ] + }, + "flags": {}, + "id": 80662, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "purge", + "parameters": [], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "overwrites": { + "name": "Storage.purge", + "target": 2344, + "type": "reference", + "ref": { + "id": 80001, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_base.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 61 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/Storage#purge" + } + }, + "parentId": 3004 + } + ], + "permalink": "/python/api/class/RequestManager#purge", + "parentId": 3001 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Add a single request to the manager and store it in underlying resource client.\n" + } + ] + }, + "decorations": [ + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 80663, + "module": "request_loaders._request_manager", + "name": "add_request", + "parsedDocstring": { + "text": "Add a single request to the manager and store it in underlying resource client.\n", + "args": { + "request": "The request object (or its string representation) to be added to the manager.", + "forefront": "Determines whether the request should be added to the beginning (if True) or the end (if False)\nof the manager.\n" + }, + "returns": "Information about the request addition to the manager or None if the request was not added." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_manager.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 32, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/request_loaders/_request_manager.py#L32" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "Information about the request addition to the manager or None if the request was not added." + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Add a single request to the manager and store it in underlying resource client.\n" + } + ] + }, + "flags": {}, + "id": 80664, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "add_request", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The request object (or its string representation) to be added to the manager." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 80665, + "kind": 32768, + "kindString": "Parameter", + "name": "request", + "type": { + "name": "str | Request", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "reference", + "name": "Request", + "target": "409", + "ref": { + "id": 78066, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_request.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 145 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/Request" + } + } + ] + }, + "parentId": 3007 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Determines whether the request should be added to the beginning (if True) or the end (if False)\nof the manager.\n" + } + ] + }, + "defaultValue": "False", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 80666, + "kind": 32768, + "kindString": "Parameter", + "name": "forefront", + "type": { + "name": "bool", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 3007 + } + ], + "type": { + "name": "ProcessedRequest | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "ProcessedRequest", + "target": "1340", + "ref": { + "id": 78997, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 140 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/ProcessedRequest" + } + }, + { + "type": "literal", + "value": {} + } + ] + }, + "overwrites": { + "name": "RequestManager.add_request", + "target": 3006, + "type": "reference", + "ref": { + "id": 80663, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_manager.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 32 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestManager#add_request" + } + }, + "parentId": 3006 + } + ], + "permalink": "/python/api/class/RequestManager#add_request", + "parentId": 3001 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Add requests to the manager in batches.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 80667, + "module": "request_loaders._request_manager", + "name": "add_requests", + "parsedDocstring": { + "text": "Add requests to the manager in batches.\n", + "args": { + "requests": "Requests to enqueue.", + "forefront": "If True, add requests to the beginning of the queue.", + "batch_size": "The number of requests to add in one batch.", + "wait_time_between_batches": "Time to wait between adding batches.", + "wait_for_all_requests_to_be_added": "If True, wait for all requests to be added before returning.", + "wait_for_all_requests_to_be_added_timeout": "Timeout for waiting for all requests to be added." + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_manager.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 49, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/request_loaders/_request_manager.py#L49" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Add requests to the manager in batches.\n" + } + ] + }, + "flags": {}, + "id": 80668, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "add_requests", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Requests to enqueue." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 80669, + "kind": 32768, + "kindString": "Parameter", + "name": "requests", + "type": { + "name": "Sequence", + "type": "reference", + "typeArguments": [ + { + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "reference", + "name": "Request", + "target": "409", + "ref": { + "id": 78066, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_request.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 145 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/Request" + } + } + ] + } + ], + "target": 0, + "ref": {} + }, + "parentId": 3011 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "If True, add requests to the beginning of the queue." + } + ] + }, + "defaultValue": "False", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 80670, + "kind": 32768, + "kindString": "Parameter", + "name": "forefront", + "type": { + "name": "bool", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 3011 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The number of requests to add in one batch." + } + ] + }, + "defaultValue": "1000", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 80671, + "kind": 32768, + "kindString": "Parameter", + "name": "batch_size", + "type": { + "name": "int", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 3011 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Time to wait between adding batches." + } + ] + }, + "defaultValue": "timedelta(seconds=1)", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 80672, + "kind": 32768, + "kindString": "Parameter", + "name": "wait_time_between_batches", + "type": { + "name": "timedelta", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 3011 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "If True, wait for all requests to be added before returning." + } + ] + }, + "defaultValue": "False", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 80673, + "kind": 32768, + "kindString": "Parameter", + "name": "wait_for_all_requests_to_be_added", + "type": { + "name": "bool", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 3011 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Timeout for waiting for all requests to be added." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 80674, + "kind": 32768, + "kindString": "Parameter", + "name": "wait_for_all_requests_to_be_added_timeout", + "type": { + "name": "timedelta | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "timedelta", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 3011 + } + ], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "overwrites": { + "name": "RequestManager.add_requests", + "target": 3010, + "type": "reference", + "ref": { + "id": 80667, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_manager.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 49 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestManager#add_requests" + } + }, + "parentId": 3010 + } + ], + "permalink": "/python/api/class/RequestManager#add_requests", + "parentId": 3001 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Reclaims a failed request back to the source, so that it can be returned for processing later again.\n\nIt is possible to modify the request data by supplying an updated request as a parameter." + } + ] + }, + "decorations": [ + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 80675, + "module": "request_loaders._request_manager", + "name": "reclaim_request", + "parsedDocstring": { + "text": "Reclaims a failed request back to the source, so that it can be returned for processing later again.\n\nIt is possible to modify the request data by supplying an updated request as a parameter." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_manager.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 77, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/request_loaders/_request_manager.py#L77" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Reclaims a failed request back to the source, so that it can be returned for processing later again.\n\nIt is possible to modify the request data by supplying an updated request as a parameter." + } + ] + }, + "flags": {}, + "id": 80676, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "reclaim_request", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 80677, + "kind": 32768, + "kindString": "Parameter", + "name": "request", + "type": { + "name": "Request", + "type": "reference", + "target": "409", + "ref": { + "id": 78066, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_request.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 145 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/Request" + } + }, + "parentId": 3019 + }, + { + "defaultValue": "False", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 80678, + "kind": 32768, + "kindString": "Parameter", + "name": "forefront", + "type": { + "name": "bool", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 3019 + } + ], + "type": { + "name": "ProcessedRequest | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "ProcessedRequest", + "target": "1340", + "ref": { + "id": 78997, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 140 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/ProcessedRequest" + } + }, + { + "type": "literal", + "value": {} + } + ] + }, + "overwrites": { + "name": "RequestManager.reclaim_request", + "target": 3018, + "type": "reference", + "ref": { + "id": 80675, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_manager.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 77 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestManager#reclaim_request" + } + }, + "parentId": 3018 + } + ], + "permalink": "/python/api/class/RequestManager#reclaim_request", + "parentId": 3001 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Get the number of requests in the loader that have been handled." + } + ] + }, + "decorations": [ + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 81800, + "module": "request_loaders._request_loader", + "name": "get_handled_count", + "parsedDocstring": { + "text": "Get the number of requests in the loader that have been handled." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 29, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/request_loaders/_request_loader.py#L29" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Get the number of requests in the loader that have been handled." + } + ] + }, + "flags": {}, + "id": 80593, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "get_handled_count", + "parameters": [], + "type": { + "name": "int", + "type": "reference", + "target": 0, + "ref": {} + }, + "overwrites": { + "name": "RequestManager.get_handled_count", + "target": 4143, + "type": "reference", + "ref": { + "id": 81800, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 29 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestManager#get_handled_count" + } + }, + "parentId": 4143 + } + ], + "inheritedFrom": { + "name": "RequestLoader.get_handled_count", + "target": 2935, + "type": "reference", + "ref": { + "id": 80592, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 29 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestLoader#get_handled_count" + } + }, + "permalink": "/python/api/class/RequestManager#get_handled_count", + "parentId": 3001 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Get an offline approximation of the total number of requests in the loader (i.e. pending + handled)." + } + ] + }, + "decorations": [ + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 81801, + "module": "request_loaders._request_loader", + "name": "get_total_count", + "parsedDocstring": { + "text": "Get an offline approximation of the total number of requests in the loader (i.e. pending + handled)." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 33, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/request_loaders/_request_loader.py#L33" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Get an offline approximation of the total number of requests in the loader (i.e. pending + handled)." + } + ] + }, + "flags": {}, + "id": 80595, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "get_total_count", + "parameters": [], + "type": { + "name": "int", + "type": "reference", + "target": 0, + "ref": {} + }, + "overwrites": { + "name": "RequestManager.get_total_count", + "target": 4144, + "type": "reference", + "ref": { + "id": 81801, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 33 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestManager#get_total_count" + } + }, + "parentId": 4144 + } + ], + "inheritedFrom": { + "name": "RequestLoader.get_total_count", + "target": 2937, + "type": "reference", + "ref": { + "id": 80594, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 33 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestLoader#get_total_count" + } + }, + "permalink": "/python/api/class/RequestManager#get_total_count", + "parentId": 3001 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Return True if there are no more requests in the loader (there might still be unfinished requests)." + } + ] + }, + "decorations": [ + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 81802, + "module": "request_loaders._request_loader", + "name": "is_empty", + "parsedDocstring": { + "text": "Return True if there are no more requests in the loader (there might still be unfinished requests)." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 37, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/request_loaders/_request_loader.py#L37" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Return True if there are no more requests in the loader (there might still be unfinished requests)." + } + ] + }, + "flags": {}, + "id": 80597, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "is_empty", + "parameters": [], + "type": { + "name": "bool", + "type": "reference", + "target": 0, + "ref": {} + }, + "overwrites": { + "name": "RequestManager.is_empty", + "target": 4145, + "type": "reference", + "ref": { + "id": 81802, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 37 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestManager#is_empty" + } + }, + "parentId": 4145 + } + ], + "inheritedFrom": { + "name": "RequestLoader.is_empty", + "target": 2939, + "type": "reference", + "ref": { + "id": 80596, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 37 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestLoader#is_empty" + } + }, + "permalink": "/python/api/class/RequestManager#is_empty", + "parentId": 3001 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Return True if all requests have been handled." + } + ] + }, + "decorations": [ + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 81803, + "module": "request_loaders._request_loader", + "name": "is_finished", + "parsedDocstring": { + "text": "Return True if all requests have been handled." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 41, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/request_loaders/_request_loader.py#L41" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Return True if all requests have been handled." + } + ] + }, + "flags": {}, + "id": 80599, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "is_finished", + "parameters": [], + "type": { + "name": "bool", + "type": "reference", + "target": 0, + "ref": {} + }, + "overwrites": { + "name": "RequestManager.is_finished", + "target": 4146, + "type": "reference", + "ref": { + "id": 81803, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 41 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestManager#is_finished" + } + }, + "parentId": 4146 + } + ], + "inheritedFrom": { + "name": "RequestLoader.is_finished", + "target": 2941, + "type": "reference", + "ref": { + "id": 80598, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 41 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestLoader#is_finished" + } + }, + "permalink": "/python/api/class/RequestManager#is_finished", + "parentId": 3001 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Return the next request to be processed, or `None` if there are no more pending requests.\n\nThe method should return `None` if and only if `is_finished` would return `True`. In other cases, the method\nshould wait until a request appears." + } + ] + }, + "decorations": [ + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 81804, + "module": "request_loaders._request_loader", + "name": "fetch_next_request", + "parsedDocstring": { + "text": "Return the next request to be processed, or `None` if there are no more pending requests.\n\nThe method should return `None` if and only if `is_finished` would return `True`. In other cases, the method\nshould wait until a request appears." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 45, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/request_loaders/_request_loader.py#L45" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Return the next request to be processed, or `None` if there are no more pending requests.\n\nThe method should return `None` if and only if `is_finished` would return `True`. In other cases, the method\nshould wait until a request appears." + } + ] + }, + "flags": {}, + "id": 80601, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "fetch_next_request", + "parameters": [], + "type": { + "name": "Request | None", + "type": "reference", + "target": 0, + "ref": {} + }, + "overwrites": { + "name": "RequestManager.fetch_next_request", + "target": 4147, + "type": "reference", + "ref": { + "id": 81804, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 45 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestManager#fetch_next_request" + } + }, + "parentId": 4147 + } + ], + "inheritedFrom": { + "name": "RequestLoader.fetch_next_request", + "target": 2943, + "type": "reference", + "ref": { + "id": 80600, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 45 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestLoader#fetch_next_request" + } + }, + "permalink": "/python/api/class/RequestManager#fetch_next_request", + "parentId": 3001 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Mark a request as handled after a successful processing (or after giving up retrying)." + } + ] + }, + "decorations": [ + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 81805, + "module": "request_loaders._request_loader", + "name": "mark_request_as_handled", + "parsedDocstring": { + "text": "Mark a request as handled after a successful processing (or after giving up retrying)." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 53, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/request_loaders/_request_loader.py#L53" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Mark a request as handled after a successful processing (or after giving up retrying)." + } + ] + }, + "flags": {}, + "id": 80603, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "mark_request_as_handled", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 80604, + "kind": 32768, + "kindString": "Parameter", + "name": "request", + "type": { + "name": "Request", + "type": "reference", + "target": "409", + "ref": { + "id": 78066, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_request.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 145 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/Request" + } + }, + "parentId": 2946 + } + ], + "type": { + "name": "ProcessedRequest | None", + "type": "reference", + "target": 0, + "ref": {} + }, + "overwrites": { + "name": "RequestManager.mark_request_as_handled", + "target": 4148, + "type": "reference", + "ref": { + "id": 81805, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 53 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestManager#mark_request_as_handled" + } + }, + "parentId": 4148 + } + ], + "inheritedFrom": { + "name": "RequestLoader.mark_request_as_handled", + "target": 2945, + "type": "reference", + "ref": { + "id": 80602, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 53 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestLoader#mark_request_as_handled" + } + }, + "permalink": "/python/api/class/RequestManager#mark_request_as_handled", + "parentId": 3001 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Combine the loader with a request manager to support adding and reclaiming requests.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 81806, + "module": "request_loaders._request_loader", + "name": "to_tandem", + "parsedDocstring": { + "text": "Combine the loader with a request manager to support adding and reclaiming requests.\n", + "args": { + "request_manager": "Request manager to combine the loader with.\nIf None is given, the default request queue is used." + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 56, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/request_loaders/_request_loader.py#L56" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Combine the loader with a request manager to support adding and reclaiming requests.\n" + } + ] + }, + "flags": {}, + "id": 80606, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "to_tandem", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Request manager to combine the loader with.\nIf None is given, the default request queue is used." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": false + }, + "id": 80607, + "kind": 32768, + "kindString": "Parameter", + "name": "request_manager", + "type": { + "name": "RequestManager | None", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2949 + } + ], + "type": { + "name": "RequestManagerTandem", + "type": "reference", + "target": "2803", + "ref": { + "id": 80460, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_manager_tandem.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 24 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/RequestManagerTandem" + } + }, + "inheritedFrom": { + "name": "RequestLoader.to_tandem", + "target": 2948, + "type": "reference", + "ref": { + "id": 80605, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 56 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestLoader#to_tandem" + } + }, + "parentId": 4149 + } + ], + "inheritedFrom": { + "name": "RequestLoader.to_tandem", + "target": 2948, + "type": "reference", + "ref": { + "id": 80605, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 56 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestLoader#to_tandem" + } + }, + "permalink": "/python/api/class/RequestManager#to_tandem", + "parentId": 3001 + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Base class that extends `RequestLoader` with the capability to enqueue new requests and reclaim failed ones." + } + ] + }, + "decorations": [ + { + "args": "('Request loaders')", + "name": "docs_group" + } + ], + "flags": {}, + "groups": [ + { + "children": [ + 80663, + 80667, + 80659, + 81804, + 81800, + 81801, + 81802, + 81803, + 81805, + 80661, + 80675, + 81806 + ], + "title": "Methods" + } + ], + "id": 80658, + "module": "request_loaders._request_manager", + "name": "RequestManager", + "parsedDocstring": { + "text": "Base class that extends `RequestLoader` with the capability to enqueue new requests and reclaim failed ones." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_manager.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 18, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/request_loaders/_request_manager.py#L18" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "extendedTypes": [ + { + "name": "RequestLoader", + "target": "2934", + "type": "reference", + "ref": { + "id": 80591, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 17 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/RequestLoader" + } + } + ], + "extendedBy": [ + { + "name": "RequestQueue", + "target": "2348", + "type": "reference", + "ref": { + "id": 80005, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_request_queue.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 33 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/RequestQueue" + } + }, + { + "name": "RequestManagerTandem", + "target": "2803", + "type": "reference", + "ref": { + "id": 80460, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_manager_tandem.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 24 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/RequestManagerTandem" + } + }, + { + "name": "ThrottlingRequestManager", + "target": "2841", + "type": "reference", + "ref": { + "id": 80498, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_throttling_request_manager.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 35 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/ThrottlingRequestManager" + } + } + ], + "permalink": "/python/api/class/RequestManager", + "categories": [] + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 84610, + "module": "request_loaders._request_manager_tandem", + "name": "__init__", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_manager_tandem.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 31, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/request_loaders/_request_manager_tandem.py#L31" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "flags": {}, + "id": 84611, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "__init__", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 84612, + "kind": 32768, + "kindString": "Parameter", + "name": "request_loader", + "type": { + "name": "RequestLoader", + "type": "reference", + "target": "2934", + "ref": { + "id": 84740, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 17 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/RequestLoader" + } + }, + "parentId": 2805 + }, + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 84613, + "kind": 32768, + "kindString": "Parameter", + "name": "request_manager", + "type": { + "name": "RequestManager", + "type": "reference", + "target": "3001", + "ref": { + "id": 84807, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_manager.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 18 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/RequestManager" + } + }, + "parentId": 2805 + } + ], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "parentId": 2804 + } + ], + "permalink": "/python/api/class/RequestManagerTandem#__init__", + "parentId": 2803 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Get the number of requests in the loader that have been handled." + } + ] + }, + "decorations": [ + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 84614, + "module": "request_loaders._request_loader", + "name": "get_handled_count", + "parsedDocstring": { + "text": "Get the number of requests in the loader that have been handled." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_manager_tandem.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 36, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/request_loaders/_request_manager_tandem.py#L36" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Get the number of requests in the loader that have been handled." + } + ] + }, + "flags": {}, + "id": 84742, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "get_handled_count", + "parameters": [], + "type": { + "name": "int", + "type": "reference", + "target": 0, + "ref": {} + }, + "overwrites": { + "name": "RequestManager.get_handled_count", + "target": 4143, + "type": "reference", + "ref": { + "id": 85949, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 29 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestManager#get_handled_count" + } + }, + "parentId": 2808 + } + ], + "overwrites": { + "name": "RequestManager.get_handled_count", + "target": 4143, + "type": "reference", + "ref": { + "id": 85949, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 29 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestManager#get_handled_count" + } + }, + "permalink": "/python/api/class/RequestManagerTandem#get_handled_count", + "parentId": 2803 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Get an offline approximation of the total number of requests in the loader (i.e. pending + handled)." + } + ] + }, + "decorations": [ + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 84616, + "module": "request_loaders._request_loader", + "name": "get_total_count", + "parsedDocstring": { + "text": "Get an offline approximation of the total number of requests in the loader (i.e. pending + handled)." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_manager_tandem.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 40, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/request_loaders/_request_manager_tandem.py#L40" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Get an offline approximation of the total number of requests in the loader (i.e. pending + handled)." + } + ] + }, + "flags": {}, + "id": 84744, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "get_total_count", + "parameters": [], + "type": { + "name": "int", + "type": "reference", + "target": 0, + "ref": {} + }, + "overwrites": { + "name": "RequestManager.get_total_count", + "target": 4144, + "type": "reference", + "ref": { + "id": 85950, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 33 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestManager#get_total_count" + } + }, + "parentId": 2810 + } + ], + "overwrites": { + "name": "RequestManager.get_total_count", + "target": 4144, + "type": "reference", + "ref": { + "id": 85950, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 33 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestManager#get_total_count" + } + }, + "permalink": "/python/api/class/RequestManagerTandem#get_total_count", + "parentId": 2803 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Return True if there are no more requests in the loader (there might still be unfinished requests)." + } + ] + }, + "decorations": [ + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 84618, + "module": "request_loaders._request_loader", + "name": "is_empty", + "parsedDocstring": { + "text": "Return True if there are no more requests in the loader (there might still be unfinished requests)." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_manager_tandem.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 44, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/request_loaders/_request_manager_tandem.py#L44" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Return True if there are no more requests in the loader (there might still be unfinished requests)." + } + ] + }, + "flags": {}, + "id": 84746, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "is_empty", + "parameters": [], + "type": { + "name": "bool", + "type": "reference", + "target": 0, + "ref": {} + }, + "overwrites": { + "name": "RequestManager.is_empty", + "target": 4145, + "type": "reference", + "ref": { + "id": 85951, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 37 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestManager#is_empty" + } + }, + "parentId": 2812 + } + ], + "overwrites": { + "name": "RequestManager.is_empty", + "target": 4145, + "type": "reference", + "ref": { + "id": 85951, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 37 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestManager#is_empty" + } + }, + "permalink": "/python/api/class/RequestManagerTandem#is_empty", + "parentId": 2803 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Return True if all requests have been handled." + } + ] + }, + "decorations": [ + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 84620, + "module": "request_loaders._request_loader", + "name": "is_finished", + "parsedDocstring": { + "text": "Return True if all requests have been handled." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_manager_tandem.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 48, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/request_loaders/_request_manager_tandem.py#L48" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Return True if all requests have been handled." + } + ] + }, + "flags": {}, + "id": 84748, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "is_finished", + "parameters": [], + "type": { + "name": "bool", + "type": "reference", + "target": 0, + "ref": {} + }, + "overwrites": { + "name": "RequestManager.is_finished", + "target": 4146, + "type": "reference", + "ref": { + "id": 85952, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 41 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestManager#is_finished" + } + }, + "parentId": 2814 + } + ], + "overwrites": { + "name": "RequestManager.is_finished", + "target": 4146, + "type": "reference", + "ref": { + "id": 85952, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 41 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestManager#is_finished" + } + }, + "permalink": "/python/api/class/RequestManagerTandem#is_finished", + "parentId": 2803 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Add a single request to the manager and store it in underlying resource client.\n" + } + ] + }, + "decorations": [ + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 84622, + "module": "request_loaders._request_manager", + "name": "add_request", + "parsedDocstring": { + "text": "Add a single request to the manager and store it in underlying resource client.\n", + "args": { + "request": "The request object (or its string representation) to be added to the manager.", + "forefront": "Determines whether the request should be added to the beginning (if True) or the end (if False)\nof the manager.\n" + }, + "returns": "Information about the request addition to the manager or None if the request was not added." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_manager_tandem.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 52, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/request_loaders/_request_manager_tandem.py#L52" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "Information about the request addition to the manager or None if the request was not added." + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Add a single request to the manager and store it in underlying resource client.\n" + } + ] + }, + "flags": {}, + "id": 84813, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "add_request", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The request object (or its string representation) to be added to the manager." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 84814, + "kind": 32768, + "kindString": "Parameter", + "name": "request", + "type": { + "name": "str | Request", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "reference", + "name": "Request", + "target": "409", + "ref": { + "id": 82215, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_request.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 145 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/Request" + } + } + ] + }, + "parentId": 3007 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Determines whether the request should be added to the beginning (if True) or the end (if False)\nof the manager.\n" + } + ] + }, + "defaultValue": "False", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 84815, + "kind": 32768, + "kindString": "Parameter", + "name": "forefront", + "type": { + "name": "bool", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 3007 + } + ], + "type": { + "name": "ProcessedRequest | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "ProcessedRequest", + "target": "1340", + "ref": { + "id": 83146, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 140 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/ProcessedRequest" + } + }, + { + "type": "literal", + "value": {} + } + ] + }, + "overwrites": { + "name": "RequestManager.add_request", + "target": 3006, + "type": "reference", + "ref": { + "id": 84812, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_manager.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 32 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestManager#add_request" + } + }, + "parentId": 2816 + } + ], + "overwrites": { + "name": "RequestManager.add_request", + "target": 3006, + "type": "reference", + "ref": { + "id": 84812, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_manager.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 32 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestManager#add_request" + } + }, + "permalink": "/python/api/class/RequestManagerTandem#add_request", + "parentId": 2803 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Add requests to the manager in batches.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 84626, + "module": "request_loaders._request_manager", + "name": "add_requests", + "parsedDocstring": { + "text": "Add requests to the manager in batches.\n", + "args": { + "requests": "Requests to enqueue.", + "forefront": "If True, add requests to the beginning of the queue.", + "batch_size": "The number of requests to add in one batch.", + "wait_time_between_batches": "Time to wait between adding batches.", + "wait_for_all_requests_to_be_added": "If True, wait for all requests to be added before returning.", + "wait_for_all_requests_to_be_added_timeout": "Timeout for waiting for all requests to be added." + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_manager_tandem.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 56, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/request_loaders/_request_manager_tandem.py#L56" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Add requests to the manager in batches.\n" + } + ] + }, + "flags": {}, + "id": 84817, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "add_requests", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Requests to enqueue." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 84818, + "kind": 32768, + "kindString": "Parameter", + "name": "requests", + "type": { + "name": "Sequence", + "type": "reference", + "typeArguments": [ + { + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "reference", + "name": "Request", + "target": "409", + "ref": { + "id": 82215, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_request.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 145 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/Request" + } + } + ] + } + ], + "target": 0, + "ref": {} + }, + "parentId": 3011 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "If True, add requests to the beginning of the queue." + } + ] + }, + "defaultValue": "False", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 84819, + "kind": 32768, + "kindString": "Parameter", + "name": "forefront", + "type": { + "name": "bool", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 3011 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The number of requests to add in one batch." + } + ] + }, + "defaultValue": "1000", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 84820, + "kind": 32768, + "kindString": "Parameter", + "name": "batch_size", + "type": { + "name": "int", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 3011 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Time to wait between adding batches." + } + ] + }, + "defaultValue": "timedelta(seconds=1)", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 84821, + "kind": 32768, + "kindString": "Parameter", + "name": "wait_time_between_batches", + "type": { + "name": "timedelta", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 3011 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "If True, wait for all requests to be added before returning." + } + ] + }, + "defaultValue": "False", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 84822, + "kind": 32768, + "kindString": "Parameter", + "name": "wait_for_all_requests_to_be_added", + "type": { + "name": "bool", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 3011 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Timeout for waiting for all requests to be added." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 84823, + "kind": 32768, + "kindString": "Parameter", + "name": "wait_for_all_requests_to_be_added_timeout", + "type": { + "name": "timedelta | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "timedelta", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 3011 + } + ], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "overwrites": { + "name": "RequestManager.add_requests", + "target": 3010, + "type": "reference", + "ref": { + "id": 84816, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_manager.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 49 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestManager#add_requests" + } + }, + "parentId": 2820 + } + ], + "overwrites": { + "name": "RequestManager.add_requests", + "target": 3010, + "type": "reference", + "ref": { + "id": 84816, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_manager.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 49 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestManager#add_requests" + } + }, + "permalink": "/python/api/class/RequestManagerTandem#add_requests", + "parentId": 2803 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Return the next request to be processed, or `None` if there are no more pending requests.\n\nThe method should return `None` if and only if `is_finished` would return `True`. In other cases, the method\nshould wait until a request appears." + } + ] + }, + "decorations": [ + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 84634, + "module": "request_loaders._request_loader", + "name": "fetch_next_request", + "parsedDocstring": { + "text": "Return the next request to be processed, or `None` if there are no more pending requests.\n\nThe method should return `None` if and only if `is_finished` would return `True`. In other cases, the method\nshould wait until a request appears." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_manager_tandem.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 76, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/request_loaders/_request_manager_tandem.py#L76" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Return the next request to be processed, or `None` if there are no more pending requests.\n\nThe method should return `None` if and only if `is_finished` would return `True`. In other cases, the method\nshould wait until a request appears." + } + ] + }, + "flags": {}, + "id": 84750, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "fetch_next_request", + "parameters": [], + "type": { + "name": "Request | None", + "type": "reference", + "target": 0, + "ref": {} + }, + "overwrites": { + "name": "RequestManager.fetch_next_request", + "target": 4147, + "type": "reference", + "ref": { + "id": 85953, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 45 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestManager#fetch_next_request" + } + }, + "parentId": 2828 + } + ], + "overwrites": { + "name": "RequestManager.fetch_next_request", + "target": 4147, + "type": "reference", + "ref": { + "id": 85953, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 45 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestManager#fetch_next_request" + } + }, + "permalink": "/python/api/class/RequestManagerTandem#fetch_next_request", + "parentId": 2803 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Reclaims a failed request back to the source, so that it can be returned for processing later again.\n\nIt is possible to modify the request data by supplying an updated request as a parameter." + } + ] + }, + "decorations": [ + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 84636, + "module": "request_loaders._request_manager", + "name": "reclaim_request", + "parsedDocstring": { + "text": "Reclaims a failed request back to the source, so that it can be returned for processing later again.\n\nIt is possible to modify the request data by supplying an updated request as a parameter." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_manager_tandem.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 102, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/request_loaders/_request_manager_tandem.py#L102" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Reclaims a failed request back to the source, so that it can be returned for processing later again.\n\nIt is possible to modify the request data by supplying an updated request as a parameter." + } + ] + }, + "flags": {}, + "id": 84825, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "reclaim_request", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 84826, + "kind": 32768, + "kindString": "Parameter", + "name": "request", + "type": { + "name": "Request", + "type": "reference", + "target": "409", + "ref": { + "id": 82215, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_request.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 145 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/Request" + } + }, + "parentId": 3019 + }, + { + "defaultValue": "False", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 84827, + "kind": 32768, + "kindString": "Parameter", + "name": "forefront", + "type": { + "name": "bool", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 3019 + } + ], + "type": { + "name": "ProcessedRequest | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "ProcessedRequest", + "target": "1340", + "ref": { + "id": 83146, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 140 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/ProcessedRequest" + } + }, + { + "type": "literal", + "value": {} + } + ] + }, + "overwrites": { + "name": "RequestManager.reclaim_request", + "target": 3018, + "type": "reference", + "ref": { + "id": 84824, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_manager.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 77 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestManager#reclaim_request" + } + }, + "parentId": 2830 + } + ], + "overwrites": { + "name": "RequestManager.reclaim_request", + "target": 3018, + "type": "reference", + "ref": { + "id": 84824, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_manager.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 77 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestManager#reclaim_request" + } + }, + "permalink": "/python/api/class/RequestManagerTandem#reclaim_request", + "parentId": 2803 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Mark a request as handled after a successful processing (or after giving up retrying)." + } + ] + }, + "decorations": [ + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 84640, + "module": "request_loaders._request_loader", + "name": "mark_request_as_handled", + "parsedDocstring": { + "text": "Mark a request as handled after a successful processing (or after giving up retrying)." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_manager_tandem.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 106, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/request_loaders/_request_manager_tandem.py#L106" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Mark a request as handled after a successful processing (or after giving up retrying)." + } + ] + }, + "flags": {}, + "id": 84752, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "mark_request_as_handled", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 84753, + "kind": 32768, + "kindString": "Parameter", + "name": "request", + "type": { + "name": "Request", + "type": "reference", + "target": "409", + "ref": { + "id": 82215, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_request.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 145 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/Request" + } + }, + "parentId": 2946 + } + ], + "type": { + "name": "ProcessedRequest | None", + "type": "reference", + "target": 0, + "ref": {} + }, + "overwrites": { + "name": "RequestManager.mark_request_as_handled", + "target": 4148, + "type": "reference", + "ref": { + "id": 85954, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 53 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestManager#mark_request_as_handled" + } + }, + "parentId": 2834 + } + ], + "overwrites": { + "name": "RequestManager.mark_request_as_handled", + "target": 4148, + "type": "reference", + "ref": { + "id": 85954, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 53 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestManager#mark_request_as_handled" + } + }, + "permalink": "/python/api/class/RequestManagerTandem#mark_request_as_handled", + "parentId": 2803 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Remove persistent state either from the Apify Cloud storage or from the local database." + } + ] + }, + "decorations": [ + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 84643, + "module": "request_loaders._request_manager", + "name": "drop", + "parsedDocstring": { + "text": "Remove persistent state either from the Apify Cloud storage or from the local database." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_manager_tandem.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 110, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/request_loaders/_request_manager_tandem.py#L110" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Remove persistent state either from the Apify Cloud storage or from the local database." + } + ] + }, + "flags": {}, + "id": 84809, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "drop", + "parameters": [], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "overwrites": { + "name": "Storage.drop", + "target": 2342, + "type": "reference", + "ref": { + "id": 84148, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_base.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 57 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/Storage#drop" + } + }, + "parentId": 2837 + } + ], + "overwrites": { + "name": "RequestManager.drop", + "target": 3002, + "type": "reference", + "ref": { + "id": 84808, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_manager.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 22 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestManager#drop" + } + }, + "permalink": "/python/api/class/RequestManagerTandem#drop", + "parentId": 2803 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Remove all queued requests from this manager while keeping it usable.\n\nDefault implementation is a no-op. Subclasses with persistent state should override to clear it." + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 85957, + "module": "request_loaders._request_manager", + "name": "purge", + "parsedDocstring": { + "text": "Remove all queued requests from this manager while keeping it usable.\n\nDefault implementation is a no-op. Subclasses with persistent state should override to clear it." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_manager.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 25, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/request_loaders/_request_manager.py#L25" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Remove all queued requests from this manager while keeping it usable.\n\nDefault implementation is a no-op. Subclasses with persistent state should override to clear it." + } + ] + }, + "flags": {}, + "id": 84811, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "purge", + "parameters": [], + "type": { + "name": "None", + "type": "reference", + "target": 0, + "ref": {} + }, + "inheritedFrom": { + "name": "RequestManager.purge", + "target": 3004, + "type": "reference", + "ref": { + "id": 84810, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_manager.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 25 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestManager#purge" + } + }, + "parentId": 4151 + } + ], + "inheritedFrom": { + "name": "RequestManager.purge", + "target": 3004, + "type": "reference", + "ref": { + "id": 84810, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_manager.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 25 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestManager#purge" + } + }, + "permalink": "/python/api/class/RequestManagerTandem#purge", + "parentId": 2803 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Combine the loader with a request manager to support adding and reclaiming requests.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 85958, + "module": "request_loaders._request_loader", + "name": "to_tandem", + "parsedDocstring": { + "text": "Combine the loader with a request manager to support adding and reclaiming requests.\n", + "args": { + "request_manager": "Request manager to combine the loader with.\nIf None is given, the default request queue is used." + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 56, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/request_loaders/_request_loader.py#L56" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Combine the loader with a request manager to support adding and reclaiming requests.\n" + } + ] + }, + "flags": {}, + "id": 84755, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "to_tandem", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Request manager to combine the loader with.\nIf None is given, the default request queue is used." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": false + }, + "id": 84756, + "kind": 32768, + "kindString": "Parameter", + "name": "request_manager", + "type": { + "name": "RequestManager | None", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2949 + } + ], + "type": { + "name": "RequestManagerTandem", + "type": "reference", + "target": "2803", + "ref": { + "id": 84609, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_manager_tandem.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 24 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/RequestManagerTandem" + } + }, + "inheritedFrom": { + "name": "RequestLoader.to_tandem", + "target": 2948, + "type": "reference", + "ref": { + "id": 84754, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 56 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestLoader#to_tandem" + } + }, + "parentId": 4152 + } + ], + "inheritedFrom": { + "name": "RequestLoader.to_tandem", + "target": 2948, + "type": "reference", + "ref": { + "id": 84754, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 56 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestLoader#to_tandem" + } + }, + "permalink": "/python/api/class/RequestManagerTandem#to_tandem", + "parentId": 2803 + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Implements a tandem behaviour for a pair of `RequestLoader` and `RequestManager`.\n\nIn this scenario, the contents of the \"loader\" get transferred into the \"manager\", allowing processing the requests\nfrom both sources and also enqueueing new requests (not possible with plain `RequestManager`)." + } + ] + }, + "decorations": [ + { + "args": "('Request loaders')", + "name": "docs_group" + } + ], + "flags": {}, + "groups": [ + { + "children": [ + 84610, + 84622, + 84626, + 84643, + 84634, + 84614, + 84616, + 84618, + 84620, + 84640, + 85957, + 84636, + 85958 + ], + "title": "Methods" + } + ], + "id": 84609, + "module": "request_loaders._request_manager_tandem", + "name": "RequestManagerTandem", + "parsedDocstring": { + "text": "Implements a tandem behaviour for a pair of `RequestLoader` and `RequestManager`.\n\nIn this scenario, the contents of the \"loader\" get transferred into the \"manager\", allowing processing the requests\nfrom both sources and also enqueueing new requests (not possible with plain `RequestManager`)." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_manager_tandem.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 24, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/request_loaders/_request_manager_tandem.py#L24" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "extendedTypes": [ + { + "name": "RequestManager", + "target": "3001", + "type": "reference", + "ref": { + "id": 84807, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_manager.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 18 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/RequestManager" + } + } + ], + "permalink": "/python/api/class/RequestManagerTandem", + "categories": [] + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Initialize the sitemap request loader.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 88922, + "module": "request_loaders._sitemap_request_loader", + "name": "__init__", + "parsedDocstring": { + "text": "Initialize the sitemap request loader.\n", + "args": { + "sitemap_urls": "Configuration options for the loader.", + "proxy_info": "Optional proxy to use for fetching sitemaps.", + "include": "List of glob or regex patterns to include URLs.", + "exclude": "List of glob or regex patterns to exclude URLs.", + "enqueue_strategy": "Strategy used to decide which sitemap-derived URLs (both nested-sitemap entries and\nURL entries) are kept relative to the parent sitemap URL. Defaults to `'same-hostname'`, matching\nthe sitemap protocol's same-host expectation and the `enqueue_links` default; pass `'all'` to\ndisable filtering. Note: regardless of `enqueue_strategy`, entries with non-`http(s)` schemes are\nalways filtered out.", + "max_buffer_size": "Maximum number of URLs to buffer in memory.", + "http_client": "the instance of `HttpClient` to use for fetching sitemaps.", + "persist_state_key": "A key for persisting the loader's state in the KeyValueStore.\nWhen provided, allows resuming from where it left off after interruption.\nIf None, no state persistence occurs.", + "transform_request_function": "An optional function to transform requests\ngenerated by the loader. It receives `RequestOptions` with `url` and should return either\nmodified `RequestOptions` or a `RequestTransformAction`." + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_sitemap_request_loader.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 109, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/request_loaders/_sitemap_request_loader.py#L109" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Initialize the sitemap request loader.\n" + } + ] + }, + "flags": {}, + "id": 88923, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "__init__", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Configuration options for the loader." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 88924, + "kind": 32768, + "kindString": "Parameter", + "name": "sitemap_urls", + "type": { + "name": "list", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + }, + "parentId": 2965 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "the instance of `HttpClient` to use for fetching sitemaps." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 88925, + "kind": 32768, + "kindString": "Parameter", + "name": "http_client", + "type": { + "name": "HttpClient", + "type": "reference", + "target": "3137", + "ref": { + "id": 89095, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/http_clients/_base.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 75 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/HttpClient" + } + }, + "parentId": 2965 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Optional proxy to use for fetching sitemaps." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 88926, + "kind": 32768, + "kindString": "Parameter", + "name": "proxy_info", + "type": { + "name": "ProxyInfo | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "ProxyInfo", + "target": "21", + "ref": { + "id": 85979, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/proxy_configuration.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 26 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/ProxyInfo" + } + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2965 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "List of glob or regex patterns to include URLs." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 88927, + "kind": 32768, + "kindString": "Parameter", + "name": "include", + "type": { + "name": "list[re.Pattern[Any] | Glob] | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "list", + "typeArguments": [ + { + "type": "union", + "types": [ + { + "type": "reference", + "name": "re.Pattern", + "typeArguments": [ + { + "type": "reference", + "name": "Any", + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + }, + { + "type": "reference", + "name": "Glob", + "target": "3253", + "ref": { + "id": 89211, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_utils/globs.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 11 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/Glob" + } + } + ] + } + ], + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2965 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "List of glob or regex patterns to exclude URLs." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 88928, + "kind": 32768, + "kindString": "Parameter", + "name": "exclude", + "type": { + "name": "list[re.Pattern[Any] | Glob] | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "list", + "typeArguments": [ + { + "type": "union", + "types": [ + { + "type": "reference", + "name": "re.Pattern", + "typeArguments": [ + { + "type": "reference", + "name": "Any", + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + }, + { + "type": "reference", + "name": "Glob", + "target": "3253", + "ref": { + "id": 89211, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_utils/globs.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 11 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/Glob" + } + } + ] + } + ], + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2965 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Strategy used to decide which sitemap-derived URLs (both nested-sitemap entries and\nURL entries) are kept relative to the parent sitemap URL. Defaults to `'same-hostname'`, matching\nthe sitemap protocol's same-host expectation and the `enqueue_links` default; pass `'all'` to\ndisable filtering. Note: regardless of `enqueue_strategy`, entries with non-`http(s)` schemes are\nalways filtered out." + } + ] + }, + "defaultValue": "'same-hostname'", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 88929, + "kind": 32768, + "kindString": "Parameter", + "name": "enqueue_strategy", + "type": { + "name": "EnqueueStrategy", + "type": "reference", + "target": "88", + "ref": { + "id": 86046, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_types.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 47 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api#EnqueueStrategy" + } + }, + "parentId": 2965 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Maximum number of URLs to buffer in memory." + } + ] + }, + "defaultValue": "200", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 88930, + "kind": 32768, + "kindString": "Parameter", + "name": "max_buffer_size", + "type": { + "name": "int", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2965 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "A key for persisting the loader's state in the KeyValueStore.\nWhen provided, allows resuming from where it left off after interruption.\nIf None, no state persistence occurs." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 88931, + "kind": 32768, + "kindString": "Parameter", + "name": "persist_state_key", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2965 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "An optional function to transform requests\ngenerated by the loader. It receives `RequestOptions` with `url` and should return either\nmodified `RequestOptions` or a `RequestTransformAction`." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 88932, + "kind": 32768, + "kindString": "Parameter", + "name": "transform_request_function", + "type": { + "name": "Callable[[RequestOptions], RequestOptions | RequestTransformAction] | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "Callable", + "typeArguments": [ + { + "type": "reference", + "name": "[RequestOptions]", + "target": 0, + "ref": {} + }, + { + "type": "union", + "types": [ + { + "type": "reference", + "name": "RequestOptions", + "target": "393", + "ref": { + "id": 86351, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_request.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 121 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/RequestOptions" + } + }, + { + "type": "reference", + "name": "RequestTransformAction", + "target": "87", + "ref": { + "id": 86045, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_types.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 45 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api#RequestTransformAction" + } + } + ] + } + ], + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2965 + } + ], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "parentId": 2964 + } + ], + "permalink": "/python/api/class/SitemapRequestLoader#__init__", + "parentId": 2963 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Enter the context manager." + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 88933, + "module": "request_loaders._sitemap_request_loader", + "name": "__aenter__", + "parsedDocstring": { + "text": "Enter the context manager." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_sitemap_request_loader.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 171, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/request_loaders/_sitemap_request_loader.py#L171" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Enter the context manager." + } + ] + }, + "flags": {}, + "id": 88934, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "__aenter__", + "parameters": [], + "type": { + "name": "SitemapRequestLoader", + "type": "reference", + "target": "2963", + "ref": { + "id": 88921, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_sitemap_request_loader.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 94 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/SitemapRequestLoader" + } + }, + "parentId": 2975 + } + ], + "permalink": "/python/api/class/SitemapRequestLoader#__aenter__", + "parentId": 2963 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Exit the context manager." + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 88935, + "module": "request_loaders._sitemap_request_loader", + "name": "__aexit__", + "parsedDocstring": { + "text": "Exit the context manager." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_sitemap_request_loader.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 176, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/request_loaders/_sitemap_request_loader.py#L176" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Exit the context manager." + } + ] + }, + "flags": {}, + "id": 88936, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "__aexit__", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 88937, + "kind": 32768, + "kindString": "Parameter", + "name": "exc_type", + "type": { + "name": "type[BaseException] | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "type", + "typeArguments": [ + { + "type": "reference", + "name": "BaseException", + "target": 0, + "ref": {} + } + ], + "target": "3422", + "ref": { + "id": 89380, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_utils/sitemap.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 62 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api/class/SitemapSource#type" + } + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2978 + }, + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 88938, + "kind": 32768, + "kindString": "Parameter", + "name": "exc_value", + "type": { + "name": "BaseException | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "BaseException", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2978 + }, + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 88939, + "kind": 32768, + "kindString": "Parameter", + "name": "exc_traceback", + "type": { + "name": "TracebackType | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "TracebackType", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2978 + } + ], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "parentId": 2977 + } + ], + "permalink": "/python/api/class/SitemapRequestLoader#__aexit__", + "parentId": 2963 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Return the total number of URLs found so far." + } + ] + }, + "decorations": [ + { + "name": "override" + } + ], + "flags": {}, + "groups": [], + "id": 88940, + "module": "request_loaders._sitemap_request_loader", + "name": "get_total_count", + "parsedDocstring": { + "text": "Return the total number of URLs found so far." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_sitemap_request_loader.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 186, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/request_loaders/_sitemap_request_loader.py#L186" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Return the total number of URLs found so far." + } + ] + }, + "flags": {}, + "id": 88941, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "get_total_count", + "parameters": [], + "type": { + "name": "int", + "type": "reference", + "target": 0, + "ref": {} + }, + "overwrites": { + "name": "RequestLoader.get_total_count", + "target": 2937, + "type": "reference", + "ref": { + "id": 88895, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 33 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestLoader#get_total_count" + } + }, + "parentId": 2982 + } + ], + "overwrites": { + "name": "RequestLoader.get_total_count", + "target": 2937, + "type": "reference", + "ref": { + "id": 88895, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 33 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestLoader#get_total_count" + } + }, + "permalink": "/python/api/class/SitemapRequestLoader#get_total_count", + "parentId": 2963 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Return the number of URLs that have been handled." + } + ] + }, + "decorations": [ + { + "name": "override" + } + ], + "flags": {}, + "groups": [], + "id": 88942, + "module": "request_loaders._sitemap_request_loader", + "name": "get_handled_count", + "parsedDocstring": { + "text": "Return the number of URLs that have been handled." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_sitemap_request_loader.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 192, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/request_loaders/_sitemap_request_loader.py#L192" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Return the number of URLs that have been handled." + } + ] + }, + "flags": {}, + "id": 88943, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "get_handled_count", + "parameters": [], + "type": { + "name": "int", + "type": "reference", + "target": 0, + "ref": {} + }, + "overwrites": { + "name": "RequestLoader.get_handled_count", + "target": 2935, + "type": "reference", + "ref": { + "id": 88893, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 29 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestLoader#get_handled_count" + } + }, + "parentId": 2984 + } + ], + "overwrites": { + "name": "RequestLoader.get_handled_count", + "target": 2935, + "type": "reference", + "ref": { + "id": 88893, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 29 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestLoader#get_handled_count" + } + }, + "permalink": "/python/api/class/SitemapRequestLoader#get_handled_count", + "parentId": 2963 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Check if there are no more URLs to process." + } + ] + }, + "decorations": [ + { + "name": "override" + } + ], + "flags": {}, + "groups": [], + "id": 88944, + "module": "request_loaders._sitemap_request_loader", + "name": "is_empty", + "parsedDocstring": { + "text": "Check if there are no more URLs to process." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_sitemap_request_loader.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 198, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/request_loaders/_sitemap_request_loader.py#L198" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Check if there are no more URLs to process." + } + ] + }, + "flags": {}, + "id": 88945, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "is_empty", + "parameters": [], + "type": { + "name": "bool", + "type": "reference", + "target": 0, + "ref": {} + }, + "overwrites": { + "name": "RequestLoader.is_empty", + "target": 2939, + "type": "reference", + "ref": { + "id": 88897, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 37 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestLoader#is_empty" + } + }, + "parentId": 2986 + } + ], + "overwrites": { + "name": "RequestLoader.is_empty", + "target": 2939, + "type": "reference", + "ref": { + "id": 88897, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 37 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestLoader#is_empty" + } + }, + "permalink": "/python/api/class/SitemapRequestLoader#is_empty", + "parentId": 2963 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Check if all URLs have been processed." + } + ] + }, + "decorations": [ + { + "name": "override" + } + ], + "flags": {}, + "groups": [], + "id": 88946, + "module": "request_loaders._sitemap_request_loader", + "name": "is_finished", + "parsedDocstring": { + "text": "Check if all URLs have been processed." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_sitemap_request_loader.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 204, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/request_loaders/_sitemap_request_loader.py#L204" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Check if all URLs have been processed." + } + ] + }, + "flags": {}, + "id": 88947, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "is_finished", + "parameters": [], + "type": { + "name": "bool", + "type": "reference", + "target": 0, + "ref": {} + }, + "overwrites": { + "name": "RequestLoader.is_finished", + "target": 2941, + "type": "reference", + "ref": { + "id": 88899, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 41 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestLoader#is_finished" + } + }, + "parentId": 2988 + } + ], + "overwrites": { + "name": "RequestLoader.is_finished", + "target": 2941, + "type": "reference", + "ref": { + "id": 88899, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 41 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestLoader#is_finished" + } + }, + "permalink": "/python/api/class/SitemapRequestLoader#is_finished", + "parentId": 2963 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Fetch the next request to process." + } + ] + }, + "decorations": [ + { + "name": "override" + } + ], + "flags": {}, + "groups": [], + "id": 88948, + "module": "request_loaders._sitemap_request_loader", + "name": "fetch_next_request", + "parsedDocstring": { + "text": "Fetch the next request to process." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_sitemap_request_loader.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 210, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/request_loaders/_sitemap_request_loader.py#L210" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Fetch the next request to process." + } + ] + }, + "flags": {}, + "id": 88949, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "fetch_next_request", + "parameters": [], + "type": { + "name": "Request | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "Request", + "target": "409", + "ref": { + "id": 86367, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_request.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 145 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/Request" + } + }, + { + "type": "literal", + "value": {} + } + ] + }, + "overwrites": { + "name": "RequestLoader.fetch_next_request", + "target": 2943, + "type": "reference", + "ref": { + "id": 88901, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 45 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestLoader#fetch_next_request" + } + }, + "parentId": 2990 + } + ], + "overwrites": { + "name": "RequestLoader.fetch_next_request", + "target": 2943, + "type": "reference", + "ref": { + "id": 88901, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 45 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestLoader#fetch_next_request" + } + }, + "permalink": "/python/api/class/SitemapRequestLoader#fetch_next_request", + "parentId": 2963 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Mark a request as successfully handled." + } + ] + }, + "decorations": [ + { + "name": "override" + } + ], + "flags": {}, + "groups": [], + "id": 88950, + "module": "request_loaders._sitemap_request_loader", + "name": "mark_request_as_handled", + "parsedDocstring": { + "text": "Mark a request as successfully handled." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_sitemap_request_loader.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 245, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/request_loaders/_sitemap_request_loader.py#L245" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Mark a request as successfully handled." + } + ] + }, + "flags": {}, + "id": 88951, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "mark_request_as_handled", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 88952, + "kind": 32768, + "kindString": "Parameter", + "name": "request", + "type": { + "name": "Request", + "type": "reference", + "target": "409", + "ref": { + "id": 86367, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_request.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 145 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/Request" + } + }, + "parentId": 2993 + } + ], + "type": { + "name": "ProcessedRequest | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "ProcessedRequest", + "target": "1340", + "ref": { + "id": 87298, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 140 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/ProcessedRequest" + } + }, + { + "type": "literal", + "value": {} + } + ] + }, + "overwrites": { + "name": "RequestLoader.mark_request_as_handled", + "target": 2945, + "type": "reference", + "ref": { + "id": 88903, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 53 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestLoader#mark_request_as_handled" + } + }, + "parentId": 2992 + } + ], + "overwrites": { + "name": "RequestLoader.mark_request_as_handled", + "target": 2945, + "type": "reference", + "ref": { + "id": 88903, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 53 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestLoader#mark_request_as_handled" + } + }, + "permalink": "/python/api/class/SitemapRequestLoader#mark_request_as_handled", + "parentId": 2963 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Start the sitemap loading process." + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 88953, + "module": "request_loaders._sitemap_request_loader", + "name": "start", + "parsedDocstring": { + "text": "Start the sitemap loading process." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_sitemap_request_loader.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 253, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/request_loaders/_sitemap_request_loader.py#L253" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Start the sitemap loading process." + } + ] + }, + "flags": {}, + "id": 88954, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "start", + "parameters": [], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "parentId": 2995 + } + ], + "permalink": "/python/api/class/SitemapRequestLoader#start", + "parentId": 2963 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Abort the sitemap loading process." + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 88955, + "module": "request_loaders._sitemap_request_loader", + "name": "abort_loading", + "parsedDocstring": { + "text": "Abort the sitemap loading process." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_sitemap_request_loader.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 259, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/request_loaders/_sitemap_request_loader.py#L259" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Abort the sitemap loading process." + } + ] + }, + "flags": {}, + "id": 88956, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "abort_loading", + "parameters": [], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "parentId": 2997 + } + ], + "permalink": "/python/api/class/SitemapRequestLoader#abort_loading", + "parentId": 2963 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Close the request loader." + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 88957, + "module": "request_loaders._sitemap_request_loader", + "name": "close", + "parsedDocstring": { + "text": "Close the request loader." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_sitemap_request_loader.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 266, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/request_loaders/_sitemap_request_loader.py#L266" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Close the request loader." + } + ] + }, + "flags": {}, + "id": 88958, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "close", + "parameters": [], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "parentId": 2999 + } + ], + "permalink": "/python/api/class/SitemapRequestLoader#close", + "parentId": 2963 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Combine the loader with a request manager to support adding and reclaiming requests.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 90100, + "module": "request_loaders._request_loader", + "name": "to_tandem", + "parsedDocstring": { + "text": "Combine the loader with a request manager to support adding and reclaiming requests.\n", + "args": { + "request_manager": "Request manager to combine the loader with.\nIf None is given, the default request queue is used." + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 56, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/request_loaders/_request_loader.py#L56" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Combine the loader with a request manager to support adding and reclaiming requests.\n" + } + ] + }, + "flags": {}, + "id": 88907, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "to_tandem", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Request manager to combine the loader with.\nIf None is given, the default request queue is used." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": false + }, + "id": 88908, + "kind": 32768, + "kindString": "Parameter", + "name": "request_manager", + "type": { + "name": "RequestManager | None", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2949 + } + ], + "type": { + "name": "RequestManagerTandem", + "type": "reference", + "target": "2803", + "ref": { + "id": 88761, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_manager_tandem.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 24 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/RequestManagerTandem" + } + }, + "inheritedFrom": { + "name": "RequestLoader.to_tandem", + "target": 2948, + "type": "reference", + "ref": { + "id": 88906, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 56 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestLoader#to_tandem" + } + }, + "parentId": 4142 + } + ], + "inheritedFrom": { + "name": "RequestLoader.to_tandem", + "target": 2948, + "type": "reference", + "ref": { + "id": 88906, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 56 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestLoader#to_tandem" + } + }, + "permalink": "/python/api/class/SitemapRequestLoader#to_tandem", + "parentId": 2963 + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "A request loader that reads URLs from sitemap(s).\n\nThe loader is designed to handle sitemaps that follow the format described in the Sitemaps protocol\n(https://www.sitemaps.org/protocol.html). It supports both XML and plain text sitemap formats.\nNote that HTML pages containing links are not supported - those should be handled by regular crawlers\nand the `enqueue_links` functionality.\n\nThe loader fetches and parses sitemaps in the background, allowing crawling to start\nbefore all URLs are loaded. It supports filtering URLs using glob and regex patterns.\n\nThe loader supports state persistence, allowing it to resume from where it left off\nafter interruption when a `persist_state_key` is provided during initialization." + } + ] + }, + "decorations": [ + { + "args": "('Request loaders')", + "name": "docs_group" + } + ], + "flags": {}, + "groups": [ + { + "children": [ + 88933, + 88935, + 88922, + 88955, + 88957, + 88948, + 88942, + 88940, + 88944, + 88946, + 88950, + 88953, + 90100 + ], + "title": "Methods" + } + ], + "id": 88921, + "module": "request_loaders._sitemap_request_loader", + "name": "SitemapRequestLoader", + "parsedDocstring": { + "text": "A request loader that reads URLs from sitemap(s).\n\nThe loader is designed to handle sitemaps that follow the format described in the Sitemaps protocol\n(https://www.sitemaps.org/protocol.html). It supports both XML and plain text sitemap formats.\nNote that HTML pages containing links are not supported - those should be handled by regular crawlers\nand the `enqueue_links` functionality.\n\nThe loader fetches and parses sitemaps in the background, allowing crawling to start\nbefore all URLs are loaded. It supports filtering URLs using glob and regex patterns.\n\nThe loader supports state persistence, allowing it to resume from where it left off\nafter interruption when a `persist_state_key` is provided during initialization." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_sitemap_request_loader.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 94, + "url": "https://github.com/apify/crawlee-python/blob/e60a2359e61ff20b194a52ab4e248cc70cc460a9//src/crawlee/request_loaders/_sitemap_request_loader.py#L94" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "extendedTypes": [ + { + "name": "RequestLoader", + "target": "2934", + "type": "reference", + "ref": { + "id": 88892, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b", + "line": 17 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/RequestLoader" + } + } + ], + "permalink": "/python/api/class/SitemapRequestLoader", + "categories": [] + } + ], + "flags": {}, + "groups": [ + { + "children": [ + 9, + 602, + 225, + 455, 211, 208, - 332 + 826 + ], + "title": "Classes" + }, + { + "children": [ + 438, + 383, + 316, + 275, + 295, + 278, + 302, + 379, + 447, + 444, + 441, + 435, + 417, + 432, + 420, + 450, + 270 + ], + "title": "Data structures" + }, + { + "children": [ + 367 + ], + "title": "Interfaces" + }, + { + "title": "Storages", + "children": [ + 3169, + 5643, + 8932, + 11356 + ] + }, + { + "title": "Storage data", + "children": [ + 14509, + 15845, + 17165, + 21381, + 25612, + 29815, + 31160, + 31574, + 32951, + 37162 + ] + }, + { + "title": "Event managers", + "children": [ + 39329, + 42822 + ] + }, + { + "title": "Events", + "children": [ + 46870 + ] + }, + { + "title": "Event data", + "children": [ + 49021, + 51163, + 53300, + 56614, + 58748 + ] + }, + { + "title": "Storage clients", + "children": [ + 61243, + 63520, + 67876, + 72346 + ] + }, + { + "title": "Request loaders", + "children": [ + 77590, + 80658, + 84609, + 88921 ] + } + ], + "id": 0, + "kind": 1, + "kindString": "Project", + "name": "apify-client", + "sources": [ + { + "character": 0, + "fileName": "src/index.ts", + "line": 1, + "gitRevision": "40a160509f9773d84e3578ab60c9f7d4df48552b" + } + ], + "symbolIdMap": { + "1": { + "qualifiedName": "ENCRYPTION_KEY_LENGTH", + "sourceFileName": "/src/apify/_crypto.py" + }, + "2": { + "qualifiedName": "ENCRYPTION_IV_LENGTH", + "sourceFileName": "/src/apify/_crypto.py" + }, + "3": { + "qualifiedName": "ENCRYPTION_AUTH_TAG_LENGTH", + "sourceFileName": "/src/apify/_crypto.py" + }, + "4": { + "qualifiedName": "CHARSET", + "sourceFileName": "/src/apify/_crypto.py" + }, + "5": { + "qualifiedName": "encode_base62", + "sourceFileName": "/src/apify/_crypto.py" + }, + "8": { + "qualifiedName": "MainReturnType", + "sourceFileName": "/src/apify/_actor.py" + }, + "9": { + "qualifiedName": "Actor", + "sourceFileName": "/src/apify/_actor.py" + }, + "10": { + "qualifiedName": "__init__", + "sourceFileName": "/src/apify/_actor.py" + }, + "15": { + "qualifiedName": "__repr__", + "sourceFileName": "/src/apify/_actor.py" + }, + "17": { + "qualifiedName": "__call__", + "sourceFileName": "/src/apify/_actor.py" + }, + "22": { + "qualifiedName": "apify_client", + "sourceFileName": "/src/apify/_actor.py" + }, + "23": { + "qualifiedName": "configuration", + "sourceFileName": "/src/apify/_actor.py" + }, + "24": { + "qualifiedName": "config", + "sourceFileName": "/src/apify/_actor.py" + }, + "25": { + "qualifiedName": "event_manager", + "sourceFileName": "/src/apify/_actor.py" + }, + "26": { + "qualifiedName": "log", + "sourceFileName": "/src/apify/_actor.py" + }, + "27": { + "qualifiedName": "init", + "sourceFileName": "/src/apify/_actor.py" + }, + "29": { + "qualifiedName": "exit", + "sourceFileName": "/src/apify/_actor.py" + }, + "35": { + "qualifiedName": "fail", + "sourceFileName": "/src/apify/_actor.py" + }, + "40": { + "qualifiedName": "new_client", + "sourceFileName": "/src/apify/_actor.py" + }, + "47": { + "qualifiedName": "open_dataset", + "sourceFileName": "/src/apify/_actor.py" + }, + "52": { + "qualifiedName": "open_key_value_store", + "sourceFileName": "/src/apify/_actor.py" + }, + "57": { + "qualifiedName": "open_request_queue", + "sourceFileName": "/src/apify/_actor.py" + }, + "62": { + "qualifiedName": "push_data", + "sourceFileName": "/src/apify/_actor.py" + }, + "66": { + "qualifiedName": "get_input", + "sourceFileName": "/src/apify/_actor.py" + }, + "68": { + "qualifiedName": "get_value", + "sourceFileName": "/src/apify/_actor.py" + }, + "72": { + "qualifiedName": "set_value", + "sourceFileName": "/src/apify/_actor.py" + }, + "77": { + "qualifiedName": "get_charging_manager", + "sourceFileName": "/src/apify/_actor.py" + }, + "79": { + "qualifiedName": "charge", + "sourceFileName": "/src/apify/_actor.py" + }, + "83": { + "qualifiedName": "on", + "sourceFileName": "/src/apify/_actor.py" + }, + "87": { + "qualifiedName": "off", + "sourceFileName": "/src/apify/_actor.py" + }, + "91": { + "qualifiedName": "is_at_home", + "sourceFileName": "/src/apify/_actor.py" + }, + "93": { + "qualifiedName": "get_env", + "sourceFileName": "/src/apify/_actor.py" + }, + "95": { + "qualifiedName": "start", + "sourceFileName": "/src/apify/_actor.py" + }, + "106": { + "qualifiedName": "abort", + "sourceFileName": "/src/apify/_actor.py" + }, + "112": { + "qualifiedName": "call", + "sourceFileName": "/src/apify/_actor.py" + }, + "124": { + "qualifiedName": "call_task", + "sourceFileName": "/src/apify/_actor.py" + }, + "134": { + "qualifiedName": "metamorph", + "sourceFileName": "/src/apify/_actor.py" + }, + "141": { + "qualifiedName": "reboot", + "sourceFileName": "/src/apify/_actor.py" + }, + "145": { + "qualifiedName": "add_webhook", + "sourceFileName": "/src/apify/_actor.py" + }, + "151": { + "qualifiedName": "set_status_message", + "sourceFileName": "/src/apify/_actor.py" + }, + "155": { + "qualifiedName": "create_proxy_configuration", + "sourceFileName": "/src/apify/_actor.py" + }, + "204": { + "qualifiedName": "Actor", + "sourceFileName": "/src/apify/_actor.py" + }, + "205": { + "qualifiedName": "APIFY_PROXY_VALUE_REGEX", + "sourceFileName": "/src/apify/_proxy_configuration.py" + }, + "206": { + "qualifiedName": "COUNTRY_CODE_REGEX", + "sourceFileName": "/src/apify/_proxy_configuration.py" + }, + "207": { + "qualifiedName": "SESSION_ID_MAX_LENGTH", + "sourceFileName": "/src/apify/_proxy_configuration.py" + }, + "208": { + "qualifiedName": "ProxyInfo", + "sourceFileName": "/src/apify/_proxy_configuration.py" + }, + "209": { + "qualifiedName": "groups", + "sourceFileName": "/src/apify/_proxy_configuration.py" + }, + "210": { + "qualifiedName": "country_code", + "sourceFileName": "/src/apify/_proxy_configuration.py" + }, + "211": { + "qualifiedName": "ProxyConfiguration", + "sourceFileName": "/src/apify/_proxy_configuration.py" + }, + "212": { + "qualifiedName": "initialize", + "sourceFileName": "/src/apify/_proxy_configuration.py" + }, + "214": { + "qualifiedName": "new_proxy_info", + "sourceFileName": "/src/apify/_proxy_configuration.py" + }, + "219": { + "qualifiedName": "EVENT_LISTENERS_TIMEOUT", + "sourceFileName": "/src/apify/_consts.py" + }, + "220": { + "qualifiedName": "BASE64_REGEXP", + "sourceFileName": "/src/apify/_consts.py" + }, + "221": { + "qualifiedName": "ENCRYPTED_STRING_VALUE_PREFIX", + "sourceFileName": "/src/apify/_consts.py" + }, + "222": { + "qualifiedName": "ENCRYPTED_JSON_VALUE_PREFIX", + "sourceFileName": "/src/apify/_consts.py" + }, + "223": { + "qualifiedName": "ENCRYPTED_INPUT_VALUE_REGEXP", + "sourceFileName": "/src/apify/_consts.py" + }, + "224": { + "qualifiedName": "logger", + "sourceFileName": "/src/apify/_configuration.py" + }, + "225": { + "qualifiedName": "Configuration", + "sourceFileName": "/src/apify/_configuration.py" + }, + "226": { + "qualifiedName": "actor_id", + "sourceFileName": "/src/apify/_configuration.py" + }, + "227": { + "qualifiedName": "actor_full_name", + "sourceFileName": "/src/apify/_configuration.py" + }, + "228": { + "qualifiedName": "actor_run_id", + "sourceFileName": "/src/apify/_configuration.py" + }, + "229": { + "qualifiedName": "actor_build_id", + "sourceFileName": "/src/apify/_configuration.py" + }, + "230": { + "qualifiedName": "actor_build_number", + "sourceFileName": "/src/apify/_configuration.py" + }, + "231": { + "qualifiedName": "actor_build_tags", + "sourceFileName": "/src/apify/_configuration.py" + }, + "232": { + "qualifiedName": "actor_task_id", + "sourceFileName": "/src/apify/_configuration.py" + }, + "233": { + "qualifiedName": "actor_events_ws_url", + "sourceFileName": "/src/apify/_configuration.py" + }, + "234": { + "qualifiedName": "api_base_url", + "sourceFileName": "/src/apify/_configuration.py" + }, + "235": { + "qualifiedName": "api_public_base_url", + "sourceFileName": "/src/apify/_configuration.py" + }, + "236": { + "qualifiedName": "dedicated_cpus", + "sourceFileName": "/src/apify/_configuration.py" + }, + "237": { + "qualifiedName": "disable_outdated_warning", + "sourceFileName": "/src/apify/_configuration.py" + }, + "238": { + "qualifiedName": "fact", + "sourceFileName": "/src/apify/_configuration.py" + }, + "239": { + "qualifiedName": "input_key", + "sourceFileName": "/src/apify/_configuration.py" + }, + "240": { + "qualifiedName": "input_secrets_private_key_file", + "sourceFileName": "/src/apify/_configuration.py" + }, + "241": { + "qualifiedName": "input_secrets_private_key_passphrase", + "sourceFileName": "/src/apify/_configuration.py" + }, + "242": { + "qualifiedName": "is_at_home", + "sourceFileName": "/src/apify/_configuration.py" + }, + "243": { + "qualifiedName": "latest_sdk_version", + "sourceFileName": "/src/apify/_configuration.py" + }, + "244": { + "qualifiedName": "log_format", + "sourceFileName": "/src/apify/_configuration.py" + }, + "245": { + "qualifiedName": "max_paid_dataset_items", + "sourceFileName": "/src/apify/_configuration.py" + }, + "246": { + "qualifiedName": "max_total_charge_usd", + "sourceFileName": "/src/apify/_configuration.py" + }, + "247": { + "qualifiedName": "test_pay_per_event", + "sourceFileName": "/src/apify/_configuration.py" + }, + "248": { + "qualifiedName": "meta_origin", + "sourceFileName": "/src/apify/_configuration.py" + }, + "249": { + "qualifiedName": "metamorph_after_sleep", + "sourceFileName": "/src/apify/_configuration.py" + }, + "250": { + "qualifiedName": "proxy_hostname", + "sourceFileName": "/src/apify/_configuration.py" + }, + "251": { + "qualifiedName": "proxy_password", + "sourceFileName": "/src/apify/_configuration.py" + }, + "252": { + "qualifiedName": "proxy_port", + "sourceFileName": "/src/apify/_configuration.py" + }, + "253": { + "qualifiedName": "proxy_status_url", + "sourceFileName": "/src/apify/_configuration.py" + }, + "254": { + "qualifiedName": "started_at", + "sourceFileName": "/src/apify/_configuration.py" + }, + "255": { + "qualifiedName": "timeout_at", + "sourceFileName": "/src/apify/_configuration.py" + }, + "256": { + "qualifiedName": "standby_port", + "sourceFileName": "/src/apify/_configuration.py" + }, + "257": { + "qualifiedName": "standby_url", + "sourceFileName": "/src/apify/_configuration.py" + }, + "258": { + "qualifiedName": "token", + "sourceFileName": "/src/apify/_configuration.py" + }, + "259": { + "qualifiedName": "user_id", + "sourceFileName": "/src/apify/_configuration.py" + }, + "260": { + "qualifiedName": "user_is_paying", + "sourceFileName": "/src/apify/_configuration.py" + }, + "261": { + "qualifiedName": "web_server_port", + "sourceFileName": "/src/apify/_configuration.py" + }, + "262": { + "qualifiedName": "web_server_url", + "sourceFileName": "/src/apify/_configuration.py" + }, + "263": { + "qualifiedName": "workflow_key", + "sourceFileName": "/src/apify/_configuration.py" + }, + "264": { + "qualifiedName": "disable_browser_sandbox_on_platform", + "sourceFileName": "/src/apify/_configuration.py" + }, + "266": { + "qualifiedName": "get_global_configuration", + "sourceFileName": "/src/apify/_configuration.py" + }, + "268": { + "qualifiedName": "logger_name", + "sourceFileName": "/src/apify/log.py" + }, + "269": { + "qualifiedName": "logger", + "sourceFileName": "/src/apify/log.py" + }, + "270": { + "qualifiedName": "Webhook", + "sourceFileName": "/src/apify/_models.py" + }, + "271": { + "qualifiedName": "__model_config__", + "sourceFileName": "/src/apify/_models.py" + }, + "272": { + "qualifiedName": "event_types", + "sourceFileName": "/src/apify/_models.py" + }, + "273": { + "qualifiedName": "request_url", + "sourceFileName": "/src/apify/_models.py" + }, + "274": { + "qualifiedName": "payload_template", + "sourceFileName": "/src/apify/_models.py" + }, + "275": { + "qualifiedName": "ActorRunMeta", + "sourceFileName": "/src/apify/_models.py" + }, + "276": { + "qualifiedName": "__model_config__", + "sourceFileName": "/src/apify/_models.py" + }, + "277": { + "qualifiedName": "origin", + "sourceFileName": "/src/apify/_models.py" + }, + "278": { + "qualifiedName": "ActorRunStats", + "sourceFileName": "/src/apify/_models.py" + }, + "279": { + "qualifiedName": "__model_config__", + "sourceFileName": "/src/apify/_models.py" + }, + "280": { + "qualifiedName": "input_body_len", + "sourceFileName": "/src/apify/_models.py" + }, + "281": { + "qualifiedName": "restart_count", + "sourceFileName": "/src/apify/_models.py" + }, + "282": { + "qualifiedName": "resurrect_count", + "sourceFileName": "/src/apify/_models.py" + }, + "283": { + "qualifiedName": "mem_avg_bytes", + "sourceFileName": "/src/apify/_models.py" + }, + "284": { + "qualifiedName": "mem_max_bytes", + "sourceFileName": "/src/apify/_models.py" + }, + "285": { + "qualifiedName": "mem_current_bytes", + "sourceFileName": "/src/apify/_models.py" + }, + "286": { + "qualifiedName": "cpu_avg_usage", + "sourceFileName": "/src/apify/_models.py" + }, + "287": { + "qualifiedName": "cpu_max_usage", + "sourceFileName": "/src/apify/_models.py" + }, + "288": { + "qualifiedName": "cpu_current_usage", + "sourceFileName": "/src/apify/_models.py" + }, + "289": { + "qualifiedName": "net_rx_bytes", + "sourceFileName": "/src/apify/_models.py" + }, + "290": { + "qualifiedName": "net_tx_bytes", + "sourceFileName": "/src/apify/_models.py" + }, + "291": { + "qualifiedName": "duration", + "sourceFileName": "/src/apify/_models.py" + }, + "292": { + "qualifiedName": "run_time", + "sourceFileName": "/src/apify/_models.py" + }, + "293": { + "qualifiedName": "metamorph", + "sourceFileName": "/src/apify/_models.py" + }, + "294": { + "qualifiedName": "compute_units", + "sourceFileName": "/src/apify/_models.py" + }, + "295": { + "qualifiedName": "ActorRunOptions", + "sourceFileName": "/src/apify/_models.py" + }, + "296": { + "qualifiedName": "__model_config__", + "sourceFileName": "/src/apify/_models.py" + }, + "297": { + "qualifiedName": "build", + "sourceFileName": "/src/apify/_models.py" + }, + "298": { + "qualifiedName": "timeout", + "sourceFileName": "/src/apify/_models.py" + }, + "299": { + "qualifiedName": "memory_mbytes", + "sourceFileName": "/src/apify/_models.py" + }, + "300": { + "qualifiedName": "disk_mbytes", + "sourceFileName": "/src/apify/_models.py" + }, + "301": { + "qualifiedName": "max_total_charge_usd", + "sourceFileName": "/src/apify/_models.py" + }, + "302": { + "qualifiedName": "ActorRunUsage", + "sourceFileName": "/src/apify/_models.py" + }, + "303": { + "qualifiedName": "__model_config__", + "sourceFileName": "/src/apify/_models.py" + }, + "304": { + "qualifiedName": "actor_compute_units", + "sourceFileName": "/src/apify/_models.py" + }, + "305": { + "qualifiedName": "dataset_reads", + "sourceFileName": "/src/apify/_models.py" + }, + "306": { + "qualifiedName": "dataset_writes", + "sourceFileName": "/src/apify/_models.py" + }, + "307": { + "qualifiedName": "key_value_store_reads", + "sourceFileName": "/src/apify/_models.py" + }, + "308": { + "qualifiedName": "key_value_store_writes", + "sourceFileName": "/src/apify/_models.py" + }, + "309": { + "qualifiedName": "key_value_store_lists", + "sourceFileName": "/src/apify/_models.py" + }, + "310": { + "qualifiedName": "request_queue_reads", + "sourceFileName": "/src/apify/_models.py" + }, + "311": { + "qualifiedName": "request_queue_writes", + "sourceFileName": "/src/apify/_models.py" + }, + "312": { + "qualifiedName": "data_transfer_internal_gbytes", + "sourceFileName": "/src/apify/_models.py" + }, + "313": { + "qualifiedName": "data_transfer_external_gbytes", + "sourceFileName": "/src/apify/_models.py" + }, + "314": { + "qualifiedName": "proxy_residential_transfer_gbytes", + "sourceFileName": "/src/apify/_models.py" + }, + "315": { + "qualifiedName": "proxy_serps", + "sourceFileName": "/src/apify/_models.py" + }, + "316": { + "qualifiedName": "ActorRun", + "sourceFileName": "/src/apify/_models.py" + }, + "317": { + "qualifiedName": "__model_config__", + "sourceFileName": "/src/apify/_models.py" + }, + "318": { + "qualifiedName": "id", + "sourceFileName": "/src/apify/_models.py" + }, + "319": { + "qualifiedName": "act_id", + "sourceFileName": "/src/apify/_models.py" + }, + "320": { + "qualifiedName": "user_id", + "sourceFileName": "/src/apify/_models.py" }, - { - "title": "Properties", - "children": [ - 158 - ] + "321": { + "qualifiedName": "actor_task_id", + "sourceFileName": "/src/apify/_models.py" }, - { - "title": "Interfaces", - "children": [ - 159 - ] + "322": { + "qualifiedName": "started_at", + "sourceFileName": "/src/apify/_models.py" }, - { - "title": "Data structures", - "children": [ - 198, - 175, - 193, - 189, - 191, - 190, - 192, - 171, - 201, - 200, - 199, - 197, - 194, - 196, - 195, - 202, - 188 - ] + "323": { + "qualifiedName": "finished_at", + "sourceFileName": "/src/apify/_models.py" }, - { - "title": "Methods", - "children": [ - 328, - 219, - 222, - 185, - 270, - 323, - 276, - 246, - 273, - 235, - 301, - 266, - 305 - ] + "324": { + "qualifiedName": "status", + "sourceFileName": "/src/apify/_models.py" }, - { - "title": "Other", - "children": [ - 296, - 248, - 280, - 309, - 238, - 226, - 227, - 228, - 232, - 233, - 234 - ] - } - ], - "sources": [ - { - "fileName": "src/index.ts", - "line": 1, - "character": 0, - "url": "http://example.com/blob/123456/src/dummy.py" + "325": { + "qualifiedName": "status_message", + "sourceFileName": "/src/apify/_models.py" + }, + "326": { + "qualifiedName": "is_status_message_terminal", + "sourceFileName": "/src/apify/_models.py" + }, + "327": { + "qualifiedName": "meta", + "sourceFileName": "/src/apify/_models.py" + }, + "328": { + "qualifiedName": "stats", + "sourceFileName": "/src/apify/_models.py" + }, + "329": { + "qualifiedName": "options", + "sourceFileName": "/src/apify/_models.py" + }, + "330": { + "qualifiedName": "build_id", + "sourceFileName": "/src/apify/_models.py" + }, + "331": { + "qualifiedName": "exit_code", + "sourceFileName": "/src/apify/_models.py" + }, + "332": { + "qualifiedName": "default_key_value_store_id", + "sourceFileName": "/src/apify/_models.py" + }, + "333": { + "qualifiedName": "default_dataset_id", + "sourceFileName": "/src/apify/_models.py" + }, + "334": { + "qualifiedName": "default_request_queue_id", + "sourceFileName": "/src/apify/_models.py" + }, + "335": { + "qualifiedName": "build_number", + "sourceFileName": "/src/apify/_models.py" + }, + "336": { + "qualifiedName": "container_url", + "sourceFileName": "/src/apify/_models.py" + }, + "337": { + "qualifiedName": "is_container_server_ready", + "sourceFileName": "/src/apify/_models.py" + }, + "338": { + "qualifiedName": "git_branch_name", + "sourceFileName": "/src/apify/_models.py" + }, + "339": { + "qualifiedName": "usage", + "sourceFileName": "/src/apify/_models.py" + }, + "340": { + "qualifiedName": "usage_total_usd", + "sourceFileName": "/src/apify/_models.py" + }, + "341": { + "qualifiedName": "usage_usd", + "sourceFileName": "/src/apify/_models.py" + }, + "342": { + "qualifiedName": "pricing_info", + "sourceFileName": "/src/apify/_models.py" + }, + "343": { + "qualifiedName": "charged_event_counts", + "sourceFileName": "/src/apify/_models.py" + }, + "344": { + "qualifiedName": "FreeActorPricingInfo", + "sourceFileName": "/src/apify/_models.py" + }, + "345": { + "qualifiedName": "pricing_model", + "sourceFileName": "/src/apify/_models.py" + }, + "346": { + "qualifiedName": "FlatPricePerMonthActorPricingInfo", + "sourceFileName": "/src/apify/_models.py" + }, + "347": { + "qualifiedName": "pricing_model", + "sourceFileName": "/src/apify/_models.py" + }, + "348": { + "qualifiedName": "trial_minutes", + "sourceFileName": "/src/apify/_models.py" + }, + "349": { + "qualifiedName": "price_per_unit_usd", + "sourceFileName": "/src/apify/_models.py" + }, + "350": { + "qualifiedName": "PricePerDatasetItemActorPricingInfo", + "sourceFileName": "/src/apify/_models.py" + }, + "351": { + "qualifiedName": "pricing_model", + "sourceFileName": "/src/apify/_models.py" + }, + "352": { + "qualifiedName": "unit_name", + "sourceFileName": "/src/apify/_models.py" + }, + "353": { + "qualifiedName": "price_per_unit_usd", + "sourceFileName": "/src/apify/_models.py" + }, + "354": { + "qualifiedName": "ActorChargeEvent", + "sourceFileName": "/src/apify/_models.py" + }, + "355": { + "qualifiedName": "event_price_usd", + "sourceFileName": "/src/apify/_models.py" + }, + "356": { + "qualifiedName": "event_title", + "sourceFileName": "/src/apify/_models.py" + }, + "357": { + "qualifiedName": "event_description", + "sourceFileName": "/src/apify/_models.py" + }, + "358": { + "qualifiedName": "PricingPerEvent", + "sourceFileName": "/src/apify/_models.py" + }, + "359": { + "qualifiedName": "actor_charge_events", + "sourceFileName": "/src/apify/_models.py" + }, + "360": { + "qualifiedName": "PayPerEventActorPricingInfo", + "sourceFileName": "/src/apify/_models.py" + }, + "361": { + "qualifiedName": "pricing_model", + "sourceFileName": "/src/apify/_models.py" + }, + "362": { + "qualifiedName": "pricing_per_event", + "sourceFileName": "/src/apify/_models.py" + }, + "363": { + "qualifiedName": "minimal_max_total_charge_usd", + "sourceFileName": "/src/apify/_models.py" + }, + "364": { + "qualifiedName": "PricingModel", + "sourceFileName": "/src/apify/_models.py" + }, + "365": { + "qualifiedName": "__version__", + "sourceFileName": "/src/apify/__init__.py" + }, + "366": { + "qualifiedName": "run_validator", + "sourceFileName": "/src/apify/_charging.py" + }, + "367": { + "qualifiedName": "ChargingManager", + "sourceFileName": "/src/apify/_charging.py" + }, + "368": { + "qualifiedName": "charge", + "sourceFileName": "/src/apify/_charging.py" + }, + "372": { + "qualifiedName": "calculate_total_charged_amount", + "sourceFileName": "/src/apify/_charging.py" + }, + "374": { + "qualifiedName": "calculate_max_event_charge_count_within_limit", + "sourceFileName": "/src/apify/_charging.py" + }, + "377": { + "qualifiedName": "get_pricing_info", + "sourceFileName": "/src/apify/_charging.py" + }, + "379": { + "qualifiedName": "ChargeResult", + "sourceFileName": "/src/apify/_charging.py" + }, + "380": { + "qualifiedName": "event_charge_limit_reached", + "sourceFileName": "/src/apify/_charging.py" + }, + "381": { + "qualifiedName": "charged_count", + "sourceFileName": "/src/apify/_charging.py" + }, + "382": { + "qualifiedName": "chargeable_within_limit", + "sourceFileName": "/src/apify/_charging.py" + }, + "383": { + "qualifiedName": "ActorPricingInfo", + "sourceFileName": "/src/apify/_charging.py" + }, + "384": { + "qualifiedName": "pricing_model", + "sourceFileName": "/src/apify/_charging.py" + }, + "385": { + "qualifiedName": "max_total_charge_usd", + "sourceFileName": "/src/apify/_charging.py" + }, + "386": { + "qualifiedName": "is_pay_per_event", + "sourceFileName": "/src/apify/_charging.py" + }, + "387": { + "qualifiedName": "per_event_prices", + "sourceFileName": "/src/apify/_charging.py" + }, + "388": { + "qualifiedName": "LOCAL_CHARGING_LOG_DATASET_NAME", + "sourceFileName": "/src/apify/_charging.py" + }, + "389": { + "qualifiedName": "__init__", + "sourceFileName": "/src/apify/_charging.py" + }, + "393": { + "qualifiedName": "__aenter__", + "sourceFileName": "/src/apify/_charging.py" + }, + "395": { + "qualifiedName": "__aexit__", + "sourceFileName": "/src/apify/_charging.py" + }, + "400": { + "qualifiedName": "charge", + "sourceFileName": "/src/apify/_charging.py" + }, + "404": { + "qualifiedName": "calculate_total_charged_amount", + "sourceFileName": "/src/apify/_charging.py" + }, + "406": { + "qualifiedName": "calculate_max_event_charge_count_within_limit", + "sourceFileName": "/src/apify/_charging.py" + }, + "409": { + "qualifiedName": "get_pricing_info", + "sourceFileName": "/src/apify/_charging.py" + }, + "411": { + "qualifiedName": "ChargingStateItem", + "sourceFileName": "/src/apify/_charging.py" + }, + "412": { + "qualifiedName": "charge_count", + "sourceFileName": "/src/apify/_charging.py" + }, + "413": { + "qualifiedName": "total_charged_amount", + "sourceFileName": "/src/apify/_charging.py" + }, + "414": { + "qualifiedName": "PricingInfoItem", + "sourceFileName": "/src/apify/_charging.py" + }, + "415": { + "qualifiedName": "price", + "sourceFileName": "/src/apify/_charging.py" + }, + "416": { + "qualifiedName": "title", + "sourceFileName": "/src/apify/_charging.py" + }, + "417": { + "qualifiedName": "PersistStateEvent", + "sourceFileName": "/src/apify/_platform_event_manager.py" + }, + "418": { + "qualifiedName": "name", + "sourceFileName": "/src/apify/_platform_event_manager.py" + }, + "419": { + "qualifiedName": "data", + "sourceFileName": "/src/apify/_platform_event_manager.py" + }, + "420": { + "qualifiedName": "SystemInfoEventData", + "sourceFileName": "/src/apify/_platform_event_manager.py" + }, + "421": { + "qualifiedName": "mem_avg_bytes", + "sourceFileName": "/src/apify/_platform_event_manager.py" + }, + "422": { + "qualifiedName": "mem_current_bytes", + "sourceFileName": "/src/apify/_platform_event_manager.py" + }, + "423": { + "qualifiedName": "mem_max_bytes", + "sourceFileName": "/src/apify/_platform_event_manager.py" + }, + "424": { + "qualifiedName": "cpu_avg_usage", + "sourceFileName": "/src/apify/_platform_event_manager.py" + }, + "425": { + "qualifiedName": "cpu_max_usage", + "sourceFileName": "/src/apify/_platform_event_manager.py" + }, + "426": { + "qualifiedName": "cpu_current_usage", + "sourceFileName": "/src/apify/_platform_event_manager.py" + }, + "427": { + "qualifiedName": "is_cpu_overloaded", + "sourceFileName": "/src/apify/_platform_event_manager.py" + }, + "428": { + "qualifiedName": "created_at", + "sourceFileName": "/src/apify/_platform_event_manager.py" + }, + "429": { + "qualifiedName": "to_crawlee_format", + "sourceFileName": "/src/apify/_platform_event_manager.py" + }, + "432": { + "qualifiedName": "SystemInfoEvent", + "sourceFileName": "/src/apify/_platform_event_manager.py" + }, + "433": { + "qualifiedName": "name", + "sourceFileName": "/src/apify/_platform_event_manager.py" + }, + "434": { + "qualifiedName": "data", + "sourceFileName": "/src/apify/_platform_event_manager.py" + }, + "435": { + "qualifiedName": "MigratingEvent", + "sourceFileName": "/src/apify/_platform_event_manager.py" + }, + "436": { + "qualifiedName": "name", + "sourceFileName": "/src/apify/_platform_event_manager.py" + }, + "437": { + "qualifiedName": "data", + "sourceFileName": "/src/apify/_platform_event_manager.py" + }, + "438": { + "qualifiedName": "AbortingEvent", + "sourceFileName": "/src/apify/_platform_event_manager.py" + }, + "439": { + "qualifiedName": "name", + "sourceFileName": "/src/apify/_platform_event_manager.py" + }, + "440": { + "qualifiedName": "data", + "sourceFileName": "/src/apify/_platform_event_manager.py" + }, + "441": { + "qualifiedName": "ExitEvent", + "sourceFileName": "/src/apify/_platform_event_manager.py" + }, + "442": { + "qualifiedName": "name", + "sourceFileName": "/src/apify/_platform_event_manager.py" + }, + "443": { + "qualifiedName": "data", + "sourceFileName": "/src/apify/_platform_event_manager.py" + }, + "444": { + "qualifiedName": "EventWithoutData", + "sourceFileName": "/src/apify/_platform_event_manager.py" + }, + "445": { + "qualifiedName": "name", + "sourceFileName": "/src/apify/_platform_event_manager.py" + }, + "446": { + "qualifiedName": "data", + "sourceFileName": "/src/apify/_platform_event_manager.py" + }, + "447": { + "qualifiedName": "DeprecatedEvent", + "sourceFileName": "/src/apify/_platform_event_manager.py" + }, + "448": { + "qualifiedName": "name", + "sourceFileName": "/src/apify/_platform_event_manager.py" + }, + "449": { + "qualifiedName": "data", + "sourceFileName": "/src/apify/_platform_event_manager.py" + }, + "450": { + "qualifiedName": "UnknownEvent", + "sourceFileName": "/src/apify/_platform_event_manager.py" + }, + "451": { + "qualifiedName": "name", + "sourceFileName": "/src/apify/_platform_event_manager.py" + }, + "452": { + "qualifiedName": "data", + "sourceFileName": "/src/apify/_platform_event_manager.py" + }, + "453": { + "qualifiedName": "EventMessage", + "sourceFileName": "/src/apify/_platform_event_manager.py" + }, + "454": { + "qualifiedName": "event_data_adapter", + "sourceFileName": "/src/apify/_platform_event_manager.py" + }, + "455": { + "qualifiedName": "PlatformEventManager", + "sourceFileName": "/src/apify/_platform_event_manager.py" + }, + "456": { + "qualifiedName": "__init__", + "sourceFileName": "/src/apify/_platform_event_manager.py" + }, + "460": { + "qualifiedName": "__aenter__", + "sourceFileName": "/src/apify/_platform_event_manager.py" + }, + "462": { + "qualifiedName": "__aexit__", + "sourceFileName": "/src/apify/_platform_event_manager.py" + }, + "467": { + "qualifiedName": "get_system_info", + "sourceFileName": "/src/apify/_utils.py" + }, + "469": { + "qualifiedName": "is_running_in_ipython", + "sourceFileName": "/src/apify/_utils.py" + }, + "471": { + "qualifiedName": "GroupName", + "sourceFileName": "/src/apify/_utils.py" + }, + "472": { + "qualifiedName": "docs_group", + "sourceFileName": "/src/apify/_utils.py" + }, + "475": { + "qualifiedName": "docs_name", + "sourceFileName": "/src/apify/_utils.py" + }, + "478": { + "qualifiedName": "get_basic_auth_header", + "sourceFileName": "/src/apify/scrapy/utils.py" + }, + "483": { + "qualifiedName": "apply_apify_settings", + "sourceFileName": "/src/apify/scrapy/utils.py" + }, + "487": { + "qualifiedName": "logger", + "sourceFileName": "/src/apify/scrapy/_async_thread.py" + }, + "488": { + "qualifiedName": "AsyncThread", + "sourceFileName": "/src/apify/scrapy/_async_thread.py" + }, + "489": { + "qualifiedName": "__init__", + "sourceFileName": "/src/apify/scrapy/_async_thread.py" + }, + "491": { + "qualifiedName": "run_coro", + "sourceFileName": "/src/apify/scrapy/_async_thread.py" + }, + "495": { + "qualifiedName": "close", + "sourceFileName": "/src/apify/scrapy/_async_thread.py" + }, + "498": { + "qualifiedName": "logger", + "sourceFileName": "/src/apify/scrapy/scheduler.py" + }, + "499": { + "qualifiedName": "ApifyScheduler", + "sourceFileName": "/src/apify/scrapy/scheduler.py" + }, + "500": { + "qualifiedName": "__init__", + "sourceFileName": "/src/apify/scrapy/scheduler.py" + }, + "502": { + "qualifiedName": "open", + "sourceFileName": "/src/apify/scrapy/scheduler.py" + }, + "505": { + "qualifiedName": "close", + "sourceFileName": "/src/apify/scrapy/scheduler.py" + }, + "508": { + "qualifiedName": "has_pending_requests", + "sourceFileName": "/src/apify/scrapy/scheduler.py" + }, + "510": { + "qualifiedName": "enqueue_request", + "sourceFileName": "/src/apify/scrapy/scheduler.py" + }, + "513": { + "qualifiedName": "next_request", + "sourceFileName": "/src/apify/scrapy/scheduler.py" + }, + "515": { + "qualifiedName": "ApifyHttpProxyMiddleware", + "sourceFileName": "/src/apify/scrapy/middlewares/apify_proxy.py" + }, + "516": { + "qualifiedName": "__init__", + "sourceFileName": "/src/apify/scrapy/middlewares/apify_proxy.py" + }, + "519": { + "qualifiedName": "from_crawler", + "sourceFileName": "/src/apify/scrapy/middlewares/apify_proxy.py" + }, + "522": { + "qualifiedName": "process_request", + "sourceFileName": "/src/apify/scrapy/middlewares/apify_proxy.py" + }, + "526": { + "qualifiedName": "process_exception", + "sourceFileName": "/src/apify/scrapy/middlewares/apify_proxy.py" + }, + "531": { + "qualifiedName": "logger", + "sourceFileName": "/src/apify/scrapy/requests.py" + }, + "532": { + "qualifiedName": "to_apify_request", + "sourceFileName": "/src/apify/scrapy/requests.py" + }, + "536": { + "qualifiedName": "to_scrapy_request", + "sourceFileName": "/src/apify/scrapy/requests.py" + }, + "540": { + "qualifiedName": "run_scrapy_actor", + "sourceFileName": "/src/apify/scrapy/_actor_runner.py" + }, + "543": { + "qualifiedName": "logger", + "sourceFileName": "/src/apify/scrapy/extensions/_httpcache.py" + }, + "544": { + "qualifiedName": "ApifyCacheStorage", + "sourceFileName": "/src/apify/scrapy/extensions/_httpcache.py" + }, + "545": { + "qualifiedName": "__init__", + "sourceFileName": "/src/apify/scrapy/extensions/_httpcache.py" + }, + "548": { + "qualifiedName": "open_spider", + "sourceFileName": "/src/apify/scrapy/extensions/_httpcache.py" + }, + "551": { + "qualifiedName": "close_spider", + "sourceFileName": "/src/apify/scrapy/extensions/_httpcache.py" + }, + "555": { + "qualifiedName": "retrieve_response", + "sourceFileName": "/src/apify/scrapy/extensions/_httpcache.py" + }, + "560": { + "qualifiedName": "store_response", + "sourceFileName": "/src/apify/scrapy/extensions/_httpcache.py" + }, + "565": { + "qualifiedName": "to_gzip", + "sourceFileName": "/src/apify/scrapy/extensions/_httpcache.py" + }, + "569": { + "qualifiedName": "from_gzip", + "sourceFileName": "/src/apify/scrapy/extensions/_httpcache.py" + }, + "572": { + "qualifiedName": "read_gzip_time", + "sourceFileName": "/src/apify/scrapy/extensions/_httpcache.py" + }, + "575": { + "qualifiedName": "get_kvs_name", + "sourceFileName": "/src/apify/scrapy/extensions/_httpcache.py" + }, + "579": { + "qualifiedName": "initialize_logging", + "sourceFileName": "/src/apify/scrapy/_logging_config.py" + }, + "581": { + "qualifiedName": "logger", + "sourceFileName": "/src/apify/scrapy/pipelines/actor_dataset_push.py" + }, + "582": { + "qualifiedName": "ActorDatasetPushPipeline", + "sourceFileName": "/src/apify/scrapy/pipelines/actor_dataset_push.py" + }, + "583": { + "qualifiedName": "process_item", + "sourceFileName": "/src/apify/scrapy/pipelines/actor_dataset_push.py" + }, + "587": { + "qualifiedName": "KeyValueStoreCollectionClient", + "sourceFileName": "/src/apify/apify_storage_client/_key_value_store_collection_client.py" + }, + "588": { + "qualifiedName": "__init__", + "sourceFileName": "/src/apify/apify_storage_client/_key_value_store_collection_client.py" + }, + "591": { + "qualifiedName": "get_or_create", + "sourceFileName": "/src/apify/apify_storage_client/_key_value_store_collection_client.py" + }, + "596": { + "qualifiedName": "list", + "sourceFileName": "/src/apify/apify_storage_client/_key_value_store_collection_client.py" + }, + "602": { + "qualifiedName": "ApifyStorageClient", + "sourceFileName": "/src/apify/apify_storage_client/_apify_storage_client.py" + }, + "603": { + "qualifiedName": "__init__", + "sourceFileName": "/src/apify/apify_storage_client/_apify_storage_client.py" + }, + "606": { + "qualifiedName": "from_config", + "sourceFileName": "/src/apify/apify_storage_client/_apify_storage_client.py" + }, + "609": { + "qualifiedName": "dataset", + "sourceFileName": "/src/apify/apify_storage_client/_apify_storage_client.py" + }, + "612": { + "qualifiedName": "datasets", + "sourceFileName": "/src/apify/apify_storage_client/_apify_storage_client.py" + }, + "614": { + "qualifiedName": "key_value_store", + "sourceFileName": "/src/apify/apify_storage_client/_apify_storage_client.py" + }, + "617": { + "qualifiedName": "key_value_stores", + "sourceFileName": "/src/apify/apify_storage_client/_apify_storage_client.py" + }, + "619": { + "qualifiedName": "request_queue", + "sourceFileName": "/src/apify/apify_storage_client/_apify_storage_client.py" + }, + "622": { + "qualifiedName": "request_queues", + "sourceFileName": "/src/apify/apify_storage_client/_apify_storage_client.py" + }, + "624": { + "qualifiedName": "purge_on_start", + "sourceFileName": "/src/apify/apify_storage_client/_apify_storage_client.py" + }, + "626": { + "qualifiedName": "get_rate_limit_errors", + "sourceFileName": "/src/apify/apify_storage_client/_apify_storage_client.py" + }, + "628": { + "qualifiedName": "DatasetCollectionClient", + "sourceFileName": "/src/apify/apify_storage_client/_dataset_collection_client.py" + }, + "629": { + "qualifiedName": "__init__", + "sourceFileName": "/src/apify/apify_storage_client/_dataset_collection_client.py" + }, + "632": { + "qualifiedName": "get_or_create", + "sourceFileName": "/src/apify/apify_storage_client/_dataset_collection_client.py" + }, + "637": { + "qualifiedName": "list", + "sourceFileName": "/src/apify/apify_storage_client/_dataset_collection_client.py" + }, + "643": { + "qualifiedName": "RequestQueueClient", + "sourceFileName": "/src/apify/apify_storage_client/_request_queue_client.py" + }, + "644": { + "qualifiedName": "__init__", + "sourceFileName": "/src/apify/apify_storage_client/_request_queue_client.py" + }, + "647": { + "qualifiedName": "get", + "sourceFileName": "/src/apify/apify_storage_client/_request_queue_client.py" + }, + "649": { + "qualifiedName": "update", + "sourceFileName": "/src/apify/apify_storage_client/_request_queue_client.py" + }, + "652": { + "qualifiedName": "delete", + "sourceFileName": "/src/apify/apify_storage_client/_request_queue_client.py" + }, + "654": { + "qualifiedName": "list_head", + "sourceFileName": "/src/apify/apify_storage_client/_request_queue_client.py" + }, + "657": { + "qualifiedName": "list_and_lock_head", + "sourceFileName": "/src/apify/apify_storage_client/_request_queue_client.py" + }, + "661": { + "qualifiedName": "add_request", + "sourceFileName": "/src/apify/apify_storage_client/_request_queue_client.py" + }, + "665": { + "qualifiedName": "get_request", + "sourceFileName": "/src/apify/apify_storage_client/_request_queue_client.py" + }, + "668": { + "qualifiedName": "update_request", + "sourceFileName": "/src/apify/apify_storage_client/_request_queue_client.py" + }, + "672": { + "qualifiedName": "delete_request", + "sourceFileName": "/src/apify/apify_storage_client/_request_queue_client.py" + }, + "675": { + "qualifiedName": "prolong_request_lock", + "sourceFileName": "/src/apify/apify_storage_client/_request_queue_client.py" + }, + "680": { + "qualifiedName": "delete_request_lock", + "sourceFileName": "/src/apify/apify_storage_client/_request_queue_client.py" + }, + "684": { + "qualifiedName": "batch_add_requests", + "sourceFileName": "/src/apify/apify_storage_client/_request_queue_client.py" + }, + "688": { + "qualifiedName": "batch_delete_requests", + "sourceFileName": "/src/apify/apify_storage_client/_request_queue_client.py" + }, + "691": { + "qualifiedName": "RequestQueueCollectionClient", + "sourceFileName": "/src/apify/apify_storage_client/_request_queue_collection_client.py" + }, + "692": { + "qualifiedName": "__init__", + "sourceFileName": "/src/apify/apify_storage_client/_request_queue_collection_client.py" + }, + "695": { + "qualifiedName": "get_or_create", + "sourceFileName": "/src/apify/apify_storage_client/_request_queue_collection_client.py" + }, + "700": { + "qualifiedName": "list", + "sourceFileName": "/src/apify/apify_storage_client/_request_queue_collection_client.py" + }, + "706": { + "qualifiedName": "KeyValueStoreClient", + "sourceFileName": "/src/apify/apify_storage_client/_key_value_store_client.py" + }, + "707": { + "qualifiedName": "__init__", + "sourceFileName": "/src/apify/apify_storage_client/_key_value_store_client.py" + }, + "711": { + "qualifiedName": "get", + "sourceFileName": "/src/apify/apify_storage_client/_key_value_store_client.py" + }, + "713": { + "qualifiedName": "update", + "sourceFileName": "/src/apify/apify_storage_client/_key_value_store_client.py" + }, + "716": { + "qualifiedName": "delete", + "sourceFileName": "/src/apify/apify_storage_client/_key_value_store_client.py" + }, + "718": { + "qualifiedName": "list_keys", + "sourceFileName": "/src/apify/apify_storage_client/_key_value_store_client.py" + }, + "722": { + "qualifiedName": "get_record", + "sourceFileName": "/src/apify/apify_storage_client/_key_value_store_client.py" + }, + "725": { + "qualifiedName": "get_record_as_bytes", + "sourceFileName": "/src/apify/apify_storage_client/_key_value_store_client.py" + }, + "728": { + "qualifiedName": "stream_record", + "sourceFileName": "/src/apify/apify_storage_client/_key_value_store_client.py" + }, + "731": { + "qualifiedName": "set_record", + "sourceFileName": "/src/apify/apify_storage_client/_key_value_store_client.py" + }, + "736": { + "qualifiedName": "delete_record", + "sourceFileName": "/src/apify/apify_storage_client/_key_value_store_client.py" + }, + "739": { + "qualifiedName": "get_public_url", + "sourceFileName": "/src/apify/apify_storage_client/_key_value_store_client.py" + }, + "742": { + "qualifiedName": "DatasetClient", + "sourceFileName": "/src/apify/apify_storage_client/_dataset_client.py" + }, + "743": { + "qualifiedName": "__init__", + "sourceFileName": "/src/apify/apify_storage_client/_dataset_client.py" + }, + "746": { + "qualifiedName": "get", + "sourceFileName": "/src/apify/apify_storage_client/_dataset_client.py" + }, + "748": { + "qualifiedName": "update", + "sourceFileName": "/src/apify/apify_storage_client/_dataset_client.py" + }, + "751": { + "qualifiedName": "delete", + "sourceFileName": "/src/apify/apify_storage_client/_dataset_client.py" + }, + "753": { + "qualifiedName": "list_items", + "sourceFileName": "/src/apify/apify_storage_client/_dataset_client.py" + }, + "766": { + "qualifiedName": "iterate_items", + "sourceFileName": "/src/apify/apify_storage_client/_dataset_client.py" + }, + "777": { + "qualifiedName": "get_items_as_bytes", + "sourceFileName": "/src/apify/apify_storage_client/_dataset_client.py" + }, + "795": { + "qualifiedName": "stream_items", + "sourceFileName": "/src/apify/apify_storage_client/_dataset_client.py" + }, + "812": { + "qualifiedName": "push_items", + "sourceFileName": "/src/apify/apify_storage_client/_dataset_client.py" + }, + "815": { + "qualifiedName": "URL_NO_COMMAS_REGEX", + "sourceFileName": "/src/apify/storages/_request_list.py" + }, + "816": { + "qualifiedName": "_RequestDetails", + "sourceFileName": "/src/apify/storages/_request_list.py" + }, + "817": { + "qualifiedName": "method", + "sourceFileName": "/src/apify/storages/_request_list.py" + }, + "818": { + "qualifiedName": "payload", + "sourceFileName": "/src/apify/storages/_request_list.py" + }, + "819": { + "qualifiedName": "headers", + "sourceFileName": "/src/apify/storages/_request_list.py" + }, + "820": { + "qualifiedName": "user_data", + "sourceFileName": "/src/apify/storages/_request_list.py" + }, + "821": { + "qualifiedName": "_RequestsFromUrlInput", + "sourceFileName": "/src/apify/storages/_request_list.py" + }, + "822": { + "qualifiedName": "requests_from_url", + "sourceFileName": "/src/apify/storages/_request_list.py" + }, + "823": { + "qualifiedName": "_SimpleUrlInput", + "sourceFileName": "/src/apify/storages/_request_list.py" + }, + "824": { + "qualifiedName": "url", + "sourceFileName": "/src/apify/storages/_request_list.py" + }, + "825": { + "qualifiedName": "url_input_adapter", + "sourceFileName": "/src/apify/storages/_request_list.py" + }, + "826": { + "qualifiedName": "RequestList", + "sourceFileName": "/src/apify/storages/_request_list.py" + }, + "827": { + "qualifiedName": "open", + "sourceFileName": "/src/apify/storages/_request_list.py" + }, + "832": { + "qualifiedName": "method", + "sourceFileName": "/src/apify/storages/_request_list.py" + }, + "833": { + "qualifiedName": "payload", + "sourceFileName": "/src/apify/storages/_request_list.py" + }, + "834": { + "qualifiedName": "headers", + "sourceFileName": "/src/apify/storages/_request_list.py" + }, + "835": { + "qualifiedName": "user_data", + "sourceFileName": "/src/apify/storages/_request_list.py" + }, + "836": { + "qualifiedName": "method", + "sourceFileName": "/src/apify/storages/_request_list.py" + }, + "837": { + "qualifiedName": "payload", + "sourceFileName": "/src/apify/storages/_request_list.py" + }, + "838": { + "qualifiedName": "headers", + "sourceFileName": "/src/apify/storages/_request_list.py" + }, + "839": { + "qualifiedName": "user_data", + "sourceFileName": "/src/apify/storages/_request_list.py" } - ] + } } \ No newline at end of file From c7d20cbe28468ea7f65784813643d5f4735d12e0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jind=C5=99ich=20B=C3=A4r?= Date: Thu, 14 May 2026 10:17:54 +0200 Subject: [PATCH 2/2] chore: fix typos and formatting --- .../versioned_docs/version-0.2/api-packages.json | 2 +- .../versioned_docs/version-0.2/api-typedoc.json | 8 ++++---- .../versioned_docs/version-1.7/api-packages.json | 2 +- .../versioned_docs/version-1.7/api-typedoc.json | 14 +++++++------- .../versioned_docs/version-2.7/api-packages.json | 2 +- 5 files changed, 14 insertions(+), 14 deletions(-) diff --git a/website/versioned_docs/version-0.2/api-packages.json b/website/versioned_docs/version-0.2/api-packages.json index 765ea7e7..e2e41f0d 100644 --- a/website/versioned_docs/version-0.2/api-packages.json +++ b/website/versioned_docs/version-0.2/api-packages.json @@ -1 +1 @@ -[{"entryPoints":{"index":{"label":"Index","path":"src/index.ts"}},"packageRoot":".","packagePath":".","packageSlug":".","packageName":"apify-sdk-python-website"}] \ No newline at end of file +[{"entryPoints":{"index":{"label":"Index","path":"src/index.ts"}},"packageRoot":".","packagePath":".","packageSlug":".","packageName":"apify-sdk-python"}] diff --git a/website/versioned_docs/version-0.2/api-typedoc.json b/website/versioned_docs/version-0.2/api-typedoc.json index 76739ea2..f2b8f0ad 100644 --- a/website/versioned_docs/version-0.2/api-typedoc.json +++ b/website/versioned_docs/version-0.2/api-typedoc.json @@ -643,7 +643,7 @@ "summary": [ { "kind": "text", - "text": "Retrive the global configuration.\n\nThe global configuration applies when you call actor methods via their static versions, e.g. `Actor.init()`.\nAlso accessible via `Actor.config`." + "text": "Retrieve the global configuration.\n\nThe global configuration applies when you call actor methods via their static versions, e.g. `Actor.init()`.\nAlso accessible via `Actor.config`." } ] }, @@ -658,7 +658,7 @@ "module": "config", "name": "get_global_configuration", "parsedDocstring": { - "text": "Retrive the global configuration.\n\nThe global configuration applies when you call actor methods via their static versions, e.g. `Actor.init()`.\nAlso accessible via `Actor.config`." + "text": "Retrieve the global configuration.\n\nThe global configuration applies when you call actor methods via their static versions, e.g. `Actor.init()`.\nAlso accessible via `Actor.config`." }, "sources": [ { @@ -678,7 +678,7 @@ "summary": [ { "kind": "text", - "text": "Retrive the global configuration.\n\nThe global configuration applies when you call actor methods via their static versions, e.g. `Actor.init()`.\nAlso accessible via `Actor.config`." + "text": "Retrieve the global configuration.\n\nThe global configuration applies when you call actor methods via their static versions, e.g. `Actor.init()`.\nAlso accessible via `Actor.config`." } ] }, @@ -56281,4 +56281,4 @@ "sourceFileName": "/src/apify/storages/request_queue.py" } } -} \ No newline at end of file +} diff --git a/website/versioned_docs/version-1.7/api-packages.json b/website/versioned_docs/version-1.7/api-packages.json index 765ea7e7..e2e41f0d 100644 --- a/website/versioned_docs/version-1.7/api-packages.json +++ b/website/versioned_docs/version-1.7/api-packages.json @@ -1 +1 @@ -[{"entryPoints":{"index":{"label":"Index","path":"src/index.ts"}},"packageRoot":".","packagePath":".","packageSlug":".","packageName":"apify-sdk-python-website"}] \ No newline at end of file +[{"entryPoints":{"index":{"label":"Index","path":"src/index.ts"}},"packageRoot":".","packagePath":".","packageSlug":".","packageName":"apify-sdk-python"}] diff --git a/website/versioned_docs/version-1.7/api-typedoc.json b/website/versioned_docs/version-1.7/api-typedoc.json index 8724eb13..332b1559 100644 --- a/website/versioned_docs/version-1.7/api-typedoc.json +++ b/website/versioned_docs/version-1.7/api-typedoc.json @@ -753,7 +753,7 @@ "summary": [ { "kind": "text", - "text": "Retrive the global configuration.\n\nThe global configuration applies when you call actor methods via their static versions, e.g. `Actor.init()`.\nAlso accessible via `Actor.config`." + "text": "Retrieve the global configuration.\n\nThe global configuration applies when you call actor methods via their static versions, e.g. `Actor.init()`.\nAlso accessible via `Actor.config`." } ] }, @@ -768,7 +768,7 @@ "module": "config", "name": "get_global_configuration", "parsedDocstring": { - "text": "Retrive the global configuration.\n\nThe global configuration applies when you call actor methods via their static versions, e.g. `Actor.init()`.\nAlso accessible via `Actor.config`." + "text": "Retrieve the global configuration.\n\nThe global configuration applies when you call actor methods via their static versions, e.g. `Actor.init()`.\nAlso accessible via `Actor.config`." }, "sources": [ { @@ -788,7 +788,7 @@ "summary": [ { "kind": "text", - "text": "Retrive the global configuration.\n\nThe global configuration applies when you call actor methods via their static versions, e.g. `Actor.init()`.\nAlso accessible via `Actor.config`." + "text": "Retrieve the global configuration.\n\nThe global configuration applies when you call actor methods via their static versions, e.g. `Actor.init()`.\nAlso accessible via `Actor.config`." } ] }, @@ -10154,7 +10154,7 @@ "summary": [ { "kind": "text", - "text": "Rename a directory. Checks for existence of soruce directory and removes destination directory if it exists." + "text": "Rename a directory. Checks for existence of source directory and removes destination directory if it exists." } ] }, @@ -10165,7 +10165,7 @@ "module": "_utils", "name": "force_rename", "parsedDocstring": { - "text": "Rename a directory. Checks for existence of soruce directory and removes destination directory if it exists." + "text": "Rename a directory. Checks for existence of source directory and removes destination directory if it exists." }, "sources": [ { @@ -10185,7 +10185,7 @@ "summary": [ { "kind": "text", - "text": "Rename a directory. Checks for existence of soruce directory and removes destination directory if it exists." + "text": "Rename a directory. Checks for existence of source directory and removes destination directory if it exists." } ] }, @@ -60546,4 +60546,4 @@ } } ] -} \ No newline at end of file +} diff --git a/website/versioned_docs/version-2.7/api-packages.json b/website/versioned_docs/version-2.7/api-packages.json index 765ea7e7..e2e41f0d 100644 --- a/website/versioned_docs/version-2.7/api-packages.json +++ b/website/versioned_docs/version-2.7/api-packages.json @@ -1 +1 @@ -[{"entryPoints":{"index":{"label":"Index","path":"src/index.ts"}},"packageRoot":".","packagePath":".","packageSlug":".","packageName":"apify-sdk-python-website"}] \ No newline at end of file +[{"entryPoints":{"index":{"label":"Index","path":"src/index.ts"}},"packageRoot":".","packagePath":".","packageSlug":".","packageName":"apify-sdk-python"}]