From a270317becadb92e680e8ebaa5988ffc0da72bb6 Mon Sep 17 00:00:00 2001 From: Lysandros Nikolaou Date: Thu, 26 Feb 2026 14:31:21 +0100 Subject: [PATCH 1/3] gh-145254: Add thread safety annotation - Implement annotation for different thread safety levels - Annotate PyMutex_* APIs --- Doc/conf.py | 1 + Doc/data/threadsafety.dat | 17 ++++++ Doc/glossary.rst | 13 +++++ Doc/tools/extensions/c_annotations.py | 78 +++++++++++++++++++++++++++ 4 files changed, 109 insertions(+) create mode 100644 Doc/data/threadsafety.dat diff --git a/Doc/conf.py b/Doc/conf.py index d7effe2572ec44..ce83c689b6c305 100644 --- a/Doc/conf.py +++ b/Doc/conf.py @@ -566,6 +566,7 @@ # Relative filename of the data files refcount_file = 'data/refcounts.dat' stable_abi_file = 'data/stable_abi.dat' +threadsafety_file = 'data/threadsafety.dat' # Options for sphinxext-opengraph # ------------------------------- diff --git a/Doc/data/threadsafety.dat b/Doc/data/threadsafety.dat new file mode 100644 index 00000000000000..19f5bc0bedf49b --- /dev/null +++ b/Doc/data/threadsafety.dat @@ -0,0 +1,17 @@ +# Thread safety annotations for C API functions. +# +# Each line has the form: +# function_name : level +# +# Where level is one of: +# incompatible -- not safe even with external locking +# compatible -- safe if the caller serializes all access with external locks +# safe -- safe for concurrent use +# +# Lines beginning with '#' are ignored. +# The function name must match the C domain identifier used in the documentation. + +# Synchronization primitives (Doc/c-api/synchronization.rst) +PyMutex_Lock:safe: +PyMutex_Unlock:safe: +PyMutex_IsLocked:safe: diff --git a/Doc/glossary.rst b/Doc/glossary.rst index 6151143a97b420..e8f0eea5d5d0db 100644 --- a/Doc/glossary.rst +++ b/Doc/glossary.rst @@ -1587,6 +1587,19 @@ Glossary See :ref:`Thread State and the Global Interpreter Lock ` for more information. + thread-compatible + A function or operation that is safe to call from multiple threads + provided the caller supplies appropriate external synchronization, for + example by holding a :term:`lock` for the duration of each call. Without + such synchronization, concurrent calls may produce :term:`race conditions + ` or :term:`data races `. + + thread-incompatible + A function or operation that cannot be made safe for concurrent use even + with external synchronization. Thread-incompatible code typically + accesses global state in an unsynchronized way and must be called from + only one thread at a time throughout the program's lifetime. + thread-safe A module, function, or class that behaves correctly when used by multiple threads concurrently. Thread-safe code uses appropriate diff --git a/Doc/tools/extensions/c_annotations.py b/Doc/tools/extensions/c_annotations.py index e04a5f144c449b..22d059750bacbf 100644 --- a/Doc/tools/extensions/c_annotations.py +++ b/Doc/tools/extensions/c_annotations.py @@ -3,10 +3,12 @@ * Reference count annotations for C API functions. * Stable ABI annotations * Limited API annotations +* Thread safety annotations for C API functions. Configuration: * Set ``refcount_file`` to the path to the reference count data file. * Set ``stable_abi_file`` to the path to stable ABI list. +* Set ``threadsafety_file`` to the path to the thread safety data file. """ from __future__ import annotations @@ -48,6 +50,15 @@ class RefCountEntry: result_refs: int | None = None +@dataclasses.dataclass(frozen=True, slots=True) +class ThreadSafetyEntry: + # Name of the function. + name: str + # Thread safety level. + # One of: 'incompatible', 'compatible', 'safe'. + level: str + + @dataclasses.dataclass(frozen=True, slots=True) class StableABIEntry: # Role of the object. @@ -113,10 +124,38 @@ def read_stable_abi_data(stable_abi_file: Path) -> dict[str, StableABIEntry]: return stable_abi_data +_VALID_THREADSAFETY_LEVELS = frozenset({ + "incompatible", + "compatible", + "safe", +}) + + +def read_threadsafety_data(threadsafety_filename: Path) -> dict[str, ThreadSafetyEntry]: + threadsafety_data = {} + for line in threadsafety_filename.read_text(encoding="utf8").splitlines(): + line = line.strip() + if not line or line.startswith("#"): + continue + # Each line is of the form: function_name : level : [comment] + parts = line.split(":", 2) + if len(parts) < 2: + raise ValueError(f"Wrong field count in {line!r}") + name, level = parts[0].strip(), parts[1].strip() + if level not in _VALID_THREADSAFETY_LEVELS: + raise ValueError( + f"Unknown thread safety level {level!r} for {name!r}. " + f"Valid levels: {sorted(_VALID_THREADSAFETY_LEVELS)}" + ) + threadsafety_data[name] = ThreadSafetyEntry(name=name, level=level) + return threadsafety_data + + def add_annotations(app: Sphinx, doctree: nodes.document) -> None: state = app.env.domaindata["c_annotations"] refcount_data = state["refcount_data"] stable_abi_data = state["stable_abi_data"] + threadsafety_data = state["threadsafety_data"] for node in doctree.findall(addnodes.desc_content): par = node.parent if par["domain"] != "c": @@ -126,6 +165,12 @@ def add_annotations(app: Sphinx, doctree: nodes.document) -> None: name = par[0]["ids"][0].removeprefix("c.") objtype = par["objtype"] + # Thread safety annotation — inserted first so it appears last (bottom-most) + # among all annotations. + if entry := threadsafety_data.get(name): + annotation = _threadsafety_annotation(entry.level) + node.insert(0, annotation) + # Stable ABI annotation. if record := stable_abi_data.get(name): if ROLE_TO_OBJECT_TYPE[record.role] != objtype: @@ -256,6 +301,35 @@ def _unstable_api_annotation() -> nodes.admonition: ) +_THREADSAFETY_DISPLAY = { + "incompatible": "Not safe to call from multiple threads.", + "compatible": "Safe to call from multiple threads with external synchronization only.", + "safe": "Safe for concurrent use.", +} + +# Maps each thread safety level to the glossary term it should link to. +_THREADSAFETY_TERM = { + "incompatible": "thread-incompatible", + "compatible": "thread-compatible", + "safe": "thread-safe", +} + + +def _threadsafety_annotation(level: str) -> nodes.emphasis: + display = sphinx_gettext(_THREADSAFETY_DISPLAY[level]) + ref_node = addnodes.pending_xref( + display, + nodes.Text(display), + refdomain="std", + reftarget=_THREADSAFETY_TERM[level], + reftype="term", + refexplicit="True", + ) + prefix = sphinx_gettext("Thread safety:") + " " + classes = [f"threadsafety-{level}"] + return nodes.emphasis("", prefix, ref_node, classes=classes) + + def _return_value_annotation(result_refs: int | None) -> nodes.emphasis: classes = ["refcount"] if result_refs is None: @@ -342,11 +416,15 @@ def init_annotations(app: Sphinx) -> None: state["stable_abi_data"] = read_stable_abi_data( Path(app.srcdir, app.config.stable_abi_file) ) + state["threadsafety_data"] = read_threadsafety_data( + Path(app.srcdir, app.config.threadsafety_file) + ) def setup(app: Sphinx) -> ExtensionMetadata: app.add_config_value("refcount_file", "", "env", types={str}) app.add_config_value("stable_abi_file", "", "env", types={str}) + app.add_config_value("threadsafety_file", "", "env", types={str}) app.add_directive("limited-api-list", LimitedAPIList) app.add_directive("corresponding-type-slot", CorrespondingTypeSlot) app.connect("builder-inited", init_annotations) From d577860ae997ff694cdee9b711f8d0439da116a9 Mon Sep 17 00:00:00 2001 From: Lysandros Nikolaou Date: Fri, 6 Mar 2026 13:29:14 +0100 Subject: [PATCH 2/3] Address first round of feedback --- Doc/tools/extensions/c_annotations.py | 39 ++++++++++++++------------- 1 file changed, 21 insertions(+), 18 deletions(-) diff --git a/Doc/tools/extensions/c_annotations.py b/Doc/tools/extensions/c_annotations.py index 22d059750bacbf..61c2ab103c2db7 100644 --- a/Doc/tools/extensions/c_annotations.py +++ b/Doc/tools/extensions/c_annotations.py @@ -131,7 +131,9 @@ def read_stable_abi_data(stable_abi_file: Path) -> dict[str, StableABIEntry]: }) -def read_threadsafety_data(threadsafety_filename: Path) -> dict[str, ThreadSafetyEntry]: +def read_threadsafety_data( + threadsafety_filename: Path, +) -> dict[str, ThreadSafetyEntry]: threadsafety_data = {} for line in threadsafety_filename.read_text(encoding="utf8").splitlines(): line = line.strip() @@ -301,32 +303,33 @@ def _unstable_api_annotation() -> nodes.admonition: ) -_THREADSAFETY_DISPLAY = { - "incompatible": "Not safe to call from multiple threads.", - "compatible": "Safe to call from multiple threads with external synchronization only.", - "safe": "Safe for concurrent use.", -} - -# Maps each thread safety level to the glossary term it should link to. -_THREADSAFETY_TERM = { - "incompatible": "thread-incompatible", - "compatible": "thread-compatible", - "safe": "thread-safe", -} - - def _threadsafety_annotation(level: str) -> nodes.emphasis: - display = sphinx_gettext(_THREADSAFETY_DISPLAY[level]) + match level: + case "incompatible": + display = sphinx_gettext("Not safe to call from multiple threads.") + reftarget = "thread-incompatible" + case "compatible": + display = sphinx_gettext( + "Safe to call from multiple threads with external synchronization only." + ) + reftarget = "thread-compatible" + case "safe": + display = sphinx_gettext("Safe for concurrent use.") + reftarget = "thread-safe" + case _: + raise AssertionError( + "Only the levels 'incompatible', 'compatible' and 'safe' are possible" + ) ref_node = addnodes.pending_xref( display, nodes.Text(display), refdomain="std", - reftarget=_THREADSAFETY_TERM[level], + reftarget=reftarget, reftype="term", refexplicit="True", ) prefix = sphinx_gettext("Thread safety:") + " " - classes = [f"threadsafety-{level}"] + classes = ["threadsafety", f"threadsafety-{level}"] return nodes.emphasis("", prefix, ref_node, classes=classes) From 3ba02827149bed898cef5b0be82c8177b25331c0 Mon Sep 17 00:00:00 2001 From: Lysandros Nikolaou Date: Fri, 6 Mar 2026 14:33:34 +0100 Subject: [PATCH 3/3] Change levels and add new section to thread safety page - Change levels to include atomic and safe on distinct objects. - Add a section explaining the levels in the thread safety page - Cross reference levels section from annotation --- Doc/data/threadsafety.dat | 10 ++-- Doc/glossary.rst | 13 ----- Doc/library/threadsafety.rst | 82 +++++++++++++++++++++++++++ Doc/tools/extensions/c_annotations.py | 34 +++++++---- 4 files changed, 111 insertions(+), 28 deletions(-) diff --git a/Doc/data/threadsafety.dat b/Doc/data/threadsafety.dat index 19f5bc0bedf49b..f063ca1360d5fb 100644 --- a/Doc/data/threadsafety.dat +++ b/Doc/data/threadsafety.dat @@ -6,12 +6,14 @@ # Where level is one of: # incompatible -- not safe even with external locking # compatible -- safe if the caller serializes all access with external locks -# safe -- safe for concurrent use +# distinct -- safe on distinct objects without external synchronization +# shared -- safe for concurrent use on the same object +# atomic -- atomic # # Lines beginning with '#' are ignored. # The function name must match the C domain identifier used in the documentation. # Synchronization primitives (Doc/c-api/synchronization.rst) -PyMutex_Lock:safe: -PyMutex_Unlock:safe: -PyMutex_IsLocked:safe: +PyMutex_Lock:shared: +PyMutex_Unlock:shared: +PyMutex_IsLocked:atomic: diff --git a/Doc/glossary.rst b/Doc/glossary.rst index e8f0eea5d5d0db..6151143a97b420 100644 --- a/Doc/glossary.rst +++ b/Doc/glossary.rst @@ -1587,19 +1587,6 @@ Glossary See :ref:`Thread State and the Global Interpreter Lock ` for more information. - thread-compatible - A function or operation that is safe to call from multiple threads - provided the caller supplies appropriate external synchronization, for - example by holding a :term:`lock` for the duration of each call. Without - such synchronization, concurrent calls may produce :term:`race conditions - ` or :term:`data races `. - - thread-incompatible - A function or operation that cannot be made safe for concurrent use even - with external synchronization. Thread-incompatible code typically - accesses global state in an unsynchronized way and must be called from - only one thread at a time throughout the program's lifetime. - thread-safe A module, function, or class that behaves correctly when used by multiple threads concurrently. Thread-safe code uses appropriate diff --git a/Doc/library/threadsafety.rst b/Doc/library/threadsafety.rst index 5b5949d4eff437..3ce1e5e1006636 100644 --- a/Doc/library/threadsafety.rst +++ b/Doc/library/threadsafety.rst @@ -13,6 +13,88 @@ For general guidance on writing thread-safe code in free-threaded Python, see :ref:`freethreading-python-howto`. +.. _threadsafety-levels: + +Thread safety levels +==================== + +The C API documentation uses the following levels to describe the thread +safety guarantees of each function. The levels are listed from least to +most safe. + +.. _threadsafety-level-incompatible: + +Incompatible +------------ + +A function or operation that cannot be made safe for concurrent use even +with external synchronization. Incompatible code typically accesses +global state in an unsynchronized way and must only be called from a single +thread throughout the program's lifetime. + +Example: a function that modifies process-wide state such as signal handlers +or environment variables, where concurrent calls from any threads, even with +external locking, can conflict with the runtime or other libraries. + +.. _threadsafety-level-compatible: + +Compatible +---------- + +A function or operation that is safe to call from multiple threads +*provided* the caller supplies appropriate external synchronization, for +example by holding a :term:`lock` for the duration of each call. Without +such synchronization, concurrent calls may produce :term:`race conditions +` or :term:`data races `. + +Example: a function that reads from or writes to an object whose internal +state is not protected by a lock. Callers must ensure that no two threads +access the same object at the same time. + +.. _threadsafety-level-distinct: + +Safe on distinct objects +------------------------ + +A function or operation that is safe to call from multiple threads without +external synchronization, as long as each thread operates on a **different** +object. Two threads may call the function at the same time, but they must +not pass the same object (or objects that share underlying state) as +arguments. + +Example: a function that modifies fields of a struct using non-atomic +writes. Two threads can each call the function on their own struct +instance safely, but concurrent calls on the *same* instance require +external synchronization. + +.. _threadsafety-level-shared: + +Safe on shared objects +---------------------- + +A function or operation that is safe for concurrent use on the **same** +object. The implementation uses internal synchronization (such as +:term:`per-object locks ` or +:ref:`critical sections `) to protect shared +mutable state, so callers do not need to supply their own locking. + +Example: :c:func:`PyMutex_Lock` can be called from multiple threads on the +same :c:type:`PyMutex` - it uses internal synchronization to serialize +access. + +.. _threadsafety-level-atomic: + +Atomic +------ + +A function or operation that appears :term:`atomic ` with +respect to other threads - it executes instantaneously from the perspective +of other threads. This is the strongest form of thread safety. + +Example: :c:func:`PyMutex_IsLocked` performs an atomic read of the mutex +state and can be called from any thread at any time. + + .. _thread-safety-list: Thread safety for list objects diff --git a/Doc/tools/extensions/c_annotations.py b/Doc/tools/extensions/c_annotations.py index 61c2ab103c2db7..58f597c2eb2d0c 100644 --- a/Doc/tools/extensions/c_annotations.py +++ b/Doc/tools/extensions/c_annotations.py @@ -127,7 +127,9 @@ def read_stable_abi_data(stable_abi_file: Path) -> dict[str, StableABIEntry]: _VALID_THREADSAFETY_LEVELS = frozenset({ "incompatible", "compatible", - "safe", + "distinct", + "shared", + "atomic", }) @@ -307,25 +309,35 @@ def _threadsafety_annotation(level: str) -> nodes.emphasis: match level: case "incompatible": display = sphinx_gettext("Not safe to call from multiple threads.") - reftarget = "thread-incompatible" + reftarget = "threadsafety-level-incompatible" case "compatible": display = sphinx_gettext( - "Safe to call from multiple threads with external synchronization only." + "Safe to call from multiple threads" + " with external synchronization only." ) - reftarget = "thread-compatible" - case "safe": - display = sphinx_gettext("Safe for concurrent use.") - reftarget = "thread-safe" - case _: - raise AssertionError( - "Only the levels 'incompatible', 'compatible' and 'safe' are possible" + reftarget = "threadsafety-level-compatible" + case "distinct": + display = sphinx_gettext( + "Safe to call without external synchronization" + " on distinct objects." + ) + reftarget = "threadsafety-level-distinct" + case "shared": + display = sphinx_gettext( + "Safe for concurrent use on the same object." ) + reftarget = "threadsafety-level-shared" + case "atomic": + display = sphinx_gettext("Atomic.") + reftarget = "threadsafety-level-atomic" + case _: + raise AssertionError(f"Unknown thread safety level {level!r}") ref_node = addnodes.pending_xref( display, nodes.Text(display), refdomain="std", reftarget=reftarget, - reftype="term", + reftype="ref", refexplicit="True", ) prefix = sphinx_gettext("Thread safety:") + " "