From 0597100827b1e927b869ca70ff2e66fe103bf8fa Mon Sep 17 00:00:00 2001 From: Okiemute Date: Thu, 19 Mar 2026 04:13:44 -0700 Subject: [PATCH 01/13] gh-146152: Fix memory leak in _json encoder recursion path Only clean up markers on the RecursionError path (PATH B), where objects accumulate during stack unwinding. Other error paths are safe because the markers dict is local and will be freed. Based on review feedback from @raminfp and @serhiy-storchaka. --- .../Library/2026-03-19-04-18-27.gh-issue-146152.Ifrd7O.rst | 3 +++ Modules/_json.c | 7 ++++++- 2 files changed, 9 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Library/2026-03-19-04-18-27.gh-issue-146152.Ifrd7O.rst diff --git a/Misc/NEWS.d/next/Library/2026-03-19-04-18-27.gh-issue-146152.Ifrd7O.rst b/Misc/NEWS.d/next/Library/2026-03-19-04-18-27.gh-issue-146152.Ifrd7O.rst new file mode 100644 index 00000000000000..cdc4d8e75254b0 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-03-19-04-18-27.gh-issue-146152.Ifrd7O.rst @@ -0,0 +1,3 @@ +Fix a memory leak in the :mod:`json` module when encoding objects with a +custom ``default()`` function that raises an exception, when a recursion +error occurs, or when nested encoding fails. diff --git a/Modules/_json.c b/Modules/_json.c index cbede8f44dc065..f2f41baf9d103d 100644 --- a/Modules/_json.c +++ b/Modules/_json.c @@ -1632,8 +1632,12 @@ encoder_listencode_obj(PyEncoderObject *s, PyUnicodeWriter *writer, } if (_Py_EnterRecursiveCall(" while encoding a JSON object")) { + if (ident != NULL) { + PyDict_DelItem(s->markers, ident); + Py_XDECREF(ident); + } Py_DECREF(newobj); - Py_XDECREF(ident); + return -1; } rv = encoder_listencode_obj(s, writer, newobj, indent_level, indent_cache); @@ -1642,6 +1646,7 @@ encoder_listencode_obj(PyEncoderObject *s, PyUnicodeWriter *writer, Py_DECREF(newobj); if (rv) { _PyErr_FormatNote("when serializing %T object", obj); + Py_XDECREF(ident); return -1; } From 9ec934841bfe851394795606439b0828be90f00e Mon Sep 17 00:00:00 2001 From: Okiemute Date: Fri, 20 Mar 2026 02:39:39 -0700 Subject: [PATCH 02/13] Add test for JSON encoder memory leak on recursion error --- Lib/test/test_json/test_recursion.py | 35 ++++++++++++++++++++++++++-- 1 file changed, 33 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_json/test_recursion.py b/Lib/test/test_json/test_recursion.py index ffd3404e6f77a0..640e1ba36ccc83 100644 --- a/Lib/test/test_json/test_recursion.py +++ b/Lib/test/test_json/test_recursion.py @@ -1,5 +1,8 @@ from test import support from test.test_json import PyTest, CTest +import weakref +import gc + class JSONTestObject: @@ -114,7 +117,35 @@ def default(self, o): with self.assertRaises(RecursionError): with support.infinite_recursion(1000): EndlessJSONEncoder(check_circular=False).encode(5j) - - + + @support.skip_if_unlimited_stack_size + @support.skip_emscripten_stack_overflow() + @support.skip_wasi_stack_overflow() + def test_memory_leak_on_recursion_error(self): + """Test that no memory leak occurs when a RecursionError is raised.""" + weak_refs = [] + class LeakTestObj: + pass + + def default(obj): + if isinstance(obj, LeakTestObj): + new_obj = LeakTestObj() + weak_refs.append(weakref.ref(new_obj)) + return new_obj + raise TypeError + + + obj = LeakTestObj() + for _ in range(1000): + obj = [obj] + + with self.assertRaises(RecursionError): + self.dumps(obj, default=default) + + gc.collect() + for i, ref in enumerate(weak_refs): + self.assertIsNone(ref(), + f"Object {i} still alive - memory leak detected!") + class TestPyRecursion(TestRecursion, PyTest): pass class TestCRecursion(TestRecursion, CTest): pass From cd9cc6f8738b79bf99dfa466faa2979243d43e58 Mon Sep 17 00:00:00 2001 From: Okiemute Date: Fri, 20 Mar 2026 02:48:01 -0700 Subject: [PATCH 03/13] Add test for JSON encoder memory leak on recursion error --- Lib/test/test_json/test_recursion.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/Lib/test/test_json/test_recursion.py b/Lib/test/test_json/test_recursion.py index 640e1ba36ccc83..fb3a34551e2920 100644 --- a/Lib/test/test_json/test_recursion.py +++ b/Lib/test/test_json/test_recursion.py @@ -2,7 +2,7 @@ from test.test_json import PyTest, CTest import weakref import gc - + class JSONTestObject: @@ -117,7 +117,7 @@ def default(self, o): with self.assertRaises(RecursionError): with support.infinite_recursion(1000): EndlessJSONEncoder(check_circular=False).encode(5j) - + @support.skip_if_unlimited_stack_size @support.skip_emscripten_stack_overflow() @support.skip_wasi_stack_overflow() @@ -126,26 +126,26 @@ def test_memory_leak_on_recursion_error(self): weak_refs = [] class LeakTestObj: pass - + def default(obj): if isinstance(obj, LeakTestObj): new_obj = LeakTestObj() weak_refs.append(weakref.ref(new_obj)) return new_obj raise TypeError - - + + obj = LeakTestObj() for _ in range(1000): obj = [obj] - + with self.assertRaises(RecursionError): self.dumps(obj, default=default) - + gc.collect() for i, ref in enumerate(weak_refs): self.assertIsNone(ref(), f"Object {i} still alive - memory leak detected!") - + class TestPyRecursion(TestRecursion, PyTest): pass class TestCRecursion(TestRecursion, CTest): pass From 5428b5e1023d73866d309a429b50f19d425086d6 Mon Sep 17 00:00:00 2001 From: Okiemute Date: Sun, 22 Mar 2026 06:42:35 -0700 Subject: [PATCH 04/13] Use support.gc_collect() in JSON encoder memory leak test --- Lib/test/test_json/test_recursion.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Lib/test/test_json/test_recursion.py b/Lib/test/test_json/test_recursion.py index fb3a34551e2920..dc9a387261d79a 100644 --- a/Lib/test/test_json/test_recursion.py +++ b/Lib/test/test_json/test_recursion.py @@ -1,7 +1,6 @@ from test import support from test.test_json import PyTest, CTest import weakref -import gc @@ -142,7 +141,7 @@ def default(obj): with self.assertRaises(RecursionError): self.dumps(obj, default=default) - gc.collect() + support.gc_collect() for i, ref in enumerate(weak_refs): self.assertIsNone(ref(), f"Object {i} still alive - memory leak detected!") From b68b52bb67046a7e551033c49e45b9f9105f0cd9 Mon Sep 17 00:00:00 2001 From: Okiemute Date: Sun, 22 Mar 2026 10:12:53 -0700 Subject: [PATCH 05/13] Trigger CI rebuild From cafe9ba35de1c6deb6d89b82240347fc7dad2a50 Mon Sep 17 00:00:00 2001 From: Okiemute Date: Sun, 22 Mar 2026 15:04:43 -0700 Subject: [PATCH 06/13] Address review feedback for JSON encoder memory leak fix - Add error checking for PyDict_DelItem return value - Remove extra blank lines in _json.c - Fix test import order and assertion message - Update NEWS entry to clearly describe the RecursionError path --- Lib/test/test_json/test_recursion.py | 23 +++++++-------- ...-03-19-04-18-27.gh-issue-146152.Ifrd7O.rst | 13 +++++++-- Modules/_json.c | 28 ++++++++++++------- 3 files changed, 40 insertions(+), 24 deletions(-) diff --git a/Lib/test/test_json/test_recursion.py b/Lib/test/test_json/test_recursion.py index dc9a387261d79a..f7fc66c88ac219 100644 --- a/Lib/test/test_json/test_recursion.py +++ b/Lib/test/test_json/test_recursion.py @@ -1,8 +1,7 @@ +import weakref +import sys from test import support from test.test_json import PyTest, CTest -import weakref - - class JSONTestObject: pass @@ -121,30 +120,32 @@ def default(self, o): @support.skip_emscripten_stack_overflow() @support.skip_wasi_stack_overflow() def test_memory_leak_on_recursion_error(self): - """Test that no memory leak occurs when a RecursionError is raised.""" - weak_refs = [] + # Test that no memory leak occurs when a RecursionError is raised. class LeakTestObj: pass + weak_refs = [] def default(obj): if isinstance(obj, LeakTestObj): new_obj = LeakTestObj() weak_refs.append(weakref.ref(new_obj)) - return new_obj + return [new_obj] raise TypeError - + depth = min(500, sys.getrecursionlimit() - 10) obj = LeakTestObj() - for _ in range(1000): + for _ in range(depth): obj = [obj] - with self.assertRaises(RecursionError): + try: self.dumps(obj, default=default) + except Exception: + pass support.gc_collect() + self.assertTrue(weak_refs, "No objects were created to track") for i, ref in enumerate(weak_refs): - self.assertIsNone(ref(), - f"Object {i} still alive - memory leak detected!") + self.assertIsNone(ref(), f"object {i} still alive") class TestPyRecursion(TestRecursion, PyTest): pass class TestCRecursion(TestRecursion, CTest): pass diff --git a/Misc/NEWS.d/next/Library/2026-03-19-04-18-27.gh-issue-146152.Ifrd7O.rst b/Misc/NEWS.d/next/Library/2026-03-19-04-18-27.gh-issue-146152.Ifrd7O.rst index cdc4d8e75254b0..8a6b68f8537904 100644 --- a/Misc/NEWS.d/next/Library/2026-03-19-04-18-27.gh-issue-146152.Ifrd7O.rst +++ b/Misc/NEWS.d/next/Library/2026-03-19-04-18-27.gh-issue-146152.Ifrd7O.rst @@ -1,3 +1,10 @@ -Fix a memory leak in the :mod:`json` module when encoding objects with a -custom ``default()`` function that raises an exception, when a recursion -error occurs, or when nested encoding fails. +Fix a memory leak in the :mod:`json` module when a RecursionError +occurs +during encoding. Previously, objects created in the `default()` +function +while recursively encoding JSON could remain alive due to being +tracked +indefinitely in the encoder’s internal circular-reference +dictionary. +This change ensures such objects are properly freed after the +exception. diff --git a/Modules/_json.c b/Modules/_json.c index f2f41baf9d103d..57c70ebdcea3bd 100644 --- a/Modules/_json.c +++ b/Modules/_json.c @@ -1633,29 +1633,37 @@ encoder_listencode_obj(PyEncoderObject *s, PyUnicodeWriter *writer, if (_Py_EnterRecursiveCall(" while encoding a JSON object")) { if (ident != NULL) { - PyDict_DelItem(s->markers, ident); - Py_XDECREF(ident); + int del_rv = PyDict_DelItem(s->markers, ident); + Py_DECREF(ident); + if (del_rv < 0) { + Py_DECREF(newobj); + return -1; + } } Py_DECREF(newobj); - return -1; } rv = encoder_listencode_obj(s, writer, newobj, indent_level, indent_cache); _Py_LeaveRecursiveCall(); - Py_DECREF(newobj); - if (rv) { - _PyErr_FormatNote("when serializing %T object", obj); - Py_XDECREF(ident); + if (rv) { + if (ident != NULL) { + int del_rv = PyDict_DelItem(s->markers, ident); + Py_XDECREF(ident); + if (del_rv < 0) { + return -1; + } + } return -1; } + if (ident != NULL) { - if (PyDict_DelItem(s->markers, ident)) { - Py_XDECREF(ident); + int del_rv = PyDict_DelItem(s->markers, ident); + Py_XDECREF(ident); + if (del_rv < 0) { return -1; } - Py_XDECREF(ident); } return rv; } From a4be61fae65f000af42c1f96767fb43a182c7578 Mon Sep 17 00:00:00 2001 From: Okiemute Date: Sun, 22 Mar 2026 15:15:18 -0700 Subject: [PATCH 07/13] Add fresh NEWS entry for JSON encoder memory leak fix --- .../2026-03-19-04-18-27.gh-issue-146152.Ifrd7O.rst | 10 ---------- .../2026-03-22-15-13-47.gh-issue-146152.0emb80.rst | 5 +++++ 2 files changed, 5 insertions(+), 10 deletions(-) delete mode 100644 Misc/NEWS.d/next/Library/2026-03-19-04-18-27.gh-issue-146152.Ifrd7O.rst create mode 100644 Misc/NEWS.d/next/Library/2026-03-22-15-13-47.gh-issue-146152.0emb80.rst diff --git a/Misc/NEWS.d/next/Library/2026-03-19-04-18-27.gh-issue-146152.Ifrd7O.rst b/Misc/NEWS.d/next/Library/2026-03-19-04-18-27.gh-issue-146152.Ifrd7O.rst deleted file mode 100644 index 8a6b68f8537904..00000000000000 --- a/Misc/NEWS.d/next/Library/2026-03-19-04-18-27.gh-issue-146152.Ifrd7O.rst +++ /dev/null @@ -1,10 +0,0 @@ -Fix a memory leak in the :mod:`json` module when a RecursionError -occurs -during encoding. Previously, objects created in the `default()` -function -while recursively encoding JSON could remain alive due to being -tracked -indefinitely in the encoder’s internal circular-reference -dictionary. -This change ensures such objects are properly freed after the -exception. diff --git a/Misc/NEWS.d/next/Library/2026-03-22-15-13-47.gh-issue-146152.0emb80.rst b/Misc/NEWS.d/next/Library/2026-03-22-15-13-47.gh-issue-146152.0emb80.rst new file mode 100644 index 00000000000000..78366e7e16736b --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-03-22-15-13-47.gh-issue-146152.0emb80.rst @@ -0,0 +1,5 @@ +Fix a memory leak in the :mod:`json` module when a RecursionError occurs +during encoding. Previously, objects created in the `default()` function +while recursively encoding JSON could remain alive due to being tracked +indefinitely in the encoder's internal circular-reference dictionary. This +change ensures such objects are properly freed after the exception. From f7bdb6fb2acaed2a0df2b7edb2ec6992b4163eb4 Mon Sep 17 00:00:00 2001 From: Okiemute Date: Sun, 22 Mar 2026 15:19:49 -0700 Subject: [PATCH 08/13] Trigger CI: all review feedback addressed From f09ce9a665f0c5d1df360b185c3d990cecc3ab30 Mon Sep 17 00:00:00 2001 From: Okiemute Date: Sun, 22 Mar 2026 15:19:49 -0700 Subject: [PATCH 09/13] Trigger CI: all review feedback addressed --- ...emb80.rst => 2026-03-22-15-25-27.gh-issue-146152.jdQhly.rst} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename Misc/NEWS.d/next/Library/{2026-03-22-15-13-47.gh-issue-146152.0emb80.rst => 2026-03-22-15-25-27.gh-issue-146152.jdQhly.rst} (79%) diff --git a/Misc/NEWS.d/next/Library/2026-03-22-15-13-47.gh-issue-146152.0emb80.rst b/Misc/NEWS.d/next/Library/2026-03-22-15-25-27.gh-issue-146152.jdQhly.rst similarity index 79% rename from Misc/NEWS.d/next/Library/2026-03-22-15-13-47.gh-issue-146152.0emb80.rst rename to Misc/NEWS.d/next/Library/2026-03-22-15-25-27.gh-issue-146152.jdQhly.rst index 78366e7e16736b..f358e286ba0f9c 100644 --- a/Misc/NEWS.d/next/Library/2026-03-22-15-13-47.gh-issue-146152.0emb80.rst +++ b/Misc/NEWS.d/next/Library/2026-03-22-15-25-27.gh-issue-146152.jdQhly.rst @@ -1,5 +1,5 @@ Fix a memory leak in the :mod:`json` module when a RecursionError occurs -during encoding. Previously, objects created in the `default()` function +during encoding. Previously, objects created in the ``default()`` function while recursively encoding JSON could remain alive due to being tracked indefinitely in the encoder's internal circular-reference dictionary. This change ensures such objects are properly freed after the exception. From 753a88e33c1026b63f25ae12c9f6f9b5613b702a Mon Sep 17 00:00:00 2001 From: Okiemute Date: Sun, 22 Mar 2026 15:43:41 -0700 Subject: [PATCH 10/13] Trigger CI: all review comments resolved From 6498f268060e9fc827cca9eca95f91f174c064b4 Mon Sep 17 00:00:00 2001 From: Okiemute Date: Sun, 22 Mar 2026 15:43:41 -0700 Subject: [PATCH 11/13] Fix test to use infinite_recursion pattern and catch RecursionError - Use support.infinite_recursion() context manager - Catch RecursionError explicitly instead of all exceptions - Fix import order with two blank lines after from imports - Remove extra blank lines in _json.c - Use Py_DECREF instead of Py_XDECREF --- Lib/test/test_json/test_recursion.py | 10 ++++------ Modules/_json.c | 6 ++---- 2 files changed, 6 insertions(+), 10 deletions(-) diff --git a/Lib/test/test_json/test_recursion.py b/Lib/test/test_json/test_recursion.py index f7fc66c88ac219..f1812969eeda10 100644 --- a/Lib/test/test_json/test_recursion.py +++ b/Lib/test/test_json/test_recursion.py @@ -3,10 +3,9 @@ from test import support from test.test_json import PyTest, CTest + class JSONTestObject: pass - - class TestRecursion: def test_listrecursion(self): x = [] @@ -137,10 +136,9 @@ def default(obj): for _ in range(depth): obj = [obj] - try: - self.dumps(obj, default=default) - except Exception: - pass + with support.infinite_recursion(): + with self.assertRaises(RecursionError): + self.dumps(obj, default=default) support.gc_collect() self.assertTrue(weak_refs, "No objects were created to track") diff --git a/Modules/_json.c b/Modules/_json.c index 57c70ebdcea3bd..ab1ea506814844 100644 --- a/Modules/_json.c +++ b/Modules/_json.c @@ -1646,21 +1646,19 @@ encoder_listencode_obj(PyEncoderObject *s, PyUnicodeWriter *writer, rv = encoder_listencode_obj(s, writer, newobj, indent_level, indent_cache); _Py_LeaveRecursiveCall(); Py_DECREF(newobj); - if (rv) { if (ident != NULL) { int del_rv = PyDict_DelItem(s->markers, ident); - Py_XDECREF(ident); + Py_DECREF(ident); if (del_rv < 0) { return -1; } } return -1; } - if (ident != NULL) { int del_rv = PyDict_DelItem(s->markers, ident); - Py_XDECREF(ident); + Py_DECREF(ident); if (del_rv < 0) { return -1; } From 81332c7e158cd5dd5bad33465673c509a95eef04 Mon Sep 17 00:00:00 2001 From: Okiemute Date: Sun, 22 Mar 2026 17:04:27 -0700 Subject: [PATCH 12/13] Fix blank lines --- Lib/test/test_json/test_recursion.py | 2 ++ Modules/_json.c | 4 ++++ 2 files changed, 6 insertions(+) diff --git a/Lib/test/test_json/test_recursion.py b/Lib/test/test_json/test_recursion.py index f1812969eeda10..ba89c76cd3f6be 100644 --- a/Lib/test/test_json/test_recursion.py +++ b/Lib/test/test_json/test_recursion.py @@ -6,6 +6,7 @@ class JSONTestObject: pass + class TestRecursion: def test_listrecursion(self): x = [] @@ -145,5 +146,6 @@ def default(obj): for i, ref in enumerate(weak_refs): self.assertIsNone(ref(), f"object {i} still alive") + class TestPyRecursion(TestRecursion, PyTest): pass class TestCRecursion(TestRecursion, CTest): pass diff --git a/Modules/_json.c b/Modules/_json.c index ab1ea506814844..261bbe5c43cc2e 100644 --- a/Modules/_json.c +++ b/Modules/_json.c @@ -1639,14 +1639,18 @@ encoder_listencode_obj(PyEncoderObject *s, PyUnicodeWriter *writer, Py_DECREF(newobj); return -1; } + } + Py_DECREF(newobj); return -1; } rv = encoder_listencode_obj(s, writer, newobj, indent_level, indent_cache); _Py_LeaveRecursiveCall(); + Py_DECREF(newobj); if (rv) { + _PyErr_FormatNote("when serializing %T object", obj); if (ident != NULL) { int del_rv = PyDict_DelItem(s->markers, ident); Py_DECREF(ident); From a9c895e67f158ccfd063b955877bec89285eff45 Mon Sep 17 00:00:00 2001 From: Okiemute Date: Sun, 22 Mar 2026 18:19:38 -0700 Subject: [PATCH 13/13] simplify error path cleanup --- Modules/_json.c | 23 +++++++---------------- 1 file changed, 7 insertions(+), 16 deletions(-) diff --git a/Modules/_json.c b/Modules/_json.c index 261bbe5c43cc2e..16fdd7b24c2296 100644 --- a/Modules/_json.c +++ b/Modules/_json.c @@ -1633,15 +1633,9 @@ encoder_listencode_obj(PyEncoderObject *s, PyUnicodeWriter *writer, if (_Py_EnterRecursiveCall(" while encoding a JSON object")) { if (ident != NULL) { - int del_rv = PyDict_DelItem(s->markers, ident); - Py_DECREF(ident); - if (del_rv < 0) { - Py_DECREF(newobj); - return -1; - } - + (void)PyDict_DelItem(s->markers, ident); } - + Py_XDECREF(ident); Py_DECREF(newobj); return -1; } @@ -1652,20 +1646,17 @@ encoder_listencode_obj(PyEncoderObject *s, PyUnicodeWriter *writer, if (rv) { _PyErr_FormatNote("when serializing %T object", obj); if (ident != NULL) { - int del_rv = PyDict_DelItem(s->markers, ident); - Py_DECREF(ident); - if (del_rv < 0) { - return -1; - } + (void)PyDict_DelItem(s->markers, ident); } + Py_XDECREF(ident); return -1; } if (ident != NULL) { - int del_rv = PyDict_DelItem(s->markers, ident); - Py_DECREF(ident); - if (del_rv < 0) { + if (PyDict_DelItem(s->markers, ident)) { + Py_DECREF(ident); return -1; } + Py_XDECREF(ident); } return rv; }