From cbde09add03535bdee354c6e7db7dd31e237c59e Mon Sep 17 00:00:00 2001 From: "rodrigo.nogueira" Date: Thu, 22 Jan 2026 14:36:40 -0300 Subject: [PATCH 01/29] Fix memory leak in pop() when key is not found The md_pop_one function in hashtable.h was missing a Py_DECREF(identity) call when the key was not found. This caused a reference leak on the identity object, which is particularly problematic for CIMultiDict where a new lowercase string is created for each lookup. Fixes: https://github.com/aio-libs/multidict/issues/1273 --- multidict/_multilib/hashtable.h | 1 + tests/isolated/multidict_pop_missing.py | 59 +++++++++++++++++++++++++ tests/test_leaks.py | 1 + 3 files changed, 61 insertions(+) create mode 100644 tests/isolated/multidict_pop_missing.py diff --git a/multidict/_multilib/hashtable.h b/multidict/_multilib/hashtable.h index 82019f9eb..4a132b4b2 100644 --- a/multidict/_multilib/hashtable.h +++ b/multidict/_multilib/hashtable.h @@ -1020,6 +1020,7 @@ md_pop_one(MultiDictObject *md, PyObject *key, PyObject **ret) } } + Py_DECREF(identity); ASSERT_CONSISTENT(md, false); return 0; fail: diff --git a/tests/isolated/multidict_pop_missing.py b/tests/isolated/multidict_pop_missing.py new file mode 100644 index 000000000..71e91279e --- /dev/null +++ b/tests/isolated/multidict_pop_missing.py @@ -0,0 +1,59 @@ + +import gc +import psutil +import os +from multidict import MultiDict, CIMultiDict + + +def trim_ram() -> None: + """Forces python garbage collection.""" + gc.collect() + + +process = psutil.Process(os.getpid()) + + +def get_memory_usage() -> float: + memory_info = process.memory_info() + return memory_info.rss / (1024 * 1024) + + +initial_memory_usage = get_memory_usage() + + +def check_for_leak() -> None: + trim_ram() + usage = get_memory_usage() - initial_memory_usage + # Threshold might need tuning, but 50MB is generous for "no leak" + # With leak it grows unboundedly. + assert usage < 50, f"Memory leaked at: {usage} MB" + + + +def _test_pop_missing(cls: type[MultiDict[str] | CIMultiDict[str]], count: int) -> None: + # Use dynamic keys for missing checks to ensure unique objects + # if there is a ref leak on identity. + d = cls() + for j in range(count): + key = f"MISSING_{j}" + try: + d.pop(key) + except KeyError: + pass + d.pop(key, None) + + +def _run_isolated_case() -> None: + # Warmup + _test_pop_missing(MultiDict, max(100, 10)) + check_for_leak() + + # Run loop + for _ in range(20): + _test_pop_missing(MultiDict, 1000) + _test_pop_missing(CIMultiDict, 1000) + check_for_leak() + + +if __name__ == "__main__": + _run_isolated_case() diff --git a/tests/test_leaks.py b/tests/test_leaks.py index 56126d4bc..24493b87c 100644 --- a/tests/test_leaks.py +++ b/tests/test_leaks.py @@ -16,6 +16,7 @@ "multidict_extend_tuple.py", "multidict_update_multidict.py", "multidict_pop.py", + "multidict_pop_missing.py", ), ) @pytest.mark.leaks From daa9f58e7a841c2665eb9daee911674681275685 Mon Sep 17 00:00:00 2001 From: "rodrigo.nogueira" Date: Fri, 23 Jan 2026 09:22:38 -0300 Subject: [PATCH 02/29] feat: Implement to_dict() method --- multidict/_abc.py | 4 ++ multidict/_multidict.c | 86 ++++++++++++++++++++++++++++++++++++++ multidict/_multidict_py.py | 18 ++++++++ tests/test_to_dict.py | 83 ++++++++++++++++++++++++++++++++++++ 4 files changed, 191 insertions(+) create mode 100644 tests/test_to_dict.py diff --git a/multidict/_abc.py b/multidict/_abc.py index 54253e9e7..897eaae77 100644 --- a/multidict/_abc.py +++ b/multidict/_abc.py @@ -42,6 +42,10 @@ def getone(self, key: str, default: _T) -> Union[_V_co, _T]: ... def getone(self, key: str, default: _T = ...) -> Union[_V_co, _T]: """Return first value for key.""" + @abc.abstractmethod + def to_dict(self) -> dict[str, list[_V_co]]: + """Return a dict with lists of all values for each key.""" + class MutableMultiMapping(MultiMapping[_V], MutableMapping[str, _V]): @abc.abstractmethod diff --git a/multidict/_multidict.c b/multidict/_multidict.c index 73c002296..e4689597e 100644 --- a/multidict/_multidict.c +++ b/multidict/_multidict.c @@ -853,8 +853,80 @@ PyDoc_STRVAR(multidict_update_doc, PyDoc_STRVAR(multidict_merge_doc, "Merge into the dictionary, adding non-existing keys."); +PyDoc_STRVAR(multidict_to_dict_doc, + "Return a dict with lists of all values for each key."); + +static PyObject * +multidict_to_dict(MultiDictObject *self) +{ + PyObject *result = PyDict_New(); + if (result == NULL) { + return NULL; + } + + PyObject *seen = PyDict_New(); + if (seen == NULL) { + Py_DECREF(result); + return NULL; + } + + md_pos_t pos; + md_init_pos(self, &pos); + PyObject *identity = NULL; + PyObject *key = NULL; + PyObject *value = NULL; + + int tmp; + while ((tmp = md_next(self, &pos, &identity, &key, &value)) > 0) { + PyObject *first_key = PyDict_GetItem(seen, identity); + if (first_key == NULL) { + PyObject *lst = PyList_New(1); + if (lst == NULL) { + goto fail; + } + PyList_SET_ITEM(lst, 0, value); + value = NULL; + if (PyDict_SetItem(seen, identity, key) < 0) { + Py_DECREF(lst); + goto fail; + } + if (PyDict_SetItem(result, key, lst) < 0) { + Py_DECREF(lst); + goto fail; + } + Py_DECREF(lst); + } else { + PyObject *lst = PyDict_GetItem(result, first_key); + if (lst == NULL || PyList_Append(lst, value) < 0) { + goto fail; + } + Py_DECREF(value); + value = NULL; + } + Py_DECREF(identity); + Py_DECREF(key); + identity = NULL; + key = NULL; + } + if (tmp < 0) { + goto fail; + } + + Py_DECREF(seen); + return result; + +fail: + Py_XDECREF(identity); + Py_XDECREF(key); + Py_XDECREF(value); + Py_DECREF(seen); + Py_DECREF(result); + return NULL; +} + PyDoc_STRVAR(sizeof__doc__, "D.__sizeof__() -> size of D in memory, in bytes"); + static PyObject * multidict_sizeof(MultiDictObject *self) { @@ -936,6 +1008,10 @@ static PyMethodDef multidict_methods[] = { METH_NOARGS, sizeof__doc__, }, + {"to_dict", + (PyCFunction)multidict_to_dict, + METH_NOARGS, + multidict_to_dict_doc}, {NULL, NULL} /* sentinel */ }; @@ -1144,6 +1220,12 @@ multidict_proxy_reduce(MultiDictProxyObject *self) return NULL; } +static PyObject * +multidict_proxy_to_dict(MultiDictProxyObject *self) +{ + return multidict_to_dict(self->md); +} + static Py_ssize_t multidict_proxy_mp_len(MultiDictProxyObject *self) { @@ -1245,6 +1327,10 @@ static PyMethodDef multidict_proxy_methods[] = { (PyCFunction)Py_GenericAlias, METH_O | METH_CLASS, NULL}, + {"to_dict", + (PyCFunction)multidict_proxy_to_dict, + METH_NOARGS, + multidict_to_dict_doc}, {NULL, NULL} /* sentinel */ }; diff --git a/multidict/_multidict_py.py b/multidict/_multidict_py.py index 6b68d52ee..faf1111e3 100644 --- a/multidict/_multidict_py.py +++ b/multidict/_multidict_py.py @@ -772,6 +772,20 @@ def __sizeof__(self) -> int: def __reduce__(self) -> tuple[type[Self], tuple[list[tuple[str, _V]]]]: return (self.__class__, (list(self.items()),)) + def to_dict(self) -> dict[str, list[_V]]: + """Return a dict with lists of all values for each key.""" + result: dict[str, list[_V]] = {} + seen_identities: dict[str, str] = {} + for e in self._keys.iter_entries(): + first_key = seen_identities.get(e.identity) + if first_key is None: + seen_identities[e.identity] = self._key(e.key) + result[self._key(e.key)] = [e.value] + else: + result[first_key].append(e.value) + return result + + def add(self, key: str, value: _V) -> None: identity = self._identity(key) hash_ = hash(identity) @@ -1212,6 +1226,10 @@ def __repr__(self) -> str: body = ", ".join(f"'{k}': {v!r}" for k, v in self.items()) return f"<{self.__class__.__name__}({body})>" + def to_dict(self) -> dict[str, list[_V]]: + """Return a dict with lists of all values for each key.""" + return self._md.to_dict() + def copy(self) -> MultiDict[_V]: """Return a copy of itself.""" return MultiDict(self._md) diff --git a/tests/test_to_dict.py b/tests/test_to_dict.py new file mode 100644 index 000000000..f2618b2f5 --- /dev/null +++ b/tests/test_to_dict.py @@ -0,0 +1,83 @@ +"""Test to_dict functionality for all multidict types.""" +import pytest + + +class BaseToDictTests: + + def test_to_dict_simple(self, cls): + d = cls([("a", 1), ("b", 2)]) + result = d.to_dict() + assert result == {"a": [1], "b": [2]} + + def test_to_dict_multi_values(self, cls): + d = cls([("a", 1), ("b", 2), ("a", 3)]) + result = d.to_dict() + assert result == {"a": [1, 3], "b": [2]} + + def test_to_dict_empty(self, cls): + d = cls() + result = d.to_dict() + assert result == {} + + def test_to_dict_returns_new_dict(self, cls): + d = cls([("a", 1)]) + result1 = d.to_dict() + result2 = d.to_dict() + assert result1 == result2 + assert result1 is not result2 + + def test_to_dict_list_is_fresh(self, cls): + d = cls([("a", 1)]) + result1 = d.to_dict() + result2 = d.to_dict() + assert result1["a"] is not result2["a"] + + +class TestMultiDictToDict(BaseToDictTests): + + @pytest.fixture + def cls(self, multidict_module): + return multidict_module.MultiDict + + +class TestCIMultiDictToDict(BaseToDictTests): + + @pytest.fixture + def cls(self, multidict_module): + return multidict_module.CIMultiDict + + def test_to_dict_case_insensitive_grouping(self, cls): + d = cls([("A", 1), ("a", 2), ("B", 3)]) + result = d.to_dict() + assert len(result) == 2 + assert "A" in result or "a" in result + assert "B" in result or "b" in result + key_a = "A" if "A" in result else "a" + key_b = "B" if "B" in result else "b" + assert result[key_a] == [1, 2] + assert result[key_b] == [3] + + +class TestMultiDictProxyToDict(BaseToDictTests): + + @pytest.fixture + def cls(self, multidict_module): + def make_proxy(*args, **kwargs): + md = multidict_module.MultiDict(*args, **kwargs) + return multidict_module.MultiDictProxy(md) + return make_proxy + + +class TestCIMultiDictProxyToDict(BaseToDictTests): + + @pytest.fixture + def cls(self, multidict_module): + def make_proxy(*args, **kwargs): + md = multidict_module.CIMultiDict(*args, **kwargs) + return multidict_module.CIMultiDictProxy(md) + return make_proxy + + def test_to_dict_case_insensitive_grouping(self, cls): + d = cls([("A", 1), ("a", 2), ("B", 3)]) + result = d.to_dict() + assert len(result) == 2 From 7a243d8d8d2fed5ace3cde0e9352456414f28dc1 Mon Sep 17 00:00:00 2001 From: "rodrigo.nogueira" Date: Fri, 23 Jan 2026 23:13:34 -0300 Subject: [PATCH 03/29] Add comprehensive psleak test suite --- tests/test_psleak_checks.py | 458 ++++++++++++++++++++++++++++++++++++ 1 file changed, 458 insertions(+) create mode 100644 tests/test_psleak_checks.py diff --git a/tests/test_psleak_checks.py b/tests/test_psleak_checks.py new file mode 100644 index 000000000..6ad169fd6 --- /dev/null +++ b/tests/test_psleak_checks.py @@ -0,0 +1,458 @@ +import pytest +import psleak +from multidict import _multidict + + +@pytest.mark.c_extension +class TestMultiDictLeaks(psleak.MemoryLeakTestCase): + def test_multidict_create(self): + def worker(): + _multidict.MultiDict() + self.execute(worker) + + def test_multidict_create_with_args(self): + def worker(): + _multidict.MultiDict({"a": 1, "b": 2}) + self.execute(worker) + + def test_multidict_add(self): + def worker(): + d = _multidict.MultiDict() + d.add("key", "value") + self.execute(worker) + + def test_multidict_pop(self): + def worker(): + d = _multidict.MultiDict({"a": 1}) + d.pop("a") + self.execute(worker) + + def test_multidict_pop_missing_with_default(self): + def worker(): + d = _multidict.MultiDict() + d.pop("missing", None) + self.execute(worker) + + def test_multidict_pop_missing_keyerror(self): + def worker(): + d = _multidict.MultiDict() + try: + d.pop("missing") + except KeyError: + pass + self.execute(worker) + + def test_multidict_popall(self): + def worker(): + d = _multidict.MultiDict([("a", 1), ("a", 2)]) + d.popall("a") + self.execute(worker) + + def test_multidict_update_dict(self): + def worker(): + d = _multidict.MultiDict() + d.update({"a": 1, "b": 2}) + self.execute(worker) + + def test_multidict_extend_tuple(self): + def worker(): + d = _multidict.MultiDict() + d.extend([("a", 1), ("b", 2)]) + self.execute(worker) + + def test_multidict_getall(self): + def worker(): + d = _multidict.MultiDict([("a", 1), ("a", 2)]) + d.getall("a") + self.execute(worker) + + def test_multidict_keys(self): + def worker(): + d = _multidict.MultiDict({"a": 1, "b": 2}) + list(d.keys()) + self.execute(worker) + + def test_multidict_values(self): + def worker(): + d = _multidict.MultiDict({"a": 1, "b": 2}) + list(d.values()) + self.execute(worker) + + def test_multidict_items(self): + def worker(): + d = _multidict.MultiDict({"a": 1, "b": 2}) + list(d.items()) + self.execute(worker) + + def test_multidict_clear(self): + def worker(): + d = _multidict.MultiDict({f"key{i}": i for i in range(100)}) + d.clear() + self.execute(worker) + + def test_multidict_copy(self): + def worker(): + d = _multidict.MultiDict({"key": "value"}) + d.copy() + self.execute(worker) + + def test_multidict_getone(self): + def worker(): + d = _multidict.MultiDict([("key", "val1"), ("key", "val2")]) + d.getone("key") + self.execute(worker) + + def test_multidict_get(self): + def worker(): + d = _multidict.MultiDict({"key": "value"}) + d.get("key") + d.get("missing") + d.get("missing", "default") + self.execute(worker) + + def test_multidict_popitem(self): + def worker(): + d = _multidict.MultiDict({"a": 1, "b": 2}) + d.popitem() + self.execute(worker) + + def test_multidict_setdefault(self): + def worker(): + d = _multidict.MultiDict() + d.setdefault("key", "default") + d.setdefault("key", "other") + self.execute(worker) + + def test_multidict_setitem(self): + def worker(): + d = _multidict.MultiDict({"key": "old"}) + d["key"] = "new" + self.execute(worker) + + def test_multidict_delitem(self): + def worker(): + d = _multidict.MultiDict({"key": "value"}) + del d["key"] + self.execute(worker) + + def test_multidict_stress(self): + def worker(): + d = _multidict.MultiDict() + for i in range(100): + d.add(f"key{i}", f"value{i}") + d.clear() + self.execute(worker) + + def test_multidict_multi_value_add(self): + def worker(): + d = _multidict.MultiDict() + for i in range(10): + d.add("same_key", f"value{i}") + self.execute(worker) + + +@pytest.mark.c_extension +class TestCIMultiDictLeaks(psleak.MemoryLeakTestCase): + def test_cimultidict_create(self): + def worker(): + _multidict.CIMultiDict() + self.execute(worker) + + def test_cimultidict_create_with_args(self): + def worker(): + _multidict.CIMultiDict({"a": 1, "b": 2}) + self.execute(worker) + + def test_cimultidict_add(self): + def worker(): + d = _multidict.CIMultiDict() + d.add("key", "value") + self.execute(worker) + + def test_cimultidict_pop(self): + def worker(): + d = _multidict.CIMultiDict({"a": 1}) + d.pop("a") + self.execute(worker) + + def test_cimultidict_pop_missing_with_default(self): + def worker(): + d = _multidict.CIMultiDict() + d.pop("missing", object()) + self.execute(worker) + + def test_cimultidict_pop_missing_keyerror(self): + def worker(): + d = _multidict.CIMultiDict() + try: + d.pop("missing") + except KeyError: + pass + self.execute(worker) + + def test_cimultidict_popall(self): + def worker(): + d = _multidict.CIMultiDict([("a", 1), ("a", 2)]) + d.popall("a") + self.execute(worker) + + def test_cimultidict_clear(self): + def worker(): + d = _multidict.CIMultiDict({f"key{i}": i for i in range(100)}) + d.clear() + self.execute(worker) + + def test_cimultidict_copy(self): + def worker(): + d = _multidict.CIMultiDict({"key": "value"}) + d.copy() + self.execute(worker) + + def test_cimultidict_getone(self): + def worker(): + d = _multidict.CIMultiDict([("key", "val1"), ("key", "val2")]) + d.getone("key") + self.execute(worker) + + def test_cimultidict_get(self): + def worker(): + d = _multidict.CIMultiDict({"key": "value"}) + d.get("key") + d.get("missing", "default") + self.execute(worker) + + def test_cimultidict_setitem(self): + def worker(): + d = _multidict.CIMultiDict({"key": "old"}) + d["key"] = "new" + self.execute(worker) + + def test_cimultidict_delitem(self): + def worker(): + d = _multidict.CIMultiDict({"key": "value"}) + del d["key"] + self.execute(worker) + + def test_cimultidict_case_insensitive_add_pop(self): + def worker(): + d = _multidict.CIMultiDict() + d.add("Key", "value1") + d.add("KEY", "value2") + d.pop("key") + self.execute(worker) + + def test_cimultidict_update(self): + def worker(): + d = _multidict.CIMultiDict() + d.update({"a": 1, "b": 2}) + self.execute(worker) + + def test_cimultidict_extend(self): + def worker(): + d = _multidict.CIMultiDict() + d.extend([("a", 1), ("b", 2)]) + self.execute(worker) + + def test_cimultidict_getall(self): + def worker(): + d = _multidict.CIMultiDict([("a", 1), ("a", 2)]) + d.getall("a") + self.execute(worker) + + def test_cimultidict_keys(self): + def worker(): + d = _multidict.CIMultiDict({"a": 1, "b": 2}) + list(d.keys()) + self.execute(worker) + + def test_cimultidict_values(self): + def worker(): + d = _multidict.CIMultiDict({"a": 1, "b": 2}) + list(d.values()) + self.execute(worker) + + def test_cimultidict_items(self): + def worker(): + d = _multidict.CIMultiDict({"a": 1, "b": 2}) + list(d.items()) + self.execute(worker) + + +@pytest.mark.c_extension +class TestMultiDictProxyLeaks(psleak.MemoryLeakTestCase): + def test_proxy_create(self): + d = _multidict.MultiDict({"a": 1}) + def worker(): + _multidict.MultiDictProxy(d) + self.execute(worker) + + def test_proxy_access(self): + d = _multidict.MultiDict({"a": 1}) + p = _multidict.MultiDictProxy(d) + def worker(): + _ = p["a"] + self.execute(worker) + + def test_proxy_getall(self): + d = _multidict.MultiDict([("a", 1), ("a", 2)]) + p = _multidict.MultiDictProxy(d) + def worker(): + p.getall("a") + self.execute(worker) + + def test_proxy_getone(self): + d = _multidict.MultiDict([("key", "val1"), ("key", "val2")]) + p = _multidict.MultiDictProxy(d) + def worker(): + p.getone("key") + self.execute(worker) + + def test_proxy_get(self): + d = _multidict.MultiDict({"key": "value"}) + p = _multidict.MultiDictProxy(d) + def worker(): + p.get("key") + p.get("missing", "default") + self.execute(worker) + + def test_proxy_keys(self): + d = _multidict.MultiDict({"a": 1, "b": 2}) + p = _multidict.MultiDictProxy(d) + def worker(): + list(p.keys()) + self.execute(worker) + + def test_proxy_values(self): + d = _multidict.MultiDict({"a": 1, "b": 2}) + p = _multidict.MultiDictProxy(d) + def worker(): + list(p.values()) + self.execute(worker) + + def test_proxy_items(self): + d = _multidict.MultiDict({"a": 1, "b": 2}) + p = _multidict.MultiDictProxy(d) + def worker(): + list(p.items()) + self.execute(worker) + + def test_proxy_copy(self): + d = _multidict.MultiDict({"key": "value"}) + p = _multidict.MultiDictProxy(d) + def worker(): + p.copy() + self.execute(worker) + + def test_proxy_after_modify(self): + d = _multidict.MultiDict() + p = _multidict.MultiDictProxy(d) + def worker(): + d.add("key", "value") + _ = p["key"] + d.clear() + self.execute(worker) + + +@pytest.mark.c_extension +class TestCIMultiDictProxyLeaks(psleak.MemoryLeakTestCase): + def test_proxy_create(self): + d = _multidict.CIMultiDict({"a": 1}) + def worker(): + _multidict.CIMultiDictProxy(d) + self.execute(worker) + + def test_proxy_access(self): + d = _multidict.CIMultiDict({"a": 1}) + p = _multidict.CIMultiDictProxy(d) + def worker(): + _ = p["a"] + self.execute(worker) + + def test_proxy_getone(self): + d = _multidict.CIMultiDict([("key", "val1"), ("key", "val2")]) + p = _multidict.CIMultiDictProxy(d) + def worker(): + p.getone("key") + self.execute(worker) + + def test_proxy_get(self): + d = _multidict.CIMultiDict({"key": "value"}) + p = _multidict.CIMultiDictProxy(d) + def worker(): + p.get("KEY") + p.get("missing", "default") + self.execute(worker) + + def test_proxy_keys(self): + d = _multidict.CIMultiDict({"a": 1, "b": 2}) + p = _multidict.CIMultiDictProxy(d) + def worker(): + list(p.keys()) + self.execute(worker) + + def test_proxy_values(self): + d = _multidict.CIMultiDict({"a": 1, "b": 2}) + p = _multidict.CIMultiDictProxy(d) + def worker(): + list(p.values()) + self.execute(worker) + + def test_proxy_items(self): + d = _multidict.CIMultiDict({"a": 1, "b": 2}) + p = _multidict.CIMultiDictProxy(d) + def worker(): + list(p.items()) + self.execute(worker) + + def test_proxy_copy(self): + d = _multidict.CIMultiDict({"key": "value"}) + p = _multidict.CIMultiDictProxy(d) + def worker(): + p.copy() + self.execute(worker) + + +@pytest.mark.c_extension +class TestIstrLeaks(psleak.MemoryLeakTestCase): + def test_istr_create(self): + def worker(): + _multidict.istr("Hello") + self.execute(worker) + + def test_istr_comparison(self): + def worker(): + s1 = _multidict.istr("Hello") + s2 = _multidict.istr("hello") + _ = s1 == s2 + self.execute(worker) + + def test_istr_lower(self): + def worker(): + s = _multidict.istr("Hello") + s.lower() + self.execute(worker) + + def test_istr_upper(self): + def worker(): + s = _multidict.istr("Hello") + s.upper() + self.execute(worker) + + def test_istr_in_dict(self): + def worker(): + d = {_multidict.istr("key"): "value"} + _ = d[_multidict.istr("key")] + self.execute(worker) + + def test_istr_concatenation(self): + def worker(): + s1 = _multidict.istr("Hello") + s2 = _multidict.istr("World") + _ = s1 + " " + s2 + self.execute(worker) + + def test_istr_hash(self): + def worker(): + s = _multidict.istr("Hello") + hash(s) + self.execute(worker) From 15000a683ad6df1b81329b505e0f3f9ac83eb45b Mon Sep 17 00:00:00 2001 From: "rodrigo.nogueira" Date: Sat, 24 Jan 2026 22:33:30 -0300 Subject: [PATCH 04/29] Enhance psleak tests with 100KB tolerance and 1M iterations --- tests/isolated/multidict_pop.py | 9 +++++++++ tests/test_psleak_checks.py | 15 ++++++++++----- 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/tests/isolated/multidict_pop.py b/tests/isolated/multidict_pop.py index 89fcce11c..72b741279 100644 --- a/tests/isolated/multidict_pop.py +++ b/tests/isolated/multidict_pop.py @@ -61,6 +61,14 @@ def _test_popone() -> None: result.popone(k) check_for_leak() +def _test_pop_with_default() -> None: + result = MultiDict() + # XXX: mypy wants an annotation so the only + # thing we can do here is pass the headers along. + result = MultiDict(headers) + for i in range(1_000_000): + result.pop(f"missing_key_{i}", None) + check_for_leak() def _test_del() -> None: for _ in range(10): @@ -76,6 +84,7 @@ def _run_isolated_case() -> None: _test_popall() _test_popone() _test_del() + _test_pop_with_default() if __name__ == "__main__": diff --git a/tests/test_psleak_checks.py b/tests/test_psleak_checks.py index 6ad169fd6..31ee46244 100644 --- a/tests/test_psleak_checks.py +++ b/tests/test_psleak_checks.py @@ -3,8 +3,13 @@ from multidict import _multidict +class MultiDictLeakTests(psleak.MemoryLeakTestCase): + tolerance = 1024*100 # Allow 100KB + times = 1000000 # More iterations + + @pytest.mark.c_extension -class TestMultiDictLeaks(psleak.MemoryLeakTestCase): +class TestMultiDictLeaks(MultiDictLeakTests): def test_multidict_create(self): def worker(): _multidict.MultiDict() @@ -152,7 +157,7 @@ def worker(): @pytest.mark.c_extension -class TestCIMultiDictLeaks(psleak.MemoryLeakTestCase): +class TestCIMultiDictLeaks(MultiDictLeakTests): def test_cimultidict_create(self): def worker(): _multidict.CIMultiDict() @@ -279,7 +284,7 @@ def worker(): @pytest.mark.c_extension -class TestMultiDictProxyLeaks(psleak.MemoryLeakTestCase): +class TestMultiDictProxyLeaks(MultiDictLeakTests): def test_proxy_create(self): d = _multidict.MultiDict({"a": 1}) def worker(): @@ -354,7 +359,7 @@ def worker(): @pytest.mark.c_extension -class TestCIMultiDictProxyLeaks(psleak.MemoryLeakTestCase): +class TestCIMultiDictProxyLeaks(MultiDictLeakTests): def test_proxy_create(self): d = _multidict.CIMultiDict({"a": 1}) def worker(): @@ -413,7 +418,7 @@ def worker(): @pytest.mark.c_extension -class TestIstrLeaks(psleak.MemoryLeakTestCase): +class TestIstrLeaks(MultiDictLeakTests): def test_istr_create(self): def worker(): _multidict.istr("Hello") From 2adb465b56d7ad813fd1181d7bee05a4960e0732 Mon Sep 17 00:00:00 2001 From: "rodrigo.nogueira" Date: Sun, 25 Jan 2026 02:05:23 -0300 Subject: [PATCH 05/29] Add to_dict leak tests for CIMultiDict and MultiDictProxy --- tests/test_psleak_checks.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/tests/test_psleak_checks.py b/tests/test_psleak_checks.py index 31ee46244..d574eb463 100644 --- a/tests/test_psleak_checks.py +++ b/tests/test_psleak_checks.py @@ -10,6 +10,12 @@ class MultiDictLeakTests(psleak.MemoryLeakTestCase): @pytest.mark.c_extension class TestMultiDictLeaks(MultiDictLeakTests): + def test_multidict_to_dict(self): + def worker(): + d = _multidict.MultiDict([("a", 1), ("b", 2)]) + d.to_dict() + self.execute(worker) + def test_multidict_create(self): def worker(): _multidict.MultiDict() @@ -158,6 +164,12 @@ def worker(): @pytest.mark.c_extension class TestCIMultiDictLeaks(MultiDictLeakTests): + def test_cimultidict_to_dict(self): + def worker(): + d = _multidict.CIMultiDict([("a", 1), ("B", 2)]) + d.to_dict() + self.execute(worker) + def test_cimultidict_create(self): def worker(): _multidict.CIMultiDict() @@ -285,6 +297,13 @@ def worker(): @pytest.mark.c_extension class TestMultiDictProxyLeaks(MultiDictLeakTests): + def test_proxy_to_dict(self): + d = _multidict.MultiDict([("a", 1), ("b", 2)]) + p = _multidict.MultiDictProxy(d) + def worker(): + p.to_dict() + self.execute(worker) + def test_proxy_create(self): d = _multidict.MultiDict({"a": 1}) def worker(): From 6f29b130d2acab4aebdfe6f5e245a53067b01cf2 Mon Sep 17 00:00:00 2001 From: "rodrigo.nogueira" Date: Sun, 25 Jan 2026 02:07:19 -0300 Subject: [PATCH 06/29] Remove psleak tests (moved to fix/psleak-expansion branch) --- tests/test_psleak_checks.py | 482 ------------------------------------ 1 file changed, 482 deletions(-) delete mode 100644 tests/test_psleak_checks.py diff --git a/tests/test_psleak_checks.py b/tests/test_psleak_checks.py deleted file mode 100644 index d574eb463..000000000 --- a/tests/test_psleak_checks.py +++ /dev/null @@ -1,482 +0,0 @@ -import pytest -import psleak -from multidict import _multidict - - -class MultiDictLeakTests(psleak.MemoryLeakTestCase): - tolerance = 1024*100 # Allow 100KB - times = 1000000 # More iterations - - -@pytest.mark.c_extension -class TestMultiDictLeaks(MultiDictLeakTests): - def test_multidict_to_dict(self): - def worker(): - d = _multidict.MultiDict([("a", 1), ("b", 2)]) - d.to_dict() - self.execute(worker) - - def test_multidict_create(self): - def worker(): - _multidict.MultiDict() - self.execute(worker) - - def test_multidict_create_with_args(self): - def worker(): - _multidict.MultiDict({"a": 1, "b": 2}) - self.execute(worker) - - def test_multidict_add(self): - def worker(): - d = _multidict.MultiDict() - d.add("key", "value") - self.execute(worker) - - def test_multidict_pop(self): - def worker(): - d = _multidict.MultiDict({"a": 1}) - d.pop("a") - self.execute(worker) - - def test_multidict_pop_missing_with_default(self): - def worker(): - d = _multidict.MultiDict() - d.pop("missing", None) - self.execute(worker) - - def test_multidict_pop_missing_keyerror(self): - def worker(): - d = _multidict.MultiDict() - try: - d.pop("missing") - except KeyError: - pass - self.execute(worker) - - def test_multidict_popall(self): - def worker(): - d = _multidict.MultiDict([("a", 1), ("a", 2)]) - d.popall("a") - self.execute(worker) - - def test_multidict_update_dict(self): - def worker(): - d = _multidict.MultiDict() - d.update({"a": 1, "b": 2}) - self.execute(worker) - - def test_multidict_extend_tuple(self): - def worker(): - d = _multidict.MultiDict() - d.extend([("a", 1), ("b", 2)]) - self.execute(worker) - - def test_multidict_getall(self): - def worker(): - d = _multidict.MultiDict([("a", 1), ("a", 2)]) - d.getall("a") - self.execute(worker) - - def test_multidict_keys(self): - def worker(): - d = _multidict.MultiDict({"a": 1, "b": 2}) - list(d.keys()) - self.execute(worker) - - def test_multidict_values(self): - def worker(): - d = _multidict.MultiDict({"a": 1, "b": 2}) - list(d.values()) - self.execute(worker) - - def test_multidict_items(self): - def worker(): - d = _multidict.MultiDict({"a": 1, "b": 2}) - list(d.items()) - self.execute(worker) - - def test_multidict_clear(self): - def worker(): - d = _multidict.MultiDict({f"key{i}": i for i in range(100)}) - d.clear() - self.execute(worker) - - def test_multidict_copy(self): - def worker(): - d = _multidict.MultiDict({"key": "value"}) - d.copy() - self.execute(worker) - - def test_multidict_getone(self): - def worker(): - d = _multidict.MultiDict([("key", "val1"), ("key", "val2")]) - d.getone("key") - self.execute(worker) - - def test_multidict_get(self): - def worker(): - d = _multidict.MultiDict({"key": "value"}) - d.get("key") - d.get("missing") - d.get("missing", "default") - self.execute(worker) - - def test_multidict_popitem(self): - def worker(): - d = _multidict.MultiDict({"a": 1, "b": 2}) - d.popitem() - self.execute(worker) - - def test_multidict_setdefault(self): - def worker(): - d = _multidict.MultiDict() - d.setdefault("key", "default") - d.setdefault("key", "other") - self.execute(worker) - - def test_multidict_setitem(self): - def worker(): - d = _multidict.MultiDict({"key": "old"}) - d["key"] = "new" - self.execute(worker) - - def test_multidict_delitem(self): - def worker(): - d = _multidict.MultiDict({"key": "value"}) - del d["key"] - self.execute(worker) - - def test_multidict_stress(self): - def worker(): - d = _multidict.MultiDict() - for i in range(100): - d.add(f"key{i}", f"value{i}") - d.clear() - self.execute(worker) - - def test_multidict_multi_value_add(self): - def worker(): - d = _multidict.MultiDict() - for i in range(10): - d.add("same_key", f"value{i}") - self.execute(worker) - - -@pytest.mark.c_extension -class TestCIMultiDictLeaks(MultiDictLeakTests): - def test_cimultidict_to_dict(self): - def worker(): - d = _multidict.CIMultiDict([("a", 1), ("B", 2)]) - d.to_dict() - self.execute(worker) - - def test_cimultidict_create(self): - def worker(): - _multidict.CIMultiDict() - self.execute(worker) - - def test_cimultidict_create_with_args(self): - def worker(): - _multidict.CIMultiDict({"a": 1, "b": 2}) - self.execute(worker) - - def test_cimultidict_add(self): - def worker(): - d = _multidict.CIMultiDict() - d.add("key", "value") - self.execute(worker) - - def test_cimultidict_pop(self): - def worker(): - d = _multidict.CIMultiDict({"a": 1}) - d.pop("a") - self.execute(worker) - - def test_cimultidict_pop_missing_with_default(self): - def worker(): - d = _multidict.CIMultiDict() - d.pop("missing", object()) - self.execute(worker) - - def test_cimultidict_pop_missing_keyerror(self): - def worker(): - d = _multidict.CIMultiDict() - try: - d.pop("missing") - except KeyError: - pass - self.execute(worker) - - def test_cimultidict_popall(self): - def worker(): - d = _multidict.CIMultiDict([("a", 1), ("a", 2)]) - d.popall("a") - self.execute(worker) - - def test_cimultidict_clear(self): - def worker(): - d = _multidict.CIMultiDict({f"key{i}": i for i in range(100)}) - d.clear() - self.execute(worker) - - def test_cimultidict_copy(self): - def worker(): - d = _multidict.CIMultiDict({"key": "value"}) - d.copy() - self.execute(worker) - - def test_cimultidict_getone(self): - def worker(): - d = _multidict.CIMultiDict([("key", "val1"), ("key", "val2")]) - d.getone("key") - self.execute(worker) - - def test_cimultidict_get(self): - def worker(): - d = _multidict.CIMultiDict({"key": "value"}) - d.get("key") - d.get("missing", "default") - self.execute(worker) - - def test_cimultidict_setitem(self): - def worker(): - d = _multidict.CIMultiDict({"key": "old"}) - d["key"] = "new" - self.execute(worker) - - def test_cimultidict_delitem(self): - def worker(): - d = _multidict.CIMultiDict({"key": "value"}) - del d["key"] - self.execute(worker) - - def test_cimultidict_case_insensitive_add_pop(self): - def worker(): - d = _multidict.CIMultiDict() - d.add("Key", "value1") - d.add("KEY", "value2") - d.pop("key") - self.execute(worker) - - def test_cimultidict_update(self): - def worker(): - d = _multidict.CIMultiDict() - d.update({"a": 1, "b": 2}) - self.execute(worker) - - def test_cimultidict_extend(self): - def worker(): - d = _multidict.CIMultiDict() - d.extend([("a", 1), ("b", 2)]) - self.execute(worker) - - def test_cimultidict_getall(self): - def worker(): - d = _multidict.CIMultiDict([("a", 1), ("a", 2)]) - d.getall("a") - self.execute(worker) - - def test_cimultidict_keys(self): - def worker(): - d = _multidict.CIMultiDict({"a": 1, "b": 2}) - list(d.keys()) - self.execute(worker) - - def test_cimultidict_values(self): - def worker(): - d = _multidict.CIMultiDict({"a": 1, "b": 2}) - list(d.values()) - self.execute(worker) - - def test_cimultidict_items(self): - def worker(): - d = _multidict.CIMultiDict({"a": 1, "b": 2}) - list(d.items()) - self.execute(worker) - - -@pytest.mark.c_extension -class TestMultiDictProxyLeaks(MultiDictLeakTests): - def test_proxy_to_dict(self): - d = _multidict.MultiDict([("a", 1), ("b", 2)]) - p = _multidict.MultiDictProxy(d) - def worker(): - p.to_dict() - self.execute(worker) - - def test_proxy_create(self): - d = _multidict.MultiDict({"a": 1}) - def worker(): - _multidict.MultiDictProxy(d) - self.execute(worker) - - def test_proxy_access(self): - d = _multidict.MultiDict({"a": 1}) - p = _multidict.MultiDictProxy(d) - def worker(): - _ = p["a"] - self.execute(worker) - - def test_proxy_getall(self): - d = _multidict.MultiDict([("a", 1), ("a", 2)]) - p = _multidict.MultiDictProxy(d) - def worker(): - p.getall("a") - self.execute(worker) - - def test_proxy_getone(self): - d = _multidict.MultiDict([("key", "val1"), ("key", "val2")]) - p = _multidict.MultiDictProxy(d) - def worker(): - p.getone("key") - self.execute(worker) - - def test_proxy_get(self): - d = _multidict.MultiDict({"key": "value"}) - p = _multidict.MultiDictProxy(d) - def worker(): - p.get("key") - p.get("missing", "default") - self.execute(worker) - - def test_proxy_keys(self): - d = _multidict.MultiDict({"a": 1, "b": 2}) - p = _multidict.MultiDictProxy(d) - def worker(): - list(p.keys()) - self.execute(worker) - - def test_proxy_values(self): - d = _multidict.MultiDict({"a": 1, "b": 2}) - p = _multidict.MultiDictProxy(d) - def worker(): - list(p.values()) - self.execute(worker) - - def test_proxy_items(self): - d = _multidict.MultiDict({"a": 1, "b": 2}) - p = _multidict.MultiDictProxy(d) - def worker(): - list(p.items()) - self.execute(worker) - - def test_proxy_copy(self): - d = _multidict.MultiDict({"key": "value"}) - p = _multidict.MultiDictProxy(d) - def worker(): - p.copy() - self.execute(worker) - - def test_proxy_after_modify(self): - d = _multidict.MultiDict() - p = _multidict.MultiDictProxy(d) - def worker(): - d.add("key", "value") - _ = p["key"] - d.clear() - self.execute(worker) - - -@pytest.mark.c_extension -class TestCIMultiDictProxyLeaks(MultiDictLeakTests): - def test_proxy_create(self): - d = _multidict.CIMultiDict({"a": 1}) - def worker(): - _multidict.CIMultiDictProxy(d) - self.execute(worker) - - def test_proxy_access(self): - d = _multidict.CIMultiDict({"a": 1}) - p = _multidict.CIMultiDictProxy(d) - def worker(): - _ = p["a"] - self.execute(worker) - - def test_proxy_getone(self): - d = _multidict.CIMultiDict([("key", "val1"), ("key", "val2")]) - p = _multidict.CIMultiDictProxy(d) - def worker(): - p.getone("key") - self.execute(worker) - - def test_proxy_get(self): - d = _multidict.CIMultiDict({"key": "value"}) - p = _multidict.CIMultiDictProxy(d) - def worker(): - p.get("KEY") - p.get("missing", "default") - self.execute(worker) - - def test_proxy_keys(self): - d = _multidict.CIMultiDict({"a": 1, "b": 2}) - p = _multidict.CIMultiDictProxy(d) - def worker(): - list(p.keys()) - self.execute(worker) - - def test_proxy_values(self): - d = _multidict.CIMultiDict({"a": 1, "b": 2}) - p = _multidict.CIMultiDictProxy(d) - def worker(): - list(p.values()) - self.execute(worker) - - def test_proxy_items(self): - d = _multidict.CIMultiDict({"a": 1, "b": 2}) - p = _multidict.CIMultiDictProxy(d) - def worker(): - list(p.items()) - self.execute(worker) - - def test_proxy_copy(self): - d = _multidict.CIMultiDict({"key": "value"}) - p = _multidict.CIMultiDictProxy(d) - def worker(): - p.copy() - self.execute(worker) - - -@pytest.mark.c_extension -class TestIstrLeaks(MultiDictLeakTests): - def test_istr_create(self): - def worker(): - _multidict.istr("Hello") - self.execute(worker) - - def test_istr_comparison(self): - def worker(): - s1 = _multidict.istr("Hello") - s2 = _multidict.istr("hello") - _ = s1 == s2 - self.execute(worker) - - def test_istr_lower(self): - def worker(): - s = _multidict.istr("Hello") - s.lower() - self.execute(worker) - - def test_istr_upper(self): - def worker(): - s = _multidict.istr("Hello") - s.upper() - self.execute(worker) - - def test_istr_in_dict(self): - def worker(): - d = {_multidict.istr("key"): "value"} - _ = d[_multidict.istr("key")] - self.execute(worker) - - def test_istr_concatenation(self): - def worker(): - s1 = _multidict.istr("Hello") - s2 = _multidict.istr("World") - _ = s1 + " " + s2 - self.execute(worker) - - def test_istr_hash(self): - def worker(): - s = _multidict.istr("Hello") - hash(s) - self.execute(worker) From 86be9e15a9aaaa78ced7c3dda2af98204fc9bda9 Mon Sep 17 00:00:00 2001 From: "rodrigo.nogueira" Date: Sun, 25 Jan 2026 02:08:32 -0300 Subject: [PATCH 07/29] Add to_dict feature with comprehensive tests and isolated leak check --- multidict/_multidict.c | 150 ++++++++++++++-------------- tests/isolated/multidict_to_dict.py | 36 +++++++ tests/test_to_dict.py | 56 +++++++++++ 3 files changed, 167 insertions(+), 75 deletions(-) create mode 100644 tests/isolated/multidict_to_dict.py diff --git a/multidict/_multidict.c b/multidict/_multidict.c index e4689597e..2a23b289a 100644 --- a/multidict/_multidict.c +++ b/multidict/_multidict.c @@ -245,6 +245,77 @@ _multidict_proxy_copy(MultiDictProxyObject *self, PyTypeObject *type) return multidict_copy(self->md); } +PyDoc_STRVAR(multidict_to_dict_doc, + "Return a dict with lists of all values for each key."); + +static PyObject * +multidict_to_dict(MultiDictObject *self) +{ + PyObject *result = PyDict_New(); + if (result == NULL) { + return NULL; + } + + PyObject *seen = PyDict_New(); + if (seen == NULL) { + Py_DECREF(result); + return NULL; + } + + md_pos_t pos; + md_init_pos(self, &pos); + PyObject *identity = NULL; + PyObject *key = NULL; + PyObject *value = NULL; + + int tmp; + while ((tmp = md_next(self, &pos, &identity, &key, &value)) > 0) { + PyObject *first_key = PyDict_GetItem(seen, identity); + if (first_key == NULL) { + PyObject *lst = PyList_New(1); + if (lst == NULL) { + goto fail; + } + PyList_SET_ITEM(lst, 0, value); + value = NULL; + if (PyDict_SetItem(seen, identity, key) < 0) { + Py_DECREF(lst); + goto fail; + } + if (PyDict_SetItem(result, key, lst) < 0) { + Py_DECREF(lst); + goto fail; + } + Py_DECREF(lst); + } else { + PyObject *lst = PyDict_GetItem(result, first_key); + if (lst == NULL || PyList_Append(lst, value) < 0) { + goto fail; + } + Py_DECREF(value); + value = NULL; + } + Py_DECREF(identity); + Py_DECREF(key); + identity = NULL; + key = NULL; + } + if (tmp < 0) { + goto fail; + } + + Py_DECREF(seen); + return result; + +fail: + Py_XDECREF(identity); + Py_XDECREF(key); + Py_XDECREF(value); + Py_DECREF(seen); + Py_DECREF(result); + return NULL; +} + /******************** Base Methods ********************/ static inline PyObject * @@ -853,77 +924,6 @@ PyDoc_STRVAR(multidict_update_doc, PyDoc_STRVAR(multidict_merge_doc, "Merge into the dictionary, adding non-existing keys."); -PyDoc_STRVAR(multidict_to_dict_doc, - "Return a dict with lists of all values for each key."); - -static PyObject * -multidict_to_dict(MultiDictObject *self) -{ - PyObject *result = PyDict_New(); - if (result == NULL) { - return NULL; - } - - PyObject *seen = PyDict_New(); - if (seen == NULL) { - Py_DECREF(result); - return NULL; - } - - md_pos_t pos; - md_init_pos(self, &pos); - PyObject *identity = NULL; - PyObject *key = NULL; - PyObject *value = NULL; - - int tmp; - while ((tmp = md_next(self, &pos, &identity, &key, &value)) > 0) { - PyObject *first_key = PyDict_GetItem(seen, identity); - if (first_key == NULL) { - PyObject *lst = PyList_New(1); - if (lst == NULL) { - goto fail; - } - PyList_SET_ITEM(lst, 0, value); - value = NULL; - if (PyDict_SetItem(seen, identity, key) < 0) { - Py_DECREF(lst); - goto fail; - } - if (PyDict_SetItem(result, key, lst) < 0) { - Py_DECREF(lst); - goto fail; - } - Py_DECREF(lst); - } else { - PyObject *lst = PyDict_GetItem(result, first_key); - if (lst == NULL || PyList_Append(lst, value) < 0) { - goto fail; - } - Py_DECREF(value); - value = NULL; - } - Py_DECREF(identity); - Py_DECREF(key); - identity = NULL; - key = NULL; - } - if (tmp < 0) { - goto fail; - } - - Py_DECREF(seen); - return result; - -fail: - Py_XDECREF(identity); - Py_XDECREF(key); - Py_XDECREF(value); - Py_DECREF(seen); - Py_DECREF(result); - return NULL; -} - PyDoc_STRVAR(sizeof__doc__, "D.__sizeof__() -> size of D in memory, in bytes"); @@ -959,6 +959,10 @@ static PyMethodDef multidict_methods[] = { METH_FASTCALL | METH_KEYWORDS, multidict_add_doc}, {"copy", (PyCFunction)multidict_copy, METH_NOARGS, multidict_copy_doc}, + {"to_dict", + (PyCFunction)multidict_to_dict, + METH_NOARGS, + multidict_to_dict_doc}, {"extend", (PyCFunction)multidict_extend, METH_VARARGS | METH_KEYWORDS, @@ -1008,10 +1012,6 @@ static PyMethodDef multidict_methods[] = { METH_NOARGS, sizeof__doc__, }, - {"to_dict", - (PyCFunction)multidict_to_dict, - METH_NOARGS, - multidict_to_dict_doc}, {NULL, NULL} /* sentinel */ }; diff --git a/tests/isolated/multidict_to_dict.py b/tests/isolated/multidict_to_dict.py new file mode 100644 index 000000000..39fa6d68c --- /dev/null +++ b/tests/isolated/multidict_to_dict.py @@ -0,0 +1,36 @@ +"""Memory leak test for to_dict().""" +import gc +import tracemalloc + +from multidict import MultiDict + + +def get_mem() -> int: + gc.collect() + gc.collect() + gc.collect() + return tracemalloc.get_traced_memory()[0] + + +def test_to_dict_leak() -> None: + tracemalloc.start() + for _ in range(100): + d = MultiDict([("a", 1), ("b", 2)]) + d.to_dict() + get_mem() + + mem_before = get_mem() + for _ in range(100_000): + d = MultiDict([("a", 1), ("b", 2)]) + d.to_dict() + mem_after = get_mem() + + tracemalloc.stop() + + growth = mem_after - mem_before + assert growth < 100_000, f"Memory grew by {growth} bytes, possible leak" + + +if __name__ == "__main__": + test_to_dict_leak() + print("PASSED: No memory leak detected in to_dict()") diff --git a/tests/test_to_dict.py b/tests/test_to_dict.py index f2618b2f5..0b4bfd40b 100644 --- a/tests/test_to_dict.py +++ b/tests/test_to_dict.py @@ -3,23 +3,28 @@ class BaseToDictTests: + """Base tests for to_dict() method, inherited by all multidict type tests.""" def test_to_dict_simple(self, cls): + """Test basic conversion with unique keys.""" d = cls([("a", 1), ("b", 2)]) result = d.to_dict() assert result == {"a": [1], "b": [2]} def test_to_dict_multi_values(self, cls): + """Test grouping multiple values under the same key.""" d = cls([("a", 1), ("b", 2), ("a", 3)]) result = d.to_dict() assert result == {"a": [1, 3], "b": [2]} def test_to_dict_empty(self, cls): + """Test conversion of an empty multidict.""" d = cls() result = d.to_dict() assert result == {} def test_to_dict_returns_new_dict(self, cls): + """Test that each call returns a new dictionary instance.""" d = cls([("a", 1)]) result1 = d.to_dict() result2 = d.to_dict() @@ -27,13 +32,36 @@ def test_to_dict_returns_new_dict(self, cls): assert result1 is not result2 def test_to_dict_list_is_fresh(self, cls): + """Test that value lists are independent between calls.""" d = cls([("a", 1)]) result1 = d.to_dict() result2 = d.to_dict() assert result1["a"] is not result2["a"] + def test_to_dict_order_preservation(self, cls): + """Test that value lists maintain insertion order.""" + d = cls([("x", 3), ("x", 1), ("x", 2)]) + result = d.to_dict() + assert result["x"] == [3, 1, 2] + + def test_to_dict_large_data(self, cls): + """Test to_dict with a large number of entries for performance.""" + items = [(f"key{i % 100}", i) for i in range(10000)] + d = cls(items) + result = d.to_dict() + assert len(result) == 100 + assert all(len(v) == 100 for v in result.values()) + + def test_to_dict_mixed_value_types(self, cls): + """Test to_dict with mixed value types (str, int) to verify generic _V.""" + d = cls([("a", 1), ("a", "two"), ("b", 3.14)]) + result = d.to_dict() + assert result["a"] == [1, "two"] + assert result["b"] == [3.14] + class TestMultiDictToDict(BaseToDictTests): + """Tests for MultiDict.to_dict().""" @pytest.fixture def cls(self, multidict_module): @@ -41,12 +69,14 @@ def cls(self, multidict_module): class TestCIMultiDictToDict(BaseToDictTests): + """Tests for CIMultiDict.to_dict().""" @pytest.fixture def cls(self, multidict_module): return multidict_module.CIMultiDict def test_to_dict_case_insensitive_grouping(self, cls): + """Test that case variants are grouped under the same key.""" d = cls([("A", 1), ("a", 2), ("B", 3)]) result = d.to_dict() assert len(result) == 2 @@ -59,6 +89,7 @@ def test_to_dict_case_insensitive_grouping(self, cls): class TestMultiDictProxyToDict(BaseToDictTests): + """Tests for MultiDictProxy.to_dict().""" @pytest.fixture def cls(self, multidict_module): @@ -67,8 +98,17 @@ def make_proxy(*args, **kwargs): return multidict_module.MultiDictProxy(md) return make_proxy + def test_to_dict_proxy_mutation_isolation(self, cls, multidict_module): + """Test that modifying returned dict does not affect the proxy.""" + md = multidict_module.MultiDict([("a", 1)]) + proxy = multidict_module.MultiDictProxy(md) + result = proxy.to_dict() + result["a"].append(999) + assert proxy.getall("a") == [1] + class TestCIMultiDictProxyToDict(BaseToDictTests): + """Tests for CIMultiDictProxy.to_dict().""" @pytest.fixture def cls(self, multidict_module): @@ -78,6 +118,22 @@ def make_proxy(*args, **kwargs): return make_proxy def test_to_dict_case_insensitive_grouping(self, cls): + """Test that case variants are grouped under the same key.""" d = cls([("A", 1), ("a", 2), ("B", 3)]) result = d.to_dict() assert len(result) == 2 + assert "A" in result or "a" in result + assert "B" in result or "b" in result + key_a = "A" if "A" in result else "a" + key_b = "B" if "B" in result else "b" + assert result[key_a] == [1, 2] + assert result[key_b] == [3] + + def test_to_dict_proxy_mutation_isolation(self, cls, multidict_module): + """Test that modifying returned dict does not affect the proxy.""" + md = multidict_module.CIMultiDict([("a", 1)]) + proxy = multidict_module.CIMultiDictProxy(md) + result = proxy.to_dict() + result["a"].append(999) + assert proxy.getall("a") == [1] + From d064ebed1b75332cf285d2f73c314253acc33f32 Mon Sep 17 00:00:00 2001 From: "rodrigo.nogueira" Date: Sun, 25 Jan 2026 02:14:17 -0300 Subject: [PATCH 08/29] Integrate isolated leak test for to_dict into CI suite --- tests/isolated/multidict_to_dict.py | 4 ++-- tests/test_leaks.py | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/isolated/multidict_to_dict.py b/tests/isolated/multidict_to_dict.py index 39fa6d68c..8d61d57e7 100644 --- a/tests/isolated/multidict_to_dict.py +++ b/tests/isolated/multidict_to_dict.py @@ -20,7 +20,7 @@ def test_to_dict_leak() -> None: get_mem() mem_before = get_mem() - for _ in range(100_000): + for _ in range(1_000_000): d = MultiDict([("a", 1), ("b", 2)]) d.to_dict() mem_after = get_mem() @@ -28,7 +28,7 @@ def test_to_dict_leak() -> None: tracemalloc.stop() growth = mem_after - mem_before - assert growth < 100_000, f"Memory grew by {growth} bytes, possible leak" + assert growth < 50_000, f"Memory grew by {growth} bytes, possible leak" if __name__ == "__main__": diff --git a/tests/test_leaks.py b/tests/test_leaks.py index 24493b87c..1517c9768 100644 --- a/tests/test_leaks.py +++ b/tests/test_leaks.py @@ -17,6 +17,7 @@ "multidict_update_multidict.py", "multidict_pop.py", "multidict_pop_missing.py", + "multidict_to_dict.py", ), ) @pytest.mark.leaks From 77b606da785b302ffaf704890cd350473ebc85ee Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sun, 25 Jan 2026 05:32:25 +0000 Subject: [PATCH 09/29] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- multidict/_multidict_py.py | 1 - tests/isolated/multidict_pop.py | 2 ++ tests/isolated/multidict_pop_missing.py | 4 +--- tests/isolated/multidict_to_dict.py | 1 + tests/test_to_dict.py | 4 +++- 5 files changed, 7 insertions(+), 5 deletions(-) diff --git a/multidict/_multidict_py.py b/multidict/_multidict_py.py index faf1111e3..ce9ec1694 100644 --- a/multidict/_multidict_py.py +++ b/multidict/_multidict_py.py @@ -785,7 +785,6 @@ def to_dict(self) -> dict[str, list[_V]]: result[first_key].append(e.value) return result - def add(self, key: str, value: _V) -> None: identity = self._identity(key) hash_ = hash(identity) diff --git a/tests/isolated/multidict_pop.py b/tests/isolated/multidict_pop.py index 72b741279..f5dde495a 100644 --- a/tests/isolated/multidict_pop.py +++ b/tests/isolated/multidict_pop.py @@ -61,6 +61,7 @@ def _test_popone() -> None: result.popone(k) check_for_leak() + def _test_pop_with_default() -> None: result = MultiDict() # XXX: mypy wants an annotation so the only @@ -70,6 +71,7 @@ def _test_pop_with_default() -> None: result.pop(f"missing_key_{i}", None) check_for_leak() + def _test_del() -> None: for _ in range(10): for _ in range(100): diff --git a/tests/isolated/multidict_pop_missing.py b/tests/isolated/multidict_pop_missing.py index 71e91279e..6e292420f 100644 --- a/tests/isolated/multidict_pop_missing.py +++ b/tests/isolated/multidict_pop_missing.py @@ -1,4 +1,3 @@ - import gc import psutil import os @@ -29,7 +28,6 @@ def check_for_leak() -> None: assert usage < 50, f"Memory leaked at: {usage} MB" - def _test_pop_missing(cls: type[MultiDict[str] | CIMultiDict[str]], count: int) -> None: # Use dynamic keys for missing checks to ensure unique objects # if there is a ref leak on identity. @@ -47,7 +45,7 @@ def _run_isolated_case() -> None: # Warmup _test_pop_missing(MultiDict, max(100, 10)) check_for_leak() - + # Run loop for _ in range(20): _test_pop_missing(MultiDict, 1000) diff --git a/tests/isolated/multidict_to_dict.py b/tests/isolated/multidict_to_dict.py index 8d61d57e7..705b51f24 100644 --- a/tests/isolated/multidict_to_dict.py +++ b/tests/isolated/multidict_to_dict.py @@ -1,4 +1,5 @@ """Memory leak test for to_dict().""" + import gc import tracemalloc diff --git a/tests/test_to_dict.py b/tests/test_to_dict.py index 0b4bfd40b..b57eadb56 100644 --- a/tests/test_to_dict.py +++ b/tests/test_to_dict.py @@ -1,4 +1,5 @@ """Test to_dict functionality for all multidict types.""" + import pytest @@ -96,6 +97,7 @@ def cls(self, multidict_module): def make_proxy(*args, **kwargs): md = multidict_module.MultiDict(*args, **kwargs) return multidict_module.MultiDictProxy(md) + return make_proxy def test_to_dict_proxy_mutation_isolation(self, cls, multidict_module): @@ -115,6 +117,7 @@ def cls(self, multidict_module): def make_proxy(*args, **kwargs): md = multidict_module.CIMultiDict(*args, **kwargs) return multidict_module.CIMultiDictProxy(md) + return make_proxy def test_to_dict_case_insensitive_grouping(self, cls): @@ -136,4 +139,3 @@ def test_to_dict_proxy_mutation_isolation(self, cls, multidict_module): result = proxy.to_dict() result["a"].append(999) assert proxy.getall("a") == [1] - From 4455db9056511593eb0d49c22feb2a9e67a3a067 Mon Sep 17 00:00:00 2001 From: "rodrigo.nogueira" Date: Sun, 25 Jan 2026 02:38:09 -0300 Subject: [PATCH 10/29] Revert unrelated changes to hashtable.h and pop tests --- multidict/_multilib/hashtable.h | 1 - tests/isolated/multidict_pop.py | 11 ----- tests/isolated/multidict_pop_missing.py | 57 ------------------------- tests/test_leaks.py | 2 - 4 files changed, 71 deletions(-) delete mode 100644 tests/isolated/multidict_pop_missing.py diff --git a/multidict/_multilib/hashtable.h b/multidict/_multilib/hashtable.h index 4a132b4b2..82019f9eb 100644 --- a/multidict/_multilib/hashtable.h +++ b/multidict/_multilib/hashtable.h @@ -1020,7 +1020,6 @@ md_pop_one(MultiDictObject *md, PyObject *key, PyObject **ret) } } - Py_DECREF(identity); ASSERT_CONSISTENT(md, false); return 0; fail: diff --git a/tests/isolated/multidict_pop.py b/tests/isolated/multidict_pop.py index f5dde495a..89fcce11c 100644 --- a/tests/isolated/multidict_pop.py +++ b/tests/isolated/multidict_pop.py @@ -62,16 +62,6 @@ def _test_popone() -> None: check_for_leak() -def _test_pop_with_default() -> None: - result = MultiDict() - # XXX: mypy wants an annotation so the only - # thing we can do here is pass the headers along. - result = MultiDict(headers) - for i in range(1_000_000): - result.pop(f"missing_key_{i}", None) - check_for_leak() - - def _test_del() -> None: for _ in range(10): for _ in range(100): @@ -86,7 +76,6 @@ def _run_isolated_case() -> None: _test_popall() _test_popone() _test_del() - _test_pop_with_default() if __name__ == "__main__": diff --git a/tests/isolated/multidict_pop_missing.py b/tests/isolated/multidict_pop_missing.py deleted file mode 100644 index 6e292420f..000000000 --- a/tests/isolated/multidict_pop_missing.py +++ /dev/null @@ -1,57 +0,0 @@ -import gc -import psutil -import os -from multidict import MultiDict, CIMultiDict - - -def trim_ram() -> None: - """Forces python garbage collection.""" - gc.collect() - - -process = psutil.Process(os.getpid()) - - -def get_memory_usage() -> float: - memory_info = process.memory_info() - return memory_info.rss / (1024 * 1024) - - -initial_memory_usage = get_memory_usage() - - -def check_for_leak() -> None: - trim_ram() - usage = get_memory_usage() - initial_memory_usage - # Threshold might need tuning, but 50MB is generous for "no leak" - # With leak it grows unboundedly. - assert usage < 50, f"Memory leaked at: {usage} MB" - - -def _test_pop_missing(cls: type[MultiDict[str] | CIMultiDict[str]], count: int) -> None: - # Use dynamic keys for missing checks to ensure unique objects - # if there is a ref leak on identity. - d = cls() - for j in range(count): - key = f"MISSING_{j}" - try: - d.pop(key) - except KeyError: - pass - d.pop(key, None) - - -def _run_isolated_case() -> None: - # Warmup - _test_pop_missing(MultiDict, max(100, 10)) - check_for_leak() - - # Run loop - for _ in range(20): - _test_pop_missing(MultiDict, 1000) - _test_pop_missing(CIMultiDict, 1000) - check_for_leak() - - -if __name__ == "__main__": - _run_isolated_case() diff --git a/tests/test_leaks.py b/tests/test_leaks.py index 1517c9768..8c04dc93e 100644 --- a/tests/test_leaks.py +++ b/tests/test_leaks.py @@ -15,8 +15,6 @@ "multidict_extend_multidict.py", "multidict_extend_tuple.py", "multidict_update_multidict.py", - "multidict_pop.py", - "multidict_pop_missing.py", "multidict_to_dict.py", ), ) From 8d46e651c65ff9500e2739888b7c7ec4210d49cb Mon Sep 17 00:00:00 2001 From: "rodrigo.nogueira" Date: Sun, 25 Jan 2026 02:47:26 -0300 Subject: [PATCH 11/29] Restore multidict_pop.py to leak tests (was accidentally removed) --- tests/test_leaks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_leaks.py b/tests/test_leaks.py index 8c04dc93e..b2259f2f4 100644 --- a/tests/test_leaks.py +++ b/tests/test_leaks.py @@ -14,7 +14,7 @@ "multidict_extend_dict.py", "multidict_extend_multidict.py", "multidict_extend_tuple.py", - "multidict_update_multidict.py", + "multidict_pop.py", "multidict_to_dict.py", ), ) From ed6d373d4c94071a4b33b578b808e49004e1f045 Mon Sep 17 00:00:00 2001 From: "rodrigo.nogueira" Date: Sun, 25 Jan 2026 02:53:05 -0300 Subject: [PATCH 12/29] Fix CI linting (mypy/clang-format) and restore update leak test --- tests/test_leaks.py | 1 + tests/test_to_dict.py | 49 ++++++++++++++++++++++--------------------- 2 files changed, 26 insertions(+), 24 deletions(-) diff --git a/tests/test_leaks.py b/tests/test_leaks.py index b2259f2f4..29949e76e 100644 --- a/tests/test_leaks.py +++ b/tests/test_leaks.py @@ -14,6 +14,7 @@ "multidict_extend_dict.py", "multidict_extend_multidict.py", "multidict_extend_tuple.py", + "multidict_update_multidict.py", "multidict_pop.py", "multidict_to_dict.py", ), diff --git a/tests/test_to_dict.py b/tests/test_to_dict.py index b57eadb56..c2d704a12 100644 --- a/tests/test_to_dict.py +++ b/tests/test_to_dict.py @@ -1,30 +1,33 @@ """Test to_dict functionality for all multidict types.""" +from typing import Any, Type import pytest +from multidict import CIMultiDict, CIMultiDictProxy, MultiDict, MultiDictProxy + class BaseToDictTests: """Base tests for to_dict() method, inherited by all multidict type tests.""" - def test_to_dict_simple(self, cls): + def test_to_dict_simple(self, cls: Any) -> None: """Test basic conversion with unique keys.""" d = cls([("a", 1), ("b", 2)]) result = d.to_dict() assert result == {"a": [1], "b": [2]} - def test_to_dict_multi_values(self, cls): + def test_to_dict_multi_values(self, cls: Any) -> None: """Test grouping multiple values under the same key.""" d = cls([("a", 1), ("b", 2), ("a", 3)]) result = d.to_dict() assert result == {"a": [1, 3], "b": [2]} - def test_to_dict_empty(self, cls): + def test_to_dict_empty(self, cls: Any) -> None: """Test conversion of an empty multidict.""" d = cls() result = d.to_dict() assert result == {} - def test_to_dict_returns_new_dict(self, cls): + def test_to_dict_returns_new_dict(self, cls: Any) -> None: """Test that each call returns a new dictionary instance.""" d = cls([("a", 1)]) result1 = d.to_dict() @@ -32,20 +35,20 @@ def test_to_dict_returns_new_dict(self, cls): assert result1 == result2 assert result1 is not result2 - def test_to_dict_list_is_fresh(self, cls): + def test_to_dict_list_is_fresh(self, cls: Any) -> None: """Test that value lists are independent between calls.""" d = cls([("a", 1)]) result1 = d.to_dict() result2 = d.to_dict() assert result1["a"] is not result2["a"] - def test_to_dict_order_preservation(self, cls): + def test_to_dict_order_preservation(self, cls: Any) -> None: """Test that value lists maintain insertion order.""" d = cls([("x", 3), ("x", 1), ("x", 2)]) result = d.to_dict() assert result["x"] == [3, 1, 2] - def test_to_dict_large_data(self, cls): + def test_to_dict_large_data(self, cls: Any) -> None: """Test to_dict with a large number of entries for performance.""" items = [(f"key{i % 100}", i) for i in range(10000)] d = cls(items) @@ -53,7 +56,7 @@ def test_to_dict_large_data(self, cls): assert len(result) == 100 assert all(len(v) == 100 for v in result.values()) - def test_to_dict_mixed_value_types(self, cls): + def test_to_dict_mixed_value_types(self, cls: Any) -> None: """Test to_dict with mixed value types (str, int) to verify generic _V.""" d = cls([("a", 1), ("a", "two"), ("b", 3.14)]) result = d.to_dict() @@ -65,18 +68,18 @@ class TestMultiDictToDict(BaseToDictTests): """Tests for MultiDict.to_dict().""" @pytest.fixture - def cls(self, multidict_module): - return multidict_module.MultiDict + def cls(self, multidict_module: Any) -> Type[MultiDict[Any]]: + return multidict_module.MultiDict # type: ignore[no-any-return] class TestCIMultiDictToDict(BaseToDictTests): """Tests for CIMultiDict.to_dict().""" @pytest.fixture - def cls(self, multidict_module): - return multidict_module.CIMultiDict + def cls(self, multidict_module: Any) -> Type[CIMultiDict[Any]]: + return multidict_module.CIMultiDict # type: ignore[no-any-return] - def test_to_dict_case_insensitive_grouping(self, cls): + def test_to_dict_case_insensitive_grouping(self, cls: Any) -> None: """Test that case variants are grouped under the same key.""" d = cls([("A", 1), ("a", 2), ("B", 3)]) result = d.to_dict() @@ -93,14 +96,13 @@ class TestMultiDictProxyToDict(BaseToDictTests): """Tests for MultiDictProxy.to_dict().""" @pytest.fixture - def cls(self, multidict_module): - def make_proxy(*args, **kwargs): + def cls(self, multidict_module: Any) -> Any: + def make_proxy(*args: Any, **kwargs: Any) -> MultiDictProxy[Any]: md = multidict_module.MultiDict(*args, **kwargs) - return multidict_module.MultiDictProxy(md) - + return multidict_module.MultiDictProxy(md) # type: ignore[no-any-return] return make_proxy - def test_to_dict_proxy_mutation_isolation(self, cls, multidict_module): + def test_to_dict_proxy_mutation_isolation(self, cls: Any, multidict_module: Any) -> None: """Test that modifying returned dict does not affect the proxy.""" md = multidict_module.MultiDict([("a", 1)]) proxy = multidict_module.MultiDictProxy(md) @@ -113,14 +115,13 @@ class TestCIMultiDictProxyToDict(BaseToDictTests): """Tests for CIMultiDictProxy.to_dict().""" @pytest.fixture - def cls(self, multidict_module): - def make_proxy(*args, **kwargs): + def cls(self, multidict_module: Any) -> Any: + def make_proxy(*args: Any, **kwargs: Any) -> CIMultiDictProxy[Any]: md = multidict_module.CIMultiDict(*args, **kwargs) - return multidict_module.CIMultiDictProxy(md) - + return multidict_module.CIMultiDictProxy(md) # type: ignore[no-any-return] return make_proxy - def test_to_dict_case_insensitive_grouping(self, cls): + def test_to_dict_case_insensitive_grouping(self, cls: Any) -> None: """Test that case variants are grouped under the same key.""" d = cls([("A", 1), ("a", 2), ("B", 3)]) result = d.to_dict() @@ -132,7 +133,7 @@ def test_to_dict_case_insensitive_grouping(self, cls): assert result[key_a] == [1, 2] assert result[key_b] == [3] - def test_to_dict_proxy_mutation_isolation(self, cls, multidict_module): + def test_to_dict_proxy_mutation_isolation(self, cls: Any, multidict_module: Any) -> None: """Test that modifying returned dict does not affect the proxy.""" md = multidict_module.CIMultiDict([("a", 1)]) proxy = multidict_module.CIMultiDictProxy(md) From 42e360596a6cd4abb3bc092a9ecfd9ce44d83654 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sun, 25 Jan 2026 05:55:11 +0000 Subject: [PATCH 13/29] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- tests/test_to_dict.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/tests/test_to_dict.py b/tests/test_to_dict.py index c2d704a12..7525ece86 100644 --- a/tests/test_to_dict.py +++ b/tests/test_to_dict.py @@ -1,4 +1,5 @@ """Test to_dict functionality for all multidict types.""" + from typing import Any, Type import pytest @@ -100,9 +101,12 @@ def cls(self, multidict_module: Any) -> Any: def make_proxy(*args: Any, **kwargs: Any) -> MultiDictProxy[Any]: md = multidict_module.MultiDict(*args, **kwargs) return multidict_module.MultiDictProxy(md) # type: ignore[no-any-return] + return make_proxy - def test_to_dict_proxy_mutation_isolation(self, cls: Any, multidict_module: Any) -> None: + def test_to_dict_proxy_mutation_isolation( + self, cls: Any, multidict_module: Any + ) -> None: """Test that modifying returned dict does not affect the proxy.""" md = multidict_module.MultiDict([("a", 1)]) proxy = multidict_module.MultiDictProxy(md) @@ -119,6 +123,7 @@ def cls(self, multidict_module: Any) -> Any: def make_proxy(*args: Any, **kwargs: Any) -> CIMultiDictProxy[Any]: md = multidict_module.CIMultiDict(*args, **kwargs) return multidict_module.CIMultiDictProxy(md) # type: ignore[no-any-return] + return make_proxy def test_to_dict_case_insensitive_grouping(self, cls: Any) -> None: @@ -133,7 +138,9 @@ def test_to_dict_case_insensitive_grouping(self, cls: Any) -> None: assert result[key_a] == [1, 2] assert result[key_b] == [3] - def test_to_dict_proxy_mutation_isolation(self, cls: Any, multidict_module: Any) -> None: + def test_to_dict_proxy_mutation_isolation( + self, cls: Any, multidict_module: Any + ) -> None: """Test that modifying returned dict does not affect the proxy.""" md = multidict_module.CIMultiDict([("a", 1)]) proxy = multidict_module.CIMultiDictProxy(md) From 20c411cb54af7b75f34d97ae8b1698fe64f799a2 Mon Sep 17 00:00:00 2001 From: "rodrigo.nogueira" Date: Sun, 25 Jan 2026 03:08:41 -0300 Subject: [PATCH 14/29] Add changelog entry and fix remaining clang-format issues --- CHANGES/783.feature | 1 + multidict/_multidict.c | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) create mode 100644 CHANGES/783.feature diff --git a/CHANGES/783.feature b/CHANGES/783.feature new file mode 100644 index 000000000..c756df555 --- /dev/null +++ b/CHANGES/783.feature @@ -0,0 +1 @@ +Added ``to_dict()`` method to export multidict contents as a standard dict with value lists. (#783) diff --git a/multidict/_multidict.c b/multidict/_multidict.c index 2a23b289a..a002494c9 100644 --- a/multidict/_multidict.c +++ b/multidict/_multidict.c @@ -926,7 +926,6 @@ PyDoc_STRVAR(multidict_merge_doc, PyDoc_STRVAR(sizeof__doc__, "D.__sizeof__() -> size of D in memory, in bytes"); - static PyObject * multidict_sizeof(MultiDictObject *self) { From 8d429f7f59517b52cce8697d7d3d7f626099019a Mon Sep 17 00:00:00 2001 From: "rodrigo.nogueira" Date: Sun, 25 Jan 2026 16:18:17 -0300 Subject: [PATCH 15/29] Fix PyPy compatibility: use psutil instead of tracemalloc --- tests/isolated/multidict_to_dict.py | 28 ++++++++++++++++------------ 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/tests/isolated/multidict_to_dict.py b/tests/isolated/multidict_to_dict.py index 705b51f24..f8bc38942 100644 --- a/tests/isolated/multidict_to_dict.py +++ b/tests/isolated/multidict_to_dict.py @@ -1,35 +1,39 @@ """Memory leak test for to_dict().""" import gc -import tracemalloc +import os +import psutil from multidict import MultiDict -def get_mem() -> int: - gc.collect() - gc.collect() +process = psutil.Process(os.getpid()) + + +def trim_ram() -> None: gc.collect() - return tracemalloc.get_traced_memory()[0] + + +def get_memory_usage() -> int: + memory_info = process.memory_info() + return memory_info.rss // (1024 * 1024) def test_to_dict_leak() -> None: - tracemalloc.start() for _ in range(100): d = MultiDict([("a", 1), ("b", 2)]) d.to_dict() - get_mem() + trim_ram() - mem_before = get_mem() + mem_before = get_memory_usage() for _ in range(1_000_000): d = MultiDict([("a", 1), ("b", 2)]) d.to_dict() - mem_after = get_mem() - - tracemalloc.stop() + trim_ram() + mem_after = get_memory_usage() growth = mem_after - mem_before - assert growth < 50_000, f"Memory grew by {growth} bytes, possible leak" + assert growth < 50, f"Memory grew by {growth} MB, possible leak" if __name__ == "__main__": From c710e1d8a5a6466eec561fc03bf5c5c098bd8198 Mon Sep 17 00:00:00 2001 From: "rodrigo.nogueira" Date: Fri, 6 Mar 2026 18:50:58 -0300 Subject: [PATCH 16/29] test: add exact type hints to to_dict tests to fix MyPy coverage --- tests/test_to_dict.py | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/tests/test_to_dict.py b/tests/test_to_dict.py index 7525ece86..a73d61f18 100644 --- a/tests/test_to_dict.py +++ b/tests/test_to_dict.py @@ -1,6 +1,7 @@ """Test to_dict functionality for all multidict types.""" -from typing import Any, Type +from typing import Any, Callable, Type +from multidict import MultiMapping import pytest @@ -10,25 +11,25 @@ class BaseToDictTests: """Base tests for to_dict() method, inherited by all multidict type tests.""" - def test_to_dict_simple(self, cls: Any) -> None: + def test_to_dict_simple(self, cls: Callable[..., MultiMapping[Any]]) -> None: """Test basic conversion with unique keys.""" d = cls([("a", 1), ("b", 2)]) result = d.to_dict() assert result == {"a": [1], "b": [2]} - def test_to_dict_multi_values(self, cls: Any) -> None: + def test_to_dict_multi_values(self, cls: Callable[..., MultiMapping[Any]]) -> None: """Test grouping multiple values under the same key.""" d = cls([("a", 1), ("b", 2), ("a", 3)]) result = d.to_dict() assert result == {"a": [1, 3], "b": [2]} - def test_to_dict_empty(self, cls: Any) -> None: + def test_to_dict_empty(self, cls: Callable[..., MultiMapping[Any]]) -> None: """Test conversion of an empty multidict.""" d = cls() result = d.to_dict() assert result == {} - def test_to_dict_returns_new_dict(self, cls: Any) -> None: + def test_to_dict_returns_new_dict(self, cls: Callable[..., MultiMapping[Any]]) -> None: """Test that each call returns a new dictionary instance.""" d = cls([("a", 1)]) result1 = d.to_dict() @@ -36,20 +37,20 @@ def test_to_dict_returns_new_dict(self, cls: Any) -> None: assert result1 == result2 assert result1 is not result2 - def test_to_dict_list_is_fresh(self, cls: Any) -> None: + def test_to_dict_list_is_fresh(self, cls: Callable[..., MultiMapping[Any]]) -> None: """Test that value lists are independent between calls.""" d = cls([("a", 1)]) result1 = d.to_dict() result2 = d.to_dict() assert result1["a"] is not result2["a"] - def test_to_dict_order_preservation(self, cls: Any) -> None: + def test_to_dict_order_preservation(self, cls: Callable[..., MultiMapping[Any]]) -> None: """Test that value lists maintain insertion order.""" d = cls([("x", 3), ("x", 1), ("x", 2)]) result = d.to_dict() assert result["x"] == [3, 1, 2] - def test_to_dict_large_data(self, cls: Any) -> None: + def test_to_dict_large_data(self, cls: Callable[..., MultiMapping[Any]]) -> None: """Test to_dict with a large number of entries for performance.""" items = [(f"key{i % 100}", i) for i in range(10000)] d = cls(items) @@ -57,7 +58,7 @@ def test_to_dict_large_data(self, cls: Any) -> None: assert len(result) == 100 assert all(len(v) == 100 for v in result.values()) - def test_to_dict_mixed_value_types(self, cls: Any) -> None: + def test_to_dict_mixed_value_types(self, cls: Callable[..., MultiMapping[Any]]) -> None: """Test to_dict with mixed value types (str, int) to verify generic _V.""" d = cls([("a", 1), ("a", "two"), ("b", 3.14)]) result = d.to_dict() @@ -80,7 +81,7 @@ class TestCIMultiDictToDict(BaseToDictTests): def cls(self, multidict_module: Any) -> Type[CIMultiDict[Any]]: return multidict_module.CIMultiDict # type: ignore[no-any-return] - def test_to_dict_case_insensitive_grouping(self, cls: Any) -> None: + def test_to_dict_case_insensitive_grouping(self, cls: Callable[..., MultiMapping[Any]]) -> None: """Test that case variants are grouped under the same key.""" d = cls([("A", 1), ("a", 2), ("B", 3)]) result = d.to_dict() @@ -105,7 +106,7 @@ def make_proxy(*args: Any, **kwargs: Any) -> MultiDictProxy[Any]: return make_proxy def test_to_dict_proxy_mutation_isolation( - self, cls: Any, multidict_module: Any + self, cls: Callable[..., MultiMapping[Any]], multidict_module: Any ) -> None: """Test that modifying returned dict does not affect the proxy.""" md = multidict_module.MultiDict([("a", 1)]) @@ -126,7 +127,7 @@ def make_proxy(*args: Any, **kwargs: Any) -> CIMultiDictProxy[Any]: return make_proxy - def test_to_dict_case_insensitive_grouping(self, cls: Any) -> None: + def test_to_dict_case_insensitive_grouping(self, cls: Callable[..., MultiMapping[Any]]) -> None: """Test that case variants are grouped under the same key.""" d = cls([("A", 1), ("a", 2), ("B", 3)]) result = d.to_dict() @@ -139,7 +140,7 @@ def test_to_dict_case_insensitive_grouping(self, cls: Any) -> None: assert result[key_b] == [3] def test_to_dict_proxy_mutation_isolation( - self, cls: Any, multidict_module: Any + self, cls: Callable[..., MultiMapping[Any]], multidict_module: Any ) -> None: """Test that modifying returned dict does not affect the proxy.""" md = multidict_module.CIMultiDict([("a", 1)]) From 06e097dddd5a8765d994c6521df949f71d871f89 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 6 Mar 2026 21:51:34 +0000 Subject: [PATCH 17/29] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- tests/test_to_dict.py | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/tests/test_to_dict.py b/tests/test_to_dict.py index a73d61f18..4d2eeb6b0 100644 --- a/tests/test_to_dict.py +++ b/tests/test_to_dict.py @@ -29,7 +29,9 @@ def test_to_dict_empty(self, cls: Callable[..., MultiMapping[Any]]) -> None: result = d.to_dict() assert result == {} - def test_to_dict_returns_new_dict(self, cls: Callable[..., MultiMapping[Any]]) -> None: + def test_to_dict_returns_new_dict( + self, cls: Callable[..., MultiMapping[Any]] + ) -> None: """Test that each call returns a new dictionary instance.""" d = cls([("a", 1)]) result1 = d.to_dict() @@ -44,7 +46,9 @@ def test_to_dict_list_is_fresh(self, cls: Callable[..., MultiMapping[Any]]) -> N result2 = d.to_dict() assert result1["a"] is not result2["a"] - def test_to_dict_order_preservation(self, cls: Callable[..., MultiMapping[Any]]) -> None: + def test_to_dict_order_preservation( + self, cls: Callable[..., MultiMapping[Any]] + ) -> None: """Test that value lists maintain insertion order.""" d = cls([("x", 3), ("x", 1), ("x", 2)]) result = d.to_dict() @@ -58,7 +62,9 @@ def test_to_dict_large_data(self, cls: Callable[..., MultiMapping[Any]]) -> None assert len(result) == 100 assert all(len(v) == 100 for v in result.values()) - def test_to_dict_mixed_value_types(self, cls: Callable[..., MultiMapping[Any]]) -> None: + def test_to_dict_mixed_value_types( + self, cls: Callable[..., MultiMapping[Any]] + ) -> None: """Test to_dict with mixed value types (str, int) to verify generic _V.""" d = cls([("a", 1), ("a", "two"), ("b", 3.14)]) result = d.to_dict() @@ -81,7 +87,9 @@ class TestCIMultiDictToDict(BaseToDictTests): def cls(self, multidict_module: Any) -> Type[CIMultiDict[Any]]: return multidict_module.CIMultiDict # type: ignore[no-any-return] - def test_to_dict_case_insensitive_grouping(self, cls: Callable[..., MultiMapping[Any]]) -> None: + def test_to_dict_case_insensitive_grouping( + self, cls: Callable[..., MultiMapping[Any]] + ) -> None: """Test that case variants are grouped under the same key.""" d = cls([("A", 1), ("a", 2), ("B", 3)]) result = d.to_dict() @@ -127,7 +135,9 @@ def make_proxy(*args: Any, **kwargs: Any) -> CIMultiDictProxy[Any]: return make_proxy - def test_to_dict_case_insensitive_grouping(self, cls: Callable[..., MultiMapping[Any]]) -> None: + def test_to_dict_case_insensitive_grouping( + self, cls: Callable[..., MultiMapping[Any]] + ) -> None: """Test that case variants are grouped under the same key.""" d = cls([("A", 1), ("a", 2), ("B", 3)]) result = d.to_dict() From 9b704c8dc91c24b8479627b411d0ca6269d3118e Mon Sep 17 00:00:00 2001 From: "rodrigo.nogueira" Date: Fri, 6 Mar 2026 19:53:29 -0300 Subject: [PATCH 18/29] tests: strict type annotations for test_to_dict for 100% coveralls precision --- tests/test_to_dict.py | 74 ++++++++++++++++++++++++++----------------- 1 file changed, 45 insertions(+), 29 deletions(-) diff --git a/tests/test_to_dict.py b/tests/test_to_dict.py index 4d2eeb6b0..1450d79d6 100644 --- a/tests/test_to_dict.py +++ b/tests/test_to_dict.py @@ -1,8 +1,24 @@ """Test to_dict functionality for all multidict types.""" -from typing import Any, Callable, Type +from typing import Any, Callable, Type, Iterable from multidict import MultiMapping + +from typing import Protocol + + +from typing import Protocol, Type +from multidict import MultiDict, CIMultiDict, MultiDictProxy, CIMultiDictProxy + +class MultidictModule(Protocol): + MultiDict: Type[MultiDict[object]] + CIMultiDict: Type[CIMultiDict[object]] + MultiDictProxy: Type[MultiDictProxy[object]] + CIMultiDictProxy: Type[CIMultiDictProxy[object]] + +class DictFactory(Protocol): + def __call__(self, arg: Iterable[tuple[str, object]] | None = None) -> MultiMapping[object]: ... + import pytest from multidict import CIMultiDict, CIMultiDictProxy, MultiDict, MultiDictProxy @@ -11,26 +27,26 @@ class BaseToDictTests: """Base tests for to_dict() method, inherited by all multidict type tests.""" - def test_to_dict_simple(self, cls: Callable[..., MultiMapping[Any]]) -> None: + def test_to_dict_simple(self, cls: DictFactory) -> None: """Test basic conversion with unique keys.""" d = cls([("a", 1), ("b", 2)]) result = d.to_dict() assert result == {"a": [1], "b": [2]} - def test_to_dict_multi_values(self, cls: Callable[..., MultiMapping[Any]]) -> None: + def test_to_dict_multi_values(self, cls: DictFactory) -> None: """Test grouping multiple values under the same key.""" d = cls([("a", 1), ("b", 2), ("a", 3)]) result = d.to_dict() assert result == {"a": [1, 3], "b": [2]} - def test_to_dict_empty(self, cls: Callable[..., MultiMapping[Any]]) -> None: + def test_to_dict_empty(self, cls: DictFactory) -> None: """Test conversion of an empty multidict.""" d = cls() result = d.to_dict() assert result == {} def test_to_dict_returns_new_dict( - self, cls: Callable[..., MultiMapping[Any]] + self, cls: DictFactory ) -> None: """Test that each call returns a new dictionary instance.""" d = cls([("a", 1)]) @@ -39,7 +55,7 @@ def test_to_dict_returns_new_dict( assert result1 == result2 assert result1 is not result2 - def test_to_dict_list_is_fresh(self, cls: Callable[..., MultiMapping[Any]]) -> None: + def test_to_dict_list_is_fresh(self, cls: DictFactory) -> None: """Test that value lists are independent between calls.""" d = cls([("a", 1)]) result1 = d.to_dict() @@ -47,14 +63,14 @@ def test_to_dict_list_is_fresh(self, cls: Callable[..., MultiMapping[Any]]) -> N assert result1["a"] is not result2["a"] def test_to_dict_order_preservation( - self, cls: Callable[..., MultiMapping[Any]] + self, cls: DictFactory ) -> None: """Test that value lists maintain insertion order.""" d = cls([("x", 3), ("x", 1), ("x", 2)]) result = d.to_dict() assert result["x"] == [3, 1, 2] - def test_to_dict_large_data(self, cls: Callable[..., MultiMapping[Any]]) -> None: + def test_to_dict_large_data(self, cls: DictFactory) -> None: """Test to_dict with a large number of entries for performance.""" items = [(f"key{i % 100}", i) for i in range(10000)] d = cls(items) @@ -63,7 +79,7 @@ def test_to_dict_large_data(self, cls: Callable[..., MultiMapping[Any]]) -> None assert all(len(v) == 100 for v in result.values()) def test_to_dict_mixed_value_types( - self, cls: Callable[..., MultiMapping[Any]] + self, cls: DictFactory ) -> None: """Test to_dict with mixed value types (str, int) to verify generic _V.""" d = cls([("a", 1), ("a", "two"), ("b", 3.14)]) @@ -76,19 +92,19 @@ class TestMultiDictToDict(BaseToDictTests): """Tests for MultiDict.to_dict().""" @pytest.fixture - def cls(self, multidict_module: Any) -> Type[MultiDict[Any]]: - return multidict_module.MultiDict # type: ignore[no-any-return] + def cls(self, multidict_module: MultidictModule) -> Type[MultiDict[object]]: + return multidict_module.MultiDict class TestCIMultiDictToDict(BaseToDictTests): """Tests for CIMultiDict.to_dict().""" @pytest.fixture - def cls(self, multidict_module: Any) -> Type[CIMultiDict[Any]]: - return multidict_module.CIMultiDict # type: ignore[no-any-return] + def cls(self, multidict_module: MultidictModule) -> Type[CIMultiDict[object]]: + return multidict_module.CIMultiDict def test_to_dict_case_insensitive_grouping( - self, cls: Callable[..., MultiMapping[Any]] + self, cls: DictFactory ) -> None: """Test that case variants are grouped under the same key.""" d = cls([("A", 1), ("a", 2), ("B", 3)]) @@ -106,19 +122,19 @@ class TestMultiDictProxyToDict(BaseToDictTests): """Tests for MultiDictProxy.to_dict().""" @pytest.fixture - def cls(self, multidict_module: Any) -> Any: - def make_proxy(*args: Any, **kwargs: Any) -> MultiDictProxy[Any]: - md = multidict_module.MultiDict(*args, **kwargs) - return multidict_module.MultiDictProxy(md) # type: ignore[no-any-return] + def cls(self, multidict_module: MultidictModule) -> DictFactory: + def make_proxy(arg: Iterable[tuple[str, object]] | None = None) -> MultiMapping[object]: + md: MultiDict[object] = multidict_module.MultiDict(arg) if arg else multidict_module.MultiDict() + return multidict_module.MultiDictProxy(md) return make_proxy def test_to_dict_proxy_mutation_isolation( - self, cls: Callable[..., MultiMapping[Any]], multidict_module: Any + self, cls: DictFactory, multidict_module: MultidictModule ) -> None: """Test that modifying returned dict does not affect the proxy.""" - md = multidict_module.MultiDict([("a", 1)]) - proxy = multidict_module.MultiDictProxy(md) + md: MultiDict[object] = multidict_module.MultiDict([("a", 1)]) + proxy: MultiMapping[object] = multidict_module.MultiDictProxy(md) result = proxy.to_dict() result["a"].append(999) assert proxy.getall("a") == [1] @@ -128,15 +144,15 @@ class TestCIMultiDictProxyToDict(BaseToDictTests): """Tests for CIMultiDictProxy.to_dict().""" @pytest.fixture - def cls(self, multidict_module: Any) -> Any: - def make_proxy(*args: Any, **kwargs: Any) -> CIMultiDictProxy[Any]: - md = multidict_module.CIMultiDict(*args, **kwargs) - return multidict_module.CIMultiDictProxy(md) # type: ignore[no-any-return] + def cls(self, multidict_module: MultidictModule) -> DictFactory: + def make_proxy(arg: Iterable[tuple[str, object]] | None = None) -> MultiMapping[object]: + md: CIMultiDict[object] = multidict_module.CIMultiDict(arg) if arg else multidict_module.CIMultiDict() + return multidict_module.CIMultiDictProxy(md) return make_proxy def test_to_dict_case_insensitive_grouping( - self, cls: Callable[..., MultiMapping[Any]] + self, cls: DictFactory ) -> None: """Test that case variants are grouped under the same key.""" d = cls([("A", 1), ("a", 2), ("B", 3)]) @@ -150,11 +166,11 @@ def test_to_dict_case_insensitive_grouping( assert result[key_b] == [3] def test_to_dict_proxy_mutation_isolation( - self, cls: Callable[..., MultiMapping[Any]], multidict_module: Any + self, cls: DictFactory, multidict_module: MultidictModule ) -> None: """Test that modifying returned dict does not affect the proxy.""" - md = multidict_module.CIMultiDict([("a", 1)]) - proxy = multidict_module.CIMultiDictProxy(md) + md: CIMultiDict[object] = multidict_module.CIMultiDict([("a", 1)]) + proxy: MultiMapping[object] = multidict_module.CIMultiDictProxy(md) result = proxy.to_dict() result["a"].append(999) assert proxy.getall("a") == [1] From 9ab86240503c822a2ca20fafb3643a23ee1d7059 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 6 Mar 2026 22:54:02 +0000 Subject: [PATCH 19/29] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- tests/test_to_dict.py | 48 +++++++++++++++++++++++-------------------- 1 file changed, 26 insertions(+), 22 deletions(-) diff --git a/tests/test_to_dict.py b/tests/test_to_dict.py index 1450d79d6..5a5744f6a 100644 --- a/tests/test_to_dict.py +++ b/tests/test_to_dict.py @@ -1,23 +1,27 @@ """Test to_dict functionality for all multidict types.""" -from typing import Any, Callable, Type, Iterable +from typing import Type, Iterable from multidict import MultiMapping from typing import Protocol -from typing import Protocol, Type from multidict import MultiDict, CIMultiDict, MultiDictProxy, CIMultiDictProxy + class MultidictModule(Protocol): MultiDict: Type[MultiDict[object]] CIMultiDict: Type[CIMultiDict[object]] MultiDictProxy: Type[MultiDictProxy[object]] CIMultiDictProxy: Type[CIMultiDictProxy[object]] + class DictFactory(Protocol): - def __call__(self, arg: Iterable[tuple[str, object]] | None = None) -> MultiMapping[object]: ... + def __call__( + self, arg: Iterable[tuple[str, object]] | None = None + ) -> MultiMapping[object]: ... + import pytest @@ -45,9 +49,7 @@ def test_to_dict_empty(self, cls: DictFactory) -> None: result = d.to_dict() assert result == {} - def test_to_dict_returns_new_dict( - self, cls: DictFactory - ) -> None: + def test_to_dict_returns_new_dict(self, cls: DictFactory) -> None: """Test that each call returns a new dictionary instance.""" d = cls([("a", 1)]) result1 = d.to_dict() @@ -62,9 +64,7 @@ def test_to_dict_list_is_fresh(self, cls: DictFactory) -> None: result2 = d.to_dict() assert result1["a"] is not result2["a"] - def test_to_dict_order_preservation( - self, cls: DictFactory - ) -> None: + def test_to_dict_order_preservation(self, cls: DictFactory) -> None: """Test that value lists maintain insertion order.""" d = cls([("x", 3), ("x", 1), ("x", 2)]) result = d.to_dict() @@ -78,9 +78,7 @@ def test_to_dict_large_data(self, cls: DictFactory) -> None: assert len(result) == 100 assert all(len(v) == 100 for v in result.values()) - def test_to_dict_mixed_value_types( - self, cls: DictFactory - ) -> None: + def test_to_dict_mixed_value_types(self, cls: DictFactory) -> None: """Test to_dict with mixed value types (str, int) to verify generic _V.""" d = cls([("a", 1), ("a", "two"), ("b", 3.14)]) result = d.to_dict() @@ -103,9 +101,7 @@ class TestCIMultiDictToDict(BaseToDictTests): def cls(self, multidict_module: MultidictModule) -> Type[CIMultiDict[object]]: return multidict_module.CIMultiDict - def test_to_dict_case_insensitive_grouping( - self, cls: DictFactory - ) -> None: + def test_to_dict_case_insensitive_grouping(self, cls: DictFactory) -> None: """Test that case variants are grouped under the same key.""" d = cls([("A", 1), ("a", 2), ("B", 3)]) result = d.to_dict() @@ -123,8 +119,12 @@ class TestMultiDictProxyToDict(BaseToDictTests): @pytest.fixture def cls(self, multidict_module: MultidictModule) -> DictFactory: - def make_proxy(arg: Iterable[tuple[str, object]] | None = None) -> MultiMapping[object]: - md: MultiDict[object] = multidict_module.MultiDict(arg) if arg else multidict_module.MultiDict() + def make_proxy( + arg: Iterable[tuple[str, object]] | None = None, + ) -> MultiMapping[object]: + md: MultiDict[object] = ( + multidict_module.MultiDict(arg) if arg else multidict_module.MultiDict() + ) return multidict_module.MultiDictProxy(md) return make_proxy @@ -145,15 +145,19 @@ class TestCIMultiDictProxyToDict(BaseToDictTests): @pytest.fixture def cls(self, multidict_module: MultidictModule) -> DictFactory: - def make_proxy(arg: Iterable[tuple[str, object]] | None = None) -> MultiMapping[object]: - md: CIMultiDict[object] = multidict_module.CIMultiDict(arg) if arg else multidict_module.CIMultiDict() + def make_proxy( + arg: Iterable[tuple[str, object]] | None = None, + ) -> MultiMapping[object]: + md: CIMultiDict[object] = ( + multidict_module.CIMultiDict(arg) + if arg + else multidict_module.CIMultiDict() + ) return multidict_module.CIMultiDictProxy(md) return make_proxy - def test_to_dict_case_insensitive_grouping( - self, cls: DictFactory - ) -> None: + def test_to_dict_case_insensitive_grouping(self, cls: DictFactory) -> None: """Test that case variants are grouped under the same key.""" d = cls([("A", 1), ("a", 2), ("B", 3)]) result = d.to_dict() From d58c07f6e025ec319ae80e977fa189c35b494d34 Mon Sep 17 00:00:00 2001 From: "rodrigo.nogueira" Date: Fri, 6 Mar 2026 20:07:27 -0300 Subject: [PATCH 20/29] Resolve formatting conflicts in test_to_dict --- tests/test_to_dict.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/tests/test_to_dict.py b/tests/test_to_dict.py index 5a5744f6a..adc084ef9 100644 --- a/tests/test_to_dict.py +++ b/tests/test_to_dict.py @@ -1,13 +1,17 @@ """Test to_dict functionality for all multidict types.""" -from typing import Type, Iterable -from multidict import MultiMapping - - -from typing import Protocol +from collections.abc import Iterable +from typing import Protocol, Type +import pytest -from multidict import MultiDict, CIMultiDict, MultiDictProxy, CIMultiDictProxy +from multidict import ( + CIMultiDict, + CIMultiDictProxy, + MultiDict, + MultiDictProxy, + MultiMapping, +) class MultidictModule(Protocol): @@ -20,12 +24,8 @@ class MultidictModule(Protocol): class DictFactory(Protocol): def __call__( self, arg: Iterable[tuple[str, object]] | None = None - ) -> MultiMapping[object]: ... - - -import pytest - -from multidict import CIMultiDict, CIMultiDictProxy, MultiDict, MultiDictProxy + ) -> MultiMapping[object]: + raise NotImplementedError class BaseToDictTests: From 1cb0497631f476648723b673b3ebc7f826882308 Mon Sep 17 00:00:00 2001 From: "rodrigo.nogueira" Date: Fri, 6 Mar 2026 20:29:05 -0300 Subject: [PATCH 21/29] tests: Use Optional instead of | Union syntax for Python 3.9 compat --- tests/test_to_dict.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/test_to_dict.py b/tests/test_to_dict.py index adc084ef9..fa4e0b8fe 100644 --- a/tests/test_to_dict.py +++ b/tests/test_to_dict.py @@ -1,7 +1,7 @@ """Test to_dict functionality for all multidict types.""" from collections.abc import Iterable -from typing import Protocol, Type +from typing import Optional, Protocol, Type import pytest @@ -23,7 +23,7 @@ class MultidictModule(Protocol): class DictFactory(Protocol): def __call__( - self, arg: Iterable[tuple[str, object]] | None = None + self, arg: Optional[Iterable[tuple[str, object]]] = None ) -> MultiMapping[object]: raise NotImplementedError @@ -120,7 +120,7 @@ class TestMultiDictProxyToDict(BaseToDictTests): @pytest.fixture def cls(self, multidict_module: MultidictModule) -> DictFactory: def make_proxy( - arg: Iterable[tuple[str, object]] | None = None, + arg: Optional[Iterable[tuple[str, object]]] = None, ) -> MultiMapping[object]: md: MultiDict[object] = ( multidict_module.MultiDict(arg) if arg else multidict_module.MultiDict() @@ -146,7 +146,7 @@ class TestCIMultiDictProxyToDict(BaseToDictTests): @pytest.fixture def cls(self, multidict_module: MultidictModule) -> DictFactory: def make_proxy( - arg: Iterable[tuple[str, object]] | None = None, + arg: Optional[Iterable[tuple[str, object]]] = None, ) -> MultiMapping[object]: md: CIMultiDict[object] = ( multidict_module.CIMultiDict(arg) From a5920ec6e8050b1dd90a0ad4ce5bd11654e62881 Mon Sep 17 00:00:00 2001 From: "rodrigo.nogueira" Date: Sat, 7 Mar 2026 00:16:50 -0300 Subject: [PATCH 22/29] tests: add pragma no cover to DictFactory Protocol --- tests/test_to_dict.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_to_dict.py b/tests/test_to_dict.py index fa4e0b8fe..b2a860acd 100644 --- a/tests/test_to_dict.py +++ b/tests/test_to_dict.py @@ -25,7 +25,7 @@ class DictFactory(Protocol): def __call__( self, arg: Optional[Iterable[tuple[str, object]]] = None ) -> MultiMapping[object]: - raise NotImplementedError + raise NotImplementedError # pragma: no cover class BaseToDictTests: From 5dda24d03f4e222eced85049574bb9bec403a84f Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sat, 2 May 2026 03:28:19 +0000 Subject: [PATCH 23/29] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- tests/test_to_dict.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/tests/test_to_dict.py b/tests/test_to_dict.py index b2a860acd..4f53cbaea 100644 --- a/tests/test_to_dict.py +++ b/tests/test_to_dict.py @@ -15,15 +15,15 @@ class MultidictModule(Protocol): - MultiDict: Type[MultiDict[object]] - CIMultiDict: Type[CIMultiDict[object]] - MultiDictProxy: Type[MultiDictProxy[object]] - CIMultiDictProxy: Type[CIMultiDictProxy[object]] + MultiDict: type[MultiDict[object]] + CIMultiDict: type[CIMultiDict[object]] + MultiDictProxy: type[MultiDictProxy[object]] + CIMultiDictProxy: type[CIMultiDictProxy[object]] class DictFactory(Protocol): def __call__( - self, arg: Optional[Iterable[tuple[str, object]]] = None + self, arg: Iterable[tuple[str, object]] | None = None ) -> MultiMapping[object]: raise NotImplementedError # pragma: no cover @@ -90,7 +90,7 @@ class TestMultiDictToDict(BaseToDictTests): """Tests for MultiDict.to_dict().""" @pytest.fixture - def cls(self, multidict_module: MultidictModule) -> Type[MultiDict[object]]: + def cls(self, multidict_module: MultidictModule) -> type[MultiDict[object]]: return multidict_module.MultiDict @@ -98,7 +98,7 @@ class TestCIMultiDictToDict(BaseToDictTests): """Tests for CIMultiDict.to_dict().""" @pytest.fixture - def cls(self, multidict_module: MultidictModule) -> Type[CIMultiDict[object]]: + def cls(self, multidict_module: MultidictModule) -> type[CIMultiDict[object]]: return multidict_module.CIMultiDict def test_to_dict_case_insensitive_grouping(self, cls: DictFactory) -> None: @@ -120,7 +120,7 @@ class TestMultiDictProxyToDict(BaseToDictTests): @pytest.fixture def cls(self, multidict_module: MultidictModule) -> DictFactory: def make_proxy( - arg: Optional[Iterable[tuple[str, object]]] = None, + arg: Iterable[tuple[str, object]] | None = None, ) -> MultiMapping[object]: md: MultiDict[object] = ( multidict_module.MultiDict(arg) if arg else multidict_module.MultiDict() @@ -146,7 +146,7 @@ class TestCIMultiDictProxyToDict(BaseToDictTests): @pytest.fixture def cls(self, multidict_module: MultidictModule) -> DictFactory: def make_proxy( - arg: Optional[Iterable[tuple[str, object]]] = None, + arg: Iterable[tuple[str, object]] | None = None, ) -> MultiMapping[object]: md: CIMultiDict[object] = ( multidict_module.CIMultiDict(arg) From 68066a361e7f52908056e91ce4a462545fbaca54 Mon Sep 17 00:00:00 2001 From: Rodrigo Nogueira Date: Sat, 2 May 2026 14:37:52 -0300 Subject: [PATCH 24/29] Address review: use conftest fixtures, collapse duplicate tests, fix changelog --- CHANGES/783.feature | 2 +- tests/test_to_dict.py | 304 ++++++++++++++++++------------------------ 2 files changed, 132 insertions(+), 174 deletions(-) diff --git a/CHANGES/783.feature b/CHANGES/783.feature index c756df555..62a6e7c3e 100644 --- a/CHANGES/783.feature +++ b/CHANGES/783.feature @@ -1 +1 @@ -Added ``to_dict()`` method to export multidict contents as a standard dict with value lists. (#783) +Added ``to_dict()`` method to export multidict contents as a standard dict with value lists. diff --git a/tests/test_to_dict.py b/tests/test_to_dict.py index 4f53cbaea..e1030c7cd 100644 --- a/tests/test_to_dict.py +++ b/tests/test_to_dict.py @@ -1,180 +1,138 @@ """Test to_dict functionality for all multidict types.""" -from collections.abc import Iterable -from typing import Optional, Protocol, Type +from __future__ import annotations import pytest -from multidict import ( - CIMultiDict, - CIMultiDictProxy, - MultiDict, - MultiDictProxy, - MultiMapping, -) +from multidict import CIMultiDict, CIMultiDictProxy, MultiDict, MultiDictProxy -class MultidictModule(Protocol): - MultiDict: type[MultiDict[object]] - CIMultiDict: type[CIMultiDict[object]] - MultiDictProxy: type[MultiDictProxy[object]] - CIMultiDictProxy: type[CIMultiDictProxy[object]] - - -class DictFactory(Protocol): - def __call__( - self, arg: Iterable[tuple[str, object]] | None = None - ) -> MultiMapping[object]: - raise NotImplementedError # pragma: no cover - - -class BaseToDictTests: - """Base tests for to_dict() method, inherited by all multidict type tests.""" - - def test_to_dict_simple(self, cls: DictFactory) -> None: - """Test basic conversion with unique keys.""" - d = cls([("a", 1), ("b", 2)]) - result = d.to_dict() - assert result == {"a": [1], "b": [2]} - - def test_to_dict_multi_values(self, cls: DictFactory) -> None: - """Test grouping multiple values under the same key.""" - d = cls([("a", 1), ("b", 2), ("a", 3)]) - result = d.to_dict() - assert result == {"a": [1, 3], "b": [2]} - - def test_to_dict_empty(self, cls: DictFactory) -> None: - """Test conversion of an empty multidict.""" - d = cls() - result = d.to_dict() - assert result == {} - - def test_to_dict_returns_new_dict(self, cls: DictFactory) -> None: - """Test that each call returns a new dictionary instance.""" - d = cls([("a", 1)]) - result1 = d.to_dict() - result2 = d.to_dict() - assert result1 == result2 - assert result1 is not result2 - - def test_to_dict_list_is_fresh(self, cls: DictFactory) -> None: - """Test that value lists are independent between calls.""" - d = cls([("a", 1)]) - result1 = d.to_dict() - result2 = d.to_dict() - assert result1["a"] is not result2["a"] - - def test_to_dict_order_preservation(self, cls: DictFactory) -> None: - """Test that value lists maintain insertion order.""" - d = cls([("x", 3), ("x", 1), ("x", 2)]) - result = d.to_dict() - assert result["x"] == [3, 1, 2] - - def test_to_dict_large_data(self, cls: DictFactory) -> None: - """Test to_dict with a large number of entries for performance.""" - items = [(f"key{i % 100}", i) for i in range(10000)] - d = cls(items) - result = d.to_dict() - assert len(result) == 100 - assert all(len(v) == 100 for v in result.values()) - - def test_to_dict_mixed_value_types(self, cls: DictFactory) -> None: - """Test to_dict with mixed value types (str, int) to verify generic _V.""" - d = cls([("a", 1), ("a", "two"), ("b", 3.14)]) - result = d.to_dict() - assert result["a"] == [1, "two"] - assert result["b"] == [3.14] - - -class TestMultiDictToDict(BaseToDictTests): - """Tests for MultiDict.to_dict().""" - - @pytest.fixture - def cls(self, multidict_module: MultidictModule) -> type[MultiDict[object]]: - return multidict_module.MultiDict - - -class TestCIMultiDictToDict(BaseToDictTests): - """Tests for CIMultiDict.to_dict().""" - - @pytest.fixture - def cls(self, multidict_module: MultidictModule) -> type[CIMultiDict[object]]: - return multidict_module.CIMultiDict - - def test_to_dict_case_insensitive_grouping(self, cls: DictFactory) -> None: - """Test that case variants are grouped under the same key.""" - d = cls([("A", 1), ("a", 2), ("B", 3)]) - result = d.to_dict() - assert len(result) == 2 - assert "A" in result or "a" in result - assert "B" in result or "b" in result - key_a = "A" if "A" in result else "a" - key_b = "B" if "B" in result else "b" - assert result[key_a] == [1, 2] - assert result[key_b] == [3] - - -class TestMultiDictProxyToDict(BaseToDictTests): - """Tests for MultiDictProxy.to_dict().""" - - @pytest.fixture - def cls(self, multidict_module: MultidictModule) -> DictFactory: - def make_proxy( - arg: Iterable[tuple[str, object]] | None = None, - ) -> MultiMapping[object]: - md: MultiDict[object] = ( - multidict_module.MultiDict(arg) if arg else multidict_module.MultiDict() - ) - return multidict_module.MultiDictProxy(md) - - return make_proxy - - def test_to_dict_proxy_mutation_isolation( - self, cls: DictFactory, multidict_module: MultidictModule - ) -> None: - """Test that modifying returned dict does not affect the proxy.""" - md: MultiDict[object] = multidict_module.MultiDict([("a", 1)]) - proxy: MultiMapping[object] = multidict_module.MultiDictProxy(md) - result = proxy.to_dict() - result["a"].append(999) - assert proxy.getall("a") == [1] - - -class TestCIMultiDictProxyToDict(BaseToDictTests): - """Tests for CIMultiDictProxy.to_dict().""" - - @pytest.fixture - def cls(self, multidict_module: MultidictModule) -> DictFactory: - def make_proxy( - arg: Iterable[tuple[str, object]] | None = None, - ) -> MultiMapping[object]: - md: CIMultiDict[object] = ( - multidict_module.CIMultiDict(arg) - if arg - else multidict_module.CIMultiDict() - ) - return multidict_module.CIMultiDictProxy(md) - - return make_proxy - - def test_to_dict_case_insensitive_grouping(self, cls: DictFactory) -> None: - """Test that case variants are grouped under the same key.""" - d = cls([("A", 1), ("a", 2), ("B", 3)]) - result = d.to_dict() - assert len(result) == 2 - assert "A" in result or "a" in result - assert "B" in result or "b" in result - key_a = "A" if "A" in result else "a" - key_b = "B" if "B" in result else "b" - assert result[key_a] == [1, 2] - assert result[key_b] == [3] - - def test_to_dict_proxy_mutation_isolation( - self, cls: DictFactory, multidict_module: MultidictModule - ) -> None: - """Test that modifying returned dict does not affect the proxy.""" - md: CIMultiDict[object] = multidict_module.CIMultiDict([("a", 1)]) - proxy: MultiMapping[object] = multidict_module.CIMultiDictProxy(md) - result = proxy.to_dict() - result["a"].append(999) - assert proxy.getall("a") == [1] +@pytest.mark.parametrize( + ("items", "expected"), + ( + pytest.param([("a", 1), ("b", 2)], {"a": [1], "b": [2]}, id="unique-keys"), + pytest.param( + [("a", 1), ("b", 2), ("a", 3)], + {"a": [1, 3], "b": [2]}, + id="multi-values", + ), + ), +) +def test_to_dict( + any_multidict_class: type[MultiDict[str]] | type[CIMultiDict[str]], + items: list[tuple[str, int]], + expected: dict[str, list[int]], +) -> None: + """Test basic to_dict conversion with unique and duplicate keys.""" + d = any_multidict_class(items) + assert d.to_dict() == expected + + +def test_to_dict_empty( + any_multidict_class: type[MultiDict[str]] | type[CIMultiDict[str]], +) -> None: + """Test conversion of an empty multidict.""" + d = any_multidict_class() + assert d.to_dict() == {} + + +def test_to_dict_returns_new_dict( + any_multidict_class: type[MultiDict[str]] | type[CIMultiDict[str]], +) -> None: + """Test that each call returns a new dictionary instance.""" + d = any_multidict_class([("a", "1")]) + result1 = d.to_dict() + result2 = d.to_dict() + assert result1 == result2 + assert result1 is not result2 + + +def test_to_dict_list_is_fresh( + any_multidict_class: type[MultiDict[str]] | type[CIMultiDict[str]], +) -> None: + """Test that value lists are independent between calls.""" + d = any_multidict_class([("a", "1")]) + result1 = d.to_dict() + result2 = d.to_dict() + assert result1["a"] is not result2["a"] + + +def test_to_dict_order_preservation( + any_multidict_class: type[MultiDict[str]] | type[CIMultiDict[str]], +) -> None: + """Test that value lists maintain insertion order.""" + d = any_multidict_class([("x", "3"), ("x", "1"), ("x", "2")]) + assert d.to_dict()["x"] == ["3", "1", "2"] + + +def test_to_dict_large_data( + any_multidict_class: type[MultiDict[str]] | type[CIMultiDict[str]], +) -> None: + """Test to_dict with a large number of entries.""" + items = [(f"key{i % 100}", str(i)) for i in range(10000)] + d = any_multidict_class(items) + result = d.to_dict() + assert len(result) == 100 + assert all(len(v) == 100 for v in result.values()) + + +def test_to_dict_mixed_value_types( + any_multidict_class: type[MultiDict[str]] | type[CIMultiDict[str]], +) -> None: + """Test to_dict with mixed value types.""" + d = any_multidict_class([("a", "1"), ("a", "two"), ("b", "3.14")]) + result = d.to_dict() + assert result["a"] == ["1", "two"] + assert result["b"] == ["3.14"] + + +def test_to_dict_case_insensitive_grouping( + case_insensitive_multidict_class: type[CIMultiDict[str]], +) -> None: + """Test that case variants are grouped under the same key.""" + d = case_insensitive_multidict_class([("A", "1"), ("a", "2"), ("B", "3")]) + result = d.to_dict() + assert len(result) == 2 + key_a = next(k for k in result if k.lower() == "a") + key_b = next(k for k in result if k.lower() == "b") + assert result[key_a] == ["1", "2"] + assert result[key_b] == ["3"] + + +def test_to_dict_proxy( + any_multidict_class: type[MultiDict[str]] | type[CIMultiDict[str]], + any_multidict_proxy_class: type[MultiDictProxy[str]] + | type[CIMultiDictProxy[str]], +) -> None: + """Test to_dict works through a proxy.""" + md = any_multidict_class([("a", "1"), ("b", "2"), ("a", "3")]) + proxy = any_multidict_proxy_class(md) + assert proxy.to_dict() == {"a": ["1", "3"], "b": ["2"]} + + +def test_to_dict_proxy_mutation_isolation( + any_multidict_class: type[MultiDict[str]] | type[CIMultiDict[str]], + any_multidict_proxy_class: type[MultiDictProxy[str]] + | type[CIMultiDictProxy[str]], +) -> None: + """Test that modifying returned dict does not affect the proxy.""" + md = any_multidict_class([("a", "1")]) + proxy = any_multidict_proxy_class(md) + result = proxy.to_dict() + result["a"].append("999") + assert proxy.getall("a") == ["1"] + + +def test_to_dict_ci_proxy_case_insensitive_grouping( + case_insensitive_multidict_class: type[CIMultiDict[str]], + case_insensitive_multidict_proxy_class: type[CIMultiDictProxy[str]], +) -> None: + """Test case-insensitive grouping through a proxy.""" + md = case_insensitive_multidict_class([("A", "1"), ("a", "2"), ("B", "3")]) + proxy = case_insensitive_multidict_proxy_class(md) + result = proxy.to_dict() + assert len(result) == 2 + key_a = next(k for k in result if k.lower() == "a") + key_b = next(k for k in result if k.lower() == "b") + assert result[key_a] == ["1", "2"] + assert result[key_b] == ["3"] From cda3b86053d708279d1adea381d394cf83358cb7 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sat, 2 May 2026 17:41:40 +0000 Subject: [PATCH 25/29] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- tests/test_to_dict.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/tests/test_to_dict.py b/tests/test_to_dict.py index e1030c7cd..375112e2f 100644 --- a/tests/test_to_dict.py +++ b/tests/test_to_dict.py @@ -101,8 +101,7 @@ def test_to_dict_case_insensitive_grouping( def test_to_dict_proxy( any_multidict_class: type[MultiDict[str]] | type[CIMultiDict[str]], - any_multidict_proxy_class: type[MultiDictProxy[str]] - | type[CIMultiDictProxy[str]], + any_multidict_proxy_class: type[MultiDictProxy[str]] | type[CIMultiDictProxy[str]], ) -> None: """Test to_dict works through a proxy.""" md = any_multidict_class([("a", "1"), ("b", "2"), ("a", "3")]) @@ -112,8 +111,7 @@ def test_to_dict_proxy( def test_to_dict_proxy_mutation_isolation( any_multidict_class: type[MultiDict[str]] | type[CIMultiDict[str]], - any_multidict_proxy_class: type[MultiDictProxy[str]] - | type[CIMultiDictProxy[str]], + any_multidict_proxy_class: type[MultiDictProxy[str]] | type[CIMultiDictProxy[str]], ) -> None: """Test that modifying returned dict does not affect the proxy.""" md = any_multidict_class([("a", "1")]) From 4c19b42e6dc7cb691117a8fa1a66dbe54e86ada6 Mon Sep 17 00:00:00 2001 From: Rodrigo Nogueira Date: Sat, 2 May 2026 14:44:12 -0300 Subject: [PATCH 26/29] Fix MyPy: use str values in parametrized test_to_dict --- tests/test_to_dict.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/tests/test_to_dict.py b/tests/test_to_dict.py index 375112e2f..0dcb89f6c 100644 --- a/tests/test_to_dict.py +++ b/tests/test_to_dict.py @@ -10,18 +10,22 @@ @pytest.mark.parametrize( ("items", "expected"), ( - pytest.param([("a", 1), ("b", 2)], {"a": [1], "b": [2]}, id="unique-keys"), pytest.param( - [("a", 1), ("b", 2), ("a", 3)], - {"a": [1, 3], "b": [2]}, + [("a", "1"), ("b", "2")], + {"a": ["1"], "b": ["2"]}, + id="unique-keys", + ), + pytest.param( + [("a", "1"), ("b", "2"), ("a", "3")], + {"a": ["1", "3"], "b": ["2"]}, id="multi-values", ), ), ) def test_to_dict( any_multidict_class: type[MultiDict[str]] | type[CIMultiDict[str]], - items: list[tuple[str, int]], - expected: dict[str, list[int]], + items: list[tuple[str, str]], + expected: dict[str, list[str]], ) -> None: """Test basic to_dict conversion with unique and duplicate keys.""" d = any_multidict_class(items) From a7b051ca1b29ff1d71e7006bbb51e31f4cf7aaa0 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sun, 5 Jul 2026 20:33:53 +0000 Subject: [PATCH 27/29] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- tests/isolated/multidict_to_dict.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/isolated/multidict_to_dict.py b/tests/isolated/multidict_to_dict.py index f8bc38942..2f0a8596e 100644 --- a/tests/isolated/multidict_to_dict.py +++ b/tests/isolated/multidict_to_dict.py @@ -4,8 +4,8 @@ import os import psutil -from multidict import MultiDict +from multidict import MultiDict process = psutil.Process(os.getpid()) From dd0113a796095e5ea56343dace9cd9f2e93627b8 Mon Sep 17 00:00:00 2001 From: Rodrigo Nogueira Date: Sun, 5 Jul 2026 18:37:10 -0300 Subject: [PATCH 28/29] Document to_dict() in the API reference --- docs/multidict.rst | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/docs/multidict.rst b/docs/multidict.rst index 7d192add2..7033192a6 100644 --- a/docs/multidict.rst +++ b/docs/multidict.rst @@ -71,6 +71,20 @@ MultiDict Return a shallow copy of the dictionary. + .. method:: to_dict() + + Return a :class:`dict` mapping each key to a list of all its + values, preserving insertion order, e.g.:: + + >>> d = MultiDict([('a', 1), ('b', 2), ('a', 3)]) + >>> d.to_dict() + {'a': [1, 3], 'b': [2]} + + The result is a new dictionary with fresh lists; mutating it + does not affect the multidict. + + .. versionadded:: 6.8 + .. method:: getone(key[, default]) Return the **first** value for *key* if *key* is in the @@ -301,6 +315,13 @@ MultiDictProxy Return a shallow copy of the underlying multidict. + .. method:: to_dict() + + Return a :class:`dict` mapping each key to a list of all its + values, preserving insertion order. + + .. versionadded:: 6.8 + .. method:: getone(key[, default]) Return the **first** value for *key* if *key* is in the From 08e83389aa44ba292734703afa59d4ed62bbe4c8 Mon Sep 17 00:00:00 2001 From: Rodrigo Nogueira Date: Sat, 8 Aug 2026 23:17:35 -0300 Subject: [PATCH 29/29] Fix the ABC break, a NULL-without-exception path and the leak test Adding to_dict() as an abstractmethod on MultiMapping made every existing downstream subclass fail to instantiate, since MultiMapping is public and documented. A class implementing everything previously required raised "Can't instantiate abstract class ... without an implementation for abstract method 'to_dict'". It is a concrete default now, documented as needing an override for mappings that fold keys. The C append path looked the list up again with PyDict_GetItem(result, first_key). That swallows any exception from hashing the key, which can be a str subclass with its own __hash__, so a miss returned NULL with nothing set and CPython would raise SystemError. The pure-Python side raised KeyError there instead, so the backends disagreed. Both now key the seen map by the list itself, which removes the second lookup rather than guarding it, and drops one dict lookup per entry on both backends. The pure-Python side also stopped building two istr objects per key. The leak script measured RSS over interned keys and small ints, so a leaked reference to the key, identity or value moved it by zero bytes. It compares refcounts against a baseline now, over non-interned keys and plain objects, and covers CIMultiDict, which is the only allocating path. Injecting a stray Py_INCREF on the append path makes it fail with "value leaked: [0, 0, 1000]"; the old script passed with that leak in place. The changelog fragment gained its .rst extension, so the docs spell check and the :user: role hook actually see it, and the docs now contrast to_dict() with dict(md) and with the getall() comprehension that gets case-insensitive keys wrong. --- CHANGES/783.feature | 1 - CHANGES/783.feature.rst | 6 ++ docs/multidict.rst | 21 +++++++ multidict/_abc.py | 18 +++++- multidict/_multidict.c | 16 ++++-- multidict/_multidict_py.py | 15 +++-- tests/isolated/multidict_to_dict.py | 87 ++++++++++++++++++----------- 7 files changed, 117 insertions(+), 47 deletions(-) delete mode 100644 CHANGES/783.feature create mode 100644 CHANGES/783.feature.rst diff --git a/CHANGES/783.feature b/CHANGES/783.feature deleted file mode 100644 index 62a6e7c3e..000000000 --- a/CHANGES/783.feature +++ /dev/null @@ -1 +0,0 @@ -Added ``to_dict()`` method to export multidict contents as a standard dict with value lists. diff --git a/CHANGES/783.feature.rst b/CHANGES/783.feature.rst new file mode 100644 index 000000000..6e788cd0b --- /dev/null +++ b/CHANGES/783.feature.rst @@ -0,0 +1,6 @@ +Added a :meth:`~multidict.MultiDict.to_dict` method returning a plain +:class:`dict` that maps every key to the list of all its values. Unlike +``dict(md)``, which keeps only the first value per key, and unlike a +``{k: md.getall(k) for k in md}`` comprehension, which emits one entry per +spelling of a case-insensitive key, it groups by key identity +-- by :user:`rodrigobnogueira`. diff --git a/docs/multidict.rst b/docs/multidict.rst index 7033192a6..1ef381914 100644 --- a/docs/multidict.rst +++ b/docs/multidict.rst @@ -83,6 +83,27 @@ MultiDict The result is a new dictionary with fresh lists; mutating it does not affect the multidict. + Every key maps to a list, even a key with a single value, so feeding + the result back does not reproduce the original: + ``MultiDict(d.to_dict())`` builds a multidict whose *values* are + lists. This differs from + :class:`dict`\\ ``(d)``, which keeps only the **first** value for each + key:: + + >>> dict(d) + {'a': 1, 'b': 2} + + For :class:`CIMultiDict` the values are grouped by key identity, so + each key appears once under its first-seen spelling. That is what a + ``{k: d.getall(k) for k in d}`` comprehension gets wrong: it emits a + separate entry for every spelling, each holding the full list:: + + >>> ci = CIMultiDict([('A', '1'), ('a', '2')]) + >>> ci.to_dict() + {'A': ['1', '2']} + >>> {k: ci.getall(k) for k in ci} + {'A': ['1', '2'], 'a': ['1', '2']} + .. versionadded:: 6.8 .. method:: getone(key[, default]) diff --git a/multidict/_abc.py b/multidict/_abc.py index 881859f6a..d9f1ce22d 100644 --- a/multidict/_abc.py +++ b/multidict/_abc.py @@ -42,9 +42,23 @@ def getone(self, key: str, default: _T) -> _V_co | _T: ... def getone(self, key: str, default: _T = ...) -> _V_co | _T: """Return first value for key.""" - @abc.abstractmethod def to_dict(self) -> dict[str, list[_V_co]]: - """Return a dict with lists of all values for each key.""" + """Return a dict with lists of all values for each key. + + Deliberately concrete, not abstract: ``MultiMapping`` is public, so + requiring a new method would stop every existing subclass outside + this project from being instantiated. Subclasses whose keys compare + equal under a normalisation the iteration does not apply, such as + case-insensitive mappings, must override this; the default would + emit one entry per spelling. + """ + result: dict[str, list[_V_co]] = {} + for key in self: + # Iteration yields a duplicated key once per occurrence, so the + # membership test is what keeps getall() from being applied twice. + if key not in result: + result[key] = list(self.getall(key)) + return result class MutableMultiMapping(MultiMapping[_V], MutableMapping[str, _V]): diff --git a/multidict/_multidict.c b/multidict/_multidict.c index a002494c9..2dc49fd82 100644 --- a/multidict/_multidict.c +++ b/multidict/_multidict.c @@ -270,15 +270,20 @@ multidict_to_dict(MultiDictObject *self) int tmp; while ((tmp = md_next(self, &pos, &identity, &key, &value)) > 0) { - PyObject *first_key = PyDict_GetItem(seen, identity); - if (first_key == NULL) { - PyObject *lst = PyList_New(1); + /* `seen` maps identity to the very list held in `result`, not to the + first key. Looking the list up again in `result` would have to + hash a key that may be a str subclass with a user-defined + __hash__, and PyDict_GetItem() swallows the exception from that, + so a miss would return NULL with nothing set. */ + PyObject *lst = PyDict_GetItem(seen, identity); + if (lst == NULL) { + lst = PyList_New(1); if (lst == NULL) { goto fail; } PyList_SET_ITEM(lst, 0, value); value = NULL; - if (PyDict_SetItem(seen, identity, key) < 0) { + if (PyDict_SetItem(seen, identity, lst) < 0) { Py_DECREF(lst); goto fail; } @@ -288,8 +293,7 @@ multidict_to_dict(MultiDictObject *self) } Py_DECREF(lst); } else { - PyObject *lst = PyDict_GetItem(result, first_key); - if (lst == NULL || PyList_Append(lst, value) < 0) { + if (PyList_Append(lst, value) < 0) { goto fail; } Py_DECREF(value); diff --git a/multidict/_multidict_py.py b/multidict/_multidict_py.py index ef10d17df..bc9819041 100644 --- a/multidict/_multidict_py.py +++ b/multidict/_multidict_py.py @@ -767,14 +767,17 @@ def __reduce__(self) -> tuple[type[Self], tuple[list[tuple[str, _V]]]]: def to_dict(self) -> dict[str, list[_V]]: """Return a dict with lists of all values for each key.""" result: dict[str, list[_V]] = {} - seen_identities: dict[str, str] = {} + # Keyed by identity so a case-insensitive multidict groups every + # spelling of a key together, and holding the list itself rather than + # the first key so the value append needs no second lookup. + seen: dict[str, list[_V]] = {} for e in self._keys.iter_entries(): - first_key = seen_identities.get(e.identity) - if first_key is None: - seen_identities[e.identity] = self._key(e.key) - result[self._key(e.key)] = [e.value] + values = seen.get(e.identity) + if values is None: + values = seen[e.identity] = [e.value] + result[self._key(e.key)] = values else: - result[first_key].append(e.value) + values.append(e.value) return result def add(self, key: str, value: _V) -> None: diff --git a/tests/isolated/multidict_to_dict.py b/tests/isolated/multidict_to_dict.py index 2f0a8596e..246f00533 100644 --- a/tests/isolated/multidict_to_dict.py +++ b/tests/isolated/multidict_to_dict.py @@ -1,41 +1,64 @@ -"""Memory leak test for to_dict().""" - import gc -import os - -import psutil - -from multidict import MultiDict - -process = psutil.Process(os.getpid()) +import sys +import sysconfig +from multidict import CIMultiDict, MultiDict -def trim_ram() -> None: - gc.collect() - +# sys.getrefcount is not meaningful under the free-threaded build: +# refcounts are biased per-thread and types may be immortalized, so +# the simple baseline/after comparison below does not apply. +FREETHREADED = bool(sysconfig.get_config_var("Py_GIL_DISABLED")) -def get_memory_usage() -> int: - memory_info = process.memory_info() - return memory_info.rss // (1024 * 1024) +if __name__ == "__main__": + if FREETHREADED: + raise SystemExit(0) -def test_to_dict_leak() -> None: - for _ in range(100): - d = MultiDict([("a", 1), ("b", 2)]) - d.to_dict() - trim_ram() - - mem_before = get_memory_usage() - for _ in range(1_000_000): - d = MultiDict([("a", 1), ("b", 2)]) - d.to_dict() - trim_ram() - mem_after = get_memory_usage() + # Distinct, non-interned keys and non-immortal values: a leaked reference + # to an interned "a" or to a small int would not move any refcount, which + # is what an RSS-growth check on such objects silently misses. + key_a = "leak-key-a".swapcase().swapcase() + key_b = "leak-key-b".swapcase().swapcase() + value_1 = object() + value_2 = object() + value_3 = object() - growth = mem_after - mem_before - assert growth < 50, f"Memory grew by {growth} MB, possible leak" + # Value leak, duplicate key: exercises both the PyList_SET_ITEM branch + # (first value for a key) and the PyList_Append branch (later values). + md = MultiDict([(key_a, value_1), (key_b, value_2), (key_a, value_3)]) + gc.collect() + baselines = [sys.getrefcount(v) for v in (value_1, value_2, value_3)] + for _ in range(1000): + _d = md.to_dict() + del _d + gc.collect() + after = [sys.getrefcount(v) for v in (value_1, value_2, value_3)] + assert after == baselines, ( + f"value leaked: {[a - b for a, b in zip(after, baselines)]}" + ) + # Key leak, same multidict. + gc.collect() + key_baselines = [sys.getrefcount(k) for k in (key_a, key_b)] + for _ in range(1000): + _d = md.to_dict() + del _d + gc.collect() + key_after = [sys.getrefcount(k) for k in (key_a, key_b)] + assert key_after == key_baselines, ( + f"key leaked: {[a - b for a, b in zip(key_after, key_baselines)]}" + ) -if __name__ == "__main__": - test_to_dict_leak() - print("PASSED: No memory leak detected in to_dict()") + # CIMultiDict takes the only allocating path: _md_ensure_key() builds a + # fresh istr and stores it back into the entry. + ci = CIMultiDict([(key_a, value_1), (key_a.upper(), value_2)]) + gc.collect() + ci_baselines = [sys.getrefcount(v) for v in (value_1, value_2)] + for _ in range(1000): + _d = ci.to_dict() + del _d + gc.collect() + ci_after = [sys.getrefcount(v) for v in (value_1, value_2)] + assert ci_after == ci_baselines, ( + f"CI value leaked: {[a - b for a, b in zip(ci_after, ci_baselines)]}" + )