diff --git a/Misc/NEWS.d/next/Library/2022-11-02-05-53-25.gh-issue-83004.qc_KHr.rst b/Misc/NEWS.d/next/Library/2022-11-02-05-53-25.gh-issue-83004.qc_KHr.rst
new file mode 100644
index 00000000000..de000634206
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2022-11-02-05-53-25.gh-issue-83004.qc_KHr.rst
@@ -0,0 +1 @@
+Clean up refleaks on failed module initialisation in in :mod:`_pickle`
diff --git a/Modules/_pickle.c b/Modules/_pickle.c
index 52704b0c59a..80bb2126de7 100644
--- a/Modules/_pickle.c
+++ b/Modules/_pickle.c
@@ -7986,16 +7986,15 @@ PyInit__pickle(void)
     if (st->UnpicklingError == NULL)
         return NULL;
 
-    Py_INCREF(st->PickleError);
-    if (PyModule_AddObject(m, "PickleError", st->PickleError) < 0)
+    if (PyModule_AddObjectRef(m, "PickleError", st->PickleError) < 0) {
         return NULL;
-    Py_INCREF(st->PicklingError);
-    if (PyModule_AddObject(m, "PicklingError", st->PicklingError) < 0)
+    }
+    if (PyModule_AddObjectRef(m, "PicklingError", st->PicklingError) < 0) {
         return NULL;
-    Py_INCREF(st->UnpicklingError);
-    if (PyModule_AddObject(m, "UnpicklingError", st->UnpicklingError) < 0)
+    }
+    if (PyModule_AddObjectRef(m, "UnpicklingError", st->UnpicklingError) < 0) {
         return NULL;
-
+    }
     if (_Pickle_InitState(st) < 0)
         return NULL;