diff --git a/Lib/test/test_json/test_float.py b/Lib/test/test_json/test_float.py
index d0c7214334d..61540a3a02c 100644
--- a/Lib/test/test_json/test_float.py
+++ b/Lib/test/test_json/test_float.py
@@ -26,7 +26,8 @@ class TestFloat:
                 res = self.loads(out)
                 self.assertEqual(len(res), 1)
                 self.assertNotEqual(res[0], res[0])
-            self.assertRaises(ValueError, self.dumps, [val], allow_nan=False)
+            msg = f'Out of range float values are not JSON compliant: {val}'
+            self.assertRaisesRegex(ValueError, msg, self.dumps, [val], allow_nan=False)
 
 
 class TestPyFloat(TestFloat, PyTest): pass
diff --git a/Misc/NEWS.d/next/Library/2022-12-01-15-44-58.gh-issue-99925.x4y6pF.rst b/Misc/NEWS.d/next/Library/2022-12-01-15-44-58.gh-issue-99925.x4y6pF.rst
new file mode 100644
index 00000000000..660635a0396
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2022-12-01-15-44-58.gh-issue-99925.x4y6pF.rst
@@ -0,0 +1,4 @@
+Unify error messages in JSON serialization between
+``json.dumps(float('nan'), allow_nan=False)`` and ``json.dumps(float('nan'),
+allow_nan=False, indent=<SOMETHING>)``. Now both include the representation
+of the value that could not be serialized.
diff --git a/Modules/_json.c b/Modules/_json.c
index 6879ad3d072..fa8e2a936d2 100644
--- a/Modules/_json.c
+++ b/Modules/_json.c
@@ -1319,9 +1319,10 @@ encoder_encode_float(PyEncoderObject *s, PyObject *obj)
     double i = PyFloat_AS_DOUBLE(obj);
     if (!Py_IS_FINITE(i)) {
         if (!s->allow_nan) {
-            PyErr_SetString(
+            PyErr_Format(
                     PyExc_ValueError,
-                    "Out of range float values are not JSON compliant"
+                    "Out of range float values are not JSON compliant: %R",
+                    obj
                     );
             return NULL;
         }