diff --git a/Lib/test/test_unicode.py b/Lib/test/test_unicode.py
index 0c23daa83e8..aeac7243705 100644
--- a/Lib/test/test_unicode.py
+++ b/Lib/test/test_unicode.py
@@ -1397,7 +1397,7 @@ class UnicodeTest(string_tests.CommonTest,
         # non-ascii format, ascii argument
         self.assertRaisesRegexp(ValueError,
             '^PyUnicode_FromFormatV\(\) expects an ASCII-encoded format '
-            'string, got a non-ascii byte: 0xe9$',
+            'string, got a non-ASCII byte: 0xe9$',
             format_unicode, b'unicode\xe9=%s', 'ascii')
 
 def test_main():
diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c
index 3b0a66ae743..92df1ce9007 100644
--- a/Objects/unicodeobject.c
+++ b/Objects/unicodeobject.c
@@ -767,9 +767,9 @@ PyUnicode_FromFormatV(const char *format, va_list vargs)
          else if (128 <= (unsigned char)*f) {
              PyErr_Format(PyExc_ValueError,
                 "PyUnicode_FromFormatV() expects an ASCII-encoded format "
-                "string, got a non-ascii byte: 0x%02x",
+                "string, got a non-ASCII byte: 0x%02x",
                 (unsigned char)*f);
-             return NULL;
+             goto fail;
          }
     }
     /* step 2: allocate memory for the results of