Marc-Andre Lemburg: support pickling Unicode objects, both in text

mode ('V') and in binary mode ('X').

Lib/pickle.py

@@ -64,6 +64,8 @@ REDUCE          = 'R'
 STRING          = 'S'
 BINSTRING       = 'T'
 SHORT_BINSTRING = 'U'
+UNICODE         = 'V'
+BINUNICODE      = 'X'
 APPEND          = 'a'
 BUILD           = 'b'
 GLOBAL          = 'c'
@@ -275,6 +277,23 @@ class Pickler:
         memo[d] = (memo_len, object)
     dispatch[StringType] = save_string
 
+    def save_unicode(self, object):
+        d = id(object)
+        memo = self.memo
+
+        if (self.bin):
+            encoding = object.encode('utf-8')
+            l = len(encoding)
+            s = mdumps(l)[1:]
+            self.write(BINUNICODE + s + encoding)
+        else:
+            self.write(UNICODE + object.encode('raw-unicode-escape') + '\n')
+
+        memo_len = len(memo)
+        self.write(self.put(memo_len))
+        memo[d] = (memo_len, object)
+    dispatch[UnicodeType] = save_unicode
+
     def save_tuple(self, object):
 
         write = self.write
@@ -566,6 +585,15 @@ class Unpickler:
         self.append(self.read(len))
     dispatch[BINSTRING] = load_binstring
 
+    def load_unicode(self):
+        self.append(unicode(self.readline()[:-1],'raw-unicode-escape'))
+    dispatch[UNICODE] = load_unicode
+
+    def load_binunicode(self):
+        len = mloads('i' + self.read(4))
+        self.append(unicode(self.read(len),'utf-8'))
+    dispatch[BINUNICODE] = load_binunicode
+
     def load_short_binstring(self):
         len = mloads('i' + self.read(1) + '\000\000\000')
         self.append(self.read(len))