From f7bc5f94553e3a98c6437862f1b5fa5fefd2e34c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Walter=20D=C3=B6rwald?= <walter@livinglogic.de>
Date: Tue, 4 Apr 2006 17:32:49 +0000
Subject: [PATCH] Change the example classes UnicodeReader and UnicodeWriter so
 that they work with all encodings. For UnicodeReader the real input stream is
 wrapped in a line iterator that reencodes the input to UTF-8. For
 UnicodeWriter the UTF-8 encoded output is written to a queue for where it is
 reencoded to the target encoding and written to the real output stream.

---
 Doc/lib/libcsv.tex | 46 +++++++++++++++++++++++++++++++++++-----------
 1 file changed, 35 insertions(+), 11 deletions(-)

diff --git a/Doc/lib/libcsv.tex b/Doc/lib/libcsv.tex
index d2203451250..65053c7f1bb 100644
--- a/Doc/lib/libcsv.tex
+++ b/Doc/lib/libcsv.tex
@@ -456,44 +456,68 @@ def utf_8_encoder(unicode_csv_data):
         yield line.encode('utf-8')
 \end{verbatim}
 
-The classes below work just like the \class{csv.reader} and
-\class{csv.writer} classes, but they add an \var{encoding} parameter
-to allow for encoded files:
+For all other encodings the following \class{UnicodeReader} and
+\class{UnicodeWriter} classes can be used. They take an additional
+\var{encoding} parameter in their constructor and make sure that the data
+passes the real reader or writer encoded as UTF-8:
 
 \begin{verbatim}
-import csv
+import csv, codecs, cStringIO
+
+class UTF8Recoder:
+    """
+    Iterator that reads an encoded stream and reencodes the input to UTF-8
+    """
+    def __init__(self, f, encoding):
+        self.reader = codecs.getreader(encoding)(f)
+
+    def __iter__(self):
+        return self
+
+    def next(self):
+        return self.reader.next().encode("utf-8")
 
 class UnicodeReader:
-
     """
     A CSV reader which will iterate over lines in the CSV file "f",
     which is encoded in the given encoding.
     """
 
     def __init__(self, f, dialect=csv.excel, encoding="utf-8", **kwds):
+        f = UTF8Recoder(f, encoding)
         self.reader = csv.reader(f, dialect=dialect, **kwds)
-        self.encoding = encoding
 
     def next(self):
         row = self.reader.next()
-        return [unicode(s, self.encoding) for s in row]
+        return [unicode(s, "utf-8") for s in row]
 
     def __iter__(self):
         return self
 
 class UnicodeWriter:
-
     """
     A CSV writer which will write rows to CSV file "f",
     which is encoded in the given encoding.
     """
 
     def __init__(self, f, dialect=csv.excel, encoding="utf-8", **kwds):
-        self.writer = csv.writer(f, dialect=dialect, **kwds)
-        self.encoding = encoding
+        # Redirect output to a queue
+        self.queue = cStringIO.StringIO()
+        self.writer = csv.writer(self.queue, dialect=dialect, **kwds)
+        self.stream = f
+        self.encoder = codecs.getincrementalencoder(encoding)()
 
     def writerow(self, row):
-        self.writer.writerow([s.encode(self.encoding) for s in row])
+        self.writer.writerow([s.encode("utf-8") for s in row])
+        # Fetch UTF-8 output from the queue ...
+        data = self.queue.getvalue()
+        data = data.decode("utf-8")
+        # ... and reencode it into the target encoding
+        data = self.encoder.encode(data)
+        # write to the target stream
+        self.stream.write(data)
+        # empty queue
+        self.queue.truncate(0)
 
     def writerows(self, rows):
         for row in rows: