From 1ee99d31d980e8a6e9c9d2379900f8bd5f98a9fe Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Martin=20v=2E=20L=C3=B6wis?= <martin@v.loewis.de>
Date: Sun, 4 Aug 2002 20:10:29 +0000
Subject: [PATCH] Make pgen compile with pydebug. Duplicate normalized names,
 as it may be longer than the old string.

---
 Parser/tokenizer.c | 8 ++++++--
 Parser/tokenizer.h | 2 ++
 2 files changed, 8 insertions(+), 2 deletions(-)

diff --git a/Parser/tokenizer.c b/Parser/tokenizer.c
index 0ae093e68ec..53c883f5e4c 100644
--- a/Parser/tokenizer.c
+++ b/Parser/tokenizer.c
@@ -128,8 +128,10 @@ tok_new(void)
 	tok->read_coding_spec = 0;
 	tok->issued_encoding_warning = 0;
 	tok->encoding = NULL;
+#ifndef PGEN
 	tok->decoding_readline = NULL;
 	tok->decoding_buffer = NULL;
+#endif
 	return tok;
 }
 
@@ -225,8 +227,8 @@ get_coding_spec(const char *s, int size)
 				char* r = new_string(begin, t - begin);
 				char* q = get_normal_name(r);
 				if (r != q) {
-					assert(strlen(r) >= strlen(q));
-					strcpy(r, q);
+					PyMem_DEL(r);
+					r = new_string(q, strlen(q));
 				}
 				return r;
 			}
@@ -584,8 +586,10 @@ PyTokenizer_Free(struct tok_state *tok)
 {
 	if (tok->encoding != NULL)
 		PyMem_DEL(tok->encoding);
+#ifndef PGEN
 	Py_XDECREF(tok->decoding_readline);
 	Py_XDECREF(tok->decoding_buffer);
+#endif
 	if (tok->fp != NULL && tok->buf != NULL)
 		PyMem_DEL(tok->buf);
 	PyMem_DEL(tok);
diff --git a/Parser/tokenizer.h b/Parser/tokenizer.h
index 9782666fd82..f3bac74acd6 100644
--- a/Parser/tokenizer.h
+++ b/Parser/tokenizer.h
@@ -45,8 +45,10 @@ struct tok_state {
 	int read_coding_spec;	/* whether 'coding:...' has been read  */
 	int issued_encoding_warning; /* whether non-ASCII warning was issued */
 	char *encoding;
+#ifndef PGEN
 	PyObject *decoding_readline; /* codecs.open(...).readline */
 	PyObject *decoding_buffer;
+#endif
 	const char* enc;
 	const char* str;
 };