traverseda
/
cpython
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Added an option to the scanner to generated marked-up HTML from the input 

file. This should make it a lot easier (I hope) to get the regular
expressions right.
This commit is contained in:
Jack Jansen 2005-07-17 00:15:46 +00:00
parent a4a8b820aa
commit 149787e7c5
1 changed files with 161 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

167
Tools/bgen/bgen/scantools.py
View File

@ -32,6 +32,76 @@ except ImportError:
Error = "scantools.Error" Error = "scantools.Error"
BEGINHTMLREPORT="""<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.0 Transitional//EN">
<html>
<head>
<style type="text/css">
.unmatched { }
.commentstripping { color: grey; text-decoration: line-through }
.comment { text-decoration: line-through }
.notcomment { color: black }
.incomplete { color: maroon }
.constant { color: green }
.pyconstant { background-color: yellow }
.blconstant { background-color: yellow; color: red }
.declaration { color: blue }
.pydeclaration { background-color: yellow }
.type { font-style: italic }
.name { font-weight: bold }
.value { font-style: italic }
.arglist { text-decoration: underline }
.blacklisted { background-color: yellow; color: red }
</style>
<title>Bgen scan report</title>
</head>
<body>
<h1>Bgen scan report</h1>
<h2>Legend</h2>
<p>This scan report is intended to help you debug the regular expressions
used by the bgen scanner. It consists of the original ".h" header file(s)
marked up to show you what the regular expressions in the bgen parser matched
for each line. NOTE: comments in the original source files may or may not be
shown.</p>
<p>The typographic conventions of this file are as follows:</p>
<dl>
<dt>comment stripping</dt>
<dd><pre><span class="commentstripping"><span class="notcomment">comment stripping is </span><span class="comment">/* marked up */</span><span class="notcomment"> and the line is repeated if needed</span></span></pre>
<p>If anything here does not appear to happen correctly look at
<tt>comment1_pat</tt> and <tt>comment2_pat</tt>.</p>
</dd>
<dt>constant definitions</dt>
<dd><pre><span class="constant">#define <span class="name">name</span> <span class="value">value</span></pre>
<p>Highlights name and value of the constant. Governed by <tt>sym_pat</tt>.</p>
</dd>
<dt>function declaration</dt>
<dd><pre><span class="declaration"><span class="type">char *</span><span class="name">rindex</span><span class="arglist">(<span class="type">const char *</span><span class="name">s</span>, <span class="type">int </span><span class="name">c</span>)</span>;</span></pre>
<p>Highlights type, name and argument list. <tt>type_pat</tt>,
<tt>name_pat</tt> and <tt>args_pat</tt> are combined into <tt>whole_pat</tt>, which
is what is used here.</p></dd>
</dd>
<dt>incomplete match for function declaration</dt>
<dd><pre><span class="incomplete"><span class="type">char *</span>foo;</span></pre>
<p>The beginning of this looked promising, but it did not match a function declaration.
In other words, it matched <tt>head_pat</tt> but not <tt>whole_pat</tt>. If the next
declaration has also been gobbled up you need to look at <tt>end_pat</tt>.</p>
</dd>
<dt>unrecognized input</dt>
<dd><pre><span class="unmatched">#include "type.h"</span></pre>
<p>If there are function declarations the scanner has missed (i.e. things
are in this class but you want them to be declarations) you need to adapt
<tt>head_pat</tt>.
</dd>
</dl>
<h2>Output</h2>
<pre>
<span class="unmatched">
"""
ENDHTMLREPORT="""</span>
</pre>
</body>
</html>
"""
class Scanner: class Scanner:
# Set to 1 in subclass to debug your scanner patterns. # Set to 1 in subclass to debug your scanner patterns.
@ -232,9 +302,11 @@ if missing: raise "Missing Types"
self.specmine = 0 self.specmine = 0
self.defsmine = 0 self.defsmine = 0
self.scanmine = 0 self.scanmine = 0
self.htmlmine = 0
self.specfile = sys.stdout self.specfile = sys.stdout
self.defsfile = None self.defsfile = None
self.scanfile = sys.stdin self.scanfile = sys.stdin
self.htmlfile = None
self.lineno = 0 self.lineno = 0
self.line = "" self.line = ""
@ -286,6 +358,7 @@ if missing: raise "Missing Types"
self.closespec() self.closespec()
self.closedefs() self.closedefs()
self.closescan() self.closescan()
self.closehtml()
def closespec(self): def closespec(self):
tmp = self.specmine and self.specfile tmp = self.specmine and self.specfile
@ -302,6 +375,12 @@ if missing: raise "Missing Types"
self.scanfile = None self.scanfile = None
if tmp: tmp.close() if tmp: tmp.close()
def closehtml(self):
if self.htmlfile: self.htmlfile.write(ENDHTMLREPORT)
tmp = self.htmlmine and self.htmlfile
self.htmlfile = None
if tmp: tmp.close()
def setoutput(self, spec, defs = None): def setoutput(self, spec, defs = None):
self.closespec() self.closespec()
self.closedefs() self.closedefs()
@ -324,6 +403,19 @@ if missing: raise "Missing Types"
self.defsfile = file self.defsfile = file
self.defsmine = mine self.defsmine = mine
def sethtmloutput(self, htmlfile):
self.closehtml()
if htmlfile:
if type(htmlfile) == StringType:
file = self.openoutput(htmlfile)
mine = 1
else:
file = htmlfile
mine = 0
self.htmlfile = file
self.htmlmine = mine
self.htmlfile.write(BEGINHTMLREPORT)
def openoutput(self, filename): def openoutput(self, filename):
try: try:
file = open(filename, 'w') file = open(filename, 'w')
@ -408,11 +500,17 @@ if missing: raise "Missing Types"
self.report("LINE: %r" % (line,)) self.report("LINE: %r" % (line,))
match = self.comment1.match(line) match = self.comment1.match(line)
if match: if match:
self.htmlreport(line, klass='commentstripping', ranges=[(
match.start('rest'), match.end('rest'), 'notcomment')])
line = match.group('rest') line = match.group('rest')
if self.debug: if self.debug:
self.report("\tafter comment1: %r" % (line,)) self.report("\tafter comment1: %r" % (line,))
match = self.comment2.match(line) match = self.comment2.match(line)
while match: while match:
if match:
self.htmlreport(line, klass='commentstripping', ranges=[
(match.start('rest1'), match.end('rest1'), 'notcomment'),
(match.start('rest2'), match.end('rest2'), 'notcomment')])
line = match.group('rest1')+match.group('rest2') line = match.group('rest1')+match.group('rest2')
if self.debug: if self.debug:
self.report("\tafter comment2: %r" % (line,)) self.report("\tafter comment2: %r" % (line,))
@ -422,7 +520,7 @@ if missing: raise "Missing Types"
if match: if match:
if self.debug: if self.debug:
self.report("\tmatches sym.") self.report("\tmatches sym.")
self.dosymdef(match) self.dosymdef(match, line)
continue continue
match = self.head.match(line) match = self.head.match(line)
if match: if match:
@ -430,19 +528,26 @@ if missing: raise "Missing Types"
self.report("\tmatches head.") self.report("\tmatches head.")
self.dofuncspec() self.dofuncspec()
continue continue
self.htmlreport(line, klass='unmatched')
except EOFError: except EOFError:
self.error("Uncaught EOF error") self.error("Uncaught EOF error")
self.reportusedtypes() self.reportusedtypes()
def dosymdef(self, match): def dosymdef(self, match, line):
name, defn = match.group('name', 'defn') name, defn = match.group('name', 'defn')
self.htmlreport(line, klass='constant', ranges=[
(match.start('name'), match.end('name'), 'name'),
(match.start('defn'), match.end('defn'), 'value')])
defn = escape8bit(defn) defn = escape8bit(defn)
if self.debug: if self.debug:
self.report("\tsym: name=%r, defn=%r" % (name, defn)) self.report("\tsym: name=%r, defn=%r" % (name, defn))
if not name in self.blacklistnames: if not name in self.blacklistnames:
self.defsfile.write("%s = %s\n" % (name, defn)) oline = "%s = %s\n" % (name, defn)
self.defsfile.write(oline)
self.htmlreport(oline, klass="pyconstant")
else: else:
self.defsfile.write("# %s = %s\n" % (name, defn)) self.defsfile.write("# %s = %s\n" % (name, defn))
self.htmlreport("** no output: name is blacklisted", klass="blconstant")
# XXXX No way to handle greylisted names # XXXX No way to handle greylisted names
def dofuncspec(self): def dofuncspec(self):
@ -473,19 +578,30 @@ if missing: raise "Missing Types"
if not match: if not match:
self.report("Bad raw spec: %r", raw) self.report("Bad raw spec: %r", raw)
if self.debug: if self.debug:
if not self.type.search(raw): match = self.type.search(raw)
if not match:
self.report("(Type already doesn't match)") self.report("(Type already doesn't match)")
self.htmlreport(raw, klass='incomplete', ranges=[(
match.start('type'), match.end('type'), 'type')])
else: else:
self.report("(but type matched)") self.report("(but type matched)")
self.htmlreport(raw, klass='incomplete')
return return
type, name, args = match.group('type', 'name', 'args') type, name, args = match.group('type', 'name', 'args')
ranges=[
(match.start('type'), match.end('type'), 'type'),
(match.start('name'), match.end('name'), 'name'),
(match.start('args'), match.end('args'), 'arglist')]
self.htmlreport(raw, klass='declaration', ranges=ranges)
modifiers = self.getmodifiers(match) modifiers = self.getmodifiers(match)
type = self.pythonizename(type) type = self.pythonizename(type)
name = self.pythonizename(name) name = self.pythonizename(name)
if self.checkduplicate(name): if self.checkduplicate(name):
self.htmlreport("*** no output generated: duplicate name", klass="blacklisted")
return return
self.report("==> %s %s <==", type, name) self.report("==> %s %s <==", type, name)
if self.blacklisted(type, name): if self.blacklisted(type, name):
self.htmlreport("*** no output generated: function name or return type blacklisted", klass="blacklisted")
self.report("*** %s %s blacklisted", type, name) self.report("*** %s %s blacklisted", type, name)
return return
returnlist = [(type, name, 'ReturnMode')] returnlist = [(type, name, 'ReturnMode')]
@ -494,6 +610,7 @@ if missing: raise "Missing Types"
arglist = self.extractarglist(args) arglist = self.extractarglist(args)
arglist = self.repairarglist(name, arglist) arglist = self.repairarglist(name, arglist)
if self.unmanageable(type, name, arglist): if self.unmanageable(type, name, arglist):
self.htmlreport("*** no output generated: some argument blacklisted", klass="blacklisted")
##for arg in arglist: ##for arg in arglist:
## self.report(" %r", arg) ## self.report(" %r", arg)
self.report("*** %s %s unmanageable", type, name) self.report("*** %s %s unmanageable", type, name)
@ -611,8 +728,12 @@ if missing: raise "Missing Types"
classname, listname = self.destination(tp, name, arglist, modifiers) classname, listname = self.destination(tp, name, arglist, modifiers)
else: else:
classname, listname = self.destination(tp, name, arglist) classname, listname = self.destination(tp, name, arglist)
if not classname or not listname: return if not classname or not listname:
if not self.specfile: return self.htmlreport("*** no output generated: self.destination() returned None", klass="blacklisted")
return
if not self.specfile:
self.htmlreport("*** no output generated: no output file specified", klass="blacklisted")
return
self.specfile.write("f = %s(%s, %r,\n" % (classname, tp, name)) self.specfile.write("f = %s(%s, %r,\n" % (classname, tp, name))
for atype, aname, amode in arglist: for atype, aname, amode in arglist:
self.typeused(atype, amode) self.typeused(atype, amode)
@ -623,6 +744,12 @@ if missing: raise "Missing Types"
self.generatemodifiers(classname, name, modifiers) self.generatemodifiers(classname, name, modifiers)
self.specfile.write(")\n") self.specfile.write(")\n")
self.specfile.write("%s.append(f)\n\n" % listname) self.specfile.write("%s.append(f)\n\n" % listname)
if self.htmlfile:
oline = "Adding to %s:\n%s(returntype=%s, name=%r" % (listname, classname, tp, name)
for atype, aname, amode in arglist:
oline += ",\n (%s, %r, %s)" % (atype, aname, amode)
oline += ")\n"
self.htmlreport(oline, klass="pydeclaration")
def destination(self, type, name, arglist): def destination(self, type, name, arglist):
return "FunctionGenerator", "functions" return "FunctionGenerator", "functions"
@ -646,6 +773,34 @@ if missing: raise "Missing Types"
return 1 return 1
return 0 return 0
def htmlreport(self, line, klass=None, ranges=None):
if not self.htmlfile: return
if ranges is None:
ranges = []
if klass:
ranges.insert(0, (0, len(line), klass))
oline = ''
i = 0
for c in line:
for b, e, name in ranges:
if b == i:
oline += '<span class="%s">' % name
if e == i:
oline += '</span>'
i += 1
if c == '<': oline += '&lt;'
elif c == '>': oline += '&gt;'
else: oline += c
for b, e, name in ranges:
if b >= i:
oline += '<span class="%s">' % name
if e >= i:
oline += '</span>'
if not line or line[-1] != '\n':
oline += '\n'
self.htmlfile.write(oline)
class Scanner_PreUH3(Scanner): class Scanner_PreUH3(Scanner):
"""Scanner for Universal Headers before release 3""" """Scanner for Universal Headers before release 3"""
def initpatterns(self): def initpatterns(self):