#!/usr/bin/env python
#
# wmlscope -- generate reports on WML macro and resource usage
#
# By Eric S. Raymond, April 2007.
#
# This tool cross-references macro definitions with macro calls, and
# resource (sound or image) files with uses of the resources in WML.
# and generates various useful reports from such cross-references.
#
# It takes a list of directories as arguments; if none is given, it
# behaves as though the current directory had been specified as a
# single argument.  Each directory is treated as a separate domain for
# macro and resource visibility purposes, except that macros and resources
# under the first directory are made visible in all later ones.  (Typically
# the first directory should point at a copy of mainline and all later
# ones at UMC.)
#
# The checking done by this tool has a couple of flaws:
#
# (1) It doesn't actually evaluate file inclusions.  Instead, any
# macro definition satisfies any macro call made under the same
# directory.  Exception: when an #undef is detected, the macro is
# tagged local and not visible outside the span of lines where it's
# defined.
#
# (2) It doesn't read [binary_path] tags, as this would require
# implementing a WML parser.  Instead, it assumes that a resource-file
# reference can be satisfied by any matching image file from anywhere
# in the same directory it came from.  The resources under the *first*
# directory argument (only) are visible everywhere.
#
# (3) A reference with embedded {}s in a macro will have the macro's
# formal args substituted in at WML evaluation time.  Instead, this
# tool treats each {} as a .* wildcard and considers the reference to
# match *every* resource filename that matches that pattern.  Under
# appropriate circumstances this might report a resource filename
# statically matching the pattern as having been referenced even
# though none of the actual macro calls would actually generate it.
#
# (4) There are some implicit references.  Notably, if an attack name
# is specified but no icon is given, the attack icon will default to
# a name generated from the attack name,
#
# Problems (1) and (2) imply that this tool might conceivably report
# that a reference has been satisfied when under actual
# WML-interpreter rules it has not.  Problem (4) means the reverse
# can also occur.
#
# The reporting format is compatible with GNU Emacs compile mode.

import sys, os, time, re, getopt, md5
from wesnoth.wmltools import *

def interpret(lines, css):
    "Interpret the ! convention for .cfg comments."
    inlisting = False
    outstr = '<p class="%s">' % css
    for line in lines:
        line = line.rstrip()
        if not inlisting and not line:
            outstr += "</p><p>"
            continue
        if not inlisting and line[0] == '!':
            outstr += "</p>\n<pre class='listing'>"
            inlisting = True
            bracketdepth = curlydepth = 0
        line = line.replace("<", "&lt;").replace(">", "&gt;").replace("&", "&amp;")
        if inlisting:
            outstr += line[1:] + "\n"
        else:
            outstr += line + "\n"
        if inlisting:
            if line and line[0] != '!':
                outstr += "</pre>\n<p>"
                inlisting = False
    if not inlisting:
        outstr += "</p>\n"
    else:
        outstr += "</pre>\n"
    outstr = outstr.replace("<p></p>", "")
    outstr = outstr.replace("\n\n</pre>", "\n</pre>")
    return outstr

class CrossRefLister(CrossRef):
    "Cross-reference generator with reporting functions"
    def xrefdump(self, pred=None):
        "Report resolved macro references."
        for name in self.xref:
            for defn in self.xref[name]:
                if pred and not pred(name, defn):
                    continue
                if defn.undef:
                    type = "local"
                else:
                    type = "global"
                nrefs = len(defn.references)
                if nrefs == 0:
                    print "%s: %s macro %s is unused" % (defn, type, name)
                else:
                    print "%s: %s macro %s is used in %d files:" % (defn, type, name, nrefs)
                defn.dump_references()
        for (name, defloc) in self.fileref.items():
            if pred and not pred(name, defloc):
                continue
            nrefs = len(defloc.references)
            if nrefs == 0:
                print "Resource %s is unused" % defloc
            else:
                print "Resource %s is used in %d files:" % (defloc, nrefs)
            defloc.dump_references()
    def unresdump(self):
        "Report unresolved references."
        if len(self.unresolved) == 0 and len(self.missing) == 0:
            print "# No unresolved references"
        else:
            #print self.fileref.keys()
            print "# Unresolved references:"
            for (name, reference) in self.unresolved + self.missing:
                print "%s -> %s" % (reference, name)
    def deflist(self, pred=None):
        "List all resource definitions."
        for name in self.xref:
            for defn in self.xref[name]:
                if not pred or pred(name, defn):
                    print name
        for (name, defloc) in self.fileref.items():
            if not pred or pred(name, defloc):
                print name
    def extracthelp(self, pref, fp):
        "Deliver all macro help comments in HTML form."
        # Bug: finds only the first definition of each macro in scope.
        doclist = self.xref.keys()
        doclist = filter(lambda x: self.xref[x][0].docstring.count("\n") > 1, doclist)
        doclist.sort(lambda x, y: cmp(self.xref[x][0], self.xref[y][0]))
        outstr = ""
        filename = None
        counted = 0
        for name in doclist:
            entry = self.xref[name][0]
            if entry.filename != filename:
                if counted:
                    outstr += "</dl>\n"
                counted += 1
                filename = entry.filename
                if filename.startswith(pref):
                    displayname = filename[len(pref):]
                else:
                    displayname = filename
                outstr += "<h1 class='file_header'>From file: " + displayname + "</h1>\n"
                hdr = []
                dfp = open(filename)
                for line in dfp:
                    if line[0] == '#':
                        hdr.append(line[1:])
                    else:
                        break
                dfp.close()
                if hdr:
                    outstr += interpret(hdr, "file_explanation")
                outstr += "<dl>\n"
            if entry.docstring:
                lines = entry.docstring.split("\n")
                header = lines.pop(0).split()
                if lines and not lines[-1]:	# Ignore trailing blank lines
                    lines.pop()
                if not lines:	# Ignore definitions without a docstring
                    continue
                outstr += "\n<dt>\n"
                outstr += "<em class='macro_name'>" + header[0] + "</em>"
                if header[1:]:
                    outstr += " <em class='macro_formals'>"+" ".join(header[1:])+"</em>"
                outstr += "\n</dt>\n"
                outstr += "<dd>\n"
                outstr += interpret(lines, "macro_explanation")
                outstr += "</dd>\n"
        outstr += "</dl>\n"
        fp.write(outstr)

if __name__ == "__main__":
    def help():
        sys.stderr.write("""\
Usage: macroscope [options] dirpath
    Options may be any of these:
    -h, --help                   Emit this help message and quit
    -c, --crossreference         Report resolved macro references (implies -w 1)
    -C, --collisions             Report duplicate resource files   
    -d, --deflist                Make definition list
    -e regexp, --exclude regexp  Ignore files matching the specified regular expression
    -f dir, --from dir           Report only on macros defined under dir
    -l, --listfiles              List files that will be processed
    -r ddd, --refcount=ddd       Report only on macros w/references in ddd files
    -u, --unresolved             Report unresolved macro references
    -w, --warnlevel              Set to 1 to warn of duplicate macro definitions
    --forced-used regexp         Ignore refcount 0 on names matching regexp
    --extracthelp                Extract help from macro definition comments.
   Options may be followed by any number of directiories to check.  If no
   directories are given, all files under the current directory are checked.
""")

    # Process options
    (options, arguments) = getopt.getopt(sys.argv[1:], "cCdhe:f:lr:uw:",
                                         [
                                          'crossreference',
                                          'collisions',
                                          'definitions',
                                          'exclude=',
                                          'extracthelp',
                                          'force-used=',
                                          'from=',
        				  'help',
                                          'listfiles',
                                          'refcount=',
                                          'unresolved',
                                          'warnlevel=',
                                          ])
    crossreference = definitions = listfiles = unresolved = extracthelp = False
    from_restrict = None
    refcount_restrict = None
    forceused = None
    exclude = []
    warnlevel = 0
    collisions = False
    for (switch, val) in options:
        if switch in ('-h', '--help'):
            help()
            sys.exit(0)
        if switch in ('-f', '--from'):
            from_restrict = val
        elif switch in ('-c', '--crossreference'):
            crossreference = True
            warnlevel = 1
        elif switch in ('-C', '--collisions'):
            collisions = True
        elif switch in ('-d', '--definitions'):
            definitions = True
        elif switch in ('-e', '--exclude'):
            exclude.append(val)
        elif switch == '--extracthelp':
            extracthelp = True
        elif switch == '--force-used':
            forceused = val
        elif switch in ('-l', '--listfiles'):
            listfiles = True
        elif switch in ('-r', '--refcount'):
            refcount_restrict = int(val)
        elif switch in ('-u', '--unresolved'):
            unresolved = True
        elif switch in ('-w', '--warnlevel'):
            warnlevel = int(val)

    if len(arguments):
        dirpath = arguments
    else:
        dirpath = ['.']
    if not extracthelp:
        print "# Macroscope reporting on %s" % time.ctime()
        print "# Invocation: %s" % " ".join(sys.argv)
        print "# Working directory: %s" % os.getcwd()
    xref = CrossRefLister(dirpath, "|".join(exclude), warnlevel)
    if extracthelp:
        xref.extracthelp(dirpath[0], sys.stdout)
    elif listfiles:
        for filename in xref.filelist.generator():
            print filename
    if collisions:
        collisions = []
        for filename in xref.filelist.generator():
            ifp = open(filename)
            collisions.append(md5.new(ifp.read()).digest())
            ifp.close()
        collisions = zip(xref.filelist.flatten(), collisions)
        hashcounts = {}
        for (n, h) in collisions:
            hashcounts[h] = hashcounts.get(h, 0) + 1
        collisions = filter(lambda (n, h): hashcounts[h] > 1, collisions)
        collisions.sort(lambda (n1, h1), (n2, h2): cmp(h1, h2))
        lasthash = None
        for (n, h) in collisions:
            if h != lasthash:
                print "%%"
                lasthash = h
            print n
    elif crossreference or definitions or listfiles or unresolved:
        def predicate(name, defloc):
            if from_restrict and not defloc.filename.startswith(from_restrict):
                return False
            if refcount_restrict!=None \
                   and len(defloc.references) != refcount_restrict \
                   or (refcount_restrict == 0 and forceused and re.search(forceused, name)):
                return False
            return True
        if crossreference:
            if xref.noxref:
                print >>sys.stderr, "macroscope: can't make cross-reference, input included a definitions file."
            else:
                xref.xrefdump(predicate)
        if definitions:
            xref.deflist(predicate)
        if unresolved:
            xref.unresdump()
    
# wmlscope ends here
