#!python
# ***** BEGIN LICENSE BLOCK *****
# Version: MPL 1.1/GPL 2.0/LGPL 2.1
# 
# The contents of this file are subject to the Mozilla Public License
# Version 1.1 (the "License"); you may not use this file except in
# compliance with the License. You may obtain a copy of the License at
# http://www.mozilla.org/MPL/
# 
# Software distributed under the License is distributed on an "AS IS"
# basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See the
# License for the specific language governing rights and limitations
# under the License.
# 
# The Original Code is Komodo code.
# 
# The Initial Developer of the Original Code is ActiveState Software Inc.
# Portions created by ActiveState Software Inc are Copyright (C) 2000-2007
# ActiveState Software Inc. All Rights Reserved.
# 
# Contributor(s):
#   ActiveState Software Inc
# 
# Alternatively, the contents of this file may be used under the terms of
# either the GNU General Public License Version 2 or later (the "GPL"), or
# the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
# in which case the provisions of the GPL or the LGPL are applicable instead
# of those above. If you wish to allow use of your version of this file only
# under the terms of either the GPL or the LGPL, and not to allow others to
# use your version of this file under the terms of the MPL, indicate your
# decision by deleting the provisions above and replace them with the notice
# and other provisions required by the GPL or the LGPL. If you do not delete
# the provisions above, a recipient may use your version of this file under
# the terms of any one of the MPL, the GPL or the LGPL.
# 
# ***** END LICENSE BLOCK *****

"""The stdlib zone of the codeintel database.
See the database/database.py module docstring for an overview.
"""


import sys
import os
from os.path import (join, dirname, exists, expanduser, splitext, basename,
                     split, abspath, isabs, isdir, isfile)
import cPickle as pickle
import threading
import time
import bisect
import fnmatch
from glob import glob
from pprint import pprint, pformat
import logging
from cStringIO import StringIO
import codecs
import copy
import weakref
import Queue

import ciElementTree as ET
from codeintel2.common import *
from codeintel2.buffer import Buffer
from codeintel2.util import dedent, safe_lang_from_lang, banner
from codeintel2.tree import tree_from_cix_path
from codeintel2.database.resource import AreaResource
from codeintel2.database.util import (rmdir, filter_blobnames_for_prefix)



#---- globals

log = logging.getLogger("codeintel.db")
#log.setLevel(logging.DEBUG)



#---- Database zone and lib implementations

class StdLib(object):
    """Singleton lib managing a particular db/stdlibs/<stdlib-name>
    area of the db.

    These are dished out via Database.get_stdlib(), which indirectly
    then is dished out by the StdLibsZone.get_lib().

    Because (1) any updating of the stdlib db area for this language has
    already been done (by StdLibsZone.get_lib()) and (2) this is a
    singleton: we shouldn't have to worry about locking.
    """
    _blob_index = None
    _toplevelname_index = None
    _toplevelprefix_index = None

    def __init__(self, db, base_dir, lang, name):
        self.db = db
        self.lang = lang
        self.name = name
        self.base_dir = base_dir
        self._import_handler = None
        self._blob_imports_from_prefix_cache = {}
        self._blob_from_blobname = {}

    def __repr__(self):
        return "<%s stdlib>" % self.name

    @property
    def import_handler(self):
        if self._import_handler is None:
            self._import_handler \
                = self.db.mgr.citadel.import_handler_from_lang(self.lang)
        return self._import_handler

    @property
    def blob_index(self):
        if self._blob_index is None:
            idxpath = join(self.base_dir, "blob_index")
            self._blob_index = self.db.load_pickle(idxpath)
        return self._blob_index

    @property
    def toplevelname_index(self):
        if self._toplevelname_index is None:
            idxpath = join(self.base_dir, "toplevelname_index")
            self._toplevelname_index = self.db.load_pickle(idxpath)
        return self._toplevelname_index

    @property
    def toplevelprefix_index(self):
        if self._toplevelprefix_index is None:
            idxpath = join(self.base_dir, "toplevelprefix_index")
            self._toplevelprefix_index = self.db.load_pickle(idxpath)
        return self._toplevelprefix_index

    def has_blob(self, blobname):
        return blobname in self.blob_index

    def get_blob(self, blobname):
        # Cache the blob once. Don't need to worry about invalidating the stdlib
        # blobs as stdlibs should not change during a Komodo session, bug 65502.
        blob = self._blob_from_blobname.get(blobname)
        if blob is None:
            try:
                dbfile = self.blob_index[blobname]
            except KeyError:
                return None
            blob = self.db.load_blob(join(self.base_dir, dbfile))
            self._blob_from_blobname[blobname] = blob
        return blob

    def get_blob_imports(self, prefix):
        """Return the set of imports under the given prefix.

            "prefix" is a tuple of import name parts. E.g. ("xml", "sax")
                for "import xml.sax." in Python. Or ("XML", "Parser") for
                "use XML::Parser::" in Perl.

        See description in database.py docstring for details.
        """
        if prefix not in self._blob_imports_from_prefix_cache:
            matches = filter_blobnames_for_prefix(self.blob_index,
                        prefix, self.import_handler.sep)
            self._blob_imports_from_prefix_cache[prefix] = matches
        return self._blob_imports_from_prefix_cache[prefix]

    def hits_from_lpath(self, lpath, ctlr=None, curr_buf=None):
        """Return all hits of the given lookup path.
        
        I.e. a symbol table lookup across all files in the dirs of this
        lib.

            "lpath" is a lookup name list, e.g. ['Casper', 'Logging']
                or ['dojo', 'animation'].
            "ctlr" (optional) is an EvalController instance. If
                specified it should be used in the normal way (logging,
                checking .is_aborted()).
            "curr_buf" (optional) is not relevant for StdLib. Used for
                other *Lib classes.

        A "hit" is (<CIX node>, <scope-ref>).  Each one represent a
        scope-tag or variable-tag hit in all of the blobs for the
        execution set buffers.

        Returns the empty list if no hits.
        """
        assert isinstance(lpath, tuple)  # common mistake to pass in a string
        hits = []
        # toplevelname_index: {ilk -> toplevelname -> blobnames}
        for blobnames_from_toplevelname in self.toplevelname_index.itervalues():
            for blobname in blobnames_from_toplevelname.get(lpath[0], ()):
                blob = self.get_blob(blobname)
                try:
                    elem = blob
                    for p in lpath:
                        #LIMITATION: *Imported* names at each scope are
                        # not being included here. This is fine while we
                        # just care about JavaScript.
                        elem = elem.names[p]
                except KeyError:
                    continue
                hits.append( (elem, (blob, list(lpath[:-1]))) )
        return hits

    def toplevel_cplns(self, prefix=None, ilk=None):
        """Return completion info for all top-level names matching the
        given prefix and ilk in all blobs in this lib.
        
            "prefix" is a 3-character prefix with which to filter top-level
                names. If None (or not specified), results are not filtered
                based on the prefix.
            "ilk" is a symbol type (e.g. "class", "variable", "function")
                with which to filter results. If None (or not specified),
                results of any ilk are returned.
            "ctlr" (optional) is an EvalController instance. If
                specified it should be used in the normal way (logging,
                checking .is_aborted()).

        Returns a list of 2-tuples: (<ilk>, <name>).

        Note: the list is not sorted, because often some special sorting
        is required for the different completion evaluators that might use
        this API.
        """
        cplns = []
        if prefix is None:
            # Use 'toplevelname_index': {ilk -> toplevelname -> blobnames}
            for i, bft in self.toplevelname_index.iteritems():
                if ilk is not None and i != ilk:
                    continue
                cplns += [(i, toplevelname) for toplevelname in bft]
        else:
            # Use 'toplevelprefix_index':
            #   {ilk -> prefix -> toplevelnames}
            if ilk is not None:
                try:
                    toplevelnames = self.toplevelprefix_index[ilk][prefix]
                except KeyError:
                    pass
                else:
                    cplns += [(ilk, t) for t in toplevelnames]
            else:
                for i, tfp in self.toplevelprefix_index.iteritems():
                    if prefix not in tfp:
                        continue
                    cplns += [(i, t) for t in tfp[prefix]]

        return cplns
        

    def reportMemory(self):
        """
        Report on memory usage from this StdLib.
        @returns {dict} memory usage; keys are the paths, values are a dict of
            "amount" -> number
            "units" -> "bytes" | "count"
            "desc" -> str description
        """
        log.debug("%s StdLib %s: reporting memory", self.lang, self.name)
        import memutils
        return {
            "explicit/python/codeintel/%s/stdlib/%s" % (self.lang, self.name): {
                "amount": memutils.memusage(self._blob_from_blobname) +
                          memutils.memusage(self._blob_imports_from_prefix_cache),
                "units": "bytes",
                "desc": "The number of bytes of %s codeintel stdlib %s blobs." % (self.lang, self.name),
            }
        }
        return total_mem_usage


class StdLibsZone(object):
    """Singleton zone managing the db/stdlibs/... area.

    Because this is a singleton we shouldn't have to worry about locking
    to prevent corruption.
    """
    _res_index = None                   # cix-path -> last-updated

    def __init__(self, db):
        self.db = db
        self.stdlibs_dir = join(dirname(dirname(__file__)), "stdlibs")
        self.base_dir = join(self.db.base_dir, "db", "stdlibs")
        self._stdlib_from_stdlib_ver_and_name = {} # cache of StdLib singletons
        self._vers_and_names_from_lang = {} # lang -> ordered list of (ver, name)

    def vers_and_names_from_lang(self, lang):
        "Returns an ordered list of (ver, name) for the given lang."
        #  _vers_and_names_from_lang = {
        #    "php": [
        #              ((4,3), "php-4.3"),
        #              ((5.0), "php-5.0"),
        #              ((5.1), "php-5.1"),
        #              ((5.2), "php-5.2"),
        #              ((5,3), "php-5.3")
        #         ],
        #    "ruby": [
        #              (None, "ruby"),
        #         ],
        #    ...
        #  }
        vers_and_names = self._vers_and_names_from_lang.get(lang)
        if vers_and_names is None:
            # Find the available stdlibs for this language.
            cix_glob = join(self.stdlibs_dir, safe_lang_from_lang(lang)+"*.cix")
            cix_paths = glob(cix_glob)
            vers_and_names = []
            for cix_path in cix_paths:
                name = splitext(basename(cix_path))[0]
                if '-' in name:
                    base, ver_str = name.split('-', 1)
                    ver = _ver_from_ver_str(ver_str)
                else:
                    base = name
                    ver = None
                if base.lower() != lang.lower():
                    # Only process when the base name matches the language.
                    # I.e. skip if base is "python3" and lang is "python".
                    continue
                vers_and_names.append((ver, name))
            vers_and_names.sort()
            self._vers_and_names_from_lang[lang] = vers_and_names
        return vers_and_names

    @property
    def res_index(self):
        "cix-path -> last-updated"
        if self._res_index is None:
            idxpath = join(self.base_dir, "res_index")
            self._res_index = self.db.load_pickle(idxpath, {})
        return self._res_index
    
    def save(self):
        if self._res_index is not None:
            self.db.save_pickle(join(self.base_dir, "res_index"),
                                self._res_index)

    def cull_mem(self):
        """Cull memory usage as appropriate.
        This is a no-op for StdLibsZone because its memory use is bounded and
        doesn't really need culling.
        """
        pass

    def reportMemory(self):
        """
        Report on memory usage from this StdLibZone.
        @returns {dict} memory usage; keys are the paths, values are a dict of
            "amount" -> number
            "units" -> "bytes" | "count"
            "desc" -> str description
        """
        log.debug("StdLibZone: reporting memory")
        result = {}
        for stdlib in self._stdlib_from_stdlib_ver_and_name.values():
            result.update(stdlib.reportMemory())
        return result

    def get_lib(self, lang, ver_str=None):
        """Return a view into the stdlibs zone for a particular language
        and version's stdlib.

            "lang" is the language, e.g. "Perl", for which to get a
                stdlib.
            "ver_str" (optional) is a specific version of the language,
                e.g. "5.8".

        On first get of a stdlib for a particular language, all
        available stdlibs for that lang are updated, if necessary.

        Returns None if there is not stdlib for this language.
        """
        vers_and_names = self.vers_and_names_from_lang(lang)
        if not vers_and_names:
            return None
        if ver_str is None:
            # Default to the latest version.
            ver = vers_and_names[-1][0]
        else:
            ver = _ver_from_ver_str(ver_str)

        # Here is something like what we have for PHP:
        #    vers_and_names = [
        #        (None, "php"),
        #        ((4,0), "php-4.0"),
        #        ((4,1), "php-4.1"),
        #        ((4,2), "php-4.2"),
        #        ((4,3), "php-4.3"),
        #        ((5,0), "php-5.0"),
        #        ((5,1), "php-5.1"),
        #    ]
        # We want to (quickly) pick the best fit stdlib for the given
        # PHP version:
        #   PHP (ver=None): php
        #   PHP 3.0:        php
        #   PHP 4.0:        php-4.0 (exact match)
        #   PHP 4.0.2:      php-4.0 (higher sub-version)
        #   PHP 4.4:        php-4.3
        #   PHP 6.0:        php-5.1
        key = (ver, "zzz") # 'zzz' > any stdlib name (e.g., 'zzz' > 'php-4.2')
        idx = max(0, bisect.bisect_right(vers_and_names, key)-1)
        log.debug("best stdlib fit for %s ver=%s in %s is %s",
                  lang, ver, vers_and_names, vers_and_names[idx])
        
        stdlib_match = vers_and_names[idx]
        stdlib_ver, stdlib_name = stdlib_match

        if stdlib_match not in self._stdlib_from_stdlib_ver_and_name:
            # TODO: This _update_lang_with_ver method should really moved into
            #       the StdLib class.
            self._update_lang_with_ver(lang, ver=stdlib_ver)
            stdlib = StdLib(self.db,
                            join(self.base_dir, stdlib_name),
                            lang, stdlib_name)
            self._stdlib_from_stdlib_ver_and_name[stdlib_match] = stdlib

        return self._stdlib_from_stdlib_ver_and_name[stdlib_match]

    def _get_preload_zip(self):
        return join(self.stdlibs_dir, "stdlibs.zip")

    def can_preload(self):
        """Return True iff can preload."""
        if exists(self.base_dir):
            log.info("can't preload stdlibs: `%s' exists", self.base_dir)
            return False
        try:
            import process
            import which
        except ImportError, ex:
            log.info("can't preload stdlibs: %s", ex)
            return False
        try:
            which.which("unzip")
        except which.WhichError, ex:
            log.info("can't preload stdlibs: %s", ex)
            return False
        preload_zip = self._get_preload_zip()
        if not exists(preload_zip):
            log.info("can't preload stdlibs: `%s' does not exist", preload_zip)
            return False
        return True

    def preload(self, progress_cb=None):
        """Pre-load the stdlibs zone, if able.

            "progress_cb" (optional) is a callable that is called as
                follows to show the progress of the update:
                    progress_cb(<desc>, <value>)
                where <desc> is a short string describing the current step
                and <value> is an integer between 0 and 100 indicating the
                level of completeness.

        Use `.can_preload()' to determine if able to pre-load.
        """
        import which
        import process

        log.debug("preloading stdlibs zone")
        if progress_cb:
            try:    progress_cb("Preloading stdlibs...", None)
            except: log.exception("error in progress_cb (ignoring)")
        preload_zip = self._get_preload_zip()
        unzip_exe = which.which("unzip")
        cmd = '"%s" -q -d "%s" "%s"'\
              % (unzip_exe, dirname(self.base_dir), preload_zip)
        p = process.ProcessOpen(cmd, stdin=None)
        stdout, stderr = p.communicate()
        retval = p.wait()
        if retval:
            raise OSError("error running '%s'" % cmd)

    #TODO: Add ver_str option (as per get_lib above) and only update
    #      the relevant stdlib.
    def remove_lang(self, lang):
        """Remove the given language from the stdlib zone."""
        log.debug("update '%s' stdlibs", lang)

        # Figure out what updates need to be done...
        cix_glob = join(self.stdlibs_dir, safe_lang_from_lang(lang)+"*.cix")
        todo = []
        for area, subpath in self.res_index:
            res = AreaResource(subpath, area)
            if fnmatch.fnmatch(res.path, cix_glob):
                todo.append(("remove", AreaResource(subpath, area)))

        # ... and then do them.
        self._handle_res_todos(lang, todo)
        self.save()

    def _update_lang_with_ver(self, lang, ver=None, progress_cb=None):
        """Import stdlib data for this lang, if necessary.
        
            "lang" is the language to update.
            "ver" (optional) is a specific version of the language,
                e.g. (5, 8).
            "progress_cb" (optional) is a callable that is called as
                follows to show the progress of the update:
                    progress_cb(<desc>, <value>)
                where <desc> is a short string describing the current step
                and <value> is an integer between 0 and 100 indicating the
                level of completeness.
        """
        log.debug("update '%s' stdlibs", lang)
        # Figure out what updates need to be done...
        if progress_cb:
            try:    progress_cb("Determining necessary updates...", 5)
            except: log.exception("error in progress_cb (ignoring)")
        if ver is not None:
            ver_str = ".".join(map(str, ver))
            cix_path = join(self.stdlibs_dir,
                            "%s-%s.cix" % (safe_lang_from_lang(lang), ver_str))
        else:
            cix_path = join(self.stdlibs_dir,
                             "%s.cix" % (safe_lang_from_lang(lang), ))

        # Need to acquire db lock, as the indexer and main thread may both be
        # calling into _update_lang_with_ver at the same time.
        self.db.acquire_lock()
        try:
            todo = []
            res = AreaResource(cix_path, "ci-pkg-dir")
            try:
                last_updated = self.res_index[res.area_path]
            except KeyError:
                todo.append(("add", res))
            else:
                mtime = os.stat(cix_path).st_mtime
                if last_updated != mtime: # epsilon? '>=' instead of '!='?
                    todo.append(("update", res))
    
            # ... and then do them.
            self._handle_res_todos(lang, todo, progress_cb)
            self.save()
        finally:
            self.db.release_lock()

    def update_lang(self, lang, progress_cb=None, ver=None):
        vers_and_names = self.vers_and_names_from_lang(lang)
        if ver is not None:
            ver = _ver_from_ver_str(ver)
            key = (ver, "zzz") # 'zzz' > any stdlib name (e.g., 'zzz' > 'php-4.2')
            idx = max(0, bisect.bisect_right(vers_and_names, key)-1)
            log.debug("update_lang: best stdlib fit for %s ver=%s in %s is %s",
                      lang, ver, vers_and_names, vers_and_names[idx])
            # Just update the one version for this language.
            vers_and_names = [vers_and_names[idx]]
        for ver, name in vers_and_names:
            self._update_lang_with_ver(lang, ver, progress_cb)

    def _handle_res_todos(self, lang, todo, progress_cb=None):
        if not todo:
            return
        for i, (action, res) in enumerate(todo):
            cix_path = res.path
            name = splitext(basename(cix_path))[0]
            if '-' in name:
                base, ver_str = name.split('-', 1)
                ver = _ver_from_ver_str(ver_str)
            else:
                base = name
                ver = None
            assert base == safe_lang_from_lang(lang)

            log.debug("%s %s stdlib: `%s'", action, name, cix_path)
            verb = {"add": "Adding", "remove": "Removing",
                    "update": "Updating"}[action]
            desc = "%s %s stdlib" % (verb, name)
            if progress_cb:
                try:    progress_cb(desc, (5 + 95/len(todo)*i))
                except: log.exception("error in progress_cb (ignoring)")
            else:
                self.db.report_event(desc)

            if action == "add":
                self._add_res(res, lang, name, ver)
            elif action == "remove":
                self._remove_res(res, lang, name, ver)
            elif action == "update":
                #XXX Bad for filesystem. Change this to do it
                #    more intelligently if possible.
                self._remove_res(res, lang, name, ver)
                self._add_res(res, lang, name, ver)

    def _remove_res(self, res, lang, name, ver):
        log.debug("%s stdlibs: remove %s", lang, res)
        del self.res_index[res.area_path]
        dbdir = join(self.base_dir, name)
        try:
            rmdir(dbdir)
        except OSError, ex:
            try:
                os.rename(dbdir, dbdir+".zombie")
            except OSError, ex2:
                log.error("could not remove %s stdlib database dir `%s' (%s): "
                          "couldn't even rename it to `%s.zombie' (%s): "
                          "giving up", name, dbdir, ex, name, ex2)
            else:
                log.warn("could not remove %s stdlib database dir `%s' (%s): "
                         "moved it to `%s.zombie'", name, dbdir, ex)

    def _add_res(self, res, lang, name, ver):
        log.debug("%s stdlibs: add %s", lang, res)
        cix_path = res.path
        try:
            tree = tree_from_cix_path(cix_path)
        except ET.XMLParserError, ex:
            log.warn("could not load %s stdlib from `%s' (%s): skipping",
                     name, cix_path, ex)
            return

        dbdir = join(self.base_dir, name)
        if exists(dbdir):
            log.warn("`db/stdlibs/%s' already exists and should not: "
                     "removing it", name)
            try:
                rmdir(dbdir)
            except OSError, ex:
                log.error("could not remove `%s' to create %s stdlib in "
                          "database (%s): skipping", dbdir, name)
        if not exists(dbdir):
            os.makedirs(dbdir)

        # Create 'blob_index' and 'toplevel*_index' and write out
        # '.blob' file.
        LEN_PREFIX = self.db.LEN_PREFIX
        is_hits_from_lpath_lang = lang in self.db.import_everything_langs
        blob_index = {} # {blobname -> dbfile}
        toplevelname_index = {} # {ilk -> toplevelname -> blobnames}
        toplevelprefix_index = {} # {ilk -> prefix -> toplevelnames}
        for blob in tree.findall("file/scope"):
            assert lang == blob.get("lang"), \
                "Adding %s resource %s to %s blob" % (lang, res, blob.get("lang"))
            blobname = blob.get("name")
            dbfile = self.db.bhash_from_blob_info(cix_path, lang, blobname)
            blob_index[blobname] = dbfile
            ET.ElementTree(blob).write(join(dbdir, dbfile+".blob"))
            for toplevelname, elem in blob.names.iteritems():
                if "__local__" in elem.get("attributes", "").split():
                    # this is internal to the stdlib
                    continue
                ilk = elem.get("ilk") or elem.tag
                bft = toplevelname_index.setdefault(ilk, {})
                if toplevelname not in bft:
                    bft[toplevelname] = set([blobname])
                else:
                    bft[toplevelname].add(blobname)
                prefix = toplevelname[:LEN_PREFIX]
                tfp = toplevelprefix_index.setdefault(ilk, {})
                if prefix not in tfp:
                    tfp[prefix] = set([toplevelname])
                else:
                    tfp[prefix].add(toplevelname)

        self.db.save_pickle(join(dbdir, "blob_index"), blob_index)
        self.db.save_pickle(join(dbdir, "toplevelname_index"),
                            toplevelname_index)
        self.db.save_pickle(join(dbdir, "toplevelprefix_index"),
                            toplevelprefix_index)

        mtime = os.stat(cix_path).st_mtime
        self.res_index[res.area_path] = mtime



#---- internal support stuff

def _ver_from_ver_str(ver_str):
    """Convert a version string to a version object as used internally
    for the "stdlibs" area of the database.
   
        >>> _ver_from_ver_str("5.8")
        (5, 8)
        >>> _ver_from_ver_str("1.8.2")
        (1, 8, 2)
        >>> _ver_from_ver_str("ecma")
        'ecma'
        >>> _ver_from_ver_str("ie")
        'ie'
    """
    ver = []
    for s in ver_str.split('.'):
        try:
            ver.append(int(s))
        except ValueError:
            ver.append(s)
    return tuple(ver)




