import sys, datetime, os, os.path, fnmatch, shutil, re, subprocess
#  import cPickle as pickle
import pickle
import pdb
from copy import deepcopy
from dateutil.rrule import *
from etmBusyFree import *
from etmParsers import *
from colorsys import hsv_to_rgb 
from platform import system

platform = system()

if platform in ('Windows', 'Microsoft'):
    allow_cur = False
else:
    allow_cur = True

try:
    from os.path import relpath
except ImportError: # python < 2.6
    from os.path import curdir, abspath, sep, commonprefix, pardir, join
    def relpath(path, start=curdir):
        """Return a relative version of a path"""
        if not path:
            raise ValueError("no path specified")
        start_list = abspath(start).split(sep)
        path_list = abspath(path).split(sep)
        # Work out how much of the filepath is shared by start and path.
        i = len(commonprefix([start_list, path_list]))
        rel_list = [pardir] * (len(start_list)-i) + path_list[i:]
        if not rel_list:
            return curdir
        return join(*rel_list)


import locale, codecs
try:
    # try using the user's default settings.
    locale.setlocale(locale.LC_ALL, '')
except:
    # use the current setting for locale.LC_ALL
    locale.setlocale(locale.LC_ALL, None)

colwdths = [15, 15, 300, 100, 100, 100]

status_format = {   # groupby date
    'task'      : '%(D)s',
    'event'     : '%(tmp_s)s',
    'reminder'  : "%4s" % '&',
    'action'    : '%(P)s',
    'note'      : "%4s" % '!'
    }
second_format = {   # groupby context, keyword, project
    'task'     : '%(D)s',
    # 'event'  : '%(P)s',
    'event'    : '%(tmp_d)s',
    'reminder' : '%4s' % '&',
    'action'   : '%4s' % '~',
    'note'     : '%4s' % '!'
}

leadingzero = re.compile(r'^0|\s+0')

contexts = {}
if contextsFile and os.path.isfile(contextsFile):
    fo = open(contextsFile, 'r')
    lines = fo.readlines()
    fo.close()
    if lines:
        for line in lines:
            l = line.strip()
            if l:
                contexts[l] = 1

keywords = {}
if keywordsFile and os.path.isfile(keywordsFile):
    fo = open(keywordsFile, 'r')
    lines = fo.readlines()
    fo.close()
    if lines:
        for line in lines:
            l = line.strip()
            if l:
                keywords[l] = 1

def cleanPkls(topdir=etmdata):
    """walk topdir looking for *.pkl for which there is no matching *.txt file and remove them."""
    for path, subdirs, names in os.walk(topdir, followlinks=True):
        for name in names:
            if fnmatch.fnmatch(name, '*.pkl'):
                pkl = os.path.join(path, name)
                os.remove(pkl)

def daycolor(num_mins):
    "vary brightness according to num_mins between min_b and max_b, holding saturation and hue constant and return the corresponding rgb color triple"
    hue = 0         # use red
    saturation = 1
    min_b = 0
    max_b = 1
    max_minutes = 480
    brightness = min(max_b, min_b + (max_b-min_b)* num_mins/float(max_minutes))
    r,g,b = hsv_to_rgb(hue, saturation, brightness)
    r = int(r*255)
    g = int(g*255)
    b = int(b*255)
    return((r,g,b))

class OL():
    # new scheme: list of lists with each component list having the form
    # [col1, col2, ..., attr]

    num_lines = 0

    def __init__(self,line=None):
        self.max_width = 74     # for str and cur output
        if line == None:
            self.line = [None, []]
        else:
            self.line = line
        OL.num_lines += 1

    def set_attr(self, attr):
        self.line[0] = attr

    def add_col(self, pos, wdth, cont):
        self.line[1].append([pos, wdth, cont])

    def com(self):
        """docstring for comma delimited"""
        attr = self.line[0]
        lst = self.line[1]

        s = '' 
        slst = []
        for col in lst:
            pos, wdth, cont = col
            if cont == None:
                cont = ''
            slst.append('"%s"' % cont.strip())

        return(", ".join(slst))

    def str(self):
        """docstring for str"""
        attr = self.line[0]
        lst = self.line[1]

        padding = u''
        s = u''
        slst = []
        tl = []
        if True:
            for col in lst[:-1]:
                pos, wdth, cont = col
                padding = u' '*(pos - len(s))
                if cont == None:
                    cont = u''
                else:
                    cont = unicode(cont[:wdth-1])
                s ="%s%s%s" % (s, padding, cont)
            # last column
            col = lst[-1]
            pos, wdth, cont = col
            padding = u' '*(pos - len(s))
            if cont == None:
                cont = u''
            if wdth > 0:
                t = unicode(cont[:wdth])
            else:
                # wrap '0' width strings if necessary
                t = unicode(cont)
                if len(t) >= self.max_width-pos:
                    tlines = text_wrap(t, self.max_width-pos-2)
                    t = tlines[0]
                    tl = tlines[1:]
            slst.append("%s%s%s" % (s, padding, t))

            if len(tl) > 0:
                for l in tl:
                    slst.append("  %s%s" % (' '*pos, l))
        return("\n".join(slst))


    def cur(self):
        """docstring for cur"""
        attr = self.line[0]
        lst = self.line[1]
        padding = u''
        s = u''
        slst = []
        tl = []
        if attr:
            for col in lst[:-1]:
                pos, wdth, cont = col
                padding = u' '*(pos - len(s))
                if cont == None:
                    cont = u''
                else:
                    cont = unicode(cont[:wdth-1])
                s ="%s%s%s" % (s, padding, cont)
            # last column
            col = lst[-1]
            pos, wdth, cont = col
            padding = u' '*(pos - len(s))
            if cont == None:
                cont = u''
            if wdth > 0:
                t = unicode(cont[:wdth])
            else:
                # wrap '0' width strings if necessary
                t = unicode(cont)
                if len(t) >= self.max_width-pos:
                    tlines = text_wrap(t, self.max_width-pos)
                    t = tlines[0]
                    tl = tlines[1:]
            try:
                slst.append("%s%s%s%s%s" % (attrs[attr], s, padding, t, all_off))
            except:
                print("except", attr, s, padding, t)

            if len(tl) > 0:
                for l in tl:
                    slst.append("  %s%s%s%s" % (' '*pos, attrs[attr],
                        unicode(l), all_off))
        return("\n".join(slst))

    def htm(self, prnt=False):
        """docstring for html table row"""
        # there should be 4 columns
        hsh = {}
        attr = self.line[0]
        hsh['beg_font'] = ''
        hsh['end_font'] = ''
        lst = self.line[1]
        num_cols = len(lst)
        empty_cols = 3 - len(lst)
        retval = ''
        if attr:
            if prnt:
                hsh['color'] = 'color = "%s"' % hexcolors[attr]
            else:
                hsh['color'] = 'color = "%s"' % hexcolors[attr]
            if attr == 'd':
                hsh['color'] += ' size="-1"'
                hsh['beg_font'] = '<i>'
                hsh['end_font'] = '</i>'
            elif attr == 'dd':
                hsh['color'] += ' size="-2" height = "2"'
            tmp = []
            if num_cols > 1:
                if empty_cols == 1:
                    tmp.append("""\
<td width="5%">
    &nbsp;
</td>""")
                elif empty_cols > 1:
                    tmp.append("""\
<td colspan="%s">
    &nbsp;
</td>""" % empty_cols)
                for col in lst[:-1]:
                    pos, wdth, cont = col
                    hsh['align'] = ''
                    if cont == None:
                        hsh['cont'] = '&nbsp;'
                    else:
                        cont = cont.strip()
                        if cont != '':
                            hsh['cont'] = cont
                            cwdth = len(cont)
                            hsh['cont'] = cont[:wdth]
                        else:
                            hsh['cont'] = '&nbsp;'
                    # if ':' in cont or 'm' in cont or ' ' in cont:
                    # if ':' in cont or ' ' in cont:
                    hsh['align'] = 'align="right"'
                    # else:
                    #     hsh['align'] = 'align="center"'
                    tmp.append("""\
<td %(align)s valign="top">
    <font %(color)s>%(cont)s</font>
</td>""" % hsh)
            # last or only column
            col = lst[-1]
            pos, wdth, cont = col
            if cont == None:
                hsh['cont'] = '&nbsp;'
            else:
                cont = cont.strip()
                if cont == '':
                    hsh['cont'] = '&nbsp;'
                else:
                    hsh['cont'] = cont
            if pos > 0:
                # last column
                if num_cols == 1:
                    tmp.append("""\
<td colspan="%s">
    &nbsp;
</td>""" % empty_cols)
                tmp.append("""\
<td valign="top" width="65%%">
    <font %(color)s>%(beg_font)s%(cont)s%(end_font)s</font>
</td>""" % hsh)
            else:
                tmp.append("""\
<td valign="top" align="left" colspan="4">
    <font %(color)s>%(beg_font)s%(cont)s%(end_font)s</font>
</td>""" % hsh)

            retval = """<tr>%s</tr>""" % "\n".join(tmp)
        return(retval)

    def pre(self, prnt=False):
        """docstring for cur"""
        hsh = {}
        attr = self.line[0]
        hsh['beg_font'] = ''
        hsh['end_font'] = ''
        if prnt:
            hsh['color'] = 'color = "%s"' % hexcolors[attr]
        else:
            hsh['color'] = 'color = "%s"' % hexcolors[attr]
        if attr == 0:
            hsh['beg_font'] = '<b>'
            hsh['end_font'] = '</b>'
        elif attr == -2:
            hsh['color'] += ' size="-1"'
            hsh['beg_font'] = '<i>'
            hsh['end_font'] = '</i>'
        before = "<font %(color)s>%(beg_font)s" % hsh
        after = "%(end_font)s</font>" % hsh
        attr = self.line[0]
        lst = self.line[1]

        padding = ''
        s = '' 
        slst = []
        tl = []
        if attr:
            for col in lst[:-1]:
                pos, wdth, cont = col
                padding = ' '*(pos - len(s))
                if cont == None:
                    cont = ''
                else:
                    cont = cont[:wdth]
                s ="%s%s%s" % (s, padding, cont)
            # last column
            col = lst[-1]
            pos, wdth, cont = col
            padding = ' '*(pos - len(s))
            if cont == None:
                cont = ''
            if wdth > 0:
                t = cont[:wdth]
            else:
                # wrap '0' width strings if necessary
                t = cont
                if len(t) >= self.max_width-pos:
                    tlines = text_wrap(t, self.max_width-pos-2)
                    t = tlines[0]
                    tl = tlines[1:]
            slst.append("%s%s%s%s%s" % (before, s, padding, t, after))

            if len(tl) > 0:
                for l in tl:
                    slst.append("  %s%s%s%s" % (before,' '*pos, l, after))
        return("\n".join(slst))


class ETMData:
    def __init__(self):
        self.alerts = []
        self.id2text = {}    # id -> entry line text
        self.events = {}     # date -> (id, start minutes, end minutes)
        self.sel_events = {}
        self.sel_days = []
        self.reckoning = {}  # date -> ()
        self.date2ids = {}   # date -> [list of ids]
        self.day2color = {}  # day num in selected month -> busy minutes
        self.proj2ids = {}   # rel_path -> [list of ids]
        self.sched_hash = {} # for busy/free
        self.contexts = contexts
        self.keywords = keywords
        self.projects = {}
        self.zones = {}
        for z in timezones:
            self.zones[z] = 0
        self.repeats = set([])
        self.beginby = []
        self.pastdue = []
        self.options = {}
        self.time_hash = {}
        self.changed = []
        self.errors = []
        self.last_modified = {}
        self.today = midnight()
        current_month = today.month
        current_year = today.year
        self.neg_search = False
        self.search_regex = None
        self.neg_context = False
        self.contex_regex = None
        self.neg_keyword = False
        self.keyword_regex = None
        self.neg_user = False
        self.user_regex = None
        self.neg_project = False
        self.project_regex = None
        self.neg_file = False
        self.file_regex = None

        # slurp items from the last 12 months through the next 12 months
        start = parse("%s-%s-01" % (current_year-1, current_month))
        stop = parse("%s-%s-01" % (current_year+1, current_month))
        start = parse_date(start)
        stop = parse_date(stop)
        self.defaultDir = {}
        for t,n in [(action, etmActions), (event, etmEvents), 
                (reminder, etmReminders), (task, etmTasks), 
                (note, etmNotes), (done, etmDone)]:
            if n:
                self.defaultDir[t] = os.path.join(etmdata,n)
            else:
                self.defaultDir[t] = etmdata
        for t,n in [('action', etmActions), ('event', etmEvents), 
                ('reminder', etmReminders), ('task', etmTasks), 
                ('note', etmNotes), ('done', etmDone)]:
            if n:
                self.defaultDir[t] = os.path.join(etmdata,n)
            else:
                self.defaultDir[t] = etmdata


        self.check_rotating_files()
        self.get_filelist()
        self.get_itemlist()
        self.select_items(start, stop)

    def get_rrule(self,hash):
        """Return the rrule for the repeated hash."""
        # The duedate for repeated tasks will be reset each time the task
        # is finished so the duedate should be for the next unfinished
        # repetition. For events (e.g. birthdays, faculty meetings, etc),
        # the duedate should be for the first event.
        mr = []
        first_d = None
        hash['rr'] = None
        try:
            frequency = frequency_names[hash['r']]
        except:
            mr.append("    unrecognized frequency '@r %s'" % hash['r'])
            frequency = ''
        until = hash.setdefault('u', None)
        include = hash.setdefault('l', None)
        exclude = hash.setdefault('x', None)
        if has(hash, 'd'):
            if type(hash['d']) == datetime.datetime:
                duedate = hash['d']
            else:
                duedate = parse_date(hash['d'])
        if has(hash, 'w'):
            # fix three character weekday abbreviations
            weekdays = hash['w'].upper()
            for x in ['MON', 'TUE', 'WED', 'THU', 'FRI', 'SAT', 'SUN']:
                weekdays = re.sub(x, x[:2], weekdays)
            hash['w'] = weekdays

        rule = [frequency]
        if 'i' in hash and int(hash['i']) != 1:
            rule.append("interval=%s" % hash['i'])
        # XXX not for list repetitions
        if hash['r'] != 'l':
            try:
                dtstart = map(int, duedate.strftime("%Y,%m,%d").split(','))
                dtstart = ",".join(map(str, dtstart))
            except:
                mr.append("    problem parsing @d in '%s'" % (hash))
                return(hash, mr)

            rule.append("dtstart=datetime.datetime(%s)" % dtstart)
            for key in by_names.keys():
                if key in hash:
                    rule.append("by%s=%s" % (by_names[key],
                        hash[key]))
            if until != None:
                try:
                    until = parse_date(until)
                    until = map(int, until.strftime("%Y,%m,%d").split(
                        ','))
                    until = ",".join(map(str, until))
                    rule.append("until=datetime.datetime(%s)" % until)

                except:
                    mr.append("    parse '@u %s' failed" % (
                    hash['u']))

            rule = ', '.join(rule)
        includedates = []
        if include != None:
            m = parens_regex.match(include)
            if m:
                include = m.group(1)
            indates = include.split(',')
            for date in indates:
                try:
                    # we need string dates for later rrule inclusion
                    dt = parse_date(date)
                    dt = map(int, dt.strftime("%Y,%m,%d").split(','))
                    dt = ",".join(map(str, dt))
                    includedates.append("%s" % dt)
                except:
                    mr.append("    parse '@l %s' failed" %(
                    date))

            if hash['r'] == 'l':
                if ('d' not in hash or hash['d'] == None):
                    hash['d'] = parse_date(indates[0])
                    duedate = hash['d']
        excludedates = []
        if exclude != None:
            m = parens_regex.match(exclude)
            if m:
                exclude = m.group(1)
            exdates = exclude.split(',')
            for date in exdates:
                try:
                    # we need string dates for later rrule inclusion
                    dt = parse_date(date)
                    dt = map(int, dt.strftime("%Y,%m,%d").split(','))
                    dt = ",".join(map(str, dt))
                    excludedates.append("%s" % dt)
                except:
                    mr.append("    parse '@x %s' failed" % (date))

        try:
            rr = rruleset()
            if hash['r'] != 'l':
                eval("rr.rrule(rrule(%s))" % rule)
        except:
            mr.append("    could not parse rule '%s'" % rule)

        if includedates:
            for dt in includedates:
                try:
                    eval("rr.rdate(datetime.datetime(%s))" % dt)
                except:
                    mr.append("    could not parse '@l %s'" % hash['l'])
                    return(False)
        if excludedates:
            for dt in excludedates:
                try:
                    eval("rr.exdate(datetime.datetime(%s))" % dt)
                except:
                    mr.append("    could not parse '@x %s'" % hash['x'])
        if len(mr) > 0:
            hash['rr'] = None
        else:
            hash['rr'] = rr
        return(hash, mr)

    def check_hash(self, i_hash, item = True, p_hash = {}, t = ''):
        """Check i_hash for presence of required fields and unknown fields and 
        for the required format of the entries, e.g., that the content of date 
        fields can be parsed as dates."""
        msgs = []
        keys = []
        typ = ''
        if 'z' in i_hash and i_hash['z'] != 'none':
            res = gettz(i_hash['z'])
            if not res:
                msgs.append("   unrecognized time zone: '@z %s'" % i_hash['z'] )

        if not item: # project
            typ = ts = i_hash['type'] = 'project'
            req = set([])
            if project_keys:
                keys = set(project_keys)
            else:
                keys = set([])
        else: # item
            req = set([])
            if not has(i_hash, 'description'):
                msgs.append("    missing description")
            if not has(i_hash, 'type'):
                keys = set(all_keys)
                msgs.append("    missing 'type' key")
                return(msgs, {})
            else:
                typ = i_hash['type']
                if  typ == 'event':
                    keys = set(event_keys)
                    if 'r' not in i_hash or i_hash['r'] != 'l':
                        req.add('d')
                    if 'a' in i_hash or 'e' in i_hash:
                        req.add('s')
                    elif not has(i_hash, 's'):
                        i_hash['s'] = ''
                    ts = 'event'
                elif  typ == 'reminder':
                    if 'r' not in i_hash or i_hash['r'] != 'l':
                        req.add('d')
                    keys = set(reminder_keys)
                    req.add('s')
                    ts = 'reminder'
                elif typ == 'task':
                    keys = set(task_keys)
                    if 'r' in i_hash:
                        req.add('d')
                    ts = 'task'
                elif typ == 'action':
                    keys = set(action_keys)
                    req.update(['d', 'p'])
                    ts = 'action'
                elif typ == 'note':
                    keys = set(note_keys)
                    if not has(i_hash, 'd'):
                        i_hash['d'] = midnight()
                    req.update(['d'])
                    ts = 'note'
        orig_keys = []
        defaults = []
        for key in keys:
            if has(i_hash, key):
                # set original keys for details
                orig_keys.append(key)
            elif has(p_hash, key):
                # Add defaults from the project line
                i_hash[key] = p_hash[key]
                if key not in special_keys:
                    defaults.append(key)
        if 'r' in i_hash and i_hash['r'] == 'l':
            req.add('l')

        for k in repeat_keys:
            if k in i_hash and k != 'r':
                req.add('r')
                break
        if 'f' in i_hash and 'd' not in i_hash:
            i_hash['d'] = i_hash['f']
        # use today when d is required but missing
        if 'd' in req and not has(i_hash, 'd'):
            i_hash['d'] = midnight()
        h_keys = set([x for x in i_hash.keys() if x not in special_keys])
        # keys we should not have but do
        extra = list(h_keys.difference(keys))
        if len(extra) > 0:
            extra.sort()
            msgs.append("    unrecognized keys: %s" % ", ".join(extra))
        # keys we should have but don't
        missing = list(req.difference(h_keys))
        if len(missing) > 0:
            missing.sort()
            msgs.append("    missing but required keys: %s" % 
                    ", ".join(missing))

        # restrict checks to the keys we should have
        h_keys.intersection_update(keys)

        # dates: ['d', 'u', 'f'] 
        for key in list(h_keys.intersection(set(date_keys))):
            try:
                i_hash[key] = parse_date(i_hash[key])
            except:
                msgs.append("    could not parse date '%s' in @%s" %
                    (i_hash[key], key))

        if has(i_hash, 'd') and has(i_hash, 'u') and i_hash['d'] > i_hash['u']:
            msgs.append("    @u %s should not be earlier than @d %s" %
                    (i_hash['u'].strftime(date_fmt), 
                        i_hash['d'].strftime(date_fmt)) )

        # goto list of files
        if has(i_hash, 'g'):
            m = parens_regex.match(i_hash['g'])
            if m:
                goto_items = m.group(1)
            else:
                goto_items = i_hash['g']
            goto_lst = goto_items.split(',')
            if len(goto_lst) > 0:
                i_hash['goto'] = {}
                for x in goto_lst:
                    parts = x.split('|')
                    if len(parts) < 2:
                        # if there is only a long part use it for the short
                        parts.append(parts[0])
                    i_hash['goto'][parts[1].strip()] = parts[0].strip()

        # list of dates: ['l', 'x']
        datelist = {}
        for key in list(h_keys.intersection(set(list_date_keys))):
            m = parens_regex.match(i_hash[key])
            if m:
                parens = True
                items = m.group(1)
            else:
                parens = False
                items = i_hash[key]
            lst = items.split(',')
            dts = []
            for d in lst:
                try:
                    dt= parse_date(d)
                    dts.append(dt.strftime(date_fmt))
                    datelist.setdefault(key, []).append(dt)
                except:
                    msgs.append("    could not parse date '%s' in @%s" % 
                            (d.strip(), key))
            if parens:
                i_hash[key] =  "(%s)" % ", ".join(dts)
            else:
                i_hash[key] = ", ".join(dts)

        # time
        has_s = False
        if typ == 'task':
            i_hash['S'] = '??'
        elif typ == 'reminder':
            if has(i_hash, 's'):
                i_hash['S'] = 0 
                m = parens_regex.match(i_hash['s'])
                if m:
                    strts = m.group(1).split(',')
                else:
                    strts = [i_hash['s']]
                lst = []
                try:
                    for st in strts:
                        st = st.strip()
                        m = time_regex.match(st)
                        if m:
                            dp= m.group(1)
                            Dp= m.group(2)
                            if ':' not in dp:
                                if len(dp) >= 3:
                                    st = "%s:%s%s" % (dp[:len(dp)-2], 
                                            dp[-2:], Dp)
                                else:
                                    st = "%s:00%s" % (dp, Dp)
                        stime = parse(st)
                        if use_ampm:
                            s = (stime.strftime(timefmt).lower())[:-1]
                            s = leadingzero.sub(' ', s)
                        else:
                            s = stime.strftime(timefmt)
                        lst.append(s)
                    i_hash['s'] = "(%s)" % ", ".join(lst)
                except:
                    msgs.append("    Could not parse time '@s %s'" % i_hash['s'])
                    del i_hash['s']

        elif typ == 'event':
            if has(i_hash, 's'):
                if i_hash['s'] == '':
                    i_hash['S'] = 0 
                else:
                    st = i_hash['s'].strip()
                    try:
                        m = time_regex.match(st)
                        if m:
                            dp= m.group(1)
                            Dp= m.group(2)
                            if ':' not in dp:
                                if len(dp) >= 3:
                                    st = "%s:%s%s" % (dp[:len(dp)-2], 
                                            dp[-2:], Dp)
                                else:
                                    st = "%s:00%s" % (dp, Dp)
                        stime = parse(st)
                        if use_ampm:
                            s = stime.strftime(timefmt).lower()
                            i_hash['s'] = (leadingzero.sub(' ', s))[:-1]
                        else:
                            i_hash['s'] = stime.strftime(timefmt)
                        i_hash['S'] = stime.strftime("%H:%M")
                        if has(i_hash, 'e'):
                            st = i_hash['e'].strip()
                            m = etime_regex.match(st)
                            if m:
                                # minutes or hours and minutes +3h; +3:30
                                minutes = 0
                                hrs = m.group(2)
                                if hrs:
                                    minutes += int(hrs)*60
                                mins = m.group(5)
                                if mins:
                                    minutes += int(mins)
                                etime = stime + minutes*oneminute
                                i_hash['e'] = etime.strftime(time_24fmt).lower()
                            else: # must be a time
                                try:
                                    m = time_regex.match(st)
                                    if m:
                                        dp= m.group(1)
                                        Dp= m.group(2)
                                        if ':' not in dp:
                                            if len(dp) >= 3:
                                                st = "%s:%s%s" % (dp[:len(dp)\
                                                        -2], 
                                                        dp[-2:], Dp)
                                            else:
                                                st = "%s:00%s" % (dp, Dp)
                                    etime = parse(st)
                                    i_hash['e'] = etime.strftime(timefmt\
                                            ).lower()
                                    i_hash['e'] = leadingzero.sub('',\
                                            i_hash['e'])
                                    # ending times should be later than 
                                    # starting times 
                                    if etime <= stime:
                                        msgs.append(
                                        "    @e (%s) should be later than @s (%s)" 
                                        % (etime.strftime(timefmt), 
                                            stime.strftime(timefmt)))
                                except:
                                    msgs.append(
                                    "    Could not parse end time '@e %s'" % i_hash['e']) 
                                    del i_hash['e']
                        else:
                            # set the missing e equal to s plus extent
                            etime = stime + extent*oneminute
                            i_hash['e'] = etime.strftime(timefmt)
                        s = (etime - stime).seconds
                        m = s//60
                        if s % 60 > 0:
                            m += 1
                        if use_ampm:
                            endtime = (etime.strftime(timefmt).lower())[:-1]
                            endtime = leadingzero.sub(' ', endtime)
                        else:
                            endtime = etime.strftime(timefmt)
                        #  endtime = etime.strftime(time_24fmt)
                        i_hash['e'] = "+%d:%02d (%s)" % (m//60, m%60, endtime.strip())
                        i_hash['p'] = m
                        i_hash['E'] = etime.strftime(time_24fmt)
                    except:
                        msgs.append("    Could not parse start time '@s %s'" % i_hash['s'])
                        del i_hash['s']

        # integer
        for key in list(h_keys.intersection(set(integer_keys))):
            try:
                i_hash[key] = int(i_hash[key])
            except:
                msgs.append("    could not convert '%s' in @%s to integer" % (i_hash[key].strip(), key))
                del i_hash[key]

        # list of integers
        for key in list(h_keys.intersection(set(list_integer_keys))):
            m = parens_regex.match(i_hash[key])
            if m:
                parens = True
                items = m.group(1)
            else:
                parens = False
                items = i_hash[key]
            lst = items.split(',')
            dts = []
            for d in lst:
                try:
                    d = int(d)
                    if type(d) == int:
                        dts.append(str(d))
                except:
                    msgs.append("    could not convert '%s' in @%s to integer"
                     % (d.strip(), key))
            if parens:
                i_hash[key] = "(%s)" % ", ".join(dts)
            else:
                i_hash[key] = ", ".join(dts)


        # add repetitions
        if has(i_hash, 'r'):
            if i_hash['r'] == 'l' and has(datelist, 'l'):
                if not has(i_hash, 'd'):
                    i_hash['d'] = datelist['l'][0]
                r = rruleset()
                for i in datelist['l']:
                    if not has(i_hash, 'd') or i >= i_hash['d']:
                        r.rdate(i)
                i_hash['rr'] = r

            else:
                i_hash, mr = self.get_rrule(i_hash)
                if mr:
                    msgs.extend(mr)
        elif has(i_hash, 'f'):
            r = rruleset()
            try:
                r.rdate(i_hash['f'])
            except:
                print('error parsing', i_hash['f'])
            i_hash['rr'] = r
        elif has(i_hash, 'd'):
            r = rruleset()
            try:
                r.rdate(i_hash['d'])
            except:
                print('error parsing', i_hash['d'])
            i_hash['rr'] = r
        if not has(i_hash, 'rr'):
            i_hash['rr'] = None
        if not has(i_hash, 'S'):
            i_hash['S'] = '??'
        if i_hash['type'] != 'project':
            i_hash['details'] = self.hash2details(t, i_hash)
        defaults_lst = []
        for key in sort_keys:
            if key in defaults and has(i_hash, key):
                defaults_lst.append(" @%s %s" % (key, i_hash[key]))
        if len(defaults_lst) > 0:
            i_hash['defaults'] = "".join(defaults_lst)
        else:
            i_hash['defaults'] = ""
        return(msgs, i_hash)

    def check_rotating_files(self):
        global thismonth, lastyear
        self.current_hash = {}
        lastyear = str(int(get_today().strftime("%Y"))-1)
        thismonth = "%s" % get_today().strftime("%m")
        # task, event, done ... are set in rc
        for t in [task, event, done, action, note, reminder]:
            currentdir = self.defaultDir[t]
            if rotate[t]:
                curstr = "%s_%s" % (thismonth, t)
            else:
                curstr = "%s" % t
            cur = "%s.txt" % (curstr)
            curfile = os.path.join(currentdir, cur)
            self.current_hash[t] = relpath(curfile, etmdata)

            if not os.path.isfile(curfile):
                fo = open(curfile, 'w')
                fo.write("%s\n" % curstr)
                fo.close()

            if rotate[t]: 
                bak = ".%s-%s" % (lastyear, curstr)
                bakfile =  os.path.join(currentdir, bak)
                if not os.path.isfile(bakfile):
                    os.rename(curfile, bakfile)
                    fo = open(curfile, 'w')
                    fo.write("%s\n" % curstr)
                    fo.close()

        if self.new_version():
            actn = "~ installed etm version %s @p 0 @d %s @g http://www.duke.edu/~dgraham/ETM/CHANGES|CHANGES\n" % (version, get_today().strftime(date_fmt))
            print "new version", actn 
            cur_actn = self.current_hash[action]
            actn_file = os.path.join(etmdata, cur_actn)
            self.lineAdd(actn, actn_file)
            cleanPkls()

    def logaction(self, file, logentry):
        pathname, ext = os.path.splitext(file)
        directory, name = os.path.split(pathname)
        logfile = os.path.join(directory, ".%s.log" % name)
        now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
        fo = codecs.open(logfile, 'a', file_encoding, 'replace')
        fo.write("%s: %s\n" % (now, logentry.rstrip()))
        fo.close()
        return(True)

    def backup(self, file):
        pathname, ext = os.path.splitext(file)
        directory, name = os.path.split(pathname)
        bak = os.path.join(directory, ".%s.bk1" % name)
        if os.path.exists(bak):
            m_secs = os.path.getmtime(bak)
            m_date = datetime.date.fromtimestamp(m_secs)
        else:
            m_date = None
        # only backup if there is no backup or it is at least one day old
        if not m_date or m_date < datetime.date.today():
            for i in range(1, numbaks):
                baknum = numbaks - i
                nextnum = baknum + 1
                bakname = os.path.join(directory,".%s.bk%d" % (name, baknum))
                if os.path.exists(bakname):
                    nextname = os.path.join(directory,".%s.bk%d" % (name,
                     nextnum))
                    shutil.move(bakname, nextname)
            shutil.copy2(file, bak)
            return(True)
        return(False)

    def lineGet(self, line, file):
        fo = codecs.open(file, 'r', file_encoding, 'replace')
        lines = fo.readlines()
        fo.close()
        if line <= len(lines):
            return(lines[line-1].rstrip())
        else:
            return None

    def lineAdd(self, line, file):
        self.backup(file)
        fo = codecs.open(file, 'r', file_encoding, 'replace')
        lines = fo.readlines()
        fo.close()
        self.changed.append(file)
        l = ["%s\n" % x.rstrip() for x in lines]
        l.append("%s\n" % line.rstrip())
        fo = codecs.open(file, 'w', file_encoding, 'replace')
        fo.writelines(l)
        fo.close()
        return(True)

    def lineReplace(self, num, line, file):
        self.backup(file)
        num = int(num)
        fo = codecs.open(file, 'r', file_encoding, 'replace')
        lines = fo.readlines()
        fo.close()
        # leave any blank lines here to keep line numbers correct
        l = ["%s\n" % x.rstrip() for x in lines]
        orig_line = l[num - 1].strip()
        l[num - 1] = "%s\n" % line.strip()
        fo = codecs.open(file, 'w', file_encoding, 'replace')
        fo.writelines(l)
        fo.close()
        self.changed.append(file)
        self.logaction(file, "\n -: %s\n +: %s" % 
                (orig_line, line))
        return(True)

    def lineDelete(self, num, file):
        self.backup(file)
        fo = codecs.open(file, 'r', file_encoding, 'replace')
        lines = fo.readlines()
        fo.close()
        l = ["%s\n" % x.rstrip() for x in lines]
        orig_line = l.pop(num - 1)
        fo = codecs.open(file, 'w', file_encoding, 'replace')
        fo.writelines(l)
        fo.close()
        self.changed.append(file)
        self.logaction(file, "\n -: %s" % orig_line)

    def update_task(self, nextdue, id, t, tsk):
        filename, linenum = id.split(':')
        retval = False
        if has(tsk,'f'):
            # self.backup(file)
            finished_copy = deepcopy(tsk)
            unfinished_copy = deepcopy(tsk)
            unfinished_copy['d'] = nextdue
            del unfinished_copy['f']
            unfinished_details = self.hash2details(t, unfinished_copy)
            for key in repeat_keys:
                if has(finished_copy, key):
                    del finished_copy[key]
            finished_details = self.hash2details(t, finished_copy)
            curfile = os.path.join(etmdata, self.current_hash[done])
            root, extension = os.path.splitext(curfile)
            cur_pkl = "%s.pkl" % root
            fullfile = os.path.join(etmdata, filename)
            root, extension = os.path.splitext(fullfile)
            full_pkl = os.path.join(etmdata, "%s.pkl" % root)
            self.lineAdd(finished_details, curfile)
            self.lineReplace(linenum, unfinished_details, fullfile)
            retval = True
        return(retval)

    def new_version(self):
        v_file = os.path.join(etmdir, 'etm_version.txt')
        if os.path.isfile(v_file):
            fo = open(v_file, 'r')
            try:
                current_version = int(fo.readline().strip())
            except:
                current_version = 0
            fo.close()
        else:
            current_version = 0
        if current_version and 'x' not in version and int(version) > current_version:
            fo = open(v_file, 'w')
            fo.write("%s" % version)
            fo.close()
            return(True)
        else:
            return(False)

    def get_filelist(self, pattern='[!.]*.txt'):
        """yield the list of files in topdir and its subdirectories whose names match pattern."""
        filelist = []
        for d in [etmdata, etmActions, etmEvents, etmReminders, 
                etmTasks, etmNotes]:
            if d:
                for path, subdirs, names in os.walk(d, followlinks=True):
                    for name in names:
                        if fnmatch.fnmatch(name, pattern):
                            full_path = os.path.join(path,name)
                            rel_path = relpath(full_path, d)
                            root, extension = os.path.splitext(full_path)
                            pkl_path = "%s.pkl" % root
                            filelist.append((rel_path, full_path, pkl_path))
        self.filelist = filelist

    def get_day(self, date):
        # memoize (cach) return values of prior calls 
        # initially the past 12 and the next 12 months will be cached
        if date in self.date2ids:
            return(self.date2ids[date])
        else:
            # slurp the month containing the uncached date requested
            month = date.month
            year = date.year
            start = parse("%s-%s-01" % (year,month))
            if month == 12:
                stop = parse("%s-%s-01" % (year+1, month)) - oneday
            else:
                stop = parse("%s-%s-01" % (year, month+1)) - oneday
            self.select_items(start, stop)
            return(self.date2ids[date])

    def prepare_data(self):
        pattern = []
        options = self.options
        if has(options,'omit'):
            pattern.append("o~'%s'" % (options['omit']))
        if has(options,'search'):
            pattern.append("s~'%s'" % (options['search']))
        if has(options,'context'):
            pattern.append("c~'%s'" % (options['context']))
        if has(options,'keyword'):
            pattern.append("k~'%s'" % (options['keyword']))
        if has(options,'project'):
            pattern.append("p~'%s'" % (options['project']))
        if has(options,'file'):
            pattern.append("f~'%s'" % (options['file']))
        if len(pattern) > 0:
            pattern =  "(%s): " % (", ".join(pattern))
        else:
            pattern = ""
        ret = []
        # blue, magenta, green, black
        atrs = {0:'4ts', 1:'3ti', 2:'0e', 3:'5t'}
        fields = ('type', 'c', 'k')
        header = {}
        for i in range(len(fields)):
            header[i] = None
        items = []
        for j, item in self.id2item.items():
            added = ''
            if has(item, 'f'):
                continue
            if not has(item, 'c') or item['c'] == None:
                item['c'] = 'None'
            if not has(item, 'k') or item['k'] == None:
                item['k'] = 'None'
            items.append(item)
        lofh = self.sort_lofh(items, 'type', 'c', 'k', 'description', 'id')
        for item in lofh:
            if has(item, 'f'):
                continue
            notempty = False
            for i in range(len(fields)):
                g = OL()
                g.set_attr(atrs[i])
                if has(item, fields[i]):
                    c = item[fields[i]]
                    if item[fields[i]] == header[i]:
                        c = ''  
                    else:
                        header[i] = item[fields[i]]
                        notempty = True
                        for j in range(i+1, len(fields)):
                            header[j] = None
                else:
                    c = ''
                if notempty:
                    g.add_col(i*4, 0, c)
                    ret.append(g)
            c = "- %s" % item['description']
            if has(item, 'n'):
                c = "%s @n %s" % (c, item['n'])
            c = "%s [%s]" % (c, item['id'])
            g = OL()
            g.set_attr(atrs[3])
            g.add_col(len(fields)*4, 0, c)
            ret.append(g)

        h = 'data view %s' % pattern
        return(h, ret) 

    def get_matching(self, search_str):
        search_regex = re.compile(r'%s' % search_str.decode(file_encoding), 
                    re.IGNORECASE | re.LOCALE) 
        matching_items = []
        for i, item in self.id2item.items():
            if search_regex.search(item['description']) or \
            (has(item, 'n') and search_regex.search(item['n'])):
                matching_items.append(item)
        show_lst = self.get_columns(matching_items, 'tmp_d', 'status', 'description', 'id', 'weekday', 'attr')
        return(show_lst) 

    def get_days(self, start, stop):
        day = start
        self.sel_events = {}
        self.time_hash.clear()
        lst = []
        self.set_filters()
        while day <= stop:
            dayitems = self.get_day(day)
            lst.extend(dayitems)
            for item in dayitems:
                # id = item['id']
                if has(self.events, day):
                    self.sel_events[day] = self.events[day]
                if (has(item, 'p') and item['tmp_d'] >= start 
                    and self.filter_hash(item)):
                    self.update_times(item, self.options)

            day += oneday
        self.sel_days = lst
        return(lst)


    def get_daycolors(self, date):
        if not has(self.date2ids, date):
            self.get_day(date)
        month = date.month
        year = date.year
        start = parse("%s-%s-01" % (year,month))
        if month == 12:
            stop = parse("%s-%s-01" % (year+1, month)) - oneday
        else:
            stop = parse("%s-%s-01" % (year, month+1)) - oneday
        day = start
        while day <= stop:
            d = day.day
            if has(self.date2ids, day):
                p = 0
                for item in self.date2ids[day]:
                    if has(item, 'p'):
                        p += item['p']
                self.day2color[d] = daycolor(p)
            else:
                self.day2color[d] = daycolor(0)
            day += oneday

    def get_itemlist(self):
        # itemlist = []
        self.id2item ={}
        # self.proj_lines = {}

        for rel_path,full_path,pkl_path in self.filelist:
            proj_id2item = self.get_projlist(rel_path, full_path, pkl_path)
            self.id2item.update(proj_id2item)

    @trace
    def get_modified(self):
        changed = 0
        update = True
        while update:
            update = False
            self.get_filelist()
            if self.changed:
                for rel_path in self.changed:
                    full_path = os.path.join(etmdata,rel_path)
                    root, extension = os.path.splitext(full_path)
                    pkl_path = "%s.pkl" % root
                    if os.path.exists(pkl_path):
                        os.remove(pkl_path)
                self.changed = []
                changed = 1
            for rel_path, full_path, pkl_path in self.filelist:
                if (not os.path.exists(pkl_path) or 
                        (os.path.getmtime(full_path) > self.last_modified[pkl_path])):
                    if has(self.proj2ids, rel_path):
                        for i in self.proj2ids[rel_path]:
                            del self.id2item[i]
                        del self.proj2ids[rel_path]
                    proj_id2item = self.get_projlist(rel_path, full_path, pkl_path)
                    self.id2item.update(proj_id2item)
                    p = open(pkl_path, 'w')
                    pickle.dump(proj_id2item, p)
                    p.close()
                    changed += 2
                    update = True
        if self.today != midnight():
            self.today = midnight()
            changed += 4
            self.check_rotating_files()
        return(changed)

    def line2hash(self, line, item=True):
        hsh = {}
        parts = []
        if item:
            m = item_regex.match(line)
            t = ''
            if m:
                t = m.group(1)
                parts = m.group(2).split(' @')
                hsh['description'] = parts.pop(0).strip()
                if t == '*':
                    item_type = 'event'
                elif t == '&':
                    item_type = 'reminder'
                elif t == '~':
                    item_type = 'action'
                elif t == '!':
                    item_type = 'note'
                elif t == '$':
                    item_type = 'project'
                else:
                    item_type = 'task'
                    level = len(t) - 1
                hsh['type'] = item_type
        else: # project
            parts = line.split(' @')
            hsh['description'] = parts.pop(0).strip()
            t = ''
        if len(parts) > 0:
            for part in parts:
                m = part_regex.match(part)
                if m:
                    k = m.group(1)
                    v = m.group(2)
                    hsh[k.strip()] = v.strip()
        return(hsh, t)

    def hash2details(self, t, hash):
        typ = hash['type']
        if  typ == 'event':
            keys = set(event_keys)
            if 'r' not in hash or hash['r'] != 'l':
                keys.add('d')
            if 'a' in hash or 'e' in hash:
                keys.add('s')
        elif  typ == 'reminder':
            keys = set(reminder_keys)
        elif typ == 'task':
            keys = set(task_keys)
            if 'r' in hash:
                keys.add('d')
        elif typ == 'action':
            keys = set(action_keys)
            keys.add('d')
            keys.add('p')
        elif typ == 'note':
            keys = set(note_keys)
            keys.add('d')
        dtlst = ["%s %s" % (t, hash['description'])]
        for key in sort_keys:
            if key in keys and has(hash, key):
                if type(hash[key]) == datetime.datetime:
                    value = hash[key].strftime(date_fmt)
                else:
                    value = hash[key]
                dtlst.append("@%s %s" % (key, value))
        return(" ".join(dtlst))

    def get_projlist(self, rel_path, full_path, pkl_path):
        global today
        events = 0
        utasks = 0 # unfinished
        ftasks = 0
        reminders = 0
        actions = 0
        notes = 0
        if (os.path.exists(pkl_path) and 
                os.path.getmtime(full_path) <= os.path.getmtime(pkl_path) and
                os.path.getsize(pkl_path) > 0):
            # pkl_path is current and can be loaded
            p = open(pkl_path)
            proj_id2item = pickle.load(p)
            p.close()
        else:
            # either pkl_path doesn't exist or full_path is newer
            f = codecs.open(full_path, 'r', file_encoding, 'replace')
            cur_prereq = []
            completed = []
            cur_level = 0
            cur_num = None
            changed = False
            groups = {}
            # process the first, project line and pop it
            proj_line = f.readline().strip()
            line = "$ %s" % proj_line
            p_hash, t = self.line2hash(line)
            if not p_hash:
                msgs = ["Could not parse '%s'" % line]
                p_hash['description'] = "none"
            else:
                p_hash['type'] = 'project'
                msgs, p_hash = self.check_hash(p_hash, item=False)
            if msgs:
                self.errors.append("%s:1" % rel_path)
                self.errors.append("  %s" % line.strip()[2:])
                self.errors.extend(msgs)
            proj_title = p_hash['description']
            # process the remainining lines
            i = 1
            projlist = []
            proj_id2item = {}
            proj_dates = {}
            for line in f:
                i += 1
                new_num = i
                hsh, t = self.line2hash(line) 
                if not hsh:
                    msgs.append("Could not parse '%s'" % line)
                else:
                    prereq = []
                    hsh['details'] = line
                    if hsh['type'] == 'task':
                        level = len(t) - 1
                        if has(hsh, 'd'):
                            proj_dates.setdefault(new_num, 
                                []).append(hsh['d'])
                        elif has(p_hash, 'd'):
                            proj_dates.setdefault(new_num, 
                                []).append(p_hash['d'])
                            hsh['d'] = p_hash['d']
                        if t[0] == '_':
                            groups.setdefault(level,[]).append(new_num)
                        if level >= cur_level + 1:
                            if (cur_level in groups):
                                cur_prereq.append(groups[cur_level])
                                groups[cur_level] = []
                            cur_prereq.append([cur_num])
                        elif level < cur_level:
                            cur_prereq = cur_prereq[:-(cur_level-level)-1]
                        for g in cur_prereq:
                            for k in g:
                                if k not in completed and k not in prereq:
                                    prereq.append(k)
                        cur_num = new_num
                        cur_level = level
                        if prereq:
                            hsh['prereq'] = prereq
                        if not has(hsh, 'd') and has(hsh, 'prereq'):
                            # use date of last prereq if possible 
                            ldate = None
                            for j in hsh['prereq']:
                                if j in proj_dates:
                                    ldate = proj_dates[j][-1]
                            if ldate:
                                hsh['d'] = ldate
                            elif has(p_hash, 'd'):
                                hsh['d'] = p_hash['d']
                        if has(hsh, 'f'):
                            completed.append(cur_num)
                    hsh['j'] = proj_title
                    ident = "%s:%s" % (rel_path, new_num)
                    hsh['id'] = ident
                    msgs, hsh = self.check_hash(hsh, item = True, p_hash = p_hash, t = t)
                    if msgs:
                        self.errors.append(ident)
                        self.errors.append("  %s" % line.strip())
                        self.errors.extend(msgs)
                    else:
                        hsh['details'] = line.strip()
                        if has(hsh, 'f'):
                            if type(hsh['f']) == datetime.datetime:
                                finished = hsh['f']
                            else:
                                try:
                                    finished = midnight(hsh['f'])
                                except:
                                    print('except', hsh['f'], 
                                            type(hsh['f']), hsh['description'])
                                    finished = midnight()
                            if has(hsh,'r'):
                                r = hsh['rr']
                                overdue = hsh.get('o', 'k')
                                today = midnight()

                                if has(hsh, 'd'):
                                    if type(hsh['d']) == datetime.datetime:
                                        due = hsh['d']
                                    else:
                                        try:
                                            due = midnight(hsh['d'])
                                        except:
                                            due = midnight()
                                            print('except', hsh['d'], 
                                                    type(hsh['d']), hsh['description'])
                                else:
                                    due = midnight()
                                if overdue == 'r':
                                    if hsh['r'] == 'l':
                                        nextdue = r.after(max(finished, due))
                                    else:
                                        hsh['d'] = finished
                                        self.get_rrule(hsh)
                                        r = hsh['rr']
                                        nextdue = r.after(finished)
                                elif overdue == 's':
                                    try:
                                        nextdue = r.after(max(today, due))
                                    except:
                                        nextdue = ''
                                elif overdue == 'k':
                                    nextdue = r.after(due)

                                if nextdue:
                                    self.update_task(nextdue, ident, t, hsh)
                                    hsh['d'] = nextdue
                        proj_id2item[ident] = hsh
            f.close()
            p = open(pkl_path, 'w')
            pickle.dump(proj_id2item, p)
            p.close()
        self.last_modified[pkl_path] = os.path.getmtime(pkl_path)
        proj_title = None
        # go through ident, item pairs and record ids in a proj -> list of ids hsh
        for ident, hsh in proj_id2item.items():
            self.proj2ids.setdefault(rel_path, []).append(ident)
            if not proj_title:
                proj_title = hsh['j']
            if hsh['type'] == 'task':
                if has(hsh, 'f'):
                    ftasks += 1
                else:
                    utasks += 1
            elif hsh['type'] == 'event':
                events += 1
            elif hsh['type'] == 'reminder':
                reminders += 1
            elif hsh['type'] == 'action':
                actions += 1
            elif hsh['type'] == 'note':
                notes += 1
            if has(hsh, 'c') and not limitContextsSelection:
                if hsh['c'] in self.contexts:
                    self.contexts[hsh['c']] += 1
                else:
                    self.contexts[hsh['c']] = 1
            if has(hsh, 'k') and not limitKeywordsSelection:
                if hsh['k'] in self.keywords:
                    self.keywords[hsh['k']] += 1
                else:
                    self.keywords[hsh['k']] = 1
            if has(hsh, 'r'):
                temp = [hsh['r']]
                for key in repeat_keys:
                    if has(hsh, key):
                        temp.append("@%s %s" % (key, hsh[key]))
                self.repeats.add(" ".join(temp))
            if has(hsh, 'z'):
                if hsh['z'] in self.zones:
                    self.zones[hsh['z']] += 1
                else:
                    self.zones[hsh['z']] = 1
        # title(24) a(3) e(3) f(3) u(3) n(3) r(3) rel_path
        if proj_title:
            # we'll only have a proj_title for files with at least one item
            self.projects[rel_path] = ('%-26s %3s %3s %3s %3s %3s %3s    %s' 
                    % (proj_title[:26], actions, events, ftasks, notes,
                        reminders, utasks, rel_path))
        return(proj_id2item)

    @trace
    def select_items(self, start, stop):
        global today
        today = midnight()
        selected = []
        start = datetime.datetime.combine(start, datetime.time())
        stop = datetime.datetime.combine(stop, datetime.time())
        self.date2ids.clear()
        self.sel_events.clear()
        self.events.clear()
        self.beginby = []
        current_end = today + 6*oneday
        ic = 0
        for id, item in self.id2item.items():
            # add the id to the item hash
            ic += 1
            try:
                item['id'] = id
            except:
                print("except id", id, item)
            if 'z' not in item or item['z'] == 'none':
                zone = None
            else:
                zone = item['z']
            slst = []
            if 's' in item and item['s']:
                m = parens_regex.match(item['s'])
                if m:
                    strts = m.group(1).split(',')
                else:
                    strts = [item['s']]
                for s in strts:
                    starttime = parse(s).strftime("%H,%M").split(',')
                    slst.append(map(int, starttime))
                starthours, startminutes = slst[0]
                item['S'] =  parse(s).strftime("%H:%M")
            else:
                # no starting time, use due_hour and due_minute
                starthours = due_hour
                startminutes = due_minute
                slst = [(due_hour, due_minute)]
                item['S'] = item['s'] = ''

            if 'd' in item:
                tmp_d = item['d']
                d0, s0 = parse_datetime(tmp_d, hour=slst[0][0],
                        minute=slst[0][1], zone = zone)
                item['tmp_d'] = d0
                item['tmp_s'] = [s0]
                item['S'] = s0.strftime("%H:%M")
                starthours, startminutes = map(int,item['S'].split(':'))
                if item['type'] == 'event' and 'p' in item:
                    etime = s0 + item['p']*oneminute 
                    item['E'] = etime.strftime(time_24fmt)

            else:
                tmp_d = datetime.datetime.now()

            if 'e_diff' in item:
                item['tmp_e'] = s + item['e_diff']
            if slst:
                lst = []
                for h,m in slst:
                    d, s = parse_datetime(tmp_d, hour=h, minute=m, zone=zone)
                    if use_ampm:
                        s = (s.strftime(timefmt).lower())[:-1]
                        s = leadingzero.sub(' ', s)
                    else:
                        s = s.strftime(timefmt)
                    lst.append(s)
                if len(lst) > 1:
                    item['tmp_s'] = "(%s)" % ", ".join(lst)
                else:
                    item['tmp_s'] = "%s" % lst[0]
            else:
                d, s = parse_datetime(tmp_d, zone=zone)
            if item['type'] == 'event':
                item['attr'] = '0e'
            elif item['type'] == 'reminder':
                item['attr'] = '0r'
            elif item['type'] == 'action':
                item['attr'] = '1a'
            elif item['type'] == 'note':
                item['attr'] = '9n'
            elif item['type'] == 'task':
                if has(item, 'f'):
                    item['attr'] = '8tf'
                elif has(item, 'prereq') and item['prereq'][0]:
                    item['attr'] = '7tw'
                else:
                    item['attr'] = '5t'
            if has(item, 'p'):
                item['P'] = " %3s'" % item['p']
            else:
                item['P'] = ''
            for field in ['c', 'k']:
                item.setdefault(field, "None") 
            if has(item, 'f'):
                item['D'] = ' %3s' % 'X'
                item['attr'] = '8tf'
                try:
                    item['tmp_d'] = parse_date(item['f'], zone = zone)
                    if item['tmp_d'] >= start and item['tmp_d'] <= stop:
                        selected.append(item)
                        self.date2ids.setdefault(item['tmp_d'], []).append(item)
                except:
                    print("except: bad tmp_d, start, stop", item)
            elif has(item, 'rr'):
                rr = item['rr']
                if item['type'] == 'task':
                    try:
                        if has(item, 'd'):
                            first = rr.after(item['d'], inc = True)
                        else:
                            first = rr[0]
                        tmp_f = parse_date(first, hour=starthours, 
                                minute=startminutes, zone=zone)
                        D = (tmp_f-today).days
                    except:
                        first = False
                        print("except: bad recurrence rule in %s" % id)
                        print("    ", item['details'])
                        print("    ", sys.exc_info())
                    if not first:
                        pass
                    elif first < today and not item in self.beginby:
                        if not has(item, 'o') or item['o'] in ['k',  'r']:
                            item['tmp_d'], item['tmp_s'] = parse_datetime(first, 
                                    hour=starthours, minute=startminutes,
                                    zone=zone)
                            item['attr'] = '2tp'
                            item['D'] = "%+4d" % D
                            self.beginby.append(item)
                        elif item['o'] == 's':
                            D = 0
                    elif has(item, 'b') and not item in self.beginby:
                        by = tmp_f - int(item['b'])*oneday
                        if by <= today:
                            item['tmp_d'] = parse_date(first, 
                                    hour=starthours, minute=startminutes,
                                    zone=zone)
                            item['S'] = D
                            item['D'] = "%+4d" % D
                            item['begby'] = by
                            if has(item, 'prereq'):
                                item['attr'] = '7tw'
                            elif by <= current_end:
                                item['attr'] = '6tb'
                            else:
                                item['attr'] = '5t'
                            self.beginby.append(item)
                    else:
                        if type(item['tmp_d']) != datetime.datetime:
                            item['tmp_d'] = item['d']
                        try:
                            D = (item['tmp_d'] - today).days
                        except:
                            print("except setting D", item['tmp_d'], today, item['d'])
                    item['S'] = D
                    if D > 0:
                        item['D'] = "%+4d" % D
                    else:
                        item['D'] = "%+4d" % D
                try:
                    for r in rr.between(start, stop, inc=True):
                        item_copy = deepcopy(item)
                        item_copy['tmp_d'] = parse_datetime(r, 
                                    hour=starthours, minute=startminutes,
                                    zone=zone)[0]
                        if 'e_diff' in item_copy:
                            item_copy['tmp_e'] = s + item_copy['e_diff']
                            et = item_copy['tmp_e'].replace(hour=0, 
                                minute=0, second=0)
                            self.date2ids.setdefault(et, []).append(item)
                        if has(item_copy, 'S') and has(item_copy, 'E'):
                            weekday = item_copy['tmp_d'].weekday()
                            if week_begin == 6:
                                # Sunday (weekday 6 ) is the first day of the 
                                # week and thus will be in column 0. This means 
                                # that Monday (weekday 0) must be in column 1, 
                                # Tuesday (weekday 1) must be in column 2 and 
                                # so forth
                                if weekday == 6:
                                    col = 0
                                else:
                                    col = weekday + 1
                            else:
                                col = weekday
                            # record busy info
                            sparts = item_copy['S'].split(':')
                            eparts = item_copy['E'].split(':')
                            if len(sparts) == 2:
                                sm = int(sparts[0])*60 + int(sparts[1])
                                em = int(eparts[0])*60 + int(eparts[1])
                                t = (item_copy['id'], col, sm, em)
                                self.events.setdefault(item_copy['tmp_d'], []).append(t)
                                item_copy['weekday'] = col
                        if item_copy['type'] == 'task':
                            if has(item, 'f'):
                                item_copy['D'] = item_copy['S'] = 'X'
                                item_copy['tmp_d'] = parse_date(item['f'])
                                item_copy['attr'] = '8tf'
                            else:
                                D = (r - today).days
                                item_copy['S'] = D
                                item_copy['D'] = "%+4d" % D
                                if D > 0:
                                    item_copy['tmp-d'] = r
                                    if has(item_copy, 'prereq'):
                                        item_copy['attr'] = '7tw'
                                    else:
                                        item_copy['attr'] = '5t'
                                elif D == 0:
                                    item_copy['tmp-d'] = r
                                    if has(item_copy, 'prereq'):
                                        item_copy['attr'] = '7tw'
                                    else:
                                        item_copy['attr'] = '3ti'
                                else:
                                    item_copy.clear()
                        if item_copy:
                            m = year_regex.search(item_copy['description'])
                            if m:
                                startyear = m.group(1)
                                endyear = r.strftime("%Y")
                                y = year2string(startyear, endyear)
                                item_copy['description'] = year_regex.sub(y,
                                        item_copy['description'])
                            selected.append(item_copy)
                            self.date2ids.setdefault(item_copy['tmp_d'], 
                                    []).append(item_copy)
                            if 'e_diff' in item_copy:
                                item_copy['tmp_e'] = s + item_copy['e_diff']
                                et = item_copy['tmp_e'].replace(hour=0, 
                                        minute=0, second=0)
                                self.date2ids.setdefault(et, set([])).add(item_copy)
                except:
                    print("except: bad start/stop")
                    print("    start:", start)
                    print("    stop:", stop)
                    print("    ", item['details'])
                    print("    ", sys.exc_info())
            else:
                if not has(item, 'prereq'):
                    item['attr'] = '5tu' # undated
                item['tmp_d'] = parse_date(0, zone=zone) 
                item['D'] = item['d'] = ''
                selected.append(item)
                z = midnight().strftime(date_fmt)
                self.date2ids.setdefault(item['tmp_d'], []).append(item)
        day = start
        while day <= stop:
            if not has(self.date2ids, day):
                self.date2ids[day] = []
            day += oneday
        for item in selected:
            if item['type'] == 'task':
                try:
                    item['S'] = int(item['D'])
                except:
                    item['S'] = item['D']
            if item['attr'] == '5t' and has(item, 'D'):
                try:
                    D = int(item['D'])
                except:
                    continue
                if D < 0:
                    item['attr'] = '2tp'
                else:
                    if has(item, 'prereq'):
                        item['attr'] = '7tw'
                    elif D == 0:
                        item['attr'] = '3ti'
                    elif D <= 6:
                        item['attr'] = '4ts'
                    else:
                        item['attr'] = '5t'
        if enable_tracing:
            return("%s items between %s and %s" % (len(selected), start.strftime(date_fmt), 
                stop.strftime(date_fmt)))
        else:
            return(True)

    def get_alerts(self):
        self.alerts = []
        today = midnight()
        if has(self.date2ids, today):
            items = self.date2ids[today]
            for item in items:
                if 'A' in item:
                    altcmd = item['A']
                else:
                    altcmd = ''
                if item['type'] == 'event': 
                    if has(item, 'a'):
                        # update alerts
                        alrts = item['a']
                        m = parens_regex.match(alrts)
                        if m:
                            alrts = m.group(1)
                        a =(item['description'],
                        parse(item['tmp_s']).strftime(datetime_fmt),
                            map(int, alrts.split(',')), altcmd)
                        self.alerts.append(a)
                elif item['type'] == 'reminder': 
                    if has(item, 'a'):
                        # update alerts
                        alrts = item['a']
                        m = parens_regex.match(alrts)
                        if m:
                            alrts = m.group(1)
                    else:
                        alrts = '0'
                    m = parens_regex.match(item['tmp_s'])
                    if m:
                        strts = [x.strip() for x in m.group(1).split(',')]
                    else:
                        strts = [item['tmp_s']]
                    for s in strts:
                        a =(item['description'],
                        parse(s).strftime(datetime_fmt),
                            map(int, alrts.split(',')), altcmd)
                        self.alerts.append(a)

    def get_columns(self, listofhashes, *fields):
        # attr will be the last field / column
        ret = []
        header = None
        current_year = midnight().year
        for item in listofhashes:
            lst = []
            if type(item['tmp_d']) == datetime.datetime:
                if fields[0] == 'tmp_d':
                    if item['tmp_d'].year == current_year:
                        fmt = weekday_fmt
                    else:
                        fmt = date_fmt
                else:
                    if item['tmp_d'].year == current_year:
                        fmt = '%b %d'
                    else:
                        fmt = date_fmt
            if 'status' in fields: # groupby date
                try:
                    item['status'] = status_format[item['type']] % item
                except:
                    print('except', status_format, item)
                    item['status'] = 'zz'
            if 'second' in fields: # groupby context, keyword, project
                try:
                    if item['type'] == 'event':
                        item['second'] = item['tmp_d'].strftime(fmt)
                    else:
                        item['second'] = second_format[item['type']] % item
                except:
                    item['second'] = 'zz'
                    print("except: bad second_format", second_format, item)

            for field in fields:
                if field in item:
                    if type(item[field]) == datetime.datetime:
                        date = item[field].strftime(fmt)
                        lst.append(leadingzero.sub(' ', date))
                    else:
                        lst.append(item[field])
                else:
                    lst.append('')
                fmt = '%b %d'
            if lst[0] == header:   # don't use first column if it repeats
                lst[0] = ''
            else:
                header = lst[0]    # different so set new header
            ret.append(lst)
        return(ret)


    def get_ol(self, listofhashes, fields, wdth = colwdths):
        # assumes attr will be the last field / column
        # widths is a list of the widths to be used
        pos = []
        pos.append(0)
        for i in range(len(wdth)):
            v = pos[i] + wdth[i]
            pos.append(v)
        ret = []
        header = None
        current_year = midnight().year
        if has(self.options, 'include_item'):
            if has(self.options, 'groupby'):
                groupby = self.options['groupby']
            else:
                groupby = 'd'
            include_str = self.options['include_item']
            include_keys = list(include_str)
        else:
            include_keys = []


        for item in listofhashes:
            lst = []
            if type(item['tmp_d']) == datetime.datetime:
                if fields[0] == 'tmp_d':
                    if item['tmp_d'].year == current_year:
                        fmt = weekday_fmt
                    else:
                        fmt = date_fmt
                else:
                    if item['tmp_d'].year == current_year:
                        fmt = '%b %d'
                    else:
                        fmt = date_fmt
            if 'status' in fields: # groupby date
                try:
                    item['status'] = status_format[item['type']] % item
                except:
                    print('except status_format item type', item)
                    print("except: no match for", item['type'], 'in status_format')
                    item['status'] = '  zz'
            if 'second' in fields: # groupby context, keyword, project
                if item['type'] == 'event':
                    if item['tmp_d'].year == current_year:
                        fmt = '%b %d'
                    else:
                        fmt = date_fmt
                    try:
                        date = item['tmp_d'].strftime(fmt)
                        item['second'] = leadingzero.sub(' ', date)
                    except:
                        item['second'] = '  zz'
                        print("except: bad second_format", second_format, item)
                else:
                    try:
                        item['second'] = second_format[item['type']] % item
                    except:
                        item['second'] = 'zz'
                        print("except: bad second_format", second_format, item)
            g = OL()
            g.set_attr(item['attr'])
            for i in range(len(fields)):
                # attr will be the last field
                if has(item, fields[i]):
                    if type(item[fields[i]]) == datetime.datetime:
                        date = item[fields[i]].strftime(fmt)
                        c = leadingzero.sub(' ', date)
                    else:
                        c = item[fields[i]]
                    if i == 0:
                        if item[fields[0]] == header:
                            c = ''  
                        else:
                            header = item[fields[0]]
                else:
                    c = ''
                fmt = '%b %d'
                g.add_col(pos[i], wdth[i], c)
            ret.append(g)
            temp = []
            for k in include_keys:
                if has(item, k) and item[k] != 'None':
                    temp.append("@%s %s" % (k, str(item[k]).strip()))
            if len(temp) > 0:
                tempstr = " ".join(temp).strip()
                h = OL()
                h.set_attr('d')
                h.add_col(pos[2], 0, tempstr)
                ret.append(h)
        return(ret)

    def show(self, view = 'h'):
        pattern = []
        options = self.options
        if has(options,'groupby'):
            pattern.append("g~'%s'" % (options['groupby']))
        if has(options,'omit'):
            pattern.append("o~'%s'" % (options['omit']))
        if has(options,'search'):
            pattern.append("s~'%s'" % (options['search']))
        if has(options,'context'):
            pattern.append("c~'%s'" % (options['context']))
        if has(options,'keyword'):
            pattern.append("k~'%s'" % (options['keyword']))
        if has(options,'project'):
            pattern.append("p~'%s'" % (options['project']))
        if has(options,'file'):
            pattern.append("f~'%s'" % (options['file']))
        if len(pattern) > 0:
            pattern =  "(%s): " % (", ".join(pattern))
        else:
            pattern = ""
        days = options.get('days', None)
        if not days:
            days = 6
        today = midnight()
        start = options.get('begin', today)
        stop = options.get('end', start + days*oneday)
        period = "%s ~ %s" % (start.strftime(date_fmt),
                stop.strftime(date_fmt))
        groupby = options.get('groupby', None)
        if not groupby:
            groupby = 'd'
        sel_items = self.get_days(start, stop)
        # beginby and pastdue
        for o_item in self.beginby:
            item = deepcopy(o_item)
            if item['tmp_d'] < start:
                # will show this one on start
                item['tmp_d'] = start
                sel_items.append(item)
            elif has(item, 'begby') and item['tmp_d'] > stop:
                if item['begby'] >= start and item['begby'] <= stop:
                    item['tmp_d'] = item['begby']
                    sel_items.append(item)
                elif item['begby'] < start:
                    item['tmp_d'] = start
                    sel_items.append(item)
        show_lst = []
        if groupby == 'd':
            sorted_list = self.sort_lofh(sel_items, 'tmp_d', 'attr', 'S', 'description')
            if view == 'm': 
                # main view in etmWX
                show_lst = self.get_columns(sorted_list, 'tmp_d', 'status',
                        'description', 'id', 'weekday', 'attr')
            else:
                show_lst = self.get_ol(sorted_list, ('tmp_d',
                    'status', 'description'), wdth=[12, 8, 70]) 
        elif groupby in ['c', 'k', 'p']:
            if groupby == 'p':
                # j is used internally since p is used for period
                groupby = 'j'
                wdth = [12, 8, 70]
            else:
                wdth = [16, 8, 66]
            sorted_list = self.sort_lofh(sel_items, groupby, 'type', 'tmp_d', 'S', 'id',
                    'description')
            if view == 'm':
                show_lst = self.get_columns(sorted_list, groupby, 'second', 'description', 'id',
                        'weekday', 'attr')
            else:
                show_lst = self.get_ol(sorted_list, (groupby,  'second', 'description'), wdth=wdth)
        if view == 'm':
            h = 'main view: %s%s' % (pattern, period)
            return(h, show_lst)
        else: # html
            h = 'item view: %s%s' % (pattern, period)
            lst = []
            sorted_list = self.sort_lofh(sel_items, groupby, 'type', 'tmp_d', 'S', 'id',
                    'description')
            if has(options, 'export'):
                if has_icalendar:
                    txt = self.make_calendar(options['export'], sorted_list)
                    lst.append(OL(['l1', [[0, 79, txt ] ]]))
                else:
                    txt = 'Export failed. Could not find the required icalendar module.'
                    lst.append(OL(['l0', [[0, 79, txt ] ]]))
            elif has(options, 'values'):
                txt = self.make_csv(options['values'], sorted_list)
                lst.append(OL(['l1', [[0, 79, txt ] ]]))
            else:
                lst = show_lst
            if view == 'h':
                ret = [x.htm() for x in lst]
            elif view == 'c':
                ret = [x.cur() for x in lst]
            elif view == 't':
                ret = [x.str() for x in lst]
            return(h, ret)

    def filter_hashes(self, listofhashes):
        self.set_filters()
        ret_loh = []
        for i in range(len(listofhashes)):
            if not listofhashes[i]:
                continue
            hsh = listofhashes[i]
            if not self.filter_hash(hsh):
                continue
            ret_loh.append(hsh)
        return(ret_loh)

    def set_filters(self):
        if has(self.options,'search'):
            if self.options['search'][0] == '!':
                self.neg_search = True
                self.search_regex = re.compile(r'%s' %
                    self.options['search'][1:].decode(file_encoding), re.IGNORECASE)
            else:
                self.neg_search = False
                self.search_regex = re.compile(r'%s' %
                    self.options['search'].decode(file_encoding), re.IGNORECASE)
        else:
            self.neg_search = False
            self.search_regex = None 
        if has(self.options,'context'):
            if self.options['context'][0] == '!':
                self.neg_context = True
                self.context_regex = re.compile(r'%s' % 
                    self.options['context'][1:].decode(file_encoding), 
                    re.IGNORECASE)
            else:
                self.neg_context = False
                self.context_regex = re.compile(r'%s' % 
                    self.options['context'].decode(file_encoding), 
                    re.IGNORECASE)
        else:
            self.neg_context = False
            self.context_regex = None
        if has(self.options,'keyword'):
            if self.options['keyword'][0] == '!':
                self.neg_keyword = True
                self.keyword_regex = re.compile(r'%s' %
                    self.options['keyword'][1:].decode(file_encoding), 
                    re.IGNORECASE)
            else:
                self.neg_keyword = False
                self.keyword_regex = re.compile(r'%s' %
                    self.options['keyword'].decode(file_encoding), 
                    re.IGNORECASE)
        else:
            self.neg_keyword = False
            self.keyword_regex = None
        if has(self.options,'user'):
            if self.options['user'][0] == '!':
                self.neg_user = True
                self.user_regex = re.compile(r'%s' %
                    self.options['user'][1:].decode(file_encoding), 
                    re.IGNORECASE)
            else:
                self.neg_user = False
                self.user_regex = re.compile(r'%s' %
                    self.options['user'].decode(file_encoding), 
                    re.IGNORECASE)
        else:
            self.neg_user = False
            self.user_regex = None
        if has(self.options,'project'):
            if self.options['project'][0] == '!':
                self.neg_project = True
                self.project_regex = re.compile(r'%s' %
                    self.options['project'][1:].decode(file_encoding), 
                    re.IGNORECASE)
            else:
                self.neg_project = False
                self.project_regex = re.compile(r'%s' %
                    self.options['project'].decode(file_encoding), 
                    re.IGNORECASE)
        else:
            self.neg_project = False
            self.project_regex = None
        if has(self.options,'file'):
            if self.options['file'][0] == '!':
                self.neg_file = True
                self.file_regex = re.compile(r'%s' %
                    self.options['file'][1:].decode(file_encoding), 
                    re.IGNORECASE)
            else:
                self.neg_file = False
                self.file_regex = re.compile(r'%s' %
                    self.options['file'].decode(file_encoding), re.IGNORECASE)
        else:
            self.neg_file = False
            self.file_regex = None

    def filter_hash(self, hash):
        if self.search_regex:
            s = hash['description'] + hash.get('n', '')
            r = self.search_regex.search(s)
            s_res = (r and r.group(0).strip())
            if (self.neg_search and s_res):
                return(False)
            if not self.neg_search and not s_res:
                return(False)
        if self.context_regex:
            if not has(hash, 'c'):
                return(False)
            r = self.context_regex.search(hash['c'])
            c_res = (r and r.group(0).strip())
            if (self.neg_context and c_res):
                return(False)
            if not self.neg_context and not c_res:
                return(False)
        if self.user_regex:
            if not has(hash, 'U'):
                return(False)
            r = self.user_regex.search(hash['U'])
            u_res = (r and r.group(0).strip())
            if (self.neg_user and u_res):
                return(False)
            if not self.neg_user and not u_res:
                return(False)
        if self.keyword_regex:
            if not has(hash, 'k'):
                return(False)
            r = self.keyword_regex.search(hash['k'])
            k_res = (r and r.group(0).strip())
            if (self.neg_keyword and k_res):
                return(False)
            if not self.neg_keyword and not k_res:
                return(False)
        if self.project_regex:
            r = self.project_regex.search(hash['j'])
            p_res = (r and r.group(0).strip())
            if (self.neg_project and p_res):
                return(False)
            if not self.neg_project and not p_res:
                return(False)
        if self.file_regex:
            filename = (hash['id'].split(':'))[0]
            r = self.file_regex.search(filename)
            f_res = (r and r.group(0).strip())
            if (self.neg_file and f_res):
                return(False)
            if not self.neg_file and not f_res:
                return(False)
        if has(self.options, 'omit'):
            omit = self.options['omit']
        else:
            omit = None
        if omit:
            item = hash['attr']
            if omit[0] == '!':
                if   'a' in item and 'a' not in omit: return(False)
                elif 'b' in item and 'b' not in omit: return(False)
                elif 'e' in item and 'e' not in omit: return(False)
                elif 'f' in item and 'f' not in omit: return(False)
                elif 'n' in item and 'n' not in omit: return(False)
                elif 'r' in item and 'r' not in omit: return(False)
                elif 't' in item and 't' not in omit: return(False)
                elif 'u' in item and 'u' not in omit: return(False)
                elif 'w' in item and 'w' not in omit: return(False)
            else:
                if   'a' in item and 'a' in omit: return(False)
                elif 'b' in item and 'b' in omit: return(False)
                elif 'e' in item and 'e' in omit: return(False)
                elif 'f' in item and 'f' in omit: return(False)
                elif 'n' in item and 'n' in omit: return(False)
                elif 'r' in item and 'r' in omit: return(False)
                elif 't' in item and 't' in omit: return(False)
                elif 'u' in item and 'u' in omit: return(False)
                elif 'w' in item and 'w' in omit: return(False)
        return(True)

    def sort_lofh(self, listofhashes, *fields):
        listofhashes = self.filter_hashes(listofhashes)
        lst = []
        thefields = []
        for item in fields:
            thefields.append(item)
        #  if has(self.options, 'omit'):
            #  omit = self.options['omit']
        #  else:
            #  omit = None
        for i in range(len(listofhashes)):
            if not listofhashes[i]:
                continue
            try:
                if 'd' in fields and not listofhashes[i]['d']:
                    listofhashes[i]['d'] = midnight()
                t = [listofhashes[i][field] for field in thefields]
                lst.append([t, i])
            except:
                print('exception in sort_lofh')
                print('  the problem hash:')
                for key in thefields:
                    if key in listofhashes[i]:
                        print("    '%s':" % key, listofhashes[i][key])
                    else:
                        print("     '%s' missing" % key)
                print("    'details':",  listofhashes[i]['details'])
                print("   ", sys.exc_info())
        lst.sort()
        indices = [l[1] for l in lst]
        return [listofhashes[i] for i in indices]

    def make_csv(self, fname, items):
        keys = ['id', 'description', 'tmp_d']
        keys.extend(sort_keys)
        lines = []
        hdrlst = ['"%s"' % x for x in keys]
        lines.append(",".join(hdrlst))
        for item in items:
            line = []
            for key in keys:
                line.append('"%s"' % item.get(key, ''))
            lines.append(",".join(line))
        out = "\n".join(lines)
        (name, ext) = os.path.splitext(fname)
        pname = os.path.join(export, "%s.csv" % name)
        fo = open(pname, 'wb')
        fo.write(out)
        fo.close()
        return('exported to %s' % pname)

    def make_calendar(self, fname, items):
        if not has_icalendar:
            return('')
        cal = Calendar()
        cal.add('prodid', '-//etm %s//dgraham.us//' % version)
        cal.add('version', '2.0')
        options = self.options
        day = options['begin']
        stop = options['end']
        for task in items:
            if 'tmp_d' in task:
                dt = ":%s" % task['tmp_d'].strftime(date_fmt)
            else:
                dt = ''
            if task['type'] == 'event':
                item = Event()
            elif task['type'] == 'task':
                item = Todo()
            elif task['type'] in ['action', 'note', 'reminder']:
                item = Journal()
            if has(task, 'description'):
                item.add('summary', task['description'])
            if 'c' in task and task['c']:
                item.add('location', task['c'])
            if 'k' in task and task['k']:
                item.add('categories', task['k'])
            if 'n' in task and task['n']:
                item.add('description', task['n'])
            if 'd' in task and task['d']:
                if task['type'] == 'task':
                    item.add('due', l2u(task['tmp_d']))
                else: # event or action entry
                    allday = True
                    if 's' in task and task['s']:
                        m = parens_regex.match(task['s'])
                        if m:
                            tlst = m.group(1)
                            times = tlst.split(',')
                            for time in times:
                                item.add('dtstart', l2u(task['tmp_d'], time.strip()))
                        else:
                            item.add('dtstart', l2u(task['tmp_d'], task['s']))
                        allday = False
                    if 'e' in task and task['e']:
                        item.add('dtend', l2u(task['tmp_d'], task['e']))
                        allday = False
                    if 'p' in task and task['p']:
                        item.add('comment', "%s minutes" % task['p'])
                    if allday:
                        item.add('dtstart', l2u(task['d']))
                        thisday = task['tmp_d']
                        nextday = thisday + oneday
                        item.add('dtend', l2u(nextday))
            if 'f' in task and task['f']:
                item.add('completed', l2u(task['f']))
            item['uid'] = 'etm%s%s' % (task['id'], dt)
            cal.add_component(item)
        (name, ext) = os.path.splitext(fname)
        pname = os.path.join(export, "%s.ics" % name)
        fo = open(pname, 'wb')
        fo.write(cal.as_string())
        fo.close()
        return('exported to %s' % pname)

    def add_time(self, key, num_mins):
        l = list(key)
        if l and l[-1] in ['no keywords', 'no context']:
            return
        self.time_hash.setdefault(key, 0)
        self.time_hash[key] += num_mins

    def update_times(self, hash, options):
        #  increment is set in rc
        #  hash must be event with @p
        include = str(self.options.get('include_ledger', '2120'))
        c_position = int(include[0])
        d_position = int(include[1])
        k_level = int(include[2])
        itemize = int(include[3])
        num_mins = 0
        if has(hash, 'p'):
            num_mins = int(hash['p'])
        else:
            return
        i = num_mins//increment
        if num_mins%increment > 0:
            i += 1
        num_mins = i*increment
        # key depends on c_position, d_position, k_level and itemized
        context = ""
        date = ""
        if itemize and has(hash, 'description'):
            title = hash['description']
        else:
            title = ""
        if c_position > 0:
            if has(hash, 'c'):
                context = hash['c']
            else:
                context = 'no context'
        if d_position > 0:
            if has(hash, 'tmp_d'):
                date = hash['tmp_d']
            else:
                date = 'no date'
        if k_level > 0:
            self.add_time(tuple(), num_mins)
            if not has(hash,'k'):
                word = ['no keywords']
                # words = ['']
            else:
                word = hash['k']
            parts = word.split(':')
            if len(parts) >= k_level:
                parts = parts[:k_level]
            k = parts
            if d_position:
                k.insert(min(len(k), d_position-1), date)
            if c_position:
                k.insert(min(len(k), c_position-1), context)
            tmp = []
            for part in k:
                if type(part) == datetime.datetime:
                    part = part.strftime(date_fmt)
                tmp.append(part)
                self.add_time(tuple(tmp), num_mins)
            if itemize:
                tmp.append(title)
                self.add_time(tuple(tmp), num_mins)
        elif k_level == 0:
            self.add_time(tuple(), num_mins)
            k = []
            tmp = []
            if d_position > 0:
                k.insert(min(len(k), d_position-1), date)
            if c_position > 0:
                k.insert(min(len(k), c_position-1), context)
            if len(k) > 0:
                tmp = []
                for part in k:
                    if type(part) == datetime.datetime:
                        part = part.strftime(date_fmt)
                    tmp.append(part)
                    self.add_time(tuple(tmp), num_mins)
            if itemize:
                tmp.append(title)
                self.add_time(tuple(tmp), num_mins)

    def prepare_ledger(self):
        """docstring for prepare_ledger"""
        pattern = []
        self.show(view = 'h')
        options = self.options
        if has(options,'omit'):
            pattern.append("o~'%s'" % (options['omit']))
        if has(options,'search'):
            pattern.append("s~'%s'" % (options['search']))
        if has(options,'context'):
            pattern.append("c~'%s'" % (options['context']))
        if has(options,'keyword'):
            pattern.append("k~'%s'" % (options['keyword']))
        if has(options,'project'):
            pattern.append("p~'%s'" % (options['project']))
        if has(options,'file'):
            pattern.append("f~'%s'" % (options['file']))
        if len(pattern) > 0:
            pattern =  "(%s): " % (", ".join(pattern))
        else:
            pattern = ""
        lst = []
        cols = 79
        if self.time_hash:
            hash = copy.deepcopy(self.time_hash)
            total = m2h(hash[()])
            del hash[()]
        else:
            total = 0
            text = "no relevant entries were found"
            lst.append(OL(['l0',
                    [
                        [0, cols-1, "%s" % text],
                    ]
                ]))
            return(total, lst)
        if not hash:
            # we only had the one key, ()
            text = "total time: %s" % total
            item = OL(['l1',
                    [
                        [0, cols-1, text],
                    ]
                ])
            lst.append(item)
            return(total, lst)
        # we have at least one entry
        keys = hash.keys()
        keys.sort()
        indent = 8 
        m_space = 7
        first = True
        len_first = 0
        # for csv export
        lines = []
        lines.append('"key","minutes"')
        for key in keys:
            # key is a tuple, we only want the last component and the hash
            # value for the key which will be the number of minutes
            item = key[-1]
            if item == 'None' and len(key) > 1:
                continue
            try:
                lines.append('"%s","%s"' % (":".join(list(key)), hash[key]))
            except:
                print("except", key, hash[key])
            if first:
                len_first = len(key)
                first = False

            if not hash[key]:
                continue
            i = len(key) - len_first
            m_pos = i*indent + 1
            t_pos = m_pos + m_space + 1
            tmp = m2h(hash[key])
            if m_pos == 1:
                attr = 'l1'
            elif m_pos == 9:
                attr = 'l2'
            else:
                attr = 'l3'
            m_text = "%*s)"% (m_space-1, tmp[:m_space])
            item = [attr,
                    [
                        [m_pos, m_space, m_text],
                        [t_pos, 0, item]
                    ]
                ]
            lst.append(OL(item))
        if has(self.options, 'values'):
            name = self.options['values']
            pname = os.path.join(export, "%s.csv" % name)
            fo = open(pname, 'wb')
            fo.write("\n".join(lines))
            fo.close()
            item = ['l1',
                    [
                        [0, 79, 'exported to %s' % pname]
                    ]
                ]
            lst = [OL(item)]
        period = "%s ~ %s" % (options['begin'].strftime(date_fmt),
                options['end'].strftime(date_fmt))
        h = 'ledger view: %s%s (%s)' % (pattern, period, total)
        return(h, lst)

    def prepare_busy(self):
        """docstring for prepare_busy"""
        pattern = []
        options = self.options
        if has(options,'search'):
            pattern.append("s~'%s'" % (options['search']))
        if has(options,'context'):
            pattern.append("c~'%s'" % (options['context']))
        if has(options,'keyword'):
            pattern.append("k~'%s'" % (options['keyword']))
        if has(options,'project'):
            pattern.append("p~'%s'" % (options['project']))
        if has(options,'file'):
            pattern.append("f~'%s'" % (options['file']))
        if len(pattern) > 0:
            pattern =  "(%s): " % (", ".join(pattern))
        else:
            pattern = ""

        day = options['begin']
        stop = options['end']
        self.sched_hash.clear()
        lst = [] 
        while day <= stop:
            lst.extend(self.get_day(day))
            day += oneday

        dayitems = self.filter_hashes(lst)
        for item in dayitems:
            if has(item, 'S') and has(item, 'E'):
                self.sched_hash.setdefault(item['tmp_d'], 
                []).append([item['S'], item['E']])

        day = options['begin']
        while day <= stop:
            if day not in self.sched_hash:
                self.sched_hash[day] = []
            day += oneday

        sched_data = []
        keys = self.sched_hash.keys()
        keys.sort()
        for key in keys:
            evnts = self.sched_hash[key]
            evnts.sort()
            sched_data.append([key, evnts])
        bb,bt,fb,ft,cols = data2busyfree(options['opening'],
                options['closing'], options['minimum'], 
                options['wrap'], options['minutes'], sched_data, True)
        ret = []
        tl = []
        i = str(options.get('include_busy', '1010'))
        b, B, f, F = map(int, [i[0], i[1], i[2], i[3]])
        if b  or B:
            attr = 'l0'
            tl.append('busy')
            if b:
                ret.append(OL([attr, [[0, cols,
            "Busy periods between %s and %s"
            % (options['opening'][:6].lower(), 
                options['closing'][:6].lower())]]]))
                for l in bb:
                    ret.append(OL([attr, [[2, cols-1, l]]]))
                ret.append(OL([attr, [[0, cols, ""]]]))
            if B:
                ret.append(OL([attr, [[0, cols, "Busy periods (anytime)"]]]))
                for l in bt:
                    ret.append(OL([attr, [[2, cols-1, l]]]))
                ret.append(OL([attr, [[0, cols, ""]]]))
        if f  or F:
            attr = 'l1'
            tl.append('free')
            if f:
                ret.append(OL([attr, [[0, cols,
                "Free periods between %s and %s (%sm minimum; %sm buffer)"
                % (options['opening'][:6].lower(),
                    options['closing'][:6].lower(),
                    options['minimum'],
                    options['wrap'])]]]))
                for l in fb:
                    ret.append(OL([attr, [[2, cols-1, l]]]))
                ret.append(OL([attr, [[0, cols, ""]]]))
            if F:
                ret.append(OL([attr, [[0, cols,
                "Free periods between %s and %s (%sm minimum; %sm buffer)"
                % (options['opening'][:6].lower(), 
                    options['closing'][:6].lower(), 
                    options['minimum'],
                    options['wrap'])]]]))
                for l in ft:
                    ret.append(OL([attr, [[2, cols-1, l]]]))
        period = "%s ~ %s" % (options['begin'].strftime(date_fmt),
                options['end'].strftime(date_fmt))
        h = 'busy view: %s%s' % (pattern, period)
        return(h, ret, cols)

def enc_print(s):
    print(s.encode(enc)) 

def main():
    global attrs, all_off, enc
    enc = term_encoding
    cmd = cmdlc = 'i'
    args = []
    if len(sys.argv) > 1:
        args = sys.argv[1:]
        cmd = args.pop(0)
        if len(cmd) == 2 and cmd[0] == '-':
            # drop the minus sign
            cmd = cmd[1:]
        cmdlc = cmd.lower()
    startdate = get_today()
    stopdate = get_today() + 6*oneday
    if cmdlc == 'h':
        enc_print(help)
    elif cmdlc == 'c':
        enc_print(date_calculator(" ".join(args))[0])
    elif cmdlc == 'n':
        enc_print(newer())
    elif cmdlc not in ['i', 'b', 'l', 'd']:
        enc_print("Unrecognized command: '%s'" % cmd)
        enc_print("Use 'h' for help.")
    else:
        for parser in [iparser, bparser, lparser, dparser]:
            parser.add_option("-t", action = "store_true",
            dest='text_output', 
            help = """Produce text (ascii) output.""")
            parser.add_option("-U", action = "store_true",
            dest='update', 
            help = """Update pkl files.""")
            if True: # file_encoding != term_encoding:
                parser.add_option("-F", action = "store_true",
                dest='file_encoding', 
                help = """If set, encode output using the setting for file_encoding, '%s'. Otherwise use the setting for term_encoding, '%s'.""" %(file_encoding, term_encoding))
        (options, err) = parse_args(cmd, args)
        if args and args[0] == '-h':
            parser = parserHash[cmd]
            return()
        if len(err) > 0:
            for e in err:
                enc_print(e)
            sys.exit()
        data = ETMData()
        if data.errors:
            for err_msg in data.errors:
                enc_print(err_msg)
        data.options = options
        if 'file_encoding' in options and options['file_encoding']:
            enc = file_encoding
        if options['text_output'] or not allow_cur:
            view = 't'
        else:
            all_off, attrs = get_attrs()
            view = 'c'
        if options['update']:
            cleanPkls()
            enc_print("Updating pkl files.")
        if cmdlc == 'i':
            h, lst = data.show(view = view)
            title = OL(['l0', [[0, 79, "%s" % h]]])
            if view == 't':
                enc_print(title.str())
            else:
                enc_print(title.cur())
            for item in lst:
                enc_print(item)
        elif cmdlc == 'b':
            h, lst, f = data.prepare_busy()
            title = OL(['l0', [[0, 79, "%s" % h]]])
            if view == 't':
                enc_print(title.str())
            else:
                enc_print(title.cur())
            for item in lst:
                if view == 't':
                    enc_print(item.str())
                else:
                    enc_print(item.cur())
        elif cmdlc == 'l':
            h, lst = data.prepare_ledger()
            title = OL(['l0', [[0, 79, "%s" % h]]])
            if view == 't':
                enc_print(title.str())
            else:
                enc_print(title.cur())
            for item in lst:
                if view == 't':
                    enc_print(item.str())
                else:
                    enc_print(item.cur())
        elif cmdlc == 'd':
            h, lst = data.prepare_data()
            title = OL(['l0', [[0, 79, "%s" % h]]])
            if view == 't':
                enc_print(title.str())
            else:
                enc_print(title.cur())
            for item in lst:
                if view == 't':
                    enc_print(item.str())
                else:
                    enc_print(item.cur())
        data.get_alerts()

if __name__ == '__main__':
    cleanPkls()
    main()