#include <assert.h>
#include <ctype.h>
#include <errno.h>
#include <stdarg.h>
#include <stdio.h>
#include <string.h>

#include "alph-in.h"
#include "array-list.h"
#include "linked-list.h"
#include "motif-in-meme-text.h"
#include "motif-spec.h"
#include "red-black-tree.h"
#include "regex-utils.h"
#include "string-builder.h"
#include "string-match.h"
#include "utils.h"

enum mtext_state {
  MTEXT_FIND_VERSION, // also look for <html> tag
  MTEXT_PRE_MOTIF,
  MTEXT_IN_ALPHABET,
  MTEXT_IN_LETTER_FREQ,
  MTEXT_IN_BACKGROUND,
  MTEXT_IN_MOTIF,
  MTEXT_IN_PSPM,
  MTEXT_IN_PSSM,
  MTEXT_PRE_C_SITES,
  MTEXT_IN_C_SITES,
  MTEXT_IN_C_SITES_BLOCKS,
  MTEXT_DONE
};
typedef enum mtext_state MTEXT_STATE_EN;

struct minfo {
  int length;
};

struct fscope {
  int options_found;
  int options_returned;
  bool is_html;
  int vmajor;
  int vminor;
  int vpatch;
  char *release;
  char *datafile;
  ALPH_T* alphabet;
  int strands;
  char *bg_source;
  ARRAY_T *background;
  ARRAY_T *letter_freqs;
  char* letters;
  int motif_count;
  struct minfo* motif_lookup;
  double scanned_sites_threshold;
  SCANNED_SITE_T *ssite_buffer;
  int ssite_allocated;
  int ssite_used;
  ARRAYLST_T *scanned_seqs;
};

struct mscope {
  int options_found;
  int options_returned;
  bool has_width;
  bool has_evalue;
  bool has_sites;
  bool started_with_bl_line;
  MOTIF_T *motif;
};

struct patterns {
  regex_t html;
  regex_t divider;
  regex_t version;
  regex_t release;
  regex_t datafile;
  regex_t alphabet;
  regex_t alphabet_start;
  regex_t alphabet_end;
  regex_t strands;
  regex_t pos_strand;
  regex_t neg_strand;
  regex_t letter_freq;
  regex_t background;
  regex_t bg_source;
  regex_t freq_pair;
  regex_t whitespace;
  regex_t motif_intro;
  regex_t motif_id2;
  regex_t motif_kv_pair;
  regex_t motif_pspm;
  regex_t motif_pssm;
  regex_t motif_num;
  regex_t motif_url;
  regex_t s_sites_header;
  regex_t s_sites_dashes;
  regex_t s_sites_titles;
  regex_t s_sites_divider;
  regex_t s_sites_seq;
  regex_t s_sites_block;
  regex_t s_sites_offset;
  regex_t s_sites_end;
};

typedef struct mtext MTEXT_T;
struct mtext {
  int options;
  char *file_name;
  short format_match;
  MTEXT_STATE_EN state;
  ALPH_READER_T *alph_rdr;
  LINKLST_T *warnings;
  LINKLST_T *errors;
  LINKLST_T *motif_queue;
  STR_T *line;
  int counter; // used for a few things, like keeping track of which position of the background I'm up to.
  double sum; // used to check sums of probabilities
  struct patterns re;
  struct fscope fscope;
  struct mscope mscope;
};

static const char * HTML_RE = "<html>";
static const char * DIVIDER_RE = "^\\*{80}$";
static const char * VERSION_RE = "^[[:space:]]*MEME[[:space:]]+version"
    "[[:space:]]+([0-9]+)(\\.([0-9]+)(\\.([0-9]+))?)?([^0-9].*)?$";
static const char * RELEASE_RE ="[[:space:]]+\\(Release[[:space:]]+date:"
    "[[:space:]]+(.+)\\)[[:space:]]*$";
static const char * DATAFILE_RE = "^[[:space:]]*DATAFILE[[:space:]]*="
    "[[:space:]]*([^[:space:]].*)$";
static const char * ALPHABET_RE = "^[[:space:]]*ALPHABET[[:space:]]*="
    "[[:space:]]*([^[:space:]]*)[[:space:]]*$";
// match: ALPHABET "name"
static const char * ALPHABET_START_RE = "^[[:space:]]*ALPHABET([[:space:]]+"
    "\"(([^\\\"]+|\\[\"\\/bfnrt]|\\u[0-9A-Fa-f]{4})*)\")?([[:space:]]+(RNA|DNA|PROTEIN)-LIKE)?[[:space:]]*$";
static const char * ALPHABET_END_RE = "^[[:space:]]*END[[:space:]]+ALPHABET[[:space:]]*$";
static const char * STRANDS_RE = "^[[:space:]]*strands[[:space:]]*:"
    "([-+[:space:]]*)$";
static const char * POS_STRAND_RE = "\\+";
static const char * NEG_STRAND_RE = "-";
static const char * LETTER_FREQ_RE = "^Letter frequencies in dataset:$";
static const char * BACKGROUND_RE = "^[[:space:]]*Background[[:space:]]+"
    "letter[[:space:]]+frequencies(:|[[:space:]].*)?$";
static const char * BG_SOURCE_RE = "^[[:space:]]+\\(from[[:space:]]+"
    "(.*)\\):.*$";
static const char * FREQ_PAIR_RE = "^[[:space:]]*([A-Za-z0-9\\.\\*\\-])[[:space:]]+"
    POS_NUM_RE "([[:space:]].*)?$";
static const char * WHITESPACE_RE = "^[[:space:]]*$";
// Can't ignore case on this one as normal meme format repeatedly
// uses Motif as the first word on a line
static const char * MOTIF_INTRO_RE = "^[[:space:]]*(BL[[:space:]]+)?MOTIF"
    "[[:space:]]*([^[:space:]]+)([[:space:]].*)?$";
// don't allow the second id to contain a '=' or be directly followed by an '='
// as that probably means it's not an id
static const char * MOTIF_ID2_RE = "^[[:space:]]*([^[:space:]=]+)"
    "([[:space:]]+([^[:space:]=].*)?)?$";
static const char * MOTIF_KV_PAIR_RE = "^[[:space:]]*([^[:space:]=]+)"
    "[[:space:]]*=[[:space:]]*([^[:space:]=]+)([[:space:]].*)?$";
static const char * MOTIF_PSPM_RE = "^[[:space:]]*letter[[:space:]]*-"
    "[[:space:]]*probability[[:space:]]+matrix[[:space:]]*:(.*)$";
static const char * MOTIF_PSSM_RE = "^[[:space:]]*log[[:space:]]*-"
    "[[:space:]]*odds[[:space:]]+matrix[[:space:]]*:(.*)$";
static const char * MOTIF_NUM_RE = "^[[:space:]]*" NUM_RE "([[:space:]].*)?$";
static const char * MOTIF_URL_RE = "^[[:space:]]*URL[[:space:]]*"
    "([^[:space:]]*)[[:space:]]*$";
// as this only really comes up in MEME output then require an exact match
static const char * C_SITES_HEADER_RE = "^\tCombined block diagrams: "
    "non-overlapping sites with p-value < " POS_NUM_RE "$";
static const char * C_SITES_DASHES_RE = "^-+$";
static const char * C_SITES_TITLES_RE = "^SEQUENCE NAME[[:space:]]+COMBINED "
    "P-VALUE[[:space:]]+MOTIF DIAGRAM$";
static const char * C_SITES_DIVIDER_RE = "^-+[[:space:]]+-+[[:space:]]+-+$";
static const char * C_SITES_SEQ_RE = "^[[:space:]]*([^[:space:]]+)[[:space:]]+"
    POS_NUM_RE "[[:space:]]+(.*)$";
static const char * C_SITES_BLOCK_RE = "^([[<])([-+]?)([0-9]+)([abc]?)\\("
    POS_NUM_RE "\\)([]>])";
static const char * C_SITES_OFFSET_RE = "^([0-9]+)";
static const char * C_SITES_END_RE = "^\\\\$";


/*
 * Compiles regular expressions
 */
static void compile_patterns(struct patterns *re) {
  regcomp_or_die("html", &(re->html), HTML_RE, REG_ICASE);
  regcomp_or_die("divider", &(re->divider), DIVIDER_RE, REG_EXTENDED);
  regcomp_or_die("version", &(re->version), VERSION_RE, REG_EXTENDED | REG_ICASE);
  regcomp_or_die("release date", &(re->release), RELEASE_RE, REG_EXTENDED | REG_ICASE);
  regcomp_or_die("data file", &(re->datafile), DATAFILE_RE, REG_EXTENDED | REG_ICASE);
  regcomp_or_die("alphabet", &(re->alphabet), ALPHABET_RE, REG_EXTENDED | REG_ICASE);
  regcomp_or_die("alphabet start", &(re->alphabet_start), ALPHABET_START_RE, REG_EXTENDED | REG_ICASE);
  regcomp_or_die("alphabet end", &(re->alphabet_end), ALPHABET_END_RE, REG_EXTENDED | REG_ICASE);
  regcomp_or_die("strands", &(re->strands), STRANDS_RE, REG_EXTENDED | REG_ICASE);
  regcomp_or_die("positive strand", &(re->pos_strand), POS_STRAND_RE, 0);
  regcomp_or_die("negative strand", &(re->neg_strand), NEG_STRAND_RE, 0);
  regcomp_or_die("letter frequency", &(re->letter_freq), LETTER_FREQ_RE, REG_EXTENDED | REG_ICASE);
  regcomp_or_die("background", &(re->background), BACKGROUND_RE, REG_EXTENDED | REG_ICASE); // note ICASE is off for HTML
  regcomp_or_die("bg source", &(re->bg_source), BG_SOURCE_RE, REG_EXTENDED | REG_ICASE);
  regcomp_or_die("freq pair", &(re->freq_pair), FREQ_PAIR_RE, REG_EXTENDED);
  regcomp_or_die("whitespace", &(re->whitespace), WHITESPACE_RE, REG_NOSUB);
  regcomp_or_die("motif intro", &(re->motif_intro), MOTIF_INTRO_RE, REG_EXTENDED);
  regcomp_or_die("motif id 2", &(re->motif_id2), MOTIF_ID2_RE, REG_EXTENDED);
  regcomp_or_die("motif kv pair", &(re->motif_kv_pair), MOTIF_KV_PAIR_RE, REG_EXTENDED);
  regcomp_or_die("motif pspm", &(re->motif_pspm), MOTIF_PSPM_RE, REG_EXTENDED | REG_ICASE);
  regcomp_or_die("motif pssm", &(re->motif_pssm), MOTIF_PSSM_RE, REG_EXTENDED | REG_ICASE);
  regcomp_or_die("motif num", &(re->motif_num), MOTIF_NUM_RE, REG_EXTENDED);
  regcomp_or_die("motif url", &(re->motif_url), MOTIF_URL_RE, REG_EXTENDED | REG_ICASE);
  regcomp_or_die("c sites header", &(re->s_sites_header), C_SITES_HEADER_RE, REG_EXTENDED);
  regcomp_or_die("c sites dashes", &(re->s_sites_dashes), C_SITES_DASHES_RE, REG_EXTENDED);
  regcomp_or_die("c sites titles", &(re->s_sites_titles), C_SITES_TITLES_RE, REG_EXTENDED);
  regcomp_or_die("c sites divider", &(re->s_sites_divider), C_SITES_DIVIDER_RE, REG_EXTENDED);
  regcomp_or_die("c sites seq", &(re->s_sites_seq), C_SITES_SEQ_RE, REG_EXTENDED);
  regcomp_or_die("c sites block", &(re->s_sites_block), C_SITES_BLOCK_RE, REG_EXTENDED);
  regcomp_or_die("c sites offset", &(re->s_sites_offset), C_SITES_OFFSET_RE, REG_EXTENDED);
  regcomp_or_die("c sites offset", &(re->s_sites_end), C_SITES_END_RE, REG_EXTENDED);
}

/*
 * frees compiled regular expressions
 */
static void free_patterns(struct patterns *re) {
  regfree(&(re->html));
  regfree(&(re->divider));
  regfree(&(re->version));
  regfree(&(re->release));
  regfree(&(re->datafile));
  regfree(&(re->alphabet));
  regfree(&(re->alphabet_start));
  regfree(&(re->alphabet_end));
  regfree(&(re->strands));
  regfree(&(re->pos_strand));
  regfree(&(re->neg_strand));
  regfree(&(re->letter_freq));
  regfree(&(re->background));
  regfree(&(re->bg_source));
  regfree(&(re->freq_pair));
  regfree(&(re->whitespace));
  regfree(&(re->motif_intro));
  regfree(&(re->motif_id2));
  regfree(&(re->motif_kv_pair));
  regfree(&(re->motif_pspm));
  regfree(&(re->motif_pssm));
  regfree(&(re->motif_num));
  regfree(&(re->motif_url));
  regfree(&(re->s_sites_header));
  regfree(&(re->s_sites_dashes));
  regfree(&(re->s_sites_titles));
  regfree(&(re->s_sites_divider));
  regfree(&(re->s_sites_seq));
  regfree(&(re->s_sites_block));
  regfree(&(re->s_sites_offset));
  regfree(&(re->s_sites_end));
}

/*
 * Initilizes file scoped data.
 * Assumes data is already zeroed.
 */
static void init_fscope(struct fscope *fscope) {
  fscope->alphabet = NULL;
}

/*
 * Frees file scoped data.
 */
static void free_fscope(struct fscope *fscope) {
  int i;
  if (fscope->release) free(fscope->release);
  if (fscope->datafile) free(fscope->datafile);
  if (fscope->bg_source) free(fscope->bg_source);
  if (fscope->letters) free(fscope->letters);
  if (fscope->alphabet) {
    alph_release(fscope->alphabet);
  }
  if (fscope->background) {
    free_array(fscope->background);
  }
  if (fscope->letter_freqs) free_array(fscope->letter_freqs);
  if (fscope->motif_lookup) free(fscope->motif_lookup);
  if (fscope->ssite_buffer) free(fscope->ssite_buffer);
}

/*
 * Adds a motif in the mscope to the queue and resets the mscope.
 * Does not validate the motif.
 */
static void enqueue_motif(MTEXT_T *parser) {
  int num;
  struct minfo **lookup_ref, *item;
  if (parser->mscope.motif) {
    // set the alphabet
    parser->mscope.motif->alph = alph_hold(parser->fscope.alphabet);
    // set the flags
    parser->mscope.motif->flags = (parser->fscope.strands == 2 ? MOTIF_BOTH_STRANDS : 0);
    // give the url a default value of empty string if it is not present
    if (!parser->mscope.motif->url) parser->mscope.motif->url = strdup("");
    // get the motif index and increment the motif count
    num = parser->fscope.motif_count++;
    // cache motif information for the scanned sites
    if (parser->options & SCANNED_SITES) {
      // to read the scanned sites I need to cache the length
      lookup_ref = &(parser->fscope.motif_lookup);
      *lookup_ref = mm_realloc(*lookup_ref, sizeof(struct minfo) * (num + 1));
      item = (*lookup_ref)+num; 
      item->length = parser->mscope.motif->length;
    }
    // make the motif avaliable
    linklst_add(parser->mscope.motif, parser->motif_queue);
  }
  memset(&(parser->mscope), 0, sizeof(struct mscope));
}

/*
 * Creates the parser state object
 */
void* mtext_create(const char *optional_filename, int options) {
  MTEXT_T *parser;
  parser = mm_malloc(sizeof(MTEXT_T));
  memset(parser, 0, sizeof(MTEXT_T));
  parser->options = options;
  parser->file_name = (optional_filename ? strdup(optional_filename) : NULL);
  parser->format_match = file_name_match("meme", "txt", optional_filename);
  parser->state = MTEXT_FIND_VERSION;
  parser->warnings = linklst_create();
  parser->errors = linklst_create();
  parser->motif_queue = linklst_create();
  parser->line = str_create(250);
  compile_patterns(&(parser->re));
  init_fscope(&(parser->fscope));
  return parser;
}

/*
 * Destroys the parser state object
 */
void mtext_destroy(void *data) {
  MTEXT_T *parser;
  parser = (MTEXT_T*)data;
  if (parser->file_name) free(parser->file_name);
  linklst_destroy_all(parser->warnings, free);
  linklst_destroy_all(parser->errors, free);
  linklst_destroy_all(parser->motif_queue, destroy_motif);
  str_destroy(parser->line, false);
  free_patterns(&(parser->re));
  free_fscope(&(parser->fscope));
  if (parser->alph_rdr != NULL) alph_reader_destroy(parser->alph_rdr);
  memset(parser, 0, sizeof(MTEXT_T));
  free(parser);
}

/*
 * Stores a parser warning message for return
 */
static void warning(MTEXT_T *parser, const char *format, ...) {
  va_list  argp;
  int len;
  char *msg;
  va_start(argp, format);
  len = vsnprintf(NULL, 0, format, argp);
  va_end(argp);
  msg = mm_malloc(sizeof(char) * (len + 1));
  va_start(argp, format);
  vsnprintf(msg, len + 1, format, argp);
  va_end(argp);
  linklst_add(msg, parser->warnings);
}

/*
 * Stores a parser error message for return
 */
static void error(MTEXT_T *parser, const char *format, ...) {
  va_list  argp;
  int len;
  char *msg;
  va_start(argp, format);
  len = vsnprintf(NULL, 0, format, argp);
  va_end(argp);
  msg = mm_malloc(sizeof(char) * (len + 1));
  va_start(argp, format);
  vsnprintf(msg, len + 1, format, argp);
  va_end(argp);
  linklst_add(msg, parser->errors);
}

/*
 * Allocate an alphabet array. If the alphabet is known then use the exact
 * size required, otherwise expand the array to fit.
 */
static void size_freqs_array(ARRAY_T **array, ALPH_T* alpha, int index) {
  // reallocate the array to the correct size
  if (alpha != NULL) {
    if (*array == NULL || get_array_length(*array) < alph_size_core(alpha)) {
      *array = resize_array(*array, alph_size_core(alpha));
    }
  } else {
    *array = resize_array(*array, index + 1);
  }
}

/*
 * Parse space delimited key=value pairs and store them in a red-black tree.
 * If there is anything left over that is not whitespace then report an error.
 * Return the red-black tree.
 */
static RBTREE_T* parse_keyvals(MTEXT_T *parser, const char *line) {
  regmatch_t matches[3];
  const char *substr;
  RBTREE_T *kvpairs;
  char *key, *value;

  kvpairs = rbtree_create(rbtree_strcasecmp, NULL, free, NULL, free);
  substr = line;
  while(regexec_or_die("key/value", &(parser->re.motif_kv_pair), substr, 3, matches, 0)) {
    key = regex_str(matches+1, substr);
    value = regex_str(matches+2, substr);
    rbtree_put(kvpairs, key, value);
    substr = substr+(matches[2].rm_eo);
  }
  if (!regexec_or_die("whitespace", &(parser->re.whitespace), substr, 0, matches, 0)) {
    error(parser, "Couldn't parse \"%s\" as key = value pairs.", substr);
  }
  return kvpairs;
}

/*
 * Check if a given red-black tree with strings for keys and values
 * contains a passed key. If it does then convert the associated value
 * which should contain the length of the alphabet. Given that DNA has
 * 4 letters and protein has 20, use this value to set the alphabet.
 * If the alphabet has already been set then validate the value.
 */
static void set_alphabet_from_params(MTEXT_T *parser, RBTREE_T *kv_pairs, 
    const char *type, const char *name, const char *alpha_key) {
  RBNODE_T *node;
  char *value, *end;
  int alen;

  if ((node = rbtree_find(kv_pairs, alpha_key)) != NULL) {
    value = (char*)rbtree_value(node);
    errno = 0;
    alen = strtol(value, &end, 10);
    if (errno != 0 || *end != '\0') {
      error(parser, "The %s of motif %s has an alphabet length value \"%s\" "
          "which is not an integer.\n", type, name, value);
    } else {
      if (parser->fscope.alphabet != NULL) {
        if (alen != alph_size_core(parser->fscope.alphabet)) {
          error(parser, "The %s of motif %s has an alphabet length value %d "
              "which does not match the %s alphabet length of %d.\n", 
              type, name, alen, alph_name(parser->fscope.alphabet), 
              alph_size_core(parser->fscope.alphabet));
        }
      } else {
        switch (alen) {
          case 4:
            parser->fscope.alphabet = alph_dna();
            break;
          case 20:
            parser->fscope.alphabet = alph_protein();
            break;
          default:
            error(parser, "The %s of motif %s has an alphabet length value %d "
                "which does not match the lengths of any supported alphabets.\n", 
                type, name, alen);
        }
      }
    }
  }
}

/*
 * Check if a given red-black tree with strings for keys and values
 * contains a passed key. If it does then convert the associated value
 * which should contain the width of the current motif. If the width
 * of the motif is already known then validate this value against it,
 * otherwise check this value is sane and use it to set the motif width.
 */
static void set_motif_width_from_params(MTEXT_T *parser, RBTREE_T *kv_pairs, 
    const char *type, const char *name, const char *width_key) {
  RBNODE_T *node;
  char *value, *end;
  int width;
  if ((node = rbtree_find(kv_pairs, width_key)) != NULL) {
    value = (char*)rbtree_value(node);
    errno = 0;
    width = strtol(value, &end, 10);
    if (errno != 0 || *end != '\0') {
      error(parser, "The %s of motif %s has an width value \"%s\" "
          "which is not an integer.\n", type, name, value);
    } else {
      if (parser->mscope.has_width) {
        if (width != get_motif_length(parser->mscope.motif)) {
          error(parser, "The %s of motif %s has a width value %d "
              "which does not match the existing width value of %d.\n",
              type, name, width, get_motif_length(parser->mscope.motif));
        }
      } else {
        if (width <= 0) {
          error(parser, "The %s of motif %s has a width value %d "
              "which is invalid as it is not larger than zero.\n", 
              type, name, width);
        } else {
          parser->mscope.has_width = true;
          parser->mscope.motif->length =  width;
        }
      }
    }
  }
}

/*
 * Check if a given red-black tree with strings for keys and values
 * contains a passed key. If it does then convert the associated value
 * which should contain the nsites of the current motif. If the nsites
 * of the motif is already known then validate this value against it,
 * otherwise check this value is sane and use it to set the motif nsites.
 */
static void set_motif_sites_from_params(MTEXT_T *parser, RBTREE_T *kv_pairs, 
    const char *type, const char *name, const char *sites_key) {
  RBNODE_T *node;
  char *value, *end;
  double sites;
  if ((node = rbtree_find(kv_pairs, sites_key)) != NULL) {
    value = (char*)rbtree_value(node);
    errno = 0;
    sites = strtod(value, &end);
    if (*end != '\0') {
      error(parser, "The %s of motif %s has a number of sites value \"%s\" "
          "which is not a number.\n", type, name, value);
    } else if (errno == ERANGE) {
      if (sites == 0) {
        error(parser, "The %s of motif %s has a number of sites value \"%s\" "
            "which is too small to represent.\n", type, name, value);
      } else {
        error(parser, "The %s of motif %s has a number of sites value \"%s\" "
            "which is too large to represent.\n", type, name, value);
      }
    } else {
      if (parser->mscope.has_sites) {
        if (sites != get_motif_nsites(parser->mscope.motif)) {
          error(parser, "The %s of motif %s has a number of sites value %g "
              "which does not match the existing value of %g.\n", type, name, 
              sites, get_motif_nsites(parser->mscope.motif));
        }
      } else {
        if (sites < 0) {
          error(parser, "The %s of motif %s has a number of sites value %g "
              "which is invalid as it is smaller than zero.\n", type, 
              name, sites);
        } else {
          parser->mscope.has_sites = true;
          parser->mscope.motif->num_sites = sites;
        }
      }
    }
  }
}

/*
 * Check if a given red-black tree with strings for keys and values
 * contains a passed key. If it does then convert the associated value
 * which should contain the evalue of the current motif. If the evalue
 * of the motif is already known then validate this value against it,
 * otherwise check this value is sane and use it to set the motif evalue.
 */
static void set_motif_evalue_from_params(MTEXT_T *parser, RBTREE_T *kv_pairs, 
    const char *type, const char *name, const char *evalue_key) {
  RBNODE_T *node;
  char *value, *end;
  double log10_evalue;
  if ((node = rbtree_find(kv_pairs, evalue_key)) != NULL) {
    value = (char*)rbtree_value(node);
    errno = 0;
    log10_evalue = log10_evalue_from_string(value);
    if (errno == EINVAL) {
      error(parser, "The %s of motif %s has an evalue value \"%s\" "
          "which is not a number.\n", type, name, value);
    } else {
      if (parser->mscope.has_evalue) {
        if (log10_evalue != get_motif_log_evalue(parser->mscope.motif)) {
          error(parser, "The %s of motif %s has an evalue value %g "
              "which does not match the existing value of %g.\n", 
              type, name, pow(10.0, log10_evalue), get_motif_evalue(parser->mscope.motif));
        }
      } else {
        parser->mscope.has_evalue = true;
        parser->mscope.motif->log_evalue = log10_evalue;
        parser->mscope.motif->evalue = pow(10.0, log10_evalue);
      }
    }
  } else {
    parser->mscope.motif->evalue = -1;
  }
}

/*
 * Given a red-black tree with strings as keys and values
 * check the alphabet, motif-width, motif-sites and motif-evalue.
 */
static void set_motif_params(MTEXT_T *parser, RBTREE_T *kv_pairs, 
    const char *type, const char *alpha_key, const char *width_key, 
    const char *sites_key, const char *evalue_key) {
  char *name;
  name = get_motif_id(parser->mscope.motif);
  set_alphabet_from_params(parser, kv_pairs, type, name, alpha_key);
  set_motif_width_from_params(parser, kv_pairs, type, name, width_key);
  set_motif_sites_from_params(parser, kv_pairs, type, name, sites_key);
  set_motif_evalue_from_params(parser, kv_pairs, type, name, evalue_key);
}

/*
 * Read an array of numbers which make up a row of either a PSPM or a PSSM
 */
static ARRAY_T* parse_nums(MTEXT_T *parser, bool probabilities, bool allow_bad, const char *line) {
  regmatch_t matches[3];
  int col, row;
  double num, sum, delta;
  const char *substr;
  char *name, *type;
  ARRAY_T *array;
  ALPH_T* alpha;
  bool problem;

  problem = false;
  name = get_motif_id(parser->mscope.motif);
  type = (probabilities ? "PSPM" : "PSSM");
  row = parser->counter;
  alpha = parser->fscope.alphabet;
  if (alpha != NULL) array = allocate_array(alph_size_core(alpha));
  else array = NULL;
  substr = line;
  col = 0;
  sum = 0;
  while (regexec_or_die("motif num", &(parser->re.motif_num), substr, 3, matches, 0)) {
    if (alpha == NULL || col >= alph_size_core(alpha)) 
      array = resize_array(array, col + 1);
    num = regex_dbl(matches+1, substr);
    if (probabilities) {
      if (num < 0 || num > 1) {
        error(parser, "The PSPM of motif %s has a number which isn't a "
            "probability on row %d column %d. The number should be in "
            "the range 0 to 1 but it was %g.\n", name, row+1, col+1, num);
        problem = true;
      }
      sum += num;
    }
    set_array_item(col, num, array);
    substr = substr+(matches[1].rm_eo);
    col++;
  }
  if (!regexec_or_die("whitespace", &(parser->re.whitespace), substr, 0, matches, 0) && 
      !(allow_bad && col == 0)) {
    error(parser, "The %s of motif %s has unparsable characters \"%s\" on "
        "the end of row %d. Only spaces or tabs are allowed on the same "
        "line as the numbers.\n", type, name, substr, row+1);
    problem = true;
  }
  if (col == 0) {
    if (array) free_array(array);  
    return NULL; // empty row
  }
  // try to guess the alphabet if it is unknown
  if (alpha == NULL) {
    switch(col) {
      case 4: // assume DNA (could be RNA but DNA is more likely)
        parser->fscope.alphabet = alph_dna();
        break;
      case 20: // AA is the only supported alphabet with 20 items
        parser->fscope.alphabet = alph_protein();
        break;
      default:
        error(parser, "The %s of motif %s has %d numbers in the %d row but "
            "this does not match with any alphabet.\n", type, name, 
            col, row+1);
        problem = true;
    }
  } else if (col != alph_size_core(alpha)) {
    error(parser, "The %s of motif %s has %d numbers in the %d row but "
        "this does not match with the %s alphabet which requires %d "
        "numbers.\n", type, name, col+1, row, alph_name(alpha), 
        alph_size_core(alpha));
    problem = true;
  }
  if (probabilities) {
    delta = sum - 1;
    if (delta < 0) delta = -delta;
    if (delta > 0.1) {
      error(parser, "The PSPM of motif %s has probabilities which don't "
          "sum to 1 on row %d.\n", name, row+1);
      problem = true;
    }
  }
  if (!problem) return array;
  else {
    if (array) free_array(array);
    return NULL;
  }
}

/*
 * Searches lines for the version declaration. The version declaration is
 * required for MEME text format. Also attempts to bail out if the file
 * looks like the new HTML format.
 */
static bool state_find_version(MTEXT_T *parser, const char *line) {
  regmatch_t matches[7];
  memset(matches, 0, sizeof(regmatch_t)*7);
  if (regexec_or_die("html", &(parser->re.html), line, 0, matches, 0)) {
    parser->fscope.is_html = true; 
  }
  memset(matches, 0, sizeof(regmatch_t)*7);
  if (regexec_or_die("version", &(parser->re.version), line, 7, matches, 0)) {
    parser->fscope.vmajor = regex_int(matches+1, line, 0);
    parser->fscope.vminor = regex_int(matches+3, line, 0);
    parser->fscope.vpatch = regex_int(matches+5, line, 0);
    // see if the release date is avaliable as well
    if (regexec_or_die("release date", &(parser->re.release), line, 2, matches, 0)) {
      parser->fscope.release = regex_str(matches+1, line);
    }
    // check if this is a more modern html file to avoid 
    // conflict with the preferred parser
    if (parser->fscope.is_html) {
      if (parser->fscope.vmajor > 4 || 
          (parser->fscope.vmajor == 4 && parser->fscope.vminor > 3) || 
          (parser->fscope.vmajor == 4 && parser->fscope.vminor == 3 && 
           parser->fscope.vpatch > 2)) { 
        error(parser, "Newer html format detected. Stopping parse to avoid "
            "conflicts with the preferred parser.\n");
      } else {
        // Turn off ignore case for the background pattern
        regfree(&(parser->re.background));
        regcomp_or_die("background", &(parser->re.background), BACKGROUND_RE, REG_EXTENDED);
      }
    }
    parser->state = MTEXT_PRE_MOTIF;
  }
  return true;
}

/*
 * Parses things that come before the motif but after the version line.
 * These include the alphabet, strands, letter frequencies and background.
 * The alphabet and strands it parses in full but for the letter frequencies
 * and background it reads the first line and changes state to parse the others.
 * When it spots the first motif it changes the state and rejects the line
 * so the motif parsing code can read it in full.
 */
static bool state_pre_motif(MTEXT_T *parser, const char *line) {
  regmatch_t matches[6];
  bool pos, neg;
  const char *substr;
  char *temp;
  int len, flags;
  ALPH_T *temp_alph;
  if (regexec_or_die("data file", &(parser->re.datafile), line, 2, matches, 0)) {
    parser->fscope.datafile = regex_str(matches+1, line);
  } else if (regexec_or_die("alphabet", &(parser->re.alphabet), line, 2, matches, 0)) {
    substr = line+(matches[1].rm_so);
    len = matches[1].rm_eo - matches[1].rm_so;
    if ((temp_alph = alph_type(substr, len)) == NULL) {
      temp = regex_str(matches+1, line);
      error(parser, "Unrecognised alphabet %s. Only the built-in alphabets "
          "are supported by this syntax. Please refer to the documentation "
          "for the syntax to use a custom alphabet.\n", temp);
      free(temp);
    } else {
      if (parser->fscope.alphabet == NULL) {
        parser->fscope.alphabet = temp_alph;
      } else if (alph_equal(parser->fscope.alphabet, temp_alph)) {
        // it's a bit weird but the same alphabet is used so ignore it.
        alph_release(temp_alph);
      } else {
        // it's a different alphabet
        error(parser, "Alphabet %s does not match previously declared %s\n",
            alph_name(temp_alph), alph_name(parser->fscope.alphabet));
      }
    }
  } else if (regexec_or_die("alphabet start", &(parser->re.alphabet_start), line, 6, matches, 0)) {
    parser->alph_rdr = alph_reader_create();
    flags = 0;
    if (regex_casecmp(matches+5, line, "RNA") == 0) {
      flags = ALPH_FLAG_EXTENDS_RNA;
    } else if (regex_casecmp(matches+5, line, "DNA") == 0) {
      flags = ALPH_FLAG_EXTENDS_DNA;
    } else if (regex_casecmp(matches+5, line, "PROTEIN") == 0) {
      flags = ALPH_FLAG_EXTENDS_PROTEIN;
    }
    temp =  regex_str(matches+2, line);
    alph_reader_header(parser->alph_rdr, 1, temp, flags);
    free(temp);
    parser->state = MTEXT_IN_ALPHABET;
    parser->counter = 0;
  } else if (regexec_or_die("strands", &(parser->re.strands), line, 2, matches, 0)) {
    substr = line+(matches[1].rm_so);
    pos = regexec_or_die("positive strand", &(parser->re.pos_strand), substr, 0, matches, 0);
    neg = regexec_or_die("negative strand", &(parser->re.neg_strand), substr, 0, matches, 0);
    if (pos && neg) {
      parser->fscope.strands = 2;
    } else if (pos) {
      parser->fscope.strands = 1;
    }
  } else if (regexec_or_die("letter frequencies", &(parser->re.letter_freq), line, 0, matches, 0)) {
    parser->counter = 0;
    parser->sum = 0;
    parser->state = MTEXT_IN_LETTER_FREQ;
  } else if (regexec_or_die("background", &(parser->re.background), line, 2, matches, 0)) {
    if (matches[1].rm_so >= 0) {			// don't parse if nothing remained
      substr = line+(matches[1].rm_so);
      if (regexec_or_die("background source", &(parser->re.bg_source), substr, 2, matches, 0)) {
	parser->fscope.bg_source = regex_str(matches+1, substr);
      }
    }
    parser->counter = 0;
    parser->sum = 0;
    parser->state = MTEXT_IN_BACKGROUND;
  } else if (regexec_or_die("motif intro", &(parser->re.motif_intro), line, 0, matches, 0)) {
    parser->state = MTEXT_IN_MOTIF;
    return false;// need to reparse this line
  } else if (regexec_or_die("combined sites header", &(parser->re.s_sites_header), line, 0, matches, 0)) {
    parser->state = MTEXT_PRE_C_SITES;
    parser->counter = 0;
    return false;
  }
  return true;
}

static bool state_in_alphabet(MTEXT_T *parser, const char *line) {
  regmatch_t matches[4];
  bool done;
  PARMSG_T *message;
  ALPH_T *temp_alph;
  done = false;
  // look for the end of the alphabet, either "END ALPHABET" or a line of "*****"
  // note that there may also be a line of "******" at the begining so we check
  // that it is not the first line
  if (regexec_or_die("alphabet end", &(parser->re.alphabet_end), line, 0, matches, 0)) {
    // seen "END ALPHABET"
    done = true;
  } else if (regexec_or_die("divider", &(parser->re.divider), line, 4, matches, 0)) {
    // seen line of "****"
    if (parser->counter > 0) done = true;
  } else {
    // try to parse line as part of the alphabet
    alph_reader_line(parser->alph_rdr, line);
  }
  if (done) alph_reader_done(parser->alph_rdr);
  // report any problems that the alphabet reader finds
  while (alph_reader_has_message(parser->alph_rdr)) {
    message = alph_reader_next_message(parser->alph_rdr);
    if (message->severity == SEVERITY_ERROR) {
      error(parser, "Alphabet error: %s.\n", message->message);
    } else {
      warning(parser, "Alphabet warning: %s.\n", message->message);
    }
    parmsg_destroy(message);
  }
  if (done) {
    // when we finish then get the parsed alphabet and go back to pre motif parsing
    temp_alph = alph_reader_alphabet(parser->alph_rdr);
    if (parser->fscope.alphabet == NULL) {
      parser->fscope.alphabet = temp_alph;
    } else if (alph_equal(parser->fscope.alphabet, temp_alph)) {
      // it shouldn't really be decared twice but as long as it's the same I'll allow it
      alph_release(temp_alph);
    } else {
      // it's a different alphabet
      error(parser, "Alphabet %s does not match previously declared %s\n",
          alph_name(temp_alph), alph_name(parser->fscope.alphabet));
    }
    alph_reader_destroy(parser->alph_rdr);
    parser->alph_rdr = NULL;
    parser->state = MTEXT_PRE_MOTIF;
  }
  // this is used to see if a "***" divider isn't on the first line
  parser->counter++;
  return true;
}

static bool state_in_freqs(MTEXT_T *parser, const char *line) {
  regmatch_t matches[5];
  char *array_name;
  const char *substr;
  char letter;
  char **letters;
  double freq, delta;
  ARRAY_T **array;
  ALPH_T **alph_p;
  int i;
  
  if (parser->state == MTEXT_IN_LETTER_FREQ) {
    array = &(parser->fscope.letter_freqs);
    array_name = "letter frequencies";
  } else {
    array = &(parser->fscope.background);
    array_name = "background frequencies";
  }
  letters = &(parser->fscope.letters);
  alph_p = &(parser->fscope.alphabet);
  substr = line;
  if (*alph_p != NULL) {
    *array = resize_array(*array, alph_size_core(*alph_p));
    *letters = mm_realloc(*letters, sizeof(char) * (alph_size_core(*alph_p) + 1));
  }
  while (regexec_or_die("freq pair", &(parser->re.freq_pair), substr, 5, matches, 0)) {
    letter = regex_chr(matches+1, substr);
    freq = regex_dbl(matches+2, substr);
    substr = substr+(matches[2].rm_eo);
    // check the letter is in the alphabet
    if (*alph_p != NULL && !alph_is_core(*alph_p, letter)) {
      error(parser, "The frequency letter %c is not a core symbol of the "
            "%s alphabet.\n", letter, alph_name(*alph_p));
    }
    // check the freq is valid
    if (freq < 0 || freq > 1) {
      error(parser, "The frequency %f associated with the letter %c is not "
          "valid as it is not in the range 0 to 1.\n", 
          freq, letter);
    }
    // store the letter and frequency
    if (*alph_p == NULL || parser->counter >= alph_size_core(*alph_p)) { // size as we go
      *array = resize_array(*array, parser->counter + 1);
      *letters = mm_realloc(*letters, sizeof(char) * (parser->counter + 2));
    }
    set_array_item(parser->counter, freq, *array);
    (*letters)[parser->counter] = letter;
    (*letters)[parser->counter + 1] = '\0';
    // track the total frequency
    parser->sum += freq;
    // move on to the next
    parser->counter++;
  }
  // now look for something that isn't a frequency pair but also isn't just whitespace
  if (!regexec_or_die("whitespace", &(parser->re.whitespace), substr, 0, matches, 0)) {
    if (substr != line) {
      // we've already parsed something from this line so 
      // there should only be whitespace left
      error(parser, "Expected only space after the letter "
          "frequencies but found \"%s\".\n", substr);
    } else {
      // the end of the alphabet frequencies
      if (*alph_p == NULL) {
        // try to guess an alphabet from the letters
        if (strlen(*letters) == 4 && 
            (strchr(*letters, 'A') || strchr(*letters, 'a')) && 
            (strchr(*letters, 'C') || strchr(*letters, 'c')) && 
            (strchr(*letters, 'G') || strchr(*letters, 'g')) && 
            (strchr(*letters, 'T') || strchr(*letters, 't'))) {
          *alph_p = alph_dna();
        } else if (strlen(*letters) == 4 && 
            (strchr(*letters, 'A') || strchr(*letters, 'a')) && 
            (strchr(*letters, 'C') || strchr(*letters, 'c')) && 
            (strchr(*letters, 'G') || strchr(*letters, 'g')) && 
            (strchr(*letters, 'U') || strchr(*letters, 'u'))) {
          *alph_p = alph_rna();
        } else {
          *alph_p = alph_protein();
        }
      }
      // check that the letters are ordered correctly
      for (i = 0; (*letters)[i] != '\0'; i++) {
        if (alph_index(*alph_p, (*letters)[i]) != i) {
          STR_T *buffer;
          buffer = str_create(100);
          error(parser, "The %s were ordered %s but for %s "
              "they must be ordered %s to ensure consistancy "
              "with the motif matrices.", array_name, *letters,
              alph_name(*alph_p), alph_string(*alph_p, buffer));
          str_destroy(buffer, false);
          break;
        }
      }
      // check that the frequencies sum to 1
      delta = parser->sum - 1;
      if (delta < 0) delta = -delta;
      if (delta > 0.1) {
        error(parser, "The frequencies do not sum to 1.\n");
      }
      // continue searching for pre motif information
      if (parser->format_match < 4) parser->format_match = 4;
      parser->state = MTEXT_PRE_MOTIF;
      return false;
    }
  }
  return true;
}

static bool state_in_motif(MTEXT_T *parser, const char *line) {
  regmatch_t matches[4];
  const char *substr;
  MOTIF_T *motif;
  RBTREE_T *kvpairs;
  bool bl_line;
  if (regexec_or_die("motif intro", &(parser->re.motif_intro), line, 4, matches, 0)) {
    bl_line = ((matches[1].rm_eo - matches[1].rm_so) >= 2);
    if (parser->mscope.motif && regex_cmp(matches+2, line, get_motif_id(parser->mscope.motif)) == 0) {
      // check if we've seen this motif intro before as the old html format 
      // has a nasty habit of repeating the intro
      if (parser->fscope.is_html) return true;
      // Additionally it's possible to have one or both of:
      // MOTIF 1 width=18 seqs=18
      // BL    MOTIF 1 width=18 seqs=18
      // So only conclude this is a new motif if we see the same type twice.
      if (parser->mscope.started_with_bl_line != bl_line) return true;
    }
    enqueue_motif(parser);// enqueue existing motifs
    // create a new motif
    parser->mscope.started_with_bl_line = bl_line;
    parser->mscope.motif = (MOTIF_T*)mm_malloc(sizeof(MOTIF_T));
    motif = parser->mscope.motif;
    memset(motif, 0, sizeof(MOTIF_T));
    // set defaults
    motif->complexity = -1;
    set_motif_strand('+', motif);
    // copy over the id
    set_motif_id(line+(matches[2].rm_so), matches[2].rm_eo - matches[2].rm_so, motif);
    // try to find a second id
    substr = line+(matches[2].rm_eo);
    if ((!parser->fscope.is_html) && 
        regexec_or_die("motif id 2", &(parser->re.motif_id2), substr, 4, matches, 0)) {
      // copy of the second id
      set_motif_id2(substr+(matches[1].rm_so), matches[1].rm_eo - matches[1].rm_so, motif);
      substr = substr+(matches[1].rm_eo);
    } else if (parser->fscope.is_html) {
      // only really old MEME html files
      set_motif_id2("MEME", 4, motif);
    }
    // should I bother with the key/value arguments?
  } else if (regexec_or_die("motif pspm", &(parser->re.motif_pspm), line, 2, matches, 0)) {
    assert(parser->mscope.motif != NULL);
    if (parser->mscope.motif->freqs != NULL) {
      error(parser, "Repeated \"letter-probability matrix\" section in motif %s.\n", 
        get_motif_id(parser->mscope.motif));
      return true;
    }
    substr = line+(matches[1].rm_so);
    // parse key / value pairs
    kvpairs = parse_keyvals(parser, substr);
    set_motif_params(parser, kvpairs, "pspm", "alength", "w", "nsites", "E");
    // Allow "P=" instead of "E=".
    if (parser->mscope.motif->evalue < 0) {
      set_motif_params(parser, kvpairs, "pspm", "alength", "w", "nsites", "P");
    }
    if (parser->mscope.motif->evalue < 0) {
      parser->mscope.motif->evalue = parser->mscope.motif->log_evalue = 0;
    }
    rbtree_destroy(kvpairs);
    parser->counter = 0;
    parser->state = MTEXT_IN_PSPM;
  } else if (regexec_or_die("motif pssm", &(parser->re.motif_pssm), line, 2, matches, 0)) {
    assert(parser->mscope.motif != NULL);
    if (parser->mscope.motif->scores != NULL) {
      error(parser, "Repeated \"log-odds matrix\" section in motif %s.\n", 
        get_motif_id(parser->mscope.motif));
      return true;
    }
    substr = line+(matches[1].rm_so);
    kvpairs = parse_keyvals(parser, substr);
    // n is not equalivent to nsites, n is weighted possible sites
    set_motif_params(parser, kvpairs, "pssm", "alength", "w", "", "E");
    rbtree_destroy(kvpairs);
    parser->counter = 0;
    parser->state = MTEXT_IN_PSSM;
  } else if (regexec_or_die("motif url", &(parser->re.motif_url), line, 2, matches, 0)) {
    parser->mscope.motif->url = regex_str(matches+1, line);
  } else if (regexec_or_die("combined sites header", &(parser->re.s_sites_header), line, 0, matches, 0)) {
    parser->state = MTEXT_PRE_C_SITES;
    parser->counter = 0;
    return false;
  }
  return true;
} // state_in_motif

static bool state_in_pspm(MTEXT_T *parser, const char *line) {
  ARRAY_T *row;
  MOTIF_T *motif;

  motif = parser->mscope.motif;
  // sanity check
  if (motif->freqs) {
    assert(parser->counter == get_num_rows(motif->freqs));
  } else {
    assert(parser->counter == 0);
  }
  // parse the line
  row = parse_nums(parser, true, !parser->mscope.has_width, line);
  if (row) {
    if (motif->freqs) {
      grow_matrix(row, motif->freqs);
    } else {
      motif->freqs = array_to_matrix(true, row);
    }
    free_array(row);
    parser->counter++;
  } else if (!parser->mscope.has_width) {
    // assume the first blank line or non-number is the end of the numbers
    motif->length = parser->counter;
    parser->mscope.has_width = true;
    parser->state = MTEXT_IN_MOTIF;
    return false;
  }
  if (parser->mscope.has_width && 
      parser->counter >= get_motif_length(parser->mscope.motif)) {
    parser->state = MTEXT_IN_MOTIF;
  }
  return true;
}

static bool state_in_pssm(MTEXT_T *parser, const char *line) {
  ARRAY_T *row;
  MOTIF_T *motif;

  motif = parser->mscope.motif;
  // sanity check
  if (motif->scores) {
    assert(parser->counter == get_num_rows(motif->scores));
  } else {
    assert(parser->counter == 0);
  }
  // parse the line
  row = parse_nums(parser, false, !parser->mscope.has_width, line);
  if (row) {
    if (motif->scores) {
      grow_matrix(row, motif->scores);
    } else {
      motif->scores = array_to_matrix(true, row);
    }
    free_array(row);
    parser->counter++;
  } else if (!parser->mscope.has_width) {
    // assume the first blank line is the end of the numbers
    motif->length = parser->counter;
    parser->mscope.has_width = true;
    parser->state = MTEXT_IN_MOTIF;
    return false;
  }
  if (parser->mscope.has_width && 
      parser->counter >= get_motif_length(parser->mscope.motif)) {
    parser->state = MTEXT_IN_MOTIF;
  }
  return true;
}

static void cleanup_in_pspm_or_pssm(MTEXT_T *parser, const char *type) {
  MOTIF_T *motif;
  motif = parser->mscope.motif;
  if (parser->mscope.has_width && parser->counter < get_motif_length(motif)) {
    error(parser, "The %s of motif %s has a detected width value %d "
        "which does not match the existing width value of %d.\n",
        type, get_motif_id(motif), parser->counter, get_motif_length(motif));
    // note that we return the motif anyway...
  }
  motif->length = parser->counter;
  parser->mscope.has_width = true;
  parser->state = MTEXT_IN_MOTIF;
}

static bool state_pre_s_sites(MTEXT_T *parser, const char *line) {
  regmatch_t matches[3];
  regex_t *patterns[4] = {
    &(parser->re.s_sites_header), 
    &(parser->re.s_sites_dashes), 
    &(parser->re.s_sites_titles), 
    &(parser->re.s_sites_divider)};

  if (regexec_or_die("c sites headers", patterns[parser->counter], line, 3, matches, 0)) {
    if (parser->counter == 0) {
      parser->fscope.scanned_sites_threshold = regex_dbl(matches+1, line);
    }
    parser->counter++;
    if (parser->counter >= 4) {
      enqueue_motif(parser);
      parser->state = MTEXT_IN_C_SITES;
      parser->counter = 0;
    }
    return true;
  } else {
    parser->state = MTEXT_DONE;
    return false;
  }
}

static bool state_in_s_sites_blocks(MTEXT_T *parser, const char *line) {
  SCANNED_SEQ_T *sseq;
  int i;
  bool continued;
  const char *substr;
  regmatch_t matches[8];
  
  continued = false;
  substr = line;
  // skip leading whitespace
  while (isspace(*substr)) substr++;
  // get the scanned sequence object to store this information
  if (parser->options & SCANNED_SITES) {
    sseq = arraylst_get(arraylst_size(parser->fscope.scanned_seqs) - 1, 
        parser->fscope.scanned_seqs);
  } else {
    sseq = NULL;
  }
  // tally up the components and check the structure
  while (*substr != '\0') {
    if (regexec_or_die("c sites block", &(parser->re.s_sites_block), substr, 8, matches, 0)) {
      int motif_i;
      // found a motif, check the brackets surrounding it match
      // left bracket + 2 = right bracket in ascii
      if ((regex_chr(matches+1, substr) + 2) != regex_chr(matches+7, substr)) {
        error(parser, "Brackets don't match!\n");
        return true;
      }
      // get the index of the motif
      motif_i = regex_int(matches+3, substr, 0) - 1; // turn into index
      if (motif_i < 0 || motif_i >= parser->fscope.motif_count) {
        error(parser, "Bad motif number %d in scanned sites.\n", motif_i +1);
        return true;
      }
      if (parser->options & SCANNED_SITES) {
        bool isstrong;
        char strand;
        double log10pvalue;
        int motif_len;
        motif_len = parser->fscope.motif_lookup[motif_i].length;
        strand = regex_chr(matches+2, substr);
        log10pvalue = regex_log10_dbl(matches+5, substr); 
        sseq_add_site(sseq, motif_i + 1, motif_len, strand, log10pvalue);
      }
      // move over the motif
      substr += matches[0].rm_eo;
    } else if (regexec_or_die("c sites offset", &(parser->re.s_sites_offset), substr, 2, matches, 0)) {
      // found a part of the sequence with no motifs
      if (parser->options & SCANNED_SITES) {
        sseq_add_spacer(sseq, regex_int(matches+1, substr, 0));
      }
      substr += matches[0].rm_eo;
    } else if (regexec_or_die("c sites end", &(parser->re.s_sites_end), substr, 0, matches, 0)) {
      // found a line continuation indicator
      continued = true;
      substr++;
      break;
    } else {
      error(parser, "Unexpected text in combined sites: %s\n", substr);
      return true;
    }
    if (*substr == '_') {
      substr++;
    } else if (*substr != '\0') {
      error(parser, "Expected underscore!\n");
      return true;
    }
  }
  if (continued) {
    parser->state = MTEXT_IN_C_SITES_BLOCKS;
  } else {
    parser->state = MTEXT_IN_C_SITES;
  }
  return true;
}

static bool state_in_s_sites(MTEXT_T *parser, const char *line) {
  regmatch_t matches[8];
  const char *blocks, *substr;
  char *seq_id;
  double seq_log10_pvalue;
  int seq_site_count, seq_length;
  SCANNED_SEQ_T *sseq;

  if (parser->options & SCANNED_SITES) {
    if (!parser->fscope.scanned_seqs) 
      parser->fscope.scanned_seqs = arraylst_create();
  }

  if (regexec_or_die("c sites seq", &(parser->re.s_sites_seq), line, 5, matches, 0)) {
    blocks = line+(matches[4].rm_so);
    if (parser->options & SCANNED_SITES)  {
      seq_id = regex_str(matches+1, line);  
      seq_log10_pvalue = regex_log10_dbl(matches+2, line);
      sseq = sseq_create2(seq_id, seq_log10_pvalue);
      arraylst_add(sseq, parser->fscope.scanned_seqs);
    }
    return state_in_s_sites_blocks(parser, blocks);
  } else if (regexec_or_die("dashes", &(parser->re.s_sites_dashes), line, 0, matches, 0)) {
    if (parser->options & SCANNED_SITES) parser->fscope.options_found |= SCANNED_SITES;
    parser->state = MTEXT_DONE;
  } else {
    error(parser, "Unrecognised line \"%s\" in scanned sites.\n", line);
    parser->state = MTEXT_DONE;
  }
  return true;
}

/*
 * When given a line of text from the file consult the current state of the 
 * parser and send the line to the approprate parsing method. That parsing
 * method may either consume the line or change the parser state and reject
 * the line. A rejected line will be sent on to another parsing method dependent
 * on the new state.
 */
static void dispatch_line(MTEXT_T *parser, const char *line, size_t size, bool end) {
  bool consumed;
  MTEXT_STATE_EN before;
  consumed = false;
  while (!consumed) {
    before = parser->state;
    switch (parser->state) {
      case MTEXT_FIND_VERSION:
        consumed = state_find_version(parser, line);
        break;
      case MTEXT_PRE_MOTIF:
        consumed = state_pre_motif(parser, line);
        break;
      case MTEXT_IN_ALPHABET:
        consumed = state_in_alphabet(parser, line);
        break;
      case MTEXT_IN_LETTER_FREQ:
        consumed = state_in_freqs(parser, line);
        break;
      case MTEXT_IN_BACKGROUND:
        consumed = state_in_freqs(parser, line);
        break;
      case MTEXT_IN_MOTIF:
        consumed = state_in_motif(parser, line);
        break;
      case MTEXT_IN_PSPM:
        consumed = state_in_pspm(parser, line);
        break;
      case MTEXT_IN_PSSM:
        consumed = state_in_pssm(parser, line);
        break;
      case MTEXT_PRE_C_SITES:
        consumed = state_pre_s_sites(parser, line);
        break;
      case MTEXT_IN_C_SITES:
        consumed = state_in_s_sites(parser, line);
        break;
      case MTEXT_IN_C_SITES_BLOCKS:
        consumed = state_in_s_sites_blocks(parser, line);
        break;
      case MTEXT_DONE:
        consumed = true;
        break;//do nothing
      default:
        die("Unknown state");
    }
    if (!consumed && parser->state == before) {
      die("Infinite loop");
    }
  }
  if (end) {
    while (parser->state != MTEXT_DONE) {
      before = parser->state;
      switch (parser->state) {
        case MTEXT_FIND_VERSION:
        case MTEXT_PRE_MOTIF:
        case MTEXT_IN_LETTER_FREQ:
        case MTEXT_IN_BACKGROUND:
        case MTEXT_IN_C_SITES:
          parser->state = MTEXT_DONE;
          break;
        case MTEXT_IN_MOTIF:
        case MTEXT_PRE_C_SITES:
          enqueue_motif(parser);
          parser->state = MTEXT_DONE;
          break;
        case MTEXT_IN_PSPM:
          cleanup_in_pspm_or_pssm(parser, "PSPM");
          break;
        case MTEXT_IN_PSSM:
          cleanup_in_pspm_or_pssm(parser, "PSSM");
          break;
        default:
          die("Unknown state");
      }
      if (parser->state == before) {
        die("Infinite loop");
      }
    }
  }
}

/*
 * Receives chunks of the file and caches or splits as necessary to get lines 
 * that can be parsed. Complete lines are sent to the dispatch_line method.
 */
#define MAX_LINE_LEN 5000
void mtext_update(void *data, const char *chunk, size_t size, short end) {
  assert(data != NULL);

  MTEXT_T *parser;
  int offset, i;

  parser = (MTEXT_T*)data;
  i = 0;
  while (i < size) {
    // handle old Mac style newlines (\n\r)
    if (str_len(parser->line) == 0 && chunk[i] == '\r') i++;
    offset = i;
    for (; i < size; ++i) {
      if (chunk[i] == '\n') break;
    }
    if (str_len(parser->line) + i > MAX_LINE_LEN) {
      error(parser, "Maximum line length exceeded.\n");
      return;
    }
    str_append(parser->line, chunk+offset, (i - offset));
    if (i < size || end) {
      // handle DOS style newlines (\r\n)
      if (str_len(parser->line) > 0 && str_char(parser->line, -1) == '\r') {
        str_truncate(parser->line, -1);
      }
      dispatch_line(parser, str_internal(parser->line), str_len(parser->line), 
        (end && i >= size - 1));
      // don't allow it to shrink the buffer
      str_ensure(parser->line, str_len(parser->line));
      str_clear(parser->line);
    }
    i++; // step over the newline
  }
  if (size == 0 && end) {
    dispatch_line(parser, str_internal(parser->line), str_len(parser->line), 1);
  }
} // mtext_update

short mtext_has_format_match(void *data) {
  return ((MTEXT_T*)data)->format_match;
}

short mtext_has_warning(void *data) {
  return linklst_size(((MTEXT_T*)data)->warnings) > 0;
}

char* mtext_next_warning(void *data) {
  return (char*)linklst_pop(((MTEXT_T*)data)->warnings);
}

short mtext_has_error(void *data) {
  return linklst_size(((MTEXT_T*)data)->errors) > 0;
}

char* mtext_next_error(void *data) {
  return (char*)linklst_pop(((MTEXT_T*)data)->errors);
}

short mtext_has_motif(void *data) {
  return linklst_size(((MTEXT_T*)data)->motif_queue) > 0;
}

MOTIF_T* mtext_next_motif(void *data) {
  return (MOTIF_T*)linklst_pop(((MTEXT_T*)data)->motif_queue);
}

/*
 * WARNING, does not "hold" alphabet. If users wish to keep the alphabet they
 * must call alph_hold on it themselves.
 */
ALPH_T* mtext_get_alphabet(void *data) {
  MTEXT_T *parser;
  parser = (MTEXT_T*)data;
  return parser->fscope.alphabet;
}

int mtext_get_strands(void *data) {
  MTEXT_T *parser;
  parser = (MTEXT_T*)data;
  return parser->fscope.strands;
}

bool mtext_get_bg(void *data, ARRAY_T **bg) {
  MTEXT_T *parser;
  parser = (MTEXT_T*)data;
  if (parser->fscope.background == NULL) return false;
  *bg = resize_array(*bg, get_array_length(parser->fscope.background));
  copy_array(parser->fscope.background, *bg);
  return true;
}

void* mtext_motif_optional(void *data, int option) {
  MTEXT_T *parser;
  parser = (MTEXT_T*)data;
  if (!(parser->options & option)) {
    die("Requested value of optional component which was not requested "
        "during construction.\n");
    return NULL;
  }
  //TODO
  return NULL;
}

void* mtext_file_optional(void *data, int option) {
  MTEXT_T *parser;
  parser = (MTEXT_T*)data;
  if (!(parser->options & option)) {
    die("Requested value of optional component which was not requested "
        "during construction.\n");
    return NULL;
  }
  if (parser->fscope.options_found & option) {
    if (parser->fscope.options_returned & option) {
      die("Sorry, optional values are only returned once. "
          "This is because we cannot guarantee that the "
          "previous caller did not deallocate the memory. "
          "Hence this is a feature to avoid memory bugs.\n");
      return NULL;
    }
    parser->fscope.options_returned |= option;
  } else {
    // Not yet found or unsupported
    return NULL;
  }
  switch (option) {
    case SCANNED_SITES:
      return parser->fscope.scanned_seqs;
    default:
      die("Option code %d does not match any single option. "
          "This means that it must contain multiple options "
          "and only one is allowed at a time\n.", option);
  }
  return NULL; //unreachable! make compiler happy
}
