<actions>
  <!-- call it with 
     xidel --template-file reddit-transpassing.xml 
  put your API keys in reddit-transpassing.keys.xml
  -->
  <action>  
    <s> keys := doc("reddit-transpassing.keys.xml")/keys,
        CACHEDIR := "transpassing/cache/"</s>    
    <s> 
    declare function local:assert($cond, $message) {
      if ($cond) then () else error(QName("xxx"), $message)
    };
    declare function local:product($v) {
      switch (count($v)) 
        case 0 return 1
        case 1 return $v
        default return head($v) * local:product(tail($v))
    };
    declare function local:matches($input, $query){
      matches($input, replace($query, "\\b", "([.:;!?() ]|$|^)"))
    };
    declare function local:json-to-sequence($v) {
      typeswitch ($v) 
        case array() return $v() ! local:json-to-sequence(.)
        case object() return {| for $key in $v() return {$key: local:json-to-sequence($v($key))} |}
        case null return ()
        default return $v
    };
    declare variable $rand-seed := seconds-from-time(current-time()) mod 7;
    declare function local:random-to($count) {
      floor((random()+$rand-seed)*$count) mod $count
    };
    declare function local:replaces-rec($sentence, $replaces, $i) {
      if ($i >= count($replaces)) then $sentence
      else local:replaces-rec(replace($sentence, $replaces[$i], $replaces[$i+1]), $replaces, $i + 2)
    };
    declare function local:replaces($sentence, $replaces) {
      local:replaces-rec($sentence, $replaces, 1)
    };
    (:
      This bot uses a "talking database" of possible phrases
      
      The talking database is stored in a minimized form as JSON object:
        * "string" -> string
        * ["a", "b", "c"] -> randomly choosen a, b, or c
        * {"concat": (a,b,c,...)} -> *a || *b || *c ...
        * {"condition": "test", "concat": value}  *value, if test or () otherwise
    :)
    (: Generates the talking database from an expression like {{Get|Bring} the {{a|b} x|c}} :)
    declare function local:create-talkings($pattern){
      let $safepattern := replace(normalize-space($pattern), "[{][?]([^}]+)[}]", "⦃$1⦄" )
      return local:merge-objects(local:extract-conditions(local:split-talkings($safepattern)))
    };
    declare function local:split-talkings($pattern){
      local:split-talkings-rec($pattern, 0)
    };  
    declare function local:split-talkings-rec($pattern, $deep){
      if (not(contains($pattern, "{"))) then 
        let $options := tokenize($pattern, "[|]") 
        return if (count($options) > 1) then [$options] else $options
      else 
        let $marker := "##" || $deep,
            $splitted := extract($pattern, "(.*)[{]([^{}]+)[}](.*)", (1,2,3)),
            $inner := local:split-talkings-rec($splitted[2], $deep),
            $outer := local:split-talkings-rec($splitted[1] || $marker || $splitted[3], $deep + 1)
        return local:apply-marker($outer, $marker, $inner)
    };
    declare function local:apply-marker($v, $marker, $inner){
      (: Replaces every $marker in a string of $v with $inner
        trace( $marker || " => " || serialize-json($inner) || " in " || serialize-json($v) , "apply")[0],
       :)
      
       if (count($v) > 1) then $v ! local:apply-marker(., $marker, $inner)
       else typeswitch($v) 
         case xs:string return
           if (not(contains($v, $marker))) then $v
           else let $concat := tail (tokenize($v, $marker) ! ( $inner, . ) ) [.] 
           return if (count($concat) > 1) then {"concat": $concat} else $concat
         case object() return 
           let $concat := one-or-more($v("concat")) ! local:apply-marker(., $marker, $inner) !
              (typeswitch(.) 
                 case xs:string return .
                 case object() return one-or-more(.("concat"))
                 case array() return .
                 default return error(QName("xxx"), "unexpected: " || .) ) [.]
           return if (count($concat) > 1) then {"concat": $concat} else $concat
          case array() return [ $v() ! local:apply-marker(., $marker, $inner) ]
          default return error(QName("xxx"), "unexpected 2: " || $v)
    };
    (: Moves conditions  ⦃$x⦄   from the strings to the condition properties :)
    declare function local:extract-conditions($v) {
      $v ! (typeswitch(.) 
         case xs:string return
           if (not(contains(., "⦃"))) then .
           else let $split1 := substring-before(., "⦃"), $split1b := substring-after(., "⦃"),
                    $split2 := substring-before($split1b, "⦄"), 
                    $split3 := substring-after($split1b, "⦄"),
           (:extract(., "([^⦃]*)⦃([^⦃⦄]+)⦄(.*)", (1,2,3)),:)
                    $rest := local:extract-conditions($split3)
           return if ($rest instance of xs:string) then
                {"condition": $split2, "concat":  $split1|| $rest  }
           else {"condition": $split2, "concat": ($split1,  $rest) }
         case object() return {"concat": one-or-more(.("concat")) ! local:extract-conditions(.)}
         case array() return [ .() ! local:extract-conditions(.) ]
         default return error(QName("xxx"), "unexpected 2: " || $v))
    };
    declare function local:merge-objects($v) {
      (: merges {"condition": A, "concat": B, { "condition": C, "concat": D } } to
                {"condition": A, C, "concat": B, D } 
       :)
      $v ! (typeswitch(.) 
         case xs:string return .
         case object() return 
           if (empty(.("concat"))) then .
           else let $concat := .("concat") ! local:merge-objects(.),
                    $objects := $concat [. instance of object()],
                    $conditions := (.("condition"), $objects("condition")),
                    $concats := if (empty($objects)) then $concat else $concat ! 
                      (if (. instance of object()) then .("concat") else .)
           return {|
             if (exists($concats)) then { "concat": $concats } else (),
             if (exists($conditions)) then { "condition": $conditions } else ()
           |}
         case array() return [ .() ! local:merge-objects(.) ]
         default return error(QName("xxx"), "unexpected in merge: " || $v))
    };
    (: Removes all entries from the talking database that have a not satisfied condition  :)
    declare function local:filter-truths($v){
      $v ! (typeswitch(.) 
         case xs:string return .
         case object() return 
           . [ every $condition in .("condition") satisfies eval($condition) ] !
           ( if (exists(.("concat"))) then {"concat": .("concat") ! local:filter-truths(.) } else () )
         case array() return [ .() ! local:filter-truths(.) ]
         default return error(QName("xxx"), "unexpected in merge: " || $v))
    };
    (: Counts how many entries the talking database contains :)
    declare function local:count-talkings($v){
      exactly-one($v) ! (typeswitch(.) 
         case xs:string return 1
         case object() return local:product( .("concat") ! local:count-talkings(.) )
         case array() return sum ( .() ! local:count-talkings(.) )
         default return error(QName("xxx"), "unexpected in merge: " || $v))
    };
    declare function local:choose-talking-from-object-concats($concats, $i){
      switch (count($concats))
        case 0 return ""
        case 1 return local:choose-talking($concats, $i)
        default return let $f := local:count-talkings(head($concats)) return
          local:choose-talking( head($concats), ($i mod $f) ) 
          || local:choose-talking-from-object-concats(tail($concats), $i idiv $f)
    };
    declare function local:choose-talking-from-alternatives($list, $i){
      switch (count($list))
        case 0 return ""
        case 1 return local:choose-talking($list, $i)
        default return let $c := local:count-talkings(head($list)) return
          if ($c > $i) then local:choose-talking(head($list), $i)
          else local:choose-talking-from-alternatives(tail($list), $i - $c)
    };
    (: Chooses the $i-th entry in the talking database :)
    declare function local:choose-talking($v, $i){
      exactly-one($v) ! (typeswitch(.) 
         case xs:string return if ($i eq 0) then . else ""
         case object() return local:choose-talking-from-object-concats(.("concat"), $i)
         case array() return local:choose-talking-from-alternatives(.(), $i)
         default return error(QName("xxx"), "unexpected in merge: " || $v))
    };
    (: Chooses a random entry in the talking database :)
    declare function local:choose-random-talking($v){
      let $count := local:count-talkings($v)
      return if ($count eq 0) then "FAILURE. No possible answer in talking database."
      else local:choose-talking($v, local:random-to($count))
    };
    (: Chooses a random entry in the talking database :)
    declare function local:enumerate-talkings($v){
      for $i in 0 to local:count-talkings($v) - 1 return local:choose-talking($v, $i)
    };
    
    
    declare function local:gender-to-number($gender) {
          switch ($gender) 
            case "male" return -1
            case "female" return +1
            default return error(QName("xxx"), "unknown gender: " || $gender)
        };  
    declare function local:average-with-confidence($seq) {
      let $fallback-confidence := 0.5,
          $confidence := sum ( $seq ! (.("confidence"), $fallback-confidence)[1] )
      return if ($confidence le 0) then 0
      else sum ( $seq ! (  .("value") * (.("confidence"), $fallback-confidence) ) ) div $confidence
    };
    declare function local:quote($s) {
      $paragraph || ">" || $s || $paragraph
    };
    declare function local:set-globals($pic){
      $ages := $pic("ages"),
      $genders := $pic("genders"),
      $multi-faces := $pic("multi-faces"),
      
      $age-values := $ages ! .("value"), 
      $age := local:average-with-confidence($ages),
      $min-age := min($age-values),
      $max-age := max($age-values),
      $age-range := $max-age - $min-age + 1,
      
      $gender-values := $genders ! .("value"), 
      $gender := local:average-with-confidence($genders),
      $gender-queer := exists($gender-values[. lt 0]) and exists($gender-values[. gt 0]) (: APIs do not agree :)
    };
    declare function local:phrase($talk){
      replace(
         local:choose-random-talking(local:filter-truths($talk)),
         " +[.]", ".")
    };
    declare function local:phrase-age($talk){
      local:replaces(
         local:choose-random-talking(local:filter-truths($talk)),
         ("\$age", $age,
         "\$min-age", $min-age,
         "\$max-age", $max-age,
         " +[.]", "."))
    };
    declare function local:phrase-certainty($v){
      if (empty($v)) then ""
      else replace(
         local:choose-random-talking(local:filter-truths($talkings-certainty)),
       "\$certainty", $v)
    };
    declare function local:full-disclosure-table($faces){
      let $apis := distinct-values($faces//api)
      return 
        $paragraph || "Image|" || join($apis, "|") || $line-ending
        || join( (0 to count($apis)) ! ":--", "|") || $line-ending
        ||  
        join( for $local-picture in $faces return (
          x"[image]({$local-picture("url")}) | { join( $apis ! local:full-disclosure-single-api($local-picture, .), "|")}"
          )
        , $line-ending)  
    };
    declare function local:full-disclosure-single-api($picture, $api){
    let $lages := local:json-to-sequence($picture("ages"))[.("api") eq $api],
         $lgenders := local:json-to-sequence($picture("genders"))[.("api") eq $api]
     return 
        join( for $i in 1 to max((count($lages), count($lgenders))) 
          let $a := $lages[$i], $g := $lgenders[$i], 
              $aconf := $lages[$i].confidence * 100, $gconf := $lgenders[$i].confidence * 100
        return (
         (if ($a) then (
            ($age := $a("value"))[0],
            local:phrase-age($talkings-age)) else "") 
         || " "
         || (if ($g) then (
             ($gender := $g("value"))[0],
             local:phrase($talkings-gender) ) else "")
         || " " 
         || (if ($aconf != $gconf) then  local:phrase-certainty(x"{$aconf}%, {$gconf}%")
            else if ( exists(($aconf, $gconf)) ) then local:phrase-certainty(x"{$aconf}%")
            else "")
       ), ", ")
    };
    (:  variables: $age in years :)
      declare variable $talkings-age-raw := "$age years {old|young{?$age le 25}}";
    (:  variables: $min-age, $max-age, $gender :)
      declare variable $talkings-gender-raw := "
        {{male|man{?$max-age gt 15}|boy{?$min-age lt 20}}{?$gender lt 0}|
         {female|woman{?$max-age gt 15}|girl{?$min-age lt 20}}{?$gender gt 0}}";
    (:  variables: $age in years, $min-age, $max-age, $age-range in years, 
                   $gender, $gender-queer,  :)
    declare variable $talkings-multi-faces-raw := "{{?not($multi-faces)} | 
              {?$multi-faces} 
              {But|Though|However} there are {multiple|many} faces {in the picture.|visible.|.} 
              {That {is|makes it} {quite|very|a little bit|} {confusing|strange|unusual|chaotic}.|} 
              {Which {one|face|of them} should I {pick|choose|look at|examine|analyze}?|} }" ;
    declare variable $talkings := local:create-talkings(
      "{{?not($gender-queer)}{I {see|classify you as} a|You {seem|appear} to be a|You {look|appear} like a} 
       
       |{?$gender-queer} {Hard too say|You {look|appear} {quite} queer|Complicated|Difficult}.
       
         {Tending towards a|{?abs($gender) > 0.6 }{You are most|Most} likely a|Perhaps a|You {might|could} be a}
        
       } 
         {{?$age-range le 10}$age years {old|young{?$min-age le 30}}|
               {?$age-range ge 10}$min-age to $max-age years {old|young{?$min-age le 30}}}

         " || $talkings-gender-raw || "
               
        {{?$gender-queer and abs($gender) le 0.3}. {Or not.|{You can|Can} never be {certain|sure}.|}|}
        
        " || $talkings-multi-faces-raw || "
      ");
      declare variable $talkings-gallery := local:create-talkings("
        {Ah|Nice} {a gallery|an album|many images|a lot of images}. 
        {{My o|The o|O}verall impression is a|
         Altogether {I {see|classify you as} a|you {seem|appear} to be a| you {look|appear} like a} 
        }
        
         {{?$age-range le 10}$age years {old|young{?$min-age le 30}}|
               {?$age-range ge 10}$min-age to $max-age years {old|young{?$min-age le 30}}}
          " || $talkings-gender-raw || "
          .
          
          " || $talkings-multi-faces-raw || "
      ");
      declare variable $talkings-gallery2 := local:create-talkings("
          {{For|On} the individual {pictures|images|photos} {the|my} impression is: }
       ");
      declare variable $talkings-gallery-image := local:create-talkings("
         {{?$age-range le 10}$age years {old|young{?$min-age le 30}}|
               {?$age-range ge 10}$min-age to $max-age years {old|young{?$min-age le 30}}}
          " || $talkings-gender-raw || "
      ");
      declare variable $talkings-age := local:create-talkings($talkings-age-raw);
      declare variable $talkings-gender := local:create-talkings($talkings-gender-raw);      
      declare variable $talkings-certainty := local:create-talkings(
        " { ( $certainty {certainty|probability|} ) |
            ( {p = |probability: |certainty: |confidence: } $certainty ) 
        } ");
      declare variable $talkings-no-help := local:create-talkings("{Sadly |||}I do not know {how {to {get better|improve your|help your} {passing|passing|blending}}}");
      declare variable $talkings-full-disclosure := local:create-talkings("
        I can tell you more precise {{what|who} I am {seeing} in your|how I {gender|see} your} {picture|photo|face}: 
      ");
      declare variable $talkings-no-clue:= local:create-talkings("
        What do you mean?|I have no idea what {you mean|that means}.|
          {I {did|could} not understand|I failed to {parse|understand}} {that|what you mean|what you are saying|your reply}.
      ");
      declare variable $talkings-bot := local:create-talkings("
        {But|Well, |}{I {identify|consider myself|identify myself} as {human|trans-redditor|trans-human}|
         I prefer {trans-redditor|trans-human}|
         I have bot-dysphoria. {Please do|do} not call me that
        }.
      ");
      declare variable $talkings-you-are-welcome := local:create-talkings("
        {You are|You're|Youre} {|very} welcome | Just doing my job 
        | Great that you {enjoyed|liked} it. | Happy to {|be of} help
        | It's my pleasure to {help you|be of help}
      ");
      declare variable $talkings-oxfort-and-sky := local:create-talkings("
        {Oxfort and Skybiometry|Skybiometry and Oxfort} are face {recognization|detection|biometry|} {API|api}s
      ");
      declare variable $talkings-oxfort := local:create-talkings("
        Oxfort is Microsoft's face {recognization|detection|biometry|} api|
        It {is|means|stands for} Microsoft's Oxfort. {A|Some kind of} face {recognization|detection|biometry|} API
      ");
      declare variable $talkings-sky := local:create-talkings("
        Skybiometry is a cloud based face {recognization|detection|biometry|} api|
        Skybiometry {is|means|stands for} a face {recognization|detection|biometry|} API
      ");
      (: what it should do one day:
          Passing report from $date 
          I have analyzed $count-posts posts of $count-pictures from /r/transpassing and /r/transtimelines.
          Of these posts $count-mtf could be identified as MTF and $count-ftm as FTM. 
          $excluded pictures were excluded due to containing multiple faces.
          Multiple pictures of the same user were averaged. (e.g. pass in 3 of 4 pictures => 0.75 passing)
          
          The overall passing rate is $passing-rate-mtf for MTF and $passing-rate-ftm for FTM.
          
          The passing rate broken down for age groups is:
                     MTF[imgur plot]      MTF^age MTF > 3 month HRT, > 6 month HRT, 9 month HRT       FTM          FTM^age
          10 - 15:
          15 - 20: 
          20 - 25: 
          25 - 30:
          ...
          
          
          
          For individual years:
                     MTF[imgur plot]      MTF^age       FTM          FTM^age
          10:
          11:
          12: 
          13:
          ...
          
          
          
          Broken down for APIs we get:
          
          Oxfort:
          wie oben
          
          Skybiometry:
          wie oben
          
          
          
       :)
      declare variable $paragraph := $line-ending || $line-ending;
      trace(serialize-json($talkings),$line-ending|| " talkings"),
      ()
    </s>
    <if test="get('skip-post')">
      <s>
        $post-cache-file := $CACHEDIR || "reddit/" || uri-encode($skip-post),
        file:write-text($post-cache-file, '{"myerror": "skipped"}'),
        fn:error(QName("x"),"done")
      </s>
    </if>
    <s>$active := empty(get('backtest', ())) and empty(get('debug-post', ())),
       $limit := (get('backtest', ()), 15)[1],
       $subreddits-active := tokenize($keys/reddit/subs/active, ","),
       $subreddits-passive := tokenize($keys/reddit/subs/passive, ","),
       $backtest-after := {}
     </s>
    <call action="tests"/>

    <if test="false()">
    <s> 
      declare function local:ground-truth-sex($s) {
        if (matches($s, "(^|[^a-z])(MTF|MFT|M2F|AMAB|assigned *male|((trans *|to +(this|that) +|first *(one)? *as *(a)? *)(girl|fem|woman|lesbian))|still *(see)? *boy|woman|lady|princess|goddess|queen|bitch)|[ /0-9][EF](strace|strogen|[,.! ()/])", "i")
            or (matches($s, "girl", "i") 
                and not(matches($s, "(from|was|ex) *girl|boy|girlfriend", "i"))
            )
        ) then -1
        else if (matches($s, "(^|[^a-z])(FTM|AFAB|FAAB|F2M|assigned *female|to *boy|testost|trans *mascul)|([ /\-\[\]()]|^)T[ .,\[\]()]", "i")) then 1
        else ()
      };
      declare variable $numbers := (1 to 70) ! format-integer(., "w");
      declare function local:parse-number($n){
        let $res := (if (empty($n)) then ()
          else let $temp := extract($n,"[0-9]+([.][0-9]+)?")
          return if ($temp) then $temp
          else let $temp := index-of($numbers, $n)
          return if ($temp) then $temp
          else let $temp := (for $i at $p in $numbers where contains($n, $i) return $p)[last()]
          return if ($temp) then $temp 
          else if (starts-with($n, "a ")) then 1
          else ()
         )
       return if ($res and matches($n, "half|1/2")) then $res + 0.5
       else $res
      };

      declare variable $is-age-regex1 := "([0-9]{2}([.][0-9]+)?|" || join((10 to 70) ! format-integer(., "w"), "|") || ") *(y/o|y(ea)?rs? *old|Mtf|ftm)";
      declare variable $is-age-regex2 := "(^|[^0-9']{2}|\[)(([0-9]{2}([.][0-9]+)?|" || join((10 to 70) ! format-integer(., "w"), "|") || ")( *[,]| -|([ |]+| *[^0-9]{2})[^0-9,.|\-/]*|$))";
      declare function local:ground-truth-age($s) {
        local:parse-number( let $ages := extract($s, $is-age-regex1, 1, "i*" )
        return if (exists($ages)) then $ages
        else let $ages2 := extract($s, $is-age-regex2, 2, "i*" )[not(matches(., "HRT|month|mo[. ]|week|[0-9]m|lbs| day", "i"))]
        return $ages2 )
      };
      
      declare function local:month-kind($m) {
        if (string-length($m) > 3) then "mmmm"
        else "mmm"
      };
      
      declare function local:parse-any-date($date) {
        let $split := tokenize(normalize-space($date), "[ ,.:/\-]+")!replace(.,"([0-9])(st|nd|rd|th)$", "$1")!normalize-space()
        return if (count($split) = 3) then
          let $format := 
            if ($split[1] castable as xs:integer and $split[1] > 1000) then 
              if ($split[2] castable as xs:integer ) then 
                if ($split[2] > 12) then "yyyy.d.m"
                else "yyyy.m.d" 
              else "yyyy.mmm.d"
            else if (not($split[2] castable as xs:integer)) then "d." || local:month-kind($split[2]) || ".y+"
            else if (not($split[1] castable as xs:integer)) then local:month-kind($split[1]) || ".d.y+"
            else if ($split[1] > 12) then "d.m.y+"
            else "m.d.y+"
          return parse-date(join($split, "."), $format)
        else if (count($split) = 2) then          
          let $format :=
            if ($split[1] castable as xs:integer) then
              if ($split[1] > 1000) then  
                if ($split[2] castable as xs:integer) then "yyyy.m"
                else "yyyy." || local:month-kind($split[2])
              else "m.y+"
            else local:month-kind($split[1]) || ".y+"
          return parse-date("15." || join($split, "."), "d." || $format)
        else fn:error("date", "invalid date:" || $date)
      };

      declare function local:parse-time-frame-factor($tf, $default){
        if (contains($tf, "month")) then 1
        else if (contains($tf, "year")) then 12
        else if (contains($tf, "week")) then 0.23013689
        else if (contains($tf, "day")) then  0.0328548
        else if (contains($tf, "m")) then 1
        else if (contains($tf, "y")) then 12
        else if (contains($tf, "w")) then 0.23013689
        else if (contains($tf, "d")) then  0.0328548
        else $default
      };

      declare function local:parse-time-frame($tf){
        let $def-factor := local:parse-time-frame-factor($tf, 1)
        return if (matches($tf, "[0-9]")) then 
          sum(extract($tf, "[0-9]+([.][0-9]+)?[^0-9]*", 0, "*") ! (local:parse-number(.) * local:parse-time-frame-factor(., $def-factor)))
         else
          local:parse-number($tf) * local:parse-time-frame-factor($tf, $def-factor)
      };
      
      declare function local:unix-time-to-date($tf){
        $tf * xs:dayTimeDuration("PT1S") + xs:date("1970-01-01")
      };
      
      declare function local:date-delta($start, $submission){
        ($submission - $start ) div xs:dayTimeDuration("P30DT9H")
      };
        
      declare variable $is-onhrt-regex1 := "(([0-9]+([.][0-9]+)?|" || join((1 to 70) ! x"({format-integer(., "w")})", "|") || ")[ -]*(ish|st|nd|\+)? *(week|mo?s?[^a-z]|month|d(ay|[^a-z]|$)|y[ears]*[a-z ]*([0-9]+ *(mo?s?[^a-z]|month))?[a-z ]*(hrt|(hor|')mone|[^a-z][ET]([ !]|$))))";
      declare variable $is-onhrt-regex2 := "((^|[^0-9])[0-9]([.][0-9]+)? *y[ears]*( *[0-9]+ *mo?s?[^t])?|a year([a-z ]*a half)|DAY *[0-9]+)";
      declare variable $month-names := join((1 to 12) ! parse-date("01-"||.||"-2016", "d-m-y+") ! (format-date(.,"[Mn,3-3]"), format-date(.,"[Mn]")) ! x"({.})", "|");
      declare variable $is-date-regex := "[0-9A-Za-z]+[.\-/][0-9]+(st|nd|rd|th)?[.\-/][0-9]+|[a-z]+ *[0-9]+(st|nd|rd|th)?[ ,]+[0-9]{4}|("||$month-names||")[-. ]*[0-9]{4}|[0-9]{4}[-. ]*("||$month-names||"|[0-9]{2})";
      declare variable $is-onhrt-regex3 := "(HRT|(?:^|[^a-z])[TE])[: (a-z\-]+?("||$is-date-regex||")";
      declare variable $is-nohrt-regex := "(pre|no)[- ]+(hrt|hormones|everything|[et][^a-z])";

      declare function local:ground-truth-hrt($s, $date) {
        let $temp := extract($s, $is-onhrt-regex1, 1, "i" ) ! replace(., $is-age-regex1, "", "i")[.]
        return if ($temp) then local:parse-time-frame($temp) else 
         let $temp := extract($s, $is-onhrt-regex2, 1, "i" )[.]
         return if ($temp) then local:parse-time-frame($temp) else
          let $date := extract($s, $is-onhrt-regex3, 2, "i" )[.]
          return if ($date) then local:date-delta(local:parse-any-date($date), $date)
          else if (matches($s, $is-nohrt-regex, "i")) then 0
          else ()
     };
    
      declare function local:ground-truth-ffs($s) {
        let $match := extract($s, "(^|[a-z]+) *FFS", (0,1))
        return if (not($match[1])) then ()
        else if (not($match[2] = ("NO", "NEED", "CAN")) ) then true()
        else false()
      };
      
      declare function local:is-pass($sex, $gender){
        if ($sex lt 0 and $gender gt 0) then true()
        else if ($sex gt 0 and $gender lt 0) then true()
        else false()
      };
      
      declare function local:compare-detections($dets, $creation-date, $birthday, $sex, $age, $hrt) {
        if (empty($dets)) then () 
        else
          let $cur := $dets[1]
          let $title := $cur/title
          let $hrt := (local:ground-truth-hrt($title, $creation-date), $hrt)[1]
          let $preage := local:ground-truth-age($title)
          let $age := if ($preage) then $age 
                      else if (exists($birthday)) then
                        let $date := extract($title, $is-date-regex, 0, "i") ! normalize-space()
                        return if ($date) then (local:parse-any-date($date) - $birthday) div xs:yearMonthDuration("P1Y")
                               else $age
                      else $age         
(:           let $api-passing := for $gender in $cur("genders") group by $api := $gender("api") 
                               return {"api": $api, "passing":  ("value") != sex => passing:)
          return ({
            "age": $age,
            "hrt": $hrt,
            "multi-faces": $cur("multi-faces"),
            "passing": local:is-pass($sex, local:average-with-confidence($cur("genders")))
          }, local:compare-detections(tail($dets), $creation-date, $birthday, $sex, $age, $hrt)
          )
      };

      trace(join(
      let $ids := file:list($CACHEDIR || "reddit/")[not(contains(., "."))] ! ($CACHEDIR || "reddit/" || .),
          $data := for $id in $ids                   
                   let $json := jn:json-doc($id)
                   let $json-detections := local:json-to-sequence(jn:json-doc($id || ".detections"))
                   let $title := join(($json/title, $json/author_flair_text)[.])
(:                   let $detections := jn:json-doc(. || ".detections"):)
                   let $gt-sex := (let $temp := local:ground-truth-sex($title) return if ($temp) then $temp else $json-detections // title ! local:ground-truth-sex(.)) [1]
                   let $gt-age := local:ground-truth-age($title)
                   let $creation-date := local:unix-time-to-date($json/created)
                   let $gt-hrt := local:ground-truth-hrt($title, $creation-date)
                   let $detections := if ($gt-sex) then local:compare-detections($json-detections, $creation-date, $creation-date - $gt-age * xs:yearMonthDuration("P1Y"), $gt-sex, $gt-age, $gt-hrt) else ()
                   return {
                     "id": $id,
                     "title": $title,
                     "sex": $gt-sex,
                     "age": $gt-age,
                     "hrt": $gt-hrt,
                     "ffs": local:ground-truth-ffs($title),
                     "detections": $detections,
                     "passing-score": 
                       let $dets := $detections[not(.("multi-faces"))], 
                           $pass := $dets[.("passing")], 
                           $no-pass := $dets[not(.("passing"))] 
                       return if ($pass + $no-pass = 0) then ()
                       else ($pass - $no-pass) div ($pass + $no-pass)
                   }
      return (:$data[exists(./ffs) ] ! (./title || "  => "|| ./ffs):)
      
      x"Total posts: { count($data) }, Total FTM: { count( $data[./sex = 1] ) }  MTF: { count( $data[./sex = -1] )}
        Usable Total FTM: { count( $data[./sex = 1 and exists(./age) and exists(./hrt) ] ) }   MTF: { count( $data[./sex = -1 and exists(./age) and exists(./hrt) and not(./ffs)] ) }
        
        {
         for $post in $data[./sex = -1 and exists(./age) and exists(./hrt) and not(./ffs)] group by $age := $post("age") order by $age
         return for $post in $post group by $hrt := $post("hrt") order by $hrt 
         return x"{$age} {$hrt}: {$post/passing-score}: {avg($post/passing-score)}" 
        }
       "

      , $line-ending), "reddit")
    </s>
    
    <s>error("end","end")</s> 
    </if>

   <!--<s>age := 60, gender := -1, 
      $curtalkings := local:filter-truths($talkings),
      trace(serialize-json($curtalkings),$line-ending|| " talkings"),
      trace(local:count-talkings($curtalkings), "count"),
      trace(local:choose-talking($curtalkings, 0), "firsr"),
      for $i in (0 to local:count-talkings($curtalkings)) return
        trace(local:choose-talking($curtalkings, $i), $i||"-th"),
      ()
    </s>        -->
    <!-- %certain-age  
    
    
    (between) 10 to 30 years old 
    
   (Probably | Tending towards | Likely | More) male. Not sure | Hard to say 
   
       <s>
       </s>
    -->
        
    
    <s> ("skybiometry2", "oxfort", "reddit", "raw") ! file:create-dir($CACHEDIR || .)</s>

    <if test="empty(get('debug-post', ()))">   
      <page url="https://www.reddit.com/api/login">
        <post name="user" value="{$keys/reddit/user}"/>
        <post name="passwd" value="{$keys/reddit/passwd}"/>
        <post name="api_type" value="json"/>
      </page>    
      <s>modhash := json("https://www.reddit.com/api/me.json")//modhash</s>
    </if>
    
    <loop test="true()">
      <s>letters := if (empty(get('debug-post', ()))) then json('https://www.reddit.com/message/unread.json')/data/children/data else ()</s>
      <loop var="letter" list="$letters">
        <s>trace($letter, "letter"),
           context := json(resolve-uri((let $context := $letter("context") return
             if (contains($context, "?")) then replace($context, "[?]", ".json?") else concat($context, ".json"))
           ,"http://reddit.com")),
           parentid :=  trace($letter("parent_id"), "parent"), (:fullname:)
           grandparentid := ($context//children)()[trace(concat(.("kind"), "_", .("data")("id")), "id") eq $parentid ]("data")("parent_id"),
           post-cache-file := trace($CACHEDIR || "reddit/" || uri-encode($grandparentid), "cache"),
           
           $sentences := extract(trace($letter/body, "letter body: "), "[^.:;!?]+([.:;!?()-]+|$)", 0, "*") ! (" " || normalize-space(.) || " ")
        </s>
        
        <s>answer := let $thankyou := ($sentences[local:matches(., "thanks|thank you|\bty\b|\bthx")])[1],
                         $oxfort := ($sentences[local:matches(., "(what|which).*oxfort.*[?]")])[1],
                         $sky := ($sentences[local:matches(., "(what|which).*sky.*[?]")])[1],
                         $bot := ($sentences[local:matches(., "\bbot\b|not human|automated")])[1] 
                      return 
                        (if (exists($thankyou)) then local:quote($thankyou) || local:phrase($talkings-you-are-welcome) else "") ||
                        (if (exists($bot)) then local:quote($bot) || local:phrase($talkings-bot) else "") ||                      
                        (if (exists($oxfort) and $oxfort = $sky) then 
                          local:quote($oxfort) || local:phrase($talkings-oxfort-and-sky) 
                        else (
                          (if (exists($oxfort)) then local:quote($oxfort) || local:phrase($talkings-oxfort) else "") ||
                          (if (exists($sky)) then local:quote($sky) || local:phrase($talkings-sky) else "") 
                        ))
        </s>
        
        <if test="file:exists($post-cache-file)">
          <!-- if that exists it is a post-->
          <s>linkpost := jn:parse-json(file:read-text($post-cache-file ))</s>
          <if test="$linkpost('author') eq $letter('author')">
            <call action="detect-linkpost"/>
            <s>answer := $answer || (
              let  $moreinfo := $sentences[local:matches(., '(what|why|improve|feature|wrong|stand out|problem\b|help).*[?]')][1]
              return if (exists($moreinfo)) then local:quote($thankyou) || local:phrase($talkings-no-help) else ""
              )
            </s>
            
            <if test="exists( $sentences[contains(., '?') and not (local:matches(., '\bbot\b|not human|automated'))  ]) ">
              <s> $answer := $answer 
                             || (if ($answer ne "") then $paragraph
                                 else local:phrase($talkings-no-clue)) || " "
                             || local:phrase($talkings-full-disclosure) </s>
              <if test="count($faces) le 2">
                <loop var="picture" list="$faces">
                  <s>local:set-globals($picture),
                     $answer := $answer || " " || join(
                       let $apis := distinct-values( $picture//api )
                       for $api in $apis
                       return (local:full-disclosure-single-api($picture, $api) || " ("||$api||")")
                     , ", ")
                  </s>
                
                </loop>
              </if>
              <else>
                $answer := $answer ||  local:full-disclosure-table($faces)
              </else>
            </if>
          </if>
        </if>
        <s>trace($answer, $line-ending || "fd")</s>
        <if test="$active and ($answer != '')">
          <page url="https://www.reddit.com/api/comment.json">
            <header name="X-Modhash" value="{$modhash}"/>
            <post name="api_type" value="json"/>
            <post name="text" value="{$answer}"/>
            <post name="thing_id" value="{$letter/name}"/>
          </page>
        </if>
      </loop>
      <page test="$active and exists($letters)" url="https://www.reddit.com/api/read_message">
        <header name="X-Modhash" value="{$modhash}"/>
        <post name="id" value="{join($letters/name, ',')}"/>
      </page>        
      <!--    <s>letters := ("/u/butler bring soda")</s>-->
      
       
     <loop var="subreddit" list="($subreddits-active, $subreddits-passive)">
      <s>
        $list := $subreddit ! json(x'https://www.reddit.com/r/{.}/.json?limit={$limit}{$backtest-after(.)}')/data/children/data,
        if (get('backtest', ())) then ( 
          $backtest-after($subreddit) := $amp || "after=" || $list[last()]/name, 
          if ($limit le 0) then error(QName("xx"), "backtest limit exceeded") else $limit := $limit - count($list) 
        ) else () 
      </s>
      <loop var="lp" 
            list="$list">
        <s>linkpost := $lp,
           post-cache-file := $CACHEDIR || "reddit/" || uri-encode(trace($linkpost("name"), "name"))
        </s>
        <if test="not(file:exists($post-cache-file)) or get('debug-post') = $linkpost('name')  ">
          <call action="detect-linkpost"/>
          <if test="count($faces) > 0 ">
            <s>(: Gives us $age and $genders, combine for talking  :) 
   
               if (count($faces) eq 1 ) then (
                 local:set-globals($faces),
                 $message := local:phrase-age($talkings)
               ) else (
                 local:set-globals( {"ages": $faces("ages"), "genders": $faces("genders"), "multi-faces": exists($faces("multi-faces")[.]) } ),
                 $message := local:phrase-age($talkings-gallery) || 
                   (if (count(distinct-values($faces("genders")("value"))) != 1 ) then
                     $paragraph ||
                       local:phrase-age($talkings-gallery2) || join (
                       for $face in $faces return (
                         local:set-globals($face)[0], local:phrase-age($talkings-gallery-image)
                       ), ", ") else "")
               ),
   
  
               trace($message, "final message: ")
            </s>
            <s>file:write-text($post-cache-file, serialize-json($linkpost))</s>
            
            <if test="$active and ($linkpost('subreddit') = $subreddits-active) and not(file:exists($post-cache-file || '.message'))"> 
              <page test="$message" url="https://www.reddit.com/api/comment.json">
                <header name="X-Modhash" value="{$modhash}"/>
                <post name="api_type" value="json"/>
                <post name="text" value="{$message}"/>
                <post name="thing_id" value="{$linkpost('name')}"/>
              </page>
              
              <s>file:write-text($post-cache-file || ".message", $message)</s>
            </if>
          </if>
        </if>
       </loop>
      </loop>
      <s>if (empty(get('backtest', ()))) then sleep(trace(1000*60*5, "sleeping")) else ()</s>
    </loop>


  </action>
  
 
  <action id="detect-linkpost">
    <s>post-cache-file := $CACHEDIR || "reddit/" || uri-encode($linkpost("name")),
       $url := $linkpost("url")</s>
    <if test="file:exists($post-cache-file || '.detections')">
       <s>faces := trace(local:json-to-sequence(jn:parse-json(file:read-text($post-cache-file || '.detections'))), "faces")</s>
    </if>
    <else>
      <choose>
        <when test="contains($url, 'imgur.com') and matches($url, '/a/|/gallery/')">
          <try>
            <page url="https://api.imgur.com/3/album/{extract($url, '/(a|gallery)/(.*)', 2)}">
              <header name="Authorization" value="Client-ID {$keys/imgur/id}"/>
            </page>
          <catch errors="http:404"><s>imgurls := (), $raw := "{}"</s></catch>
          <catch errors="http:403"><s>file:append( $CACHEDIR || "imgur-failed.log", $linkpost("name") || ": " || $linkpost("url")), imgurls := (), $raw := "{}"</s></catch>
          </try>
          <s>imgurls :=jn:parse-json($raw)("data")("images")()!({"title": title,"url":link})</s>
        </when>
        <when test="matches($url, 'imgur.com.*,')">
          <s> imgurls := tokenize(substring-after($url, "imgur.com/"), ",") ! (
                let $img := "http://i.imgur.com/" || . 
                return if (matches($img, "[.](jpe?g|png)")) then $img else ($img || ".jpg") )</s>
        </when>
        <when test="contains($url, 'picpaste.com') and not( contains($url, '/pics'))">
          <s>$imgurls := resolve-uri(exactly-one(doc($url)/css('div.picture img')[1])/@src, $url)</s>
        </when>
        <when test="matches($url, '[.](jpe?g|png)')">
          <s> imgurls := $url</s>
        </when>
        <when test="contains($url, 'm.imgur.com')">
          <s>$imgurls := replace($url, "m[.]imgur", "i.imgur") || ".jpg"</s>
        </when>
        <when test="contains($url, 'imgur.com')">
          <s>$imgurls := replace($url, "/r/[^/]+/", "/") || ".jpg"</s>
        </when>
        <otherwise>
          <s>imgurls := ()</s>
        </otherwise>
      </choose>
      
      <s>faces := ()</s>
      <loop var="local-url" list="$imgurls">
        <s>imgurl := replace(
                         if ($local-url instance of object()) 
                         then $local-url("url") 
                         else $local-url, $amp || "amp;", $amp)
        </s>
        <call action="detect"/>
        <s>faces := ($faces, {|
          if ($local-url instance of object()) then $local-url else {"url": $local-url} , 
          {"ages": $ages, 
          "genders": $genders, 
          "multi-faces": $multi-faces} |} )</s>
      </loop>
      <s>file:write-text($post-cache-file || '.detections', serialize-json($faces))</s>
      <s>
       file:write-text($post-cache-file || '.author', serialize-json(
         try {
           json("https://www.reddit.com/user/"||$linkpost("author")||".json?limit=25")
         } catch err:FODC0002 {
           try {
             sleep(4*1000),
             json("https://www.reddit.com/user/"||$linkpost("author")||".json?limit=25")
           } catch err:FODC0002 {
             file:append( $CACHEDIR || "reddit/failed.log", $linkpost("name") || ": " || "https://www.reddit.com/user/"||$linkpost("author") || " : "|| $err:description ),
             {"myerror": $err:description  }
           }
         }
       ))
       </s>
    </else>
  </action>

  <action id="detect">
    <s> genders := (), ages := (), multi-faces := false() </s>
    <s> api := "oxfort" </s> <call action="{$api}"/>
    <s> api := "skybiometry2" </s> <call action="{$api}"/>
    <s>trace($genders, "genders: "), trace($ages, "ages: ")</s>
  </action>  
    
  <action id="cache-load">
    <s>cache-file := $CACHEDIR || $api || "/" || uri-encode($imgurl),
       cached := file:exists($cache-file),
       if ($cached) then $response := jn:parse-json(file:read-text($cache-file)) else ()
    </s>    
  </action>
  <action id="cache-save">
    <s> if (not($cached)) then (
      response := jn:parse-json($raw),
      file:write-text($cache-file, serialize-json($response))
    ) else () </s>
  </action>

  <action id="skybiometry2"> 
    <call action="cache-load"/>
    <if test="not($cached)">
      <s>temp-url := uri-combine("http://api.skybiometry.com/fc/faces/detect.json?",
         {"api_key": $keys/skybiometry/key,
          "api_secret": $keys/skybiometry/secret,
          "urls": $imgurl,
          "detector": "aggressive",
          "attributes": "gender,age"
         })
      </s>
      <page url="{$temp-url}"><error-handling value="400=accept"/></page>
      <s>response := jn:parse-json($raw)</s>
      <loop test="($response).status = 'failure'">
        <choose>
          <when test="($response).error_code = 33 and matches($imgurl, 'imgur.*[^m][.][^.]+$')">
            <!-- too big -->
            <s>temp-url := uri-combine($temp-url, {"urls": replace($imgurl, '[.]([^.]+)$', 'm.$1')})</s> <!-- imgur thumbnail-->
          </when>
          <when test="(($response).error_code = 202)"> <!-- too many requests -->
            <s>sleep(1000*60*(60 + 1))</s>
          </when>
          <when test="(($response).error_code = 30) and matches($imgurl, '[.]gif')"> <!-- invalid image format -->
            <s>$temp-url := (), $raw := serialize-json({"myerror": $response}) </s>
          </when>
          <otherwise>
            <s>$raw := '{"myerror": "fail400", "theirerror": ' || $raw || '}',  
               $temp-url := (),
               file:append( $CACHEDIR || $api || "/failed.log", $linkpost("name") || ": " || $imgurl || " => "||$response("error_message") || $line-ending)</s>
          </otherwise>
        </choose>
        <page url="{$temp-url}"><error-handling value="400=accept"/></page>          
        <s>response := jn:parse-json($raw)</s>
      </loop>
      
      <!-- 
      
      {"status":"failure","photos":[{"error_code":33,"error_message":"DOWNLOAD_ERROR_FILE_TOO_LARGE - max image size is more than 2.0 MB"}],"error_code":33,"error_message":"DOWNLOAD_ERROR_FILE_TOO_LARGE - max image size is more than 2.0 MB","usage":{"used":38,"remaining":62,"limit":100,"reset_time":1456704789,"reset_time_text":"Mon, 29 February 2016 00:13:09 +0000"},"operation_id":"35fb87e39d1f4f4bb6e28f4e804a3581"}
      
      202? too many request?
      
      -->	
    </if>
      
    <call action="cache-save"/>

    <s>let $attributes := ($response).photos().tags().attributes
         return (
           $genders[] := ($attributes).gender ! { 
             "value": local:gender-to-number((.).value),
             "confidence": (.).confidence div 100,
             "api": $api
             },
           $ages[] := ($attributes).age_est ! {
             "value": number((.).value),
              "confidence": (.).confidence div 100,
              "api": $api
            }, 
           $multi-faces := $multi-faces or (count($attributes) > 1)
         )
    </s>
  </action>
 
  <action id="do-oxfort"> 
    <page url="https://api.projectoxford.ai/face/v1.0/detect?returnFaceAttributes=age,gender">
      <header name="Ocp-Apim-Subscription-Key" value="{$keys/oxfort}"/>
      <header>Content-Type: application/json; charset=utf-8</header>
      <header>Accept: application/json</header>
      <method>POST</method>
      <post>{{ "url": "{($temp-url)}" }}</post>
    </page>
  </action>
  
  <action id="oxfort"> 
    <call action="cache-load"/>
    <if test="not($cached)">
      <s>$temp-url:=$imgurl</s>
      <try>
        <call action="do-oxfort"/>
        <catch errors="http:400">
          <choose>
            <when test="matches($imgurl, 'imgur.*[^m][.][^.]+$')">
              <!-- imgur thumbnail-->
              <s>temp-url := replace($imgurl, '[.]([^.]+)$', 'm.$1')</s> 
            </when>
            <otherwise><s>sleep(60*1000)</s></otherwise>
          </choose>          
          <!-- try again -->
          <try>
            <call action="do-oxfort"/>
            <catch errors="http:400"> 
              <s>$raw := '[{"myerror": "fail400"}]',  
                 file:append( $CACHEDIR || "oxfort/failed.log", $linkpost("name") || ": " || $imgurl || $line-ending)</s>
            </catch>
          </try>
        </catch>
      </try>
    </if>
    <call action="cache-save"/>
    
    <s>let $attributes := ($response)().faceAttributes
         return (
           $genders[] := ($attributes).gender ! {
               "value": local:gender-to-number(.),
   	           "api": $api
             } ,
           $ages[] := ($attributes).age ! {
               "value": round(.),
   	           "api": $api 
               }, 
           $multi-faces := $multi-faces or (count($attributes) > 1)
         )
    
    </s>
  </action>
 
  
<!--  <action id="lambda">  does not gender, despite saying so in the example
    <page url="https://lambda-face-recognition.p.mashape.com/detect">
      <header name="X-Mashape-Key" value="{trace($keys/mashape, 'key')}"/>
      <header name="Accept" value="application/json"/>
      <post name="urls" value="{$imgurl}"/>
    </page>
    <s>trace($raw, "abc") </s>
  </action>
-->
   <!-- 
  <action id="face++">fails to open imgur links 
    <page url="https://faceplusplus-faceplusplus.p.mashape.com/detection/detect?attribute=gender%2Cage&url={uri-encode($imgurl)}">
      <header name="X-Mashape-Key" value="{trace($keys/mashape, 'key')}"/>
      <header name="Accept" value="application/json"/>
    </page>
  </action>-->
  
  <action id="tests">
     <s>
      testtrue := true(),
      testfalse := false(),
      for $test in ( 
		  ["foobar", "foobar", 1],
		  ["foo|bar", ["foo", "bar"], 2],
		  ["{x}", "x", 1],
		  ["{{x}}", "x", 1],
		  ["a{b}c", {"concat": ["a", "b", "c"]}, 1],
		  ["a{b}c{e}f", {"concat": ["a", "b", "c", "e", "f"]}, 1],
		  ["a{b}c{e|f|g}h", {"concat": ["a", "b", "c", ["e", "f", "g"], "h"]}, 3],
		  ["a{{b}}c", {"concat": ["a", "b", "c"]}, 1], 
		  ["a{b|c}d{e|f}|x", [{"concat": ["a", ["b", "c"], "d", ["e", "f"]]}, "x"], 5],
		  ["a{?$testtrue}", {"concat": "a"}, 1] ,
		  ["b{?$testfalse}|c", ["c"], 1] 
		  ) 
		  let $talk := local:filter-truths(local:create-talkings($test(1)))
  return
    if (serialize-json($talk) ne serialize-json($test(2))) then
      error(QName("xxx"), "test fail: " || $test(1) || " got " || serialize-json($talk))
    else if (local:count-talkings($talk) ne $test(3)) then
      error(QName("xxx"), "count test fail: " || $test(1) || " got " || local:count-talkings($talk))
    else ("ok"),
    
    local:assert(count(local:json-to-sequence(jn:null())) = 0, "null"),
    local:assert(count(local:json-to-sequence([1,2,3])) = 3, "array"),
    local:assert(count(local:json-to-sequence({"a": [1,2,3,4]}).a) = 4, "array"),
    local:assert(count(local:json-to-sequence([{"a": [1,2,3,4]}]).a) = 4, "obj"),
    local:assert(count(local:json-to-sequence([[[{"a": [1,2,3,4]}]]]).a) = 4, "array2"),
    local:assert(count(local:json-to-sequence([[{"x": [{"a": [1,2]}]}]]).x.a) = 2, "array3")
    
    </s>
  </action>
</actions>
