# -*- coding: ascii -*-

###########################################################################
# clive, video extraction utility
# Copyright (C) 2007-2008 Toni Gundogdu
#
# clive is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 0.1.2-1307 USA
###########################################################################

## The "nomad" core classes

import sys
import os
import gzip
import time
import math
import subprocess
import string
import re
import urllib2

try:
    from cStringIO import StringIO
except ImportError:
    import StringIO

import clive as _clive
from clive.console import Console
from clive.parse import PageParser
from clive.config import ConfigParser
from clive.opts import Options
from clive.update import Update
from clive.progress import Progress, ProgressVerify
from clive.cache import Cache
from clive.recall import Recall
from clive.login import Login
from clive.rss import FeedParser
from clive.error import CliveError, CliveNoMediaError
try:
    from clive.urlgrabber.grabber import URLGrabber, URLGrabError
except ImportError, err:
    raise SystemExit('error: %s' % err)

__all__ = ['Nomad']


## The "nomad" engine and core class of clive
class Nomad:

    ## Entry point for the engine
    # \param self The object pointer
    # \param opts The parsed runtime options returned by optparse.OptionParser
    # \param args The parsed command line arguments (optparse.OptionParser)
    # \param say The pointer to a stdout method used throughout life-time
    def run(self, opts, args, say):
        self._opts = opts
        self._args = args
        self._say = say

        self._cache = Cache()

        self._found_urls = []
        self._error_urls = []
        self._dld_videos = []
        self._encoded_videos = []

        self._cookie_opener = None

        if opts.check_updates:
            self._check_updates()

        if opts.write_conf:
            self._write_conf()

        raw_urls = self._get_raw_urls(args)

        if opts.enable_rss:
            # raw_urls: RSS URLs (input) -> Video URLs (user selection)
            raw_urls = self._parse_rss(raw_urls)

        self._login_as_needed(raw_urls)
        self._check_raw_urls(raw_urls)

        if opts.enable_extract and not opts.emit_csv:
            self._dl()
            if opts.reencode_format:
                self._re_encode()
            if opts.play_format:
                self._play()
        else:
            if not opts.enable_extract:
                self._no_extract()
            elif opts.emit_csv:
                self._emit_csv()

    def _parse_rss(self, raw_urls):
        p = FeedParser(self._opts, self._say, self._get_proxy())
        return p.parse(raw_urls)

    def _login_as_needed(self, raw_urls):
        self._cookie_opener = \
            urllib2.build_opener(urllib2.HTTPCookieProcessor())

        l = Login(self._opts, self._cookie_opener,
            self._get_proxy, self._say)

        a = [('youtube.', l.youtube),
            ('metacafe.', l.metacafe),
            ('dailymotion.', l.dmotion)]

        found = 0
        for (host,method) in a:
            for url in raw_urls:
                if url.lower().find(host) != -1:
                    method(); found=1; break;
        if not found:
            self._cookie_opener = None

    def _check_updates(self):
        Update(self._say, self._opts, self._get_proxy()).check()
        raise SystemExit

    def _write_conf(self):
        ConfigParser().rewrite_config(self._say)
        raise SystemExit

    def _no_extract(self):
        if len(self._found_urls) > 0:
            self._say('==> Passed:', show_always=1)

        for v_info in self._found_urls:
            url = v_info['url']
            if len(url) > 40: url = url[:38] + '..'
            text = url.ljust(40)
            fn = os.path.basename(v_info['output_file'])
            if len(fn) > 28: fn = fn[:24] + '..'
            text += fn.rjust(28)
            text += v_info['length'].rjust(10)
            self._say(text, show_always=1)

        if len(self._error_urls) > 0:            
            self._say('==> Failed:', show_always=1)

        for video in self._error_urls:
            (url, err) = video
            if len(url) > 40: url = url[:38] + '..'
            text = url.ljust(40)
            if len(err) > 38: err = err[:33] + '..'
            text += err.rjust(38)
            self._say(text, show_always=1)

    def _emit_csv(self):
        if len(self._found_urls) > 0:
            self._say('==> Passed:', show_always=1)

        for v_info in self._found_urls:
            self._say('OK: "%s","%s","%s","%s"\n' % (v_info['url'],
                v_info['xurl'], os.path.basename(v_info['output_file']),
                v_info['length']), show_always=1)

        if len(self._error_urls) > 0:            
            self._say('==> Failed:', show_always=1)

        for video in self._error_urls:
            self._say('FAILED: "%s","%s"\n' % (video[0],video[1]),
                show_always=1)

    def _dl(self):
        if len(self._found_urls) == 0:
            raise SystemExit('error: nothing to extract.')

        for v_info in self._found_urls:
            try:
                if not v_info['skip_extraction']:
                    self._dl_video(v_info)
                self._dld_videos.append((v_info['xurl'], v_info['output_file']))
            except URLGrabError, err:
                self._say('%s [%s]' % (err.strerror, v_info['url']), is_error=1)
                self._error_urls.append((v_info['url'], err.strerror))
    
    def _re_encode(self):
        try:
            for (url, file) in self._dld_videos:
                if file.lower().endswith('.%s' % self._opts.reencode_format):
                    s = 'warn: %s already %s, skipping re-encoding' % (
                        os.path.basename(file), self._opts.reencode_format
                    )
                    self._say(s)
                    continue

                output_file = '%s.%s' % (file, self._opts.reencode_format)
                output_file_stderr = '%s.stderr' % output_file

                self._say('re-encode: %s -> %s' % (
                    os.path.basename(file), os.path.basename(output_file)))

                cmd = self._opts.ffmpeg.replace('%','$')
                d = {'i':'"%s"'%file, 'o':'"%s"'%output_file}
                cmd = string.Template(cmd).substitute(d)
                cmd = cmd + ' >/dev/null'
                cmd = cmd + ' 2>"%s"' % output_file_stderr

                rc = subprocess.call(cmd, shell=1)
                if rc != 0:
                    self._say('warn: ffmpeg returned an error, ' +
                        'dumping stderr (%s).' % output_file_stderr)
                else:
                    try: os.remove('%s' % output_file_stderr)
                    except: pass
                    self._encoded_videos.append((file, output_file))
        except OSError, e:
            self._say(str(e), is_error=1)
    
    def _play(self):
        play_videos = self._dld_videos

        if self._opts.play_format != 'src':
            play_videos = self._encoded_videos

        for (ignore, file) in play_videos:
            self._say('play: %s' % os.path.basename(file))

            cmd = self._opts.player.replace('%','$')
            cmd = string.Template(cmd).substitute({'i':'"%s"'%file})
            cmd = cmd + ' >/dev/null'
            cmd = cmd + ' 2>&1'

            rc = subprocess.call(cmd, shell=1)
            if rc != 0:
                self._say('warn: player returned an error (%d).' % rc)

    def _get_raw_urls(self, args):
        if not args:
            if self._opts.enable_rss and not self._opts.enable_xclip_paste:
                return self._opts.url_feed
            return Console(self._opts, self._say).read()
        else:
            return args
    
    def _check_raw_urls(self, raw_urls):
        for (index, url) in enumerate(raw_urls):
            self._check_url(url, (index,len(raw_urls)))
        self._show_queue(raw_urls)

    def _check_host(self, url):
        found = 0
        for host in PageParser()._supported_hosts:
            if url.lower().find(host[0]) != -1:
                found = 1
                break
        if not found:
            raise CliveError('error: unsupported host (%s)' % url)

    def _check_url(self, url, index):
        if not url.startswith('http://'):
            url = 'http://' + url

        # ytube
        url = url.replace('/v/','/watch?v=')

        # vgoogle
        url = url.replace('/googleplayer.swf?docId=',
            '/videoplay?docid=')

        try:
            if url.find('last.fm') != -1:
                id_start = url.rfind('/') + 1
                if url[id_start:id_start+3] == '+1-':
                    url = ''.join(('http://youtube.com/watch?v=',
                        url[id_start+3:len(url)]))
                else:
                    raise CliveError('error: only youtube format lastfm ' \
                        'videos are supported (%s); see clive(1) NOTES' % url)

            if url.find('youtube.') != -1:
                if not self._opts.enable_low_quality:
                    url += '&fmt=18'

            self._check_host(url)

            cache_row = None
            xurl_expired = False

            if self._opts.enable_cache:
                cache_row = self._cache.read(url)
                xurl_expired = self._cache.has_xurl_expired(url)
                if xurl_expired:
                    cache_row = None

            title = data = xurl = v_id  = ''
            file_length = -1

            low_quality = self._opts.enable_low_quality
            if cache_row: # Compare cache entry with current option
                if cache_row[4] != low_quality: # [4]=low_quality
                    cache_row = None # Different, disable cache use

            if not cache_row:
                # Fetch video page
                g = URLGrabber(
                    user_agent = self._opts.http_agent,
                    http_headers = (('accept-encoding','gzip'),),
                    progress_obj = ProgressVerify(self._opts, index),
                    throttle = self._opts.http_throttle,
                    proxies = self._get_proxy(),
                    opener = self._cookie_opener)
                o = g.urlopen(url)
                data = o.read()
                if o.hdr.get('content-encoding') == 'gzip':
                    data = gzip.GzipFile(fileobj=StringIO(data)).read()
            else:
                # Use cache values
                (title, file_length, xurl, v_id, low_quality) = cache_row
                # NOTE: Using unicode strings with urlgrabber will
                # cause bugs that can be difficult to trace. Convert
                # all unicode strings fetched from the sqlite3 cache file
                # to either utf-8 or ASCII strings.
                title = title.encode('utf-8')
                v_id = v_id.encode('utf-8')
                xurl = str(xurl)
                ProgressVerify(self._opts, index).from_cache(url, cache_row)

            url_data = {
                'url':url,
                'data':data,
                'title':title,
                'file_length':file_length,
                'xurl':xurl,
                'v_id':v_id,
                'low_quality':low_quality,
                'file_length_callb':self._callb_check_file_len,
                'say_callb':self._say,
                'opts':self._opts,
            }

            self._say("\n")
            v_info = PageParser().parse(url_data, self._found_urls)

            if self._opts.enable_cache and xurl_expired:
                self._cache.update_expired_xurl(url, v_info['xurl'])

            if self._opts.enable_extract and not self._opts.emit_csv:
                fn = os.path.basename(v_info['output_file'])
                if len(fn) > 60: fn = fn[:56] + '..'
                text = fn.ljust(60)
                text += Progress(self._opts, v_info)._byteshuman( \
                    v_info['length_bytes']).rjust(8)
                self._say(text)
            self._found_urls.append(v_info)
        except URLGrabError, err:
            self._say('%s [%s]' % (err.strerror,url), is_error=1)
            self._error_urls.append((url, err.strerror))
        except CliveError, err:
            self._say(str(err), is_error=1)
            self._error_urls.append((url, str(err)))

    def _callb_check_file_len(self, xurl):
        g = URLGrabber(
            user_agent = self._opts.http_agent,
            throttle = self._opts.http_throttle,
            proxies = self._get_proxy())
            #opener = self._cookie_opener)
        bytes = 0
        try:
            o = g.urlopen(xurl)
            bytes = long(o.hdr.get('content-length'))
        except URLGrabError, e:
            if xurl.find('youtube.') != -1:
                if e.errno == 14 and e.code == 415:
                    raise CliveNoMediaError
            else:
                raise e
        if bytes == 0:
            raise CliveError('error: invalid file length -- check url')
        mbytes = '%.2fMB' % (float(bytes) / (1024*1024))
        return (mbytes, bytes)

    def _show_queue(self, raw_urls):
        opts = Options()
        total_bytes = 0
        total_count = 0
        total_skipped = 0

        for (index, v_info) in enumerate(self._found_urls):
            if not v_info['skip_extraction']:
                total_count += 1
                total_bytes += v_info['length_bytes']
                if self._opts.enable_cache:
                    self._cache.write(self._found_urls[index])
            else:
                total_skipped += 1

        self._say('=> %d (%s), failed: %d, skipped: %d.' % (
            total_count,
            Progress(self._opts, None)._byteshuman(total_bytes), 
            len(self._error_urls), total_skipped))

        if self._opts.enable_recall:
            if total_count > 0:
                Recall().write(raw_urls)
            else:
                if total_skipped > 0:
                    Recall().write(raw_urls)

    def _dl_video(self, v_info, omit_say=0):
        g = URLGrabber(
            user_agent = self._opts.http_agent,
            throttle = self._opts.http_throttle,
            progress_obj = Progress(self._opts, v_info),
            proxies = self._get_proxy(),
            reget = v_info['urlg_reget'])
            #opener = self._cookie_opener)
        g.urlgrab(v_info['xurl'], filename=v_info['output_file'])

    def _get_proxy(self):
        # Wraps proxy in a dictionary. None return value disables
        # the use of a proxy explicitly.
        if self._opts.http_proxy:
            return {'http':self._opts.http_proxy}
        return None


