# -*- coding: ascii -*-

###########################################################################
# clive, video extraction utility
# Copyright (C) 2007-2008 Toni Gundogdu
#
# This file is part of clive.
#
# clive is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# clive is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with clive.  If not, see <http://www.gnu.org/licenses/>.
###########################################################################

## The "nomad" core classes

__all__ = ['Nomad']

import clive as _clive
from clive.console import Console
from clive.parse import PageParser
from clive.config import ConfigParser
from clive.opts import Options
from clive.update import Update
from clive.progress import Progress, ProgressVerify
from clive.cache import Cache, Browse
from clive.recall import Recall
from clive.login import Login
from clive.rss import FeedParser
from clive.scan import Scan
from clive.util import transl_embed_url
from clive.modules import Modules
from clive.error import CliveError, CliveNoMediaError

## The "nomad" engine and core class of clive
class Nomad:

    ## Entry point for the engine
    def run(self, say):
        self._say = say
        o = Options()
        self._opts = o._opts
        self._args = o._args
        (self._URLGrabber, self._URLGrabError) = Modules().getinst('urlgrabber')
        self._os = Modules().getinst('os')
        self._subprocess = Modules().getinst('subprocess')
        self._string = Modules().getinst('string')
        self._cache = Cache(say)
        self._found_urls = []
        self._error_urls = []
        self._dld_videos = []
        self._encoded_videos = []
        self._cookie_opener = None
        self._run_terminate = False # If true, terminate before stdin read

        if self._opts.check_updates: # Check for updates
            self._check_updates()

        if self._opts.write_conf: # Write conf
            self._write_conf()

        if self._opts.clear_cache: # Clear cache
            self._clear_cache()

        if self._opts.clear_last: # Clear recall
            self._run_terminate = 1
            self._clear_last()

        if self._run_terminate: # Terminate if flagged
            raise SystemExit

        if self._opts.enable_cache_browse: # Browse cache -- grab URLs
            # raw_urls: User selection from cache
            raw_urls = self._browse_cache()
        else: # Otherwise -- grab URLs elsewhere
            raw_urls = self._get_raw_urls(self._args)

        if self._opts.enable_rss: # We're parsing feeds
            # raw_urls: RSS URLs (input) -> Video URLs (user selection)
            raw_urls = self._parse_rss(raw_urls)

        if self._opts.enable_scan: # We're scanning URLs for video links
            self._scan(raw_urls)
        else: # Not scanning: Login if info is provided and check grabbed URLs
            self._login_if_needed(raw_urls)
            self._check_raw_urls(raw_urls)

        if self._opts.enable_extract and not self._opts.emit_csv:
            # We're extracting
            self._dl()
            if self._opts.reencode_format:
                self._re_encode()
            if self._opts.play_format:
                self._play()
        else:
            # Not extracting: only pretend or emit as csv
            if not self._opts.enable_extract:
                self._no_extract()
            elif self._opts.emit_csv:
                self._emit_csv()

    def _parse_rss(self, raw_urls):
        p = FeedParser(self._say, self._get_proxy())
        return p.parse(raw_urls)

    def _reset_found_urls(self):
        self._found_urls = []

    def _get_found_urls(self):
        return self._found_urls

    def _browse_cache(self):
        return Browse(self._say).browse()

    def _scan(self, raw_urls):
        callbacks = (self._check_url, self._reset_found_urls,
            self._get_found_urls, self._login_if_needed)
        s = Scan(self._say, self._get_proxy(),
            callbacks, self._cache)
        self._found_urls = s.scan(raw_urls)
        self._show_queue()

    def _login_if_needed(self, raw_urls):
        urllib2 = Modules().getinst('urllib2')
        self._cookie_opener = \
            urllib2.build_opener(urllib2.HTTPCookieProcessor())
        l = Login(self._cookie_opener,
            self._get_proxy, self._say)
        a = [('youtube.', l.youtube),
            ('metacafe.', l.metacafe),
            ('dailymotion.', l.dmotion)]
        found = 0
        for (host,method) in a:
            for url in raw_urls:
                if host in url.lower():
                    method(); found=1; break;
        if not found:
            self._cookie_opener = None

    def _check_updates(self):
        self._run_terminate = 1
        Update(self._say, self._get_proxy()).check()

    def _write_conf(self):
        self._run_terminate = 1
        ConfigParser().rewrite_config(self._say, self._opts.enable_confirm)

    def _clear_cache(self):
        self._run_terminate = 1
        self._cache.clear(self._opts.enable_confirm)

    def _clear_last(self):
        self._run_terminate = 1
        Recall().clear(self._say, self._opts.enable_confirm)

    def _no_extract(self):
        if len(self._found_urls) > 0:
            self._say('==> Passed:', show_always=1)
        for v_info in self._found_urls:
            text = v_info['url'][:40].ljust(40)
            fn = self._os.path.basename(v_info['output_file'])
            text += fn[:27].rjust(28)
            text += v_info['length'].rjust(11)
            self._say(text, show_always=1)
        if len(self._error_urls) > 0:            
            self._say('==> Failed:', show_always=1)
        for video in self._error_urls:
            (url, err) = video
            text = url[:45].ljust(46)
            text += err[:32].rjust(33)
            self._say(text, show_always=1)

    def _emit_csv(self):
        if len(self._found_urls) > 0:
            self._say('==> Passed:', show_always=1)
        for v_info in self._found_urls:
            self._say('OK: "%s","%s","%s","%s"\n' % (v_info['url'],
                v_info['xurl'], self._os.path.basename(v_info['output_file']),
                v_info['length']), show_always=1)
        if len(self._error_urls) > 0:            
            self._say('==> Failed:', show_always=1)
        for video in self._error_urls:
            self._say('FAILED: "%s","%s"\n' % (video[0],video[1]),
                show_always=1)

    def _dl(self):
        if len(self._found_urls) == 0:
            raise SystemExit('error: nothing to extract.')
        for v_info in self._found_urls:
            try:
                if not v_info['skip_extraction']:
                    self._dl_video(v_info)
                self._dld_videos.append((v_info['xurl'], v_info['output_file']))
            except self._URLGrabError, err:
                self._say('%s [%s]' % (err.strerror, v_info['url']), is_error=1)
                self._error_urls.append((v_info['url'], err.strerror))
    
    def _re_encode(self):
        for (url, file) in self._dld_videos:
            output_file = '%s.%s' % (file, self._opts.reencode_format)
            output_file_stderr = '%s.stderr' % output_file
            self._say('re-encode: %s -> %s' % (
                self._os.path.basename(file),
                self._os.path.basename(output_file)))
            cmd = self._opts.ffmpeg.replace('%','$')
            d = {'i':'"%s"'%file, 'o':'"%s"'%output_file}
            cmd = self._string.Template(cmd).substitute(d)
            cmd += ' >/dev/null'
            cmd += ' 2>"%s"' % output_file_stderr
            rc = self._subprocess.call(cmd, shell=1)
            if rc != 0:
                self._say('warn: ffmpeg returned an error, see %s' % ( \
                    output_file_stderr))
            else:
                try: self._os.remove('%s' % output_file_stderr)
                except: pass
                self._encoded_videos.append((file, output_file))

    def _play(self):
        play_videos = self._dld_videos
        if self._opts.play_format != 'src':
            play_videos = self._encoded_videos
        for (ignore, file) in play_videos:
            self._say('play: %s' % self._os.path.basename(file))
            cmd = self._opts.player.replace('%','$')
            cmd = self._string.Template(cmd).substitute({'i':'"%s"'%file})
            cmd += ' >/dev/null'
            cmd += ' 2>&1'
            rc = self._subprocess.call(cmd, shell=1)
            if rc != 0:
                self._say('warn: player returned error %d' % rc)

    def _get_raw_urls(self, args):
        if not args:
            if self._opts.enable_rss and not self._opts.enable_xclip_paste:
                return self._opts.feed_url
            return Console(self._say).read()
        else:
            return args
    
    def _check_raw_urls(self, raw_urls):
        for (index, url) in enumerate(raw_urls):
            self._check_url(url, (index,len(raw_urls)))
        self._show_queue(raw_urls)

    def _check_host(self, url):
        found = 0
        for host in PageParser()._supported_hosts:
            if host[0] in url.lower():
                found = 1; break
        if not found:
            raise CliveError('error: unsupported host (%s)' % url)

    def _check_url(self, url, index):
        # Append "http://' if it's missing
        if not url.lower().startswith('http://'):
            url = 'http://' + url
        # Convert embedded video URL to video page URL
        url = transl_embed_url(url)
        try:
            if 'last.fm' in url.lower():
                id_start = url.rfind('/') + 1
                if url[id_start:id_start+3] == '+1-':
                    url = ''.join(('http://youtube.com/watch?v=',
                        url[id_start+3:len(url)]))
                else:
                    raise CliveError('error: only youtube format lastfm ' \
                        'videos are supported (%s); see clive(1) HOSTS' % url)
            if 'youtube.' in url.lower():
                if not self._opts.enable_low_quality:
                    url += '&fmt=18'
            self._check_host(url)
            cache_row = None
            xurl_expired = False
            if self._opts.enable_cache:
                cache_row = self._cache.read(url)
                if cache_row:
                    xurl_expired = self._cache.has_xurl_expired(url)
                    if xurl_expired:
                        cache_row = None
            title = data = xurl = v_id  = ''
            file_length = -1
            low_quality = self._opts.enable_low_quality
            if cache_row: # Compare cache entry with current option
                if cache_row['cache_lowq'] != low_quality:
                    cache_row = None # Different, disable cache use
            if not cache_row:
                # Fetch video page
                g = self._URLGrabber(
                    user_agent = self._opts.http_agent,
                    http_headers = (('accept-encoding','gzip'),),
                    progress_obj = ProgressVerify(index),
                    throttle = self._opts.http_throttle,
                    proxies = self._get_proxy(),
                    opener = self._cookie_opener)
                o = g.urlopen(url)
                data = o.read()
                if o.hdr.get('content-encoding') == 'gzip':
                    gzip = Modules().getinst('gzip')
                    StringIO = Modules().getinst('StringIO')
                    data = gzip.GzipFile(fileobj=StringIO(data)).read()
            else:
                # Use cache values
                title = cache_row['cache_title']
                file_length = cache_row['cache_length']
                xurl = cache_row['cache_xurl']
                v_id = cache_row['cache_vid']
                low_quality = cache_row['cache_lowq']
                ProgressVerify(index).from_cache(url, cache_row)
            url_data = {
                'page_url':url,
                'page_data':data,
                'page_title':title,
                'page_xurl':xurl,
                'video_id':v_id,
                'video_lowq':low_quality,
                'file_length':file_length,
                'callb_filelen':self._callb_check_file_len,
                'callb_say':self._say}
            self._say("\n")
            v_info = PageParser().parse(url_data,
                self._found_urls, self._get_proxy())
            if self._opts.enable_cache and xurl_expired:
                self._cache.update_expired_xurl(url, v_info['xurl'])
            if self._opts.enable_extract and not self._opts.emit_csv:
                fn = self._os.path.basename(v_info['output_file'])
                text = fn[:70].ljust(70)
                text += Progress(v_info)._byteshuman( \
                    v_info['length_bytes']).rjust(9)
                self._say(text)
            self._found_urls.append(v_info)
        except self._URLGrabError, err:
            self._say('%s [%s]' % (err.strerror,url), is_error=1)
            self._error_urls.append((url, err.strerror))
        except CliveError, err:
            self._say(str(err), is_error=1)
            self._error_urls.append((url, str(err)))

    def _callb_check_file_len(self, xurl):
        g = self._URLGrabber(
            user_agent = self._opts.http_agent,
            throttle = self._opts.http_throttle,
            proxies = self._get_proxy())
            #opener = self._cookie_opener)
        bytes = -1
        hdrs = None
        try:
            o = g.urlopen(xurl)
            hdrs = o.hdr.items()
            l = o.hdr.get('content-length')
            if l: bytes = long(l)
        except self._URLGrabError, e:
            if 'youtube.' in xurl.lower():
                if e.errno == 14 and e.code == 415:
                    raise CliveNoMediaError
            else:
                raise e
        if bytes <= 0:
            if hdrs:
                err = 'error: no-content: zero length - dumping headers:\n'
                for hdr in hdrs: err += (repr(hdr) + '\n')
            else:
                if 'youtube.' in xurl.lower():
                    if '&fmt=18' in xurl.lower():
                        raise CliveNoMediaError # Try low quality
                err = 'error: no-content: host returned zero length (timeout?)'
            raise CliveError(err)
        mbytes = '%.2fMB' % (float(bytes) / (1024*1024))
        return (mbytes, bytes)

    def _show_queue(self, raw_urls=None):
        total_bytes = 0
        total_count = 0
        total_skipped = 0
        for (index, v_info) in enumerate(self._found_urls):
            if not v_info['skip_extraction']:
                total_count += 1
                total_bytes += v_info['length_bytes']
                if self._opts.enable_cache:
                    self._cache.write(self._found_urls[index])
            else:
                total_skipped += 1
        self._say('=> %d (%s), failed: %d, skipped: %d' % (
            total_count,
            Progress(None)._byteshuman(total_bytes), 
            len(self._error_urls), total_skipped))
        if self._opts.enable_recall:
            if total_count > 0 and raw_urls:
                Recall().write(raw_urls)
            else:
                if total_skipped > 0 and raw_urls:
                    Recall().write(raw_urls)

    def _dl_video(self, v_info, omit_say=0):
        g = self._URLGrabber(
            user_agent = self._opts.http_agent,
            throttle = self._opts.http_throttle,
            progress_obj = Progress(v_info),
            proxies = self._get_proxy(),
            reget = v_info['urlg_reget'])
            #opener = self._cookie_opener)
        g.urlgrab(v_info['xurl'],
            filename=v_info['output_file'])

    def _get_proxy(self):
        # Wraps proxy in a dictionary. None return value disables
        # the use of a proxy explicitly.
        if self._opts.http_proxy:
            return {'http':self._opts.http_proxy}
        return None


