/*
 *  Copyright (c) 2003-2008 Mark Kretschmann <kretschmann@kde.org>
 *  Copyright (c) 2007 Maximilian Kossick <maximilian.kossick@googlemail.com>
 *  Copyright (c) 2007 Casey Link <unnamedrambler@gmail.com>
 *  Copyright (c) 2008 Leo Franchi <lfranchi@kde.org>
 *
 *  This program is free software; you can redistribute it and/or modify
 *  it under the terms of the GNU General Public License as published by
 *  the Free Software Foundation; either version 2 of the License, or
 *  (at your option) any later version.
 *
 *  This program is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *  GNU General Public License for more details.
 *
 *  You should have received a copy of the GNU General Public License
 *  along with this program; if not, write to the Free Software
 *  Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
 */

#include "ScanManager.h"

#include "App.h"
#include "Debug.h"
#include "MountPointManager.h"
#include "ScanResultProcessor.h"
#include "SqlCollection.h"
#include "SqlCollectionDBusHandler.h"
#include "amarokconfig.h"
#include "meta/MetaConstants.h"
#include "meta/MetaUtility.h"
#include "playlistmanager/PlaylistManager.h"
#include "statusbar/StatusBar.h"

#include <QFileInfo>
#include <QListIterator>
#include <QStringList>
#include <QVariant>
#include <QTextCodec>
#include <QXmlStreamAttributes>

#include <KMessageBox>
#include <KStandardDirs>

#include <threadweaver/ThreadWeaver.h>

#include <unistd.h>

static const int MAX_RESTARTS = 80;
static const int MAX_FAILURE_PERCENTAGE = 5;
static const int WATCH_INTERVAL = 60 * 1000; // = 60 seconds


ScanManager::ScanManager( SqlCollection *parent )
    : QObject( parent )
    , m_collection( parent )
    , m_dbusHandler( 0 )
    , m_scanner( 0 )
    , m_parser( 0 )
    , m_restartCount( 0 )
    , m_isIncremental( false )
    , m_blockScan( false )
{
    DEBUG_BLOCK

    // If Amarok is not installed in standard directory
    const QString binDir = KStandardDirs::findExe( "amarok" );
    m_amarokCollectionScanDir = binDir.left( binDir.lastIndexOf( '/' ) + 1 );

    QTimer *watchFoldersTimer = new QTimer( this );
    connect( watchFoldersTimer, SIGNAL( timeout() ), SLOT( slotWatchFolders() ) ); 
    watchFoldersTimer->start( WATCH_INTERVAL );
}

ScanManager::~ScanManager()
{
    DEBUG_BLOCK

    stopParser();
}


void
ScanManager::startFullScan()
{
    DEBUG_BLOCK
    if( m_parser )
    {
        debug() << "scanner already running";
        return;
    }
    if( m_blockScan )
    {
        debug() << "scanning currently blocked";
        return;
    }
    cleanTables();
    m_scanner = new AmarokProcess( this );
    *m_scanner << m_amarokCollectionScanDir + "amarokcollectionscanner" << "--nocrashhandler" << "-p";
    if( AmarokConfig::scanRecursively() )
        *m_scanner << "-r";
    *m_scanner << MountPointManager::instance()->collectionFolders();
    m_scanner->setOutputChannelMode( KProcess::OnlyStdoutChannel );
    connect( m_scanner, SIGNAL( readyReadStandardOutput() ), this, SLOT( slotReadReady() ) );
    connect( m_scanner, SIGNAL( finished( int ) ), SLOT( slotFinished(  ) ) );
    connect( m_scanner, SIGNAL( error( QProcess::ProcessError ) ), SLOT( slotError( QProcess::ProcessError ) ) );
    m_scanner->start();
    if( m_parser )
    {
        stopParser();
    }
    m_parser = new XmlParseJob( this, m_collection );
    m_parser->setIsIncremental( false );
    m_isIncremental = false;
    connect( m_parser, SIGNAL( done( ThreadWeaver::Job* ) ), SLOT( slotJobDone() ) );
    ThreadWeaver::Weaver::instance()->enqueue( m_parser );
}

void ScanManager::startIncrementalScan()
{
    DEBUG_BLOCK
    if( m_parser )
    {
        debug() << "scanner already running";
        return;
    }
    if( m_blockScan )
    {
        debug() << "scanning currently blocked";
        return;
    }
    const QStringList dirs = getDirsToScan();

    debug() << "GOING TO SCAN:";
    foreach( const QString &dir, dirs )
        debug() << "    " << dir;

    if( dirs.isEmpty() )
    {
        debug() << "Scanning nothing, return.";
        return;
    }
    if( !m_dbusHandler )
    {
        m_dbusHandler = new SqlCollectionDBusHandler( m_collection );
    }
    m_scanner = new AmarokProcess( this );
    *m_scanner << m_amarokCollectionScanDir + "amarokcollectionscanner" << "--nocrashhandler" << "-i" << "--collectionid" << m_collection->collectionId();
    if( AmarokConfig::scanRecursively() )
        *m_scanner << "-r";
    if( pApp->isUniqueInstance() ) 
        *m_scanner << "--pid" << QString::number( QApplication::applicationPid() );
    *m_scanner << dirs;
    m_scanner->setOutputChannelMode( KProcess::OnlyStdoutChannel );
    connect( m_scanner, SIGNAL( readyReadStandardOutput() ), this, SLOT( slotReadReady() ) );
    connect( m_scanner, SIGNAL( finished( int ) ), SLOT( slotFinished() ) );
    connect( m_scanner, SIGNAL( error( QProcess::ProcessError ) ), SLOT( slotError( QProcess::ProcessError ) ) );
    m_scanner->start();
    if( m_parser )
    {
        stopParser();
    }
    m_parser = new XmlParseJob( this, m_collection );
    m_parser->setIsIncremental( true );
    m_isIncremental = true;
    connect( m_parser, SIGNAL( done( ThreadWeaver::Job* ) ), SLOT( slotJobDone() ) );
    ThreadWeaver::Weaver::instance()->enqueue( m_parser );
}

bool
ScanManager::isDirInCollection( QString path )
{
    // In the database all directories have a trailing slash, so we must add that
    if ( !path.endsWith( '/' ) )
        path += '/';

    const int deviceid = MountPointManager::instance()->getIdForUrl( path );
    const QString rpath = MountPointManager::instance()->getRelativePath( deviceid, path );

    const QStringList values =
            m_collection->query( QString( "SELECT changedate FROM directories WHERE dir = '%2' AND deviceid = %1;" )
            .arg( QString::number( deviceid ), m_collection->escape( rpath ) ) );

    //debug() << "dir " << rpath << " is in collection? " << !values.isEmpty();
    return !values.isEmpty();
}

bool
ScanManager::isFileInCollection( const QString &url  )
{
    const int deviceid = MountPointManager::instance()->getIdForUrl( url );
    const QString rpath = MountPointManager::instance()->getRelativePath( deviceid, url );

    QString sql = QString( "SELECT id FROM urls WHERE rpath = '%2' AND deviceid = %1" )
            .arg( QString::number( deviceid ),  m_collection->escape( rpath ) );

    if ( deviceid == -1 )
    {
        sql += ';';
    }
    else
    {
        QString rpath2 = '.' + url;
        sql += QString( " OR rpath = '%1' AND deviceid = -1;" )
                .arg( m_collection->escape( rpath2 ) );
    }

    const QStringList values = m_collection->query( sql );

    //debug() << "File " << rpath << " is in collection? " << !values.isEmpty();
    return !values.isEmpty();
}

void
ScanManager::setBlockScan( bool blockScan )
{
    m_blockScan = blockScan;
    if( m_parser )
    {
        warning() << "Scanner is running while scan got blocked";
    }
    //TODO what happens if the collection scanner is currently running?
}

void
ScanManager::slotWatchFolders()
{
    if( AmarokConfig::monitorChanges() )
        startIncrementalScan();
}

void
ScanManager::slotReadReady()
{
    QByteArray line;
    QString newData;
    line = m_scanner->readLine();

    while( !line.isEmpty() )
    {
        // amarokcollectionscanner outputs UTF-8 regardless of local encoding
        QString data = QTextCodec::codecForName( "UTF-8" )->toUnicode( line );
        if( !data.startsWith( "exepath=" ) ) // skip binary location info from scanner
            newData += data;
        line = m_scanner->readLine();
    }
    //debug() << "Parsing all the following data:\n" << newData;
    if( m_parser )
        m_parser->addNewXmlData( newData );
}

void
ScanManager::slotFinished( )
{
    DEBUG_BLOCK

    //make sure that we read the complete buffer
    slotReadReady();
    m_scanner->deleteLater();
    m_scanner = 0;
    m_restartCount = 0;
    if( m_dbusHandler )
    {
        m_dbusHandler->deleteLater();
        m_dbusHandler = 0;
    }
}

void
ScanManager::slotError( QProcess::ProcessError error )
{
    DEBUG_BLOCK

    debug() << "Error: " << error;

    if( error == QProcess::Crashed )
    {
        handleRestart();
    }
}

QStringList
ScanManager::getDirsToScan() const
{
    DEBUG_BLOCK

    const IdList list = MountPointManager::instance()->getMountedDeviceIds();
    QString deviceIds;
    foreach( int id, list )
    {
        if ( !deviceIds.isEmpty() ) deviceIds += ',';
        deviceIds += QString::number( id );
    }
    
    const QStringList values = m_collection->query(
            QString( "SELECT id, deviceid, dir, changedate FROM directories WHERE deviceid IN (%1);" )
            .arg( deviceIds ) );

    QList<int> changedFolderIds;

    QStringList result;
    for( QListIterator<QString> iter( values ); iter.hasNext(); )
    {
        int id = iter.next().toInt();
        int deviceid = iter.next().toInt();
        const QString folder = MountPointManager::instance()->getAbsolutePath( deviceid, iter.next() );
        const uint mtime = iter.next().toUInt();

        QFileInfo info( folder );
        if( info.exists() )
        {
            if( info.lastModified().toTime_t() != mtime )
            {
                result << folder;
                changedFolderIds << id;
            }
        }
        else
        {
            // this folder has been removed
            result << folder;
            changedFolderIds << id;
        }
    }
    {
        QString ids;
        foreach( int id, changedFolderIds )
        {
            if( !ids.isEmpty() )
                ids += ',';
            ids += QString::number( id );
        }
        if( !ids.isEmpty() )
        {
            QString query = QString( "SELECT id FROM urls WHERE directory IN ( %1 );" ).arg( ids );
            QStringList urlIds = m_collection->query( query );
            ids.clear();
            foreach( const QString &id, urlIds )
            {
                if( !ids.isEmpty() )
                    ids += ',';
                ids += id;
            }
            if( !ids.isEmpty() )
            {
                QString sql = QString( "DELETE FROM tracks WHERE url IN ( %1 );" ).arg( ids );
                m_collection->query( sql );
            }
        }
    }
    //debug() << "Scanning the following dirs: " << result;
    return result;
}

void
ScanManager::slotJobDone()
{
    m_parser->deleteLater();
    m_parser = 0;
}

void
ScanManager::handleRestart()
{
    DEBUG_BLOCK

    m_restartCount++;
    debug() << "Collection scanner crashed, restart count is " << m_restartCount;

    slotReadReady(); //make sure that we read the complete buffer

    disconnect( m_scanner, SIGNAL( readyReadStandardOutput() ), this, SLOT( slotReadReady() ) );
    disconnect( m_scanner, SIGNAL( finished( int ) ), this, SLOT( slotFinished(  ) ) );
    disconnect( m_scanner, SIGNAL( error( QProcess::ProcessError ) ), this, SLOT( slotError( QProcess::ProcessError ) ) );
    m_scanner->deleteLater();
    m_scanner = 0;

    if( m_restartCount >= MAX_RESTARTS )
    {
        KMessageBox::error( 0, i18n( "<p>Sorry, the collection scan had to be aborted.</p><p>Too many errors were encountered during the scan.</p>" ),
                            i18n( "Collection Scan Error" ) );
        stopParser();
        return;
    }

    QTimer::singleShot( 0, this, SLOT( restartScanner() ) );
}

void
ScanManager::restartScanner()
{
    DEBUG_BLOCK

    m_scanner = new AmarokProcess( this );
    *m_scanner << m_amarokCollectionScanDir + "amarokcollectionscanner" << "--nocrashhandler";
    if( m_isIncremental )
    {
        *m_scanner << "-i" << "--collectionid" << m_collection->collectionId();
        if( pApp->isUniqueInstance() )
            *m_scanner << "--pid" << QString::number( QApplication::applicationPid() );
    }

    *m_scanner << "-s"; // "--restart"
    m_scanner->setOutputChannelMode( KProcess::OnlyStdoutChannel );
    connect( m_scanner, SIGNAL( readyReadStandardOutput() ), this, SLOT( slotReadReady() ) );
    connect( m_scanner, SIGNAL( finished( int ) ), SLOT( slotFinished(  ) ) );
    connect( m_scanner, SIGNAL( error( QProcess::ProcessError ) ), SLOT( slotError( QProcess::ProcessError ) ) );
    m_scanner->start();
}

void
ScanManager::cleanTables()
{
    DEBUG_BLOCK
    m_collection->query( "DELETE FROM tracks;" );
    m_collection->query( "DELETE FROM genres;" );
    m_collection->query( "DELETE FROM years;" );
    m_collection->query( "DELETE FROM composers;" );
    m_collection->query( "DELETE FROM albums;" );
    m_collection->query( "DELETE FROM artists;" );
}

void
ScanManager::stopParser()
{
    DEBUG_BLOCK

    if( m_parser )
    {
        ThreadWeaver::Weaver::instance()->dequeue( m_parser );

        m_parser->requestAbort();
        while( !m_parser->isFinished() )
            usleep( 100000 ); // Sleep 100 msec

        m_parser->deleteLater();
        m_parser = 0;
    }
}


///////////////////////////////////////////////////////////////////////////////
// class XmlParseJob
///////////////////////////////////////////////////////////////////////////////

XmlParseJob::XmlParseJob( ScanManager *parent, SqlCollection *collection )
    : ThreadWeaver::Job( parent )
    , m_collection( collection )
    , m_abortRequested( false )
    , m_isIncremental( false )
{
    DEBUG_BLOCK
    The::statusBar()->newProgressOperation( this, i18n( "Scanning music" ) )
        ->setAbortSlot( parent, SLOT( deleteLater() ) );

    connect( this, SIGNAL( incrementProgress() ), The::statusBar(), SLOT( incrementProgress() ), Qt::QueuedConnection );
}

XmlParseJob::~XmlParseJob()
{
    DEBUG_BLOCK
    The::statusBar()->endProgressOperation( this );
}

void
XmlParseJob::setIsIncremental( bool incremental )
{
    m_isIncremental = incremental;
}

void
XmlParseJob::run()
{
    DEBUG_BLOCK

    QList<QVariantMap > directoryData;
    bool firstTrack = true;
    QString currentDir;

    ScanResultProcessor processor( m_collection );
    if( m_isIncremental )
    {
        processor.setScanType( ScanResultProcessor::IncrementalScan );
    }
    else
    {
        processor.setScanType( ScanResultProcessor::FullScan );
    }
    
    do
    {
        m_abortMutex.lock();
        const bool abort = m_abortRequested;
        m_abortMutex.unlock();

        if( abort )
            break;
        
        //debug() << "Get new xml data or wait till new xml data is available";

        m_mutex.lock();
        if( m_nextData.isEmpty() )
        {
            m_wait.wait( &m_mutex );
        }
        
        if( m_nextData.isEmpty() )
            break;

        m_reader.addData( m_nextData );
        m_nextData.clear();
        m_mutex.unlock();

        while( !m_reader.atEnd() )
        {
            m_reader.readNext();
            if( m_reader.isStartElement() )
            {
                QStringRef localname = m_reader.name();
                if( localname == "dud" || localname == "tags" || localname == "playlist" || localname == "image" )
                {
                    emit incrementProgress();
                }

                if( localname == "itemcount" )
                {
                    The::statusBar()->incrementProgressTotalSteps( this, m_reader.attributes().value( "count" ).toString().toInt() );
                }
                else if( localname == "tags" )
                {
                    //debug() << "Parsing FILE:\n";
                    QXmlStreamAttributes attrs = m_reader.attributes();
                    QList<QXmlStreamAttribute> list = attrs.toList();

                    //foreach( QXmlStreamAttribute l, list )
                    //    debug() << " TAG: " << l.name().toString() << '\t' << l.value().toString() << '\n';
                    //debug() << "End FILE";

                    QVariantMap data;
                    data.insert( Meta::Field::URL, attrs.value( "path" ).toString() );
                    data.insert( Meta::Field::TITLE, attrs.value( "title" ).toString() );
                    data.insert( Meta::Field::ARTIST, attrs.value( "artist" ).toString() );
                    data.insert( Meta::Field::COMPOSER, attrs.value( "composer" ).toString() );
                    data.insert( Meta::Field::ALBUM, attrs.value( "album" ).toString() );
                    data.insert( Meta::Field::COMMENT, attrs.value( "comment" ).toString() );
                    data.insert( Meta::Field::GENRE, attrs.value( "genre" ).toString() );
                    data.insert( Meta::Field::YEAR, attrs.value( "year" ).toString() );
                    data.insert( Meta::Field::TRACKNUMBER, attrs.value( "track" ).toString() );
                    data.insert( Meta::Field::DISCNUMBER, attrs.value( "discnumber" ).toString() );
                    data.insert( Meta::Field::BPM, attrs.value( "bpm" ).toString() );
                    //filetype and uniqueid are missing in the fields, compilation is not used here
                    if( attrs.value( "audioproperties" ) == "true" )
                    {
                        data.insert( Meta::Field::BITRATE, attrs.value( "bitrate" ).toString() );
                        data.insert( Meta::Field::LENGTH, attrs.value( "length" ).toString() );
                        data.insert( Meta::Field::SAMPLERATE, attrs.value( "samplerate" ).toString() );
                    }
                    if( !attrs.value( "filesize" ).isEmpty() )
                        data.insert( Meta::Field::FILESIZE, attrs.value( "filesize" ).toString() );

                    data.insert( Meta::Field::UNIQUEID, attrs.value( "uniqueid" ).toString() );

                    KUrl url( data.value( Meta::Field::URL ).toString() );
                    if( firstTrack )
                    {
                        currentDir = url.directory();
                        firstTrack = false;
                    }

                    if( url.directory() == currentDir )
                    {
                        directoryData.append( data );
                    }
                    else
                    {
                        processor.processDirectory( directoryData );
                        directoryData.clear();
                        directoryData.append( data );
                        currentDir = url.directory();
                    }
                }
                else if( localname == "folder" )
                {
                    //debug() << "Parsing FOLDER:\n";
                    QXmlStreamAttributes attrs = m_reader.attributes();
                    QList<QXmlStreamAttribute> list = attrs.toList();

                    //foreach( QXmlStreamAttribute l, list )
                    //    debug() << " ATTRIBUTE: " << l.name().toString() << '\t' << l.value().toString() << '\n';
                    //debug() << "End FOLDER";

                    const QString folder = attrs.value( "path" ).toString();
                    const QFileInfo info( folder );

                    processor.addDirectory( folder, info.lastModified().toTime_t() );
                }
                else if( localname == "playlist" )
                {
                    //TODO check for duplicates
                    //debug() << "Saving playlist with path: " << m_reader.attributes().value( "path" ).toString();
                    The::playlistManager()->save( m_reader.attributes().value( "path" ).toString() );
                }
                else if( localname == "image" )
                {
                    debug() << "Parsing IMAGE:\n";
                    QXmlStreamAttributes attrs = m_reader.attributes();
                    QList<QXmlStreamAttribute> thisList = attrs.toList();

                    //foreach( QXmlStreamAttribute l, thisList )
                    //    debug() << " ATTR: " << l.name().toString() << '\t' << l.value().toString() << '\n';
                    //debug() << "End IMAGE";

                    // Deserialize CoverBundle list
                    QStringList list = attrs.value( "list" ).toString().split( "AMAROK_MAGIC" );
                    QList< QPair<QString, QString> > covers;
                   
                    // Don't iterate if the list only has one element
                    if( list.size() > 1 )
                    {
                        for( int i = 0; i < list.count(); i += 2 )
                            covers += qMakePair( list[i], list[i + 1] );

                        processor.addImage( attrs.value( "path" ).toString(), covers );
                    }
                }
            }
        }
        if( m_reader.error() != QXmlStreamReader::PrematureEndOfDocumentError )
        {
            debug() << "do-while done with error: " << m_reader.error();
            //the error cannot be PrematureEndOfDocumentError, so handle an unrecoverable error here

            // At this point, most likely the scanner has crashed and is about to get restarted.
            // Reset the XML-reader and try to get new data:
            debug() << "Trying to restart the QXmlStreamReader.."; 
            m_reader.clear();
            continue;
        }
    } while( m_reader.error() == QXmlStreamReader::PrematureEndOfDocumentError );

    if( m_abortRequested )
    {
        debug() << "Abort requested.";
        processor.rollback();
    }
    else
    {
        debug() << "Success. Committing result to database.";
        if( !directoryData.isEmpty() )
            processor.processDirectory( directoryData );
        processor.commit();
    } 
}

void
XmlParseJob::addNewXmlData( const QString &data )
{
    m_mutex.lock();
    //append the new xml data because the parser thread
    //might not have retrieved all xml data yet
    m_nextData += data;
    m_wait.wakeOne();
    m_mutex.unlock();
}

void
XmlParseJob::requestAbort()
{
    DEBUG_BLOCK

    m_abortMutex.lock();
    m_abortRequested = true;
    m_abortMutex.unlock();
    m_wait.wakeOne();
}

#include "ScanManager.moc"

