src/collectionscanner/collectionscanner.cpp

   1 /***************************************************************************
   2  *   Copyright (C) 2003-2005 by The Amarok Developers                      *
   3  *                                                                         *
   4  *   This program is free software; you can redistribute it and/or modify  *
   5  *   it under the terms of the GNU General Public License as published by  *
   6  *   the Free Software Foundation; either version 2 of the License, or     *
   7  *   (at your option) any later version.                                   *
   8  *                                                                         *
   9  *   This program is distributed in the hope that it will be useful,       *
  10  *   but WITHOUT ANY WARRANTY; without even the implied warranty of        *
  11  *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the         *
  12  *   GNU General Public License for more details.                          *
  13  *                                                                         *
  14  *   You should have received a copy of the GNU General Public License     *
  15  *   along with this program; if not, write to the                         *
  16  *   Free Software Foundation, Inc.,                                       *
  17  *   51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.          *
  18  ***************************************************************************/
  19
  20 #define DEBUG_PREFIX "CollectionScanner"
  21
  22 #include "amarok.h"
  23 #include "collectionscanner.h"
  24 #include "debug.h"
  25
  26 #include <cerrno>
  27 #include <iostream>
  28
  29 #include <limits.h>    //PATH_MAX
  30 #include <stdlib.h>    //realpath
  31
  32 #include <audioproperties.h>
  33 #include <fileref.h>
  34 #include <tag.h>
  35 #include <tstring.h>
  36
  37 #include <QByteArray>
  38 #include <QDir>
  39 #include <QDBusReply>
  40 #include <QFile>
  41 #include <QTimer>
  42 #include <qdom.h>
  43
  44 #include <kglobal.h>
  45 #include <klocale.h>
  46
  47 #include <amarok_collection_interface.h>
  48
  49 CollectionScanner::CollectionScanner( const QStringList& folders,
  50                                       bool recursive,
  51                                       bool incremental,
  52                                       bool importPlaylists,
  53                                       bool restart )
  54         : KApplication( /*GUIenabled*/ false )
  55         , m_importPlaylists( importPlaylists )
  56         , m_folders( folders )
  57         , m_recursively( recursive )
  58         , m_incremental( incremental )
  59         , m_restart( restart )
  60         , m_logfile( Amarok::saveLocation( QString() ) + "collection_scan.log"  )
  61 {
  62     amarokCollectionInterface = new OrgKdeAmarokCollectionInterface("org.kde.amarok", "/Collection", QDBusConnection::sessionBus());
  63     kapp->setObjectName( QString( "amarokcollectionscanner" ).toAscii() );
  64     if( !restart )
  65         QFile::remove( m_logfile );
  66
  67     QTimer::singleShot( 0, this, SLOT( doJob() ) );
  68 }
  69
  70
  71 CollectionScanner::~CollectionScanner()
  72 {
  73     DEBUG_BLOCK
  74     delete amarokCollectionInterface;
  75 }
  76
  77
  78 void
  79 CollectionScanner::doJob() //SLOT
  80 {
  81     std::cout << "<?xml version=\"1.0\" encoding=\"UTF-8\" ?>";
  82     std::cout << "<scanner>";
  83
  84
  85     QStringList entries;
  86
  87     if( m_restart ) {
  88         QFile logFile( m_logfile );
  89         QString lastFile;
  90         if ( !logFile.open( QIODevice::ReadOnly ) )
  91             warning() << "Failed to open log file " << logFile.fileName() << " read-only"
  92            ;
  93         else {
  94             QTextStream logStream;
  95             logStream.setDevice(&logFile);
  96             logStream.setCodec(QTextCodec::codecForName( "UTF-8" ) );
  97             lastFile = logStream.readAll();
  98             logFile.close();
  99         }
 100
 101         QFile folderFile( Amarok::saveLocation( QString() ) + "collection_scan.files"   );
 102         if ( !folderFile.open( QIODevice::ReadOnly ) )
 103             warning() << "Failed to open folder file " << folderFile.fileName()
 104             << " read-only";
 105         else {
 106             QTextStream folderStream;
 107             folderStream.setDevice(&folderFile);
 108             folderStream.setCodec( QTextCodec::codecForName( "UTF-8" ) );
 109             entries = folderStream.readAll().split( "\n" );
 110         }
 111
 112         for( int count = entries.indexOf( lastFile ) + 1; count; --count )
 113             entries.pop_front();
 114
 115     }
 116     else {
 117         foreach( QString dir, m_folders ) {
 118             if( dir.isEmpty() )
 119                 //apparently somewhere empty strings get into the mix
 120                 //which results in a full-system scan! Which we can't allow
 121                 continue;
 122
 123             if( !dir.endsWith( '/' ) )
 124                 dir += '/';
 125             readDir( dir, entries );
 126         }
 127
 128         QFile folderFile( Amarok::saveLocation( QString() ) + "collection_scan.files"   );
 129         folderFile.open( QIODevice::WriteOnly );
 130         QTextStream stream( &folderFile );
 131         stream.setCodec( QTextCodec::codecForName("UTF-8") );
 132         stream << entries.join( "\n" );
 133         folderFile.close();
 134     }
 135
 136     if( !entries.isEmpty() ) {
 137         if( !m_restart ) {
 138             AttributeMap attributes;
 139             attributes["count"] = QString::number( entries.count() );
 140             writeElement( "itemcount", attributes );
 141         }
 142
 143         scanFiles( entries );
 144     }
 145
 146     std::cout << "</scanner>" << std::endl;
 147
 148     quit();
 149 }
 150
 151 void
 152 CollectionScanner::readDir( const QString& dir, QStringList& entries )
 153 {
 154     // linux specific, but this fits the 90% rule
 155     if( dir.startsWith( "/dev" ) || dir.startsWith( "/sys" ) || dir.startsWith( "/proc" ) )
 156         return;
 157     QDir d( dir );
 158     m_scannedFolders << d.canonicalPath();
 159
 160     if( !d.exists() )
 161        return;
 162
 163     AttributeMap attributes;
 164     attributes["path"] = dir;
 165     writeElement( "folder", attributes );
 166     d.setFilter( QDir::NoDotAndDotDot | QDir::Dirs | QDir::Files | QDir::Readable );
 167     foreach( QFileInfo f, d.entryInfoList() )
 168     {
 169         if( !f.exists() )
 170             break;
 171
 172         if( f.isSymLink() )
 173             f = QFileInfo(f.symLinkTarget() );
 174
 175         if( f.isDir() && m_recursively && !m_scannedFolders.contains( f.canonicalFilePath() ) )
 176         {
 177             readDir( f.absoluteFilePath() + '/', entries );
 178         }
 179         else if( f.isFile() )
 180             entries.append( f.absoluteFilePath() );
 181     }
 182 }
 183
 184
 185 void
 186 CollectionScanner::scanFiles( const QStringList& entries )
 187 {
 188     DEBUG_BLOCK
 189
 190     typedef QPair<QString, QString> CoverBundle;
 191
 192     QStringList validImages;    validImages    << "jpg" << "png" << "gif" << "jpeg";
 193     QStringList validPlaylists; validPlaylists << "m3u" << "pls";
 194
 195     QList<CoverBundle> covers;
 196     QStringList images;
 197
 198     int itemCount = 0;
 199
 200     oldForeachType( QStringList, entries ) {
 201         const QString path = *it;
 202         const QString ext  = extension( path );
 203         const QString dir  = directory( path );
 204
 205         itemCount++;
 206
 207         // Write path to logfile
 208         if( !m_logfile.isEmpty() ) {
 209             QFile log( m_logfile );
 210             if( log.open( QIODevice::WriteOnly ) ) {
 211                 QByteArray cPath = path.toUtf8();
 212                 log.write( cPath, cPath.length() );
 213                 log.close();
 214             }
 215         }
 216
 217         if( validImages.contains( ext ) )
 218             images += path;
 219
 220         else if( m_importPlaylists && validPlaylists.contains( ext ) ) {
 221             AttributeMap attributes;
 222             attributes["path"] = path;
 223             writeElement( "playlist", attributes );
 224         }
 225
 226         else {
 227             MetaBundle::EmbeddedImageList images;
 228             MetaBundle mb( KUrl( path ), true, TagLib::AudioProperties::Fast, &images );
 229             const AttributeMap attributes = readTags( mb );
 230
 231             if( !attributes.empty() ) {
 232                 writeElement( "tags", attributes );
 233
 234                 CoverBundle cover( attributes["artist"], attributes["album"] );
 235
 236                 if( !covers.contains( cover ) )
 237                     covers += cover;
 238
 239                 oldForeachType( MetaBundle::EmbeddedImageList, images ) {
 240                     AttributeMap attributes;
 241                     attributes["path"] = path;
 242                     attributes["hash"] = (*it).hash();
 243                     attributes["description"] = (*it).description();
 244                     writeElement( "embed", attributes );
 245                 }
 246             }
 247         }
 248
 249         // Update Compilation-flag, when this is the last loop-run
 250         // or we're going to switch to another dir in the next run
 251         QStringList::ConstIterator itTemp( it );
 252         ++itTemp;
 253         if( path == entries.last() || dir != directory( *itTemp ) )
 254         {
 255             // we entered the next directory
 256             oldForeachType( QStringList, images ) {
 257                 // Serialize CoverBundle list with AMAROK_MAGIC as separator
 258                 QString string;
 259
 260                 for( QList<CoverBundle>::ConstIterator it2 = covers.begin(); it2 != covers.end(); ++it2 )
 261                     string += (*it2).first + "AMAROK_MAGIC" + (*it2).second + "AMAROK_MAGIC";
 262
 263                 AttributeMap attributes;
 264                 attributes["path"] = *it;
 265                 attributes["list"] = string;
 266                 writeElement( "image", attributes );
 267             }
 268
 269             AttributeMap attributes;
 270             attributes["path"] = dir;
 271             writeElement( "compilation", attributes );
 272
 273             // clear now because we've processed them
 274             covers.clear();
 275             images.clear();
 276         }
 277     }
 278 }
 279
 280
 281 AttributeMap
 282 CollectionScanner::readTags( const MetaBundle& mb )
 283 {
 284     // Tests reveal the following:
 285     //
 286     // TagLib::AudioProperties   Relative Time Taken
 287     //
 288     //  No AudioProp Reading        1
 289     //  Fast                        1.18
 290     //  Average                     Untested
 291     //  Accurate                    Untested
 292
 293     AttributeMap attributes;
 294
 295     if ( !mb.isValidMedia() ) {
 296         std::cout << "<dud/>";
 297         return attributes;
 298     }
 299
 300     attributes["path"]    = mb.url().path();
 301     attributes["title"]   = mb.title();
 302     attributes["artist"]  = mb.artist();
 303     attributes["composer"]= mb.composer();
 304     attributes["album"]   = mb.album();
 305     attributes["comment"] = mb.comment();
 306     attributes["genre"]   = mb.genre();
 307     attributes["year"]    = mb.year() ? QString::number( mb.year() ) : QString();
 308     attributes["track"]   = mb.track() ? QString::number( mb.track() ) : QString();
 309     attributes["discnumber"]   = mb.discNumber() ? QString::number( mb.discNumber() ) : QString();
 310     attributes["bpm"]   = mb.bpm() ? QString::number( mb.bpm() ) : QString();
 311     attributes["filetype"]  = QString::number( mb.fileType() );
 312     attributes["uniqueid"] = mb.uniqueId();
 313     attributes["compilation"] = QString::number( mb.compilation() );
 314
 315     if ( mb.audioPropertiesUndetermined() )
 316         attributes["audioproperties"] = "false";
 317     else {
 318         attributes["audioproperties"] = "true";
 319         attributes["bitrate"]         = QString::number( mb.bitrate() );
 320         attributes["length"]          = QString::number( mb.length() );
 321         attributes["samplerate"]      = QString::number( mb.sampleRate() );
 322     }
 323
 324     if ( mb.filesize() >= 0 )
 325         attributes["filesize"] = QString::number( mb.filesize() );
 326
 327     return attributes;
 328 }
 329
 330
 331 void
 332 CollectionScanner::writeElement( const QString& name, const AttributeMap& attributes )
 333 {
 334     QDomDocument doc; // A dummy. We don't really use DOM, but SAX2
 335     QDomElement element = doc.createElement( name );
 336
 337     oldForeachType( AttributeMap, attributes )
 338     {
 339         // There are at least some characters that Qt cannot categorize which make the resulting
 340         // xml document ill-formed and prevent the parser from processing the remaining document.
 341         // Because of this we skip attributes containing characters not belonging to any category.
 342         QString data = it.value();
 343         const unsigned len = data.length();
 344         bool nonPrint = false;
 345         for( unsigned i = 0; i < len; i++ )
 346         {
 347             if( data[i].category() == QChar::NoCategory )
 348             {
 349                 nonPrint = true;
 350                 break;
 351             }
 352         }
 353
 354         if( nonPrint )
 355             continue;
 356
 357         element.setAttribute( it.key(), it.value() );
 358     }
 359
 360     QString text;
 361     QTextStream stream( &text, QIODevice::WriteOnly );
 362     element.save( stream, 0 );
 363
 364
 365     std::cout << text.toUtf8().data() << std::endl;
 366 }
 367
 368
 369 #include "collectionscanner.moc"
 370