search/indexer.php

   1 <?php
   2 /**
   3 * Global Search Engine for Moodle
   4 *
   5 * @package search
   6 * @category core
   7 * @subpackage search_engine
   8 * @author Michael Champanis (mchampan) [cynnical@gmail.com], Valery Fremaux [valery.fremaux@club-internet.fr] > 1.8
   9 * @date 2008/03/31
  10 * @license http://www.gnu.org/copyleft/gpl.html GNU Public License
  11 *
  12 * The indexer logic -
  13 *
  14 * Look through each installed module's or block's search document class file (/search/documents)
  15 * for necessary search functions, and if they're present add the content to the index.
  16 * Repeat this for blocks.
  17 *
  18 * Because the iterator/retrieval functions are now stored in /search/documents/<mod>_document.php,
  19 * /mod/mod/lib.php doesn't have to be modified - and thus the search module becomes quite
  20 * self-sufficient. URL's are now stored in the index, stopping us from needing to require
  21 * the class files to generate a results page.
  22 *
  23 * Along with the index data, each document's summary gets stored in the database
  24 * and synchronised to the index (flat file) via the primary key ('id') which is mapped
  25 * to the 'dbid' field in the index
  26 * */
  27
  28 //this'll take some time, set up the environment
  29 @set_time_limit(0);
  30 @ob_implicit_flush(true);
  31 @ob_end_flush();
  32
  33 /**
  34 * includes and requires
  35 */
  36 require_once('../config.php');
  37 require_once("$CFG->dirroot/search/lib.php");
  38
  39 /// only administrators can index the moodle installation, because access to all pages is required
  40
  41     require_login();
  42
  43     if (empty($CFG->enableglobalsearch)) {
  44         error(get_string('globalsearchdisabled', 'search'));
  45     }
  46
  47     if (!isadmin()) {
  48         error(get_string('beadmin', 'search'), "$CFG->wwwroot/login/index.php");
  49     }
  50
  51 /// confirmation flag to prevent accidental reindexing (indexersplash.php is the correct entry point)
  52
  53     $sure = strtolower(optional_param('areyousure', '', PARAM_ALPHA));
  54
  55     if ($sure != 'yes') {
  56         mtrace("<pre>Sorry, you need to confirm indexing via <a href='indexersplash.php'>indexersplash.php</a>"
  57               .". (<a href='index.php'>Back to query page</a>).</pre>");
  58
  59         exit(0);
  60     }
  61
  62 /// check for php5 (lib.php)
  63
  64     if (!search_check_php5()) {
  65         $phpversion = phpversion();
  66         mtrace("Sorry, global search requires PHP 5.0.0 or later (currently using version ".phpversion().")");
  67         exit(0);
  68     }
  69
  70     //php5 found, continue including php5-only files
  71     //require_once("$CFG->dirroot/search/Zend/Search/Lucene.php");
  72     require_once("$CFG->dirroot/search/indexlib.php");
  73
  74     mtrace('<html><head><meta http-equiv="content-type" content="text/html; charset=utf-8" /></head><body>');
  75     mtrace('<pre>Server Time: '.date('r',time())."\n");
  76
  77     if (isset($CFG->search_indexer_busy) && $CFG->search_indexer_busy == '1') {
  78         //means indexing was not finished previously
  79         mtrace("Warning: Indexing was not successfully completed last time, restarting.\n");
  80     }
  81
  82 /// turn on busy flag
  83
  84     set_config('search_indexer_busy', '1');
  85
  86     //paths
  87     $index_path = SEARCH_INDEX_PATH;
  88     $index_db_file = "{$CFG->dirroot}/search/db/$CFG->dbtype.sql";
  89     $dbcontrol = new IndexDBControl();
  90
  91 /// setup directory in data root
  92
  93     if (!file_exists($index_path)) {
  94         mtrace("Data directory ($index_path) does not exist, attempting to create.");
  95         if (!mkdir($index_path)) {
  96             search_pexit("Error creating data directory at: $index_path. Please correct.");
  97         }  else {
  98             mtrace("Directory successfully created.");
  99         }
 100     } else {
 101         mtrace("Using $index_path as data directory.");
 102     }
 103
 104     $index = new Zend_Search_Lucene($index_path, true);
 105
 106     /*
 107     OBSOLETE REGENERATION - DB installs with search block by now
 108     if (!$dbcontrol->checkDB()) {
 109         search_pexit("Database error. Please check settings/files.");
 110     }
 111     */
 112
 113 /// New regeneration
 114
 115     mtrace('Deleting old index entries.');
 116     delete_records(SEARCH_DATABASE_TABLE);
 117
 118 /// begin timer
 119
 120     search_stopwatch();
 121     mtrace("Starting activity modules\n");
 122
 123     //the presence of the required search functions -
 124     // * mod_iterator
 125     // * mod_get_content_for_index
 126     //are the sole basis for including a module in the index at the moment.
 127     $searchables = array();
 128
 129 /// collects modules
 130
 131     if ($mods = get_records('modules', '', '', '', 'id,name')) {
 132         $searchables = array_merge($searchables, $mods);
 133     }
 134     mtrace(count($searchables).' modules found.');
 135
 136     // collects blocks as indexable information may be found in blocks either
 137     if ($blocks = get_records('block', '', '', '', 'id,name')) {
 138         // prepend the "block_" prefix to discriminate document type plugins
 139         foreach(array_keys($blocks) as $aBlockId){
 140             $blocks[$aBlockId]->name = 'block_'.$blocks[$aBlockId]->name;
 141         }
 142         $searchables = array_merge($searchables, $blocks);
 143         mtrace(count($blocks).' blocks found.');
 144     }
 145
 146 /// add virtual modules onto the back of the array
 147
 148     $searchables = array_merge($searchables, search_get_additional_modules());
 149     if ($searchables){
 150         foreach ($searchables as $mod) {
 151             $class_file = $CFG->dirroot.'/search/documents/'.$mod->name.'_document.php';
 152
 153             if (file_exists($class_file)) {
 154                 include_once($class_file);
 155
 156                 //build function names
 157                 $iter_function = $mod->name.'_iterator';
 158                 $index_function = $mod->name.'_get_content_for_index';
 159                 $counter = 0;
 160                 if (function_exists($index_function) && function_exists($iter_function)) {
 161                     mtrace("Processing module function $index_function ...");
 162                     $sources = $iter_function();
 163                     if ($sources){
 164                         foreach ($sources as $i) {
 165                             $documents = $index_function($i);
 166
 167                             //begin transaction
 168                             if ($documents){
 169                                 foreach($documents as $document) {
 170                                     $counter++;
 171
 172                                     //object to insert into db
 173                                     $dbid = $dbcontrol->addDocument($document);
 174
 175                                     //synchronise db with index
 176                                     $document->addField(Zend_Search_Lucene_Field::Keyword('dbid', $dbid));
 177
 178                                     //add document to index
 179                                     $index->addDocument($document);
 180
 181                                     //commit every x new documents, and print a status message
 182                                     if (($counter % 2000) == 0) {
 183                                         $index->commit();
 184                                         mtrace(".. $counter");
 185                                     }
 186                                 }
 187                             }
 188                             //end transaction
 189                         }
 190                     }
 191
 192                     //commit left over documents, and finish up
 193                     $index->commit();
 194
 195                     mtrace("-- $counter documents indexed");
 196                     mtrace("done.\n");
 197                 }
 198             }
 199         }
 200     }
 201
 202 /// finished modules
 203
 204     mtrace('Finished activity modules');
 205     search_stopwatch();
 206
 207     mtrace(".<br/><a href='index.php'>Back to query page</a>.");
 208     mtrace('</pre>');
 209
 210 /// finished, turn busy flag off
 211
 212     set_config('search_indexer_busy', '0');
 213
 214 /// mark the time we last updated
 215
 216     set_config('search_indexer_run_date', time());
 217
 218 /// and the index size
 219
 220     set_config('search_index_size', (int)$index->count());
 221
 222 ?>