search/indexer.php

   1 <?php
   2 /**
   3 * Global Search Engine for Moodle
   4 * Michael Champanis (mchampan) [cynnical@gmail.com]
   5 * review 1.8+ : Valery Fremaux [valery.fremaux@club-internet.fr]
   6 * 2007/08/02
   7 *
   8 * The indexer logic -
   9 *
  10 * Look through each installed module's or block's search document class file (/search/documents)
  11 * for necessary search functions, and if they're present add the content to the index.
  12 * Repeat this for blocks.
  13 *
  14 * Because the iterator/retrieval functions are now stored in /search/documents/<mod>_document.php,
  15 * /mod/mod/lib.php doesn't have to be modified - and thus the search module becomes quite
  16 * self-sufficient. URL's are now stored in the index, stopping us from needing to require
  17 * the class files to generate a results page.
  18 *
  19 * Along with the index data, each document's summary gets stored in the database
  20 * and synchronised to the index (flat file) via the primary key ('id') which is mapped
  21 * to the 'dbid' field in the index
  22 * */
  23
  24 //this'll take some time, set up the environment
  25 @set_time_limit(0);
  26 @ob_implicit_flush(true);
  27 @ob_end_flush();
  28
  29 require_once('../config.php');
  30 require_once("$CFG->dirroot/search/lib.php");
  31
  32 //only administrators can index the moodle installation, because access to all pages is required
  33 require_login();
  34
  35 if (empty($CFG->enableglobalsearch)) {
  36     error(get_string('globalsearchdisabled', 'search'));
  37 }
  38
  39 if (!isadmin()) {
  40     error(get_string('beadmin', 'search'), "$CFG->wwwroot/login/index.php");
  41 } //if
  42
  43 //confirmation flag to prevent accidental reindexing (indexersplash.php is the correct entry point)
  44 $sure = strtolower(optional_param('areyousure', '', PARAM_ALPHA));
  45
  46 if ($sure != 'yes') {
  47     mtrace("<pre>Sorry, you need to confirm indexing via <a href='indexersplash.php'>indexersplash.php</a>"
  48           .". (<a href='index.php'>Back to query page</a>).</pre>");
  49
  50     exit(0);
  51 } //if
  52
  53 //check for php5 (lib.php)
  54 if (!search_check_php5()) {
  55     $phpversion = phpversion();
  56     mtrace("Sorry, global search requires PHP 5.0.0 or later (currently using version $phpversion)");
  57     exit(0);
  58 }
  59
  60 //php5 found, continue including php5-only files
  61 //require_once("$CFG->dirroot/search/Zend/Search/Lucene.php");
  62 require_once("$CFG->dirroot/search/indexlib.php");
  63
  64 mtrace('<html><head><meta http-equiv="content-type" content="text/html; charset=utf-8" /></head><body>');
  65 mtrace('<pre>Server Time: '.date('r',time())."\n");
  66
  67 if (isset($CFG->search_indexer_busy) && $CFG->search_indexer_busy == '1') {
  68     //means indexing was not finished previously
  69     mtrace("Warning: Indexing was not successfully completed last time, restarting.\n");
  70 }
  71
  72 //turn on busy flag
  73 set_config('search_indexer_busy', '1');
  74
  75 //paths
  76 $index_path = SEARCH_INDEX_PATH;
  77 $index_db_file = "{$CFG->dirroot}/search/db/$CFG->dbtype.sql";
  78 $dbcontrol = new IndexDBControl();
  79
  80 //setup directory in data root
  81 if (!file_exists($index_path)) {
  82     mtrace("Data directory ($index_path) does not exist, attempting to create.");
  83     if (!mkdir($index_path)) {
  84         search_pexit("Error creating data directory at: $index_path. Please correct.");
  85     }
  86     else {
  87         mtrace("Directory successfully created.");
  88     }
  89 }
  90 else {
  91     mtrace("Using $index_path as data directory.");
  92 }
  93
  94 $index = new Zend_Search_Lucene($index_path, true);
  95
  96 /*
  97 OBSOLETE REGENERATION - DB installs with search block by now
  98 if (!$dbcontrol->checkDB()) {
  99     search_pexit("Database error. Please check settings/files.");
 100 }
 101 */
 102 // New regeneration
 103 mtrace('Deleting old index entries.');
 104 delete_records(SEARCH_DATABASE_TABLE);
 105
 106 //begin timer
 107 search_stopwatch();
 108 mtrace("Starting activity modules\n");
 109
 110 //the presence of the required search functions -
 111 // * mod_iterator
 112 // * mod_get_content_for_index
 113 //are the sole basis for including a module in the index at the moment.
 114 $searchables = array();
 115
 116 // collects modules
 117 if ($mods = get_records('modules', '', '', '', 'id,name')) {
 118     $searchables = array_merge($searchables, $mods);
 119 }
 120 mtrace(count($searchables).' modules found.');
 121
 122 // collects blocks as indexable information may be found in blocks either
 123 if ($blocks = get_records('block', '', '', '', 'id,name')) {
 124     // prepend the "block_" prefix to discriminate document type plugins
 125     foreach(array_keys($blocks) as $aBlockId){
 126         $blocks[$aBlockId]->name = 'block_'.$blocks[$aBlockId]->name;
 127     }
 128     $searchables = array_merge($searchables, $blocks);
 129     mtrace(count($blocks).' blocks found.');
 130 }
 131
 132 //add virtual modules onto the back of the array
 133 $searchables = array_merge($searchables, search_get_additional_modules());
 134 if ($searchables){
 135     foreach ($searchables as $mod) {
 136         $class_file = $CFG->dirroot.'/search/documents/'.$mod->name.'_document.php';
 137
 138         if (file_exists($class_file)) {
 139             include_once($class_file);
 140
 141             //build function names
 142             $iter_function = $mod->name.'_iterator';
 143             $index_function = $mod->name.'_get_content_for_index';
 144             $counter = 0;
 145             if (function_exists($index_function) && function_exists($iter_function)) {
 146                 mtrace("Processing module function $index_function ...");
 147                 $sources = $iter_function();
 148                 if ($sources){
 149                     foreach ($sources as $i) {
 150                         $documents = $index_function($i);
 151
 152                         //begin transaction
 153                         if ($documents){
 154                             foreach($documents as $document) {
 155                                 $counter++;
 156
 157                                 //object to insert into db
 158                                 $dbid = $dbcontrol->addDocument($document);
 159
 160                                 //synchronise db with index
 161                                 $document->addField(Zend_Search_Lucene_Field::Keyword('dbid', $dbid));
 162
 163                                 //add document to index
 164                                 $index->addDocument($document);
 165
 166                                 //commit every x new documents, and print a status message
 167                                 if (($counter % 2000) == 0) {
 168                                     $index->commit();
 169                                     mtrace(".. $counter");
 170                                 }
 171                             }
 172                         }
 173                         //end transaction
 174                     }
 175                 }
 176
 177                 //commit left over documents, and finish up
 178                 $index->commit();
 179
 180                 mtrace("-- $counter documents indexed");
 181                 mtrace("done.\n");
 182             }
 183         }
 184     }
 185 }
 186
 187 //finished modules
 188 mtrace('Finished activity modules');
 189 search_stopwatch();
 190
 191 mtrace(".<br/><a href='index.php'>Back to query page</a>.");
 192 mtrace('</pre>');
 193
 194 //finished, turn busy flag off
 195 set_config('search_indexer_busy', '0');
 196
 197 //mark the time we last updated
 198 set_config('search_indexer_run_date', time());
 199
 200 //and the index size
 201 set_config('search_index_size', (int)$index->count());
 202
 203 ?>