beagled/Lucene.Net/Search/ParallelMultiSearcher.cs

   1 /*
   2  * Copyright 2004 The Apache Software Foundation
   3  *
   4  * Licensed under the Apache License, Version 2.0 (the "License");
   5  * you may not use this file except in compliance with the License.
   6  * You may obtain a copy of the License at
   7  *
   8  * http://www.apache.org/licenses/LICENSE-2.0
   9  *
  10  * Unless required by applicable law or agreed to in writing, software
  11  * distributed under the License is distributed on an "AS IS" BASIS,
  12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13  * See the License for the specific language governing permissions and
  14  * limitations under the License.
  15  */
  16
  17 using System;
  18 using Term = Lucene.Net.Index.Term;
  19 using PriorityQueue = Lucene.Net.Util.PriorityQueue;
  20
  21 namespace Lucene.Net.Search
  22 {
  23
  24         /// <summary>Implements parallel search over a set of <code>Searchables</code>.
  25         ///
  26         /// <p>Applications usually need only call the inherited {@link #Search(Query)}
  27         /// or {@link #Search(Query,Filter)} methods.
  28         /// </summary>
  29         public class ParallelMultiSearcher : MultiSearcher
  30         {
  31                 private class AnonymousClassHitCollector1 : HitCollector
  32                 {
  33                         public AnonymousClassHitCollector1(Lucene.Net.Search.HitCollector results, int start, ParallelMultiSearcher enclosingInstance)
  34                         {
  35                                 InitBlock(results, start, enclosingInstance);
  36                         }
  37                         private void  InitBlock(Lucene.Net.Search.HitCollector results, int start, ParallelMultiSearcher enclosingInstance)
  38                         {
  39                                 this.results = results;
  40                                 this.start = start;
  41                                 this.enclosingInstance = enclosingInstance;
  42                         }
  43                         private Lucene.Net.Search.HitCollector results;
  44                         private int start;
  45                         private ParallelMultiSearcher enclosingInstance;
  46                         public ParallelMultiSearcher Enclosing_Instance
  47                         {
  48                                 get
  49                                 {
  50                                         return enclosingInstance;
  51                                 }
  52
  53                         }
  54                         public override void  Collect(int doc, float score)
  55                         {
  56                                 results.Collect(doc + start, score);
  57                         }
  58                 }
  59
  60                 private Lucene.Net.Search.Searchable[] searchables;
  61                 private int[] starts;
  62
  63                 /// <summary>Creates a searcher which searches <i>searchables</i>. </summary>
  64                 public ParallelMultiSearcher(Lucene.Net.Search.Searchable[] searchables) : base(searchables)
  65                 {
  66                         this.searchables = searchables;
  67                         this.starts = GetStarts();
  68                 }
  69
  70                 /// <summary> TODO: parallelize this one too</summary>
  71                 public override int DocFreq(Term term)
  72                 {
  73                         return base.DocFreq(term);
  74                 }
  75
  76                 /// <summary> A search implementation which spans a new thread for each
  77                 /// Searchable, waits for each search to complete and merge
  78                 /// the results back together.
  79                 /// </summary>
  80                 public override TopDocs Search(Weight weight, Filter filter, int nDocs)
  81                 {
  82                         HitQueue hq = new HitQueue(nDocs);
  83                         int totalHits = 0;
  84                         MultiSearcherThread[] msta = new MultiSearcherThread[searchables.Length];
  85                         for (int i = 0; i < searchables.Length; i++)
  86                         {
  87                                 // search each searcher
  88                                 // Assume not too many searchables and cost of creating a thread is by far inferior to a search
  89                                 msta[i] = new MultiSearcherThread(searchables[i], weight, filter, nDocs, hq, i, starts, "MultiSearcher thread #" + (i + 1));
  90                                 msta[i].Start();
  91                         }
  92
  93                         for (int i = 0; i < searchables.Length; i++)
  94                         {
  95                                 try
  96                                 {
  97                                         msta[i].Join();
  98                                 }
  99                                 catch (System.Threading.ThreadInterruptedException)
 100                                 {
 101                                         ; // TODO: what should we do with this???
 102                                 }
 103                                 System.IO.IOException ioe = msta[i].GetIOException();
 104                                 if (ioe == null)
 105                                 {
 106                                         totalHits += msta[i].Hits();
 107                                 }
 108                                 else
 109                                 {
 110                                         // if one search produced an IOException, rethrow it
 111                                         throw ioe;
 112                                 }
 113                         }
 114
 115                         ScoreDoc[] scoreDocs = new ScoreDoc[hq.Size()];
 116                         for (int i = hq.Size() - 1; i >= 0; i--)
 117                         // put docs in array
 118                                 scoreDocs[i] = (ScoreDoc) hq.Pop();
 119
 120                         float maxScore = (totalHits == 0) ? System.Single.NegativeInfinity : scoreDocs[0].score;
 121
 122                         return new TopDocs(totalHits, scoreDocs, maxScore);
 123                 }
 124
 125                 /// <summary> A search implementation allowing sorting which spans a new thread for each
 126                 /// Searchable, waits for each search to complete and merges
 127                 /// the results back together.
 128                 /// </summary>
 129                 public override TopFieldDocs Search(Weight weight, Filter filter, int nDocs, Sort sort)
 130                 {
 131                         // don't specify the fields - we'll wait to do this until we get results
 132                         FieldDocSortedHitQueue hq = new FieldDocSortedHitQueue(null, nDocs);
 133                         int totalHits = 0;
 134                         MultiSearcherThread[] msta = new MultiSearcherThread[searchables.Length];
 135                         for (int i = 0; i < searchables.Length; i++)
 136                         {
 137                                 // search each searcher
 138                                 // Assume not too many searchables and cost of creating a thread is by far inferior to a search
 139                                 msta[i] = new MultiSearcherThread(searchables[i], weight, filter, nDocs, hq, sort, i, starts, "MultiSearcher thread #" + (i + 1));
 140                                 msta[i].Start();
 141                         }
 142
 143                         float maxScore = System.Single.NegativeInfinity;
 144
 145                         for (int i = 0; i < searchables.Length; i++)
 146                         {
 147                                 try
 148                                 {
 149                                         msta[i].Join();
 150                                 }
 151                                 catch (System.Threading.ThreadInterruptedException)
 152                                 {
 153                                         ; // TODO: what should we do with this???
 154                                 }
 155                                 System.IO.IOException ioe = msta[i].GetIOException();
 156                                 if (ioe == null)
 157                                 {
 158                                         totalHits += msta[i].Hits();
 159                                         maxScore = System.Math.Max(maxScore, msta[i].GetMaxScore());
 160                                 }
 161                                 else
 162                                 {
 163                                         // if one search produced an IOException, rethrow it
 164                                         throw ioe;
 165                                 }
 166                         }
 167
 168                         ScoreDoc[] scoreDocs = new ScoreDoc[hq.Size()];
 169                         for (int i = hq.Size() - 1; i >= 0; i--)
 170                         // put docs in array
 171                                 scoreDocs[i] = (ScoreDoc) hq.Pop();
 172
 173                         return new TopFieldDocs(totalHits, scoreDocs, hq.GetFields(), maxScore);
 174                 }
 175
 176                 /// <summary>Lower-level search API.
 177                 ///
 178                 /// <p>{@link HitCollector#Collect(int,float)} is called for every non-zero
 179                 /// scoring document.
 180                 ///
 181                 /// <p>Applications should only use this if they need <i>all</i> of the
 182                 /// matching documents.  The high-level search API ({@link
 183                 /// Searcher#Search(Query)}) is usually more efficient, as it skips
 184                 /// non-high-scoring hits.
 185                 ///
 186                 /// </summary>
 187                 /// <param name="weight">to match documents
 188                 /// </param>
 189                 /// <param name="filter">if non-null, a bitset used to eliminate some documents
 190                 /// </param>
 191                 /// <param name="results">to receive hits
 192                 ///
 193                 /// </param>
 194                 /// <todo>  parallelize this one too </todo>
 195                 public override void  Search(Weight weight, Filter filter, HitCollector results)
 196                 {
 197                         for (int i = 0; i < searchables.Length; i++)
 198                         {
 199
 200                                 int start = starts[i];
 201
 202                                 searchables[i].Search(weight, filter, new AnonymousClassHitCollector1(results, start, this));
 203                         }
 204                 }
 205
 206                 /*
 207                 * TODO: this one could be parallelized too
 208                 * @see Lucene.Net.search.Searchable#rewrite(Lucene.Net.search.Query)
 209                 */
 210                 public override Query Rewrite(Query original)
 211                 {
 212                         return base.Rewrite(original);
 213                 }
 214         }
 215
 216         /// <summary> A thread subclass for searching a single searchable </summary>
 217         class MultiSearcherThread : SupportClass.ThreadClass
 218         {
 219
 220                 private Lucene.Net.Search.Searchable searchable;
 221                 private Weight weight;
 222                 private Filter filter;
 223                 private int nDocs;
 224                 private TopDocs docs;
 225                 private int i;
 226                 private PriorityQueue hq;
 227                 private int[] starts;
 228                 private System.IO.IOException ioe;
 229                 private Sort sort;
 230
 231                 public MultiSearcherThread(Lucene.Net.Search.Searchable searchable, Weight weight, Filter filter, int nDocs, HitQueue hq, int i, int[] starts, System.String name):base(name)
 232                 {
 233                         this.searchable = searchable;
 234                         this.weight = weight;
 235                         this.filter = filter;
 236                         this.nDocs = nDocs;
 237                         this.hq = hq;
 238                         this.i = i;
 239                         this.starts = starts;
 240                 }
 241
 242                 public MultiSearcherThread(Lucene.Net.Search.Searchable searchable, Weight weight, Filter filter, int nDocs, FieldDocSortedHitQueue hq, Sort sort, int i, int[] starts, System.String name):base(name)
 243                 {
 244                         this.searchable = searchable;
 245                         this.weight = weight;
 246                         this.filter = filter;
 247                         this.nDocs = nDocs;
 248                         this.hq = hq;
 249                         this.i = i;
 250                         this.starts = starts;
 251                         this.sort = sort;
 252                 }
 253
 254                 override public void  Run()
 255                 {
 256                         try
 257                         {
 258                                 docs = (sort == null) ? searchable.Search(weight, filter, nDocs) : searchable.Search(weight, filter, nDocs, sort);
 259                         }
 260                         // Store the IOException for later use by the caller of this thread
 261                         catch (System.IO.IOException ioe)
 262                         {
 263                                 this.ioe = ioe;
 264                         }
 265                         if (this.ioe == null)
 266                         {
 267                                 // if we are sorting by fields, we need to tell the field sorted hit queue
 268                                 // the actual type of fields, in case the original list contained AUTO.
 269                                 // if the searchable returns null for fields, we'll have problems.
 270                                 if (sort != null)
 271                                 {
 272                                         ((FieldDocSortedHitQueue) hq).SetFields(((TopFieldDocs) docs).fields);
 273                                 }
 274                                 ScoreDoc[] scoreDocs = docs.scoreDocs;
 275                                 for (int j = 0; j < scoreDocs.Length; j++)
 276                                 {
 277                                         // merge scoreDocs into hq
 278                                         ScoreDoc scoreDoc = scoreDocs[j];
 279                                         scoreDoc.doc += starts[i]; // convert doc
 280                                         //it would be so nice if we had a thread-safe insert
 281                                         lock (hq)
 282                                         {
 283                                                 if (!hq.Insert(scoreDoc))
 284                                                         break;
 285                                         } // no more scores > minScore
 286                                 }
 287                         }
 288                 }
 289
 290                 public virtual int Hits()
 291                 {
 292                         return docs.totalHits;
 293                 }
 294
 295                 public virtual float GetMaxScore()
 296                 {
 297                         return docs.GetMaxScore();
 298                 }
 299
 300                 public virtual System.IO.IOException GetIOException()
 301                 {
 302                         return ioe;
 303                 }
 304         }
 305 }