beagled/Lucene.Net/Index/MultiReader.cs

   1 /*
   2  * Copyright 2004 The Apache Software Foundation
   3  *
   4  * Licensed under the Apache License, Version 2.0 (the "License");
   5  * you may not use this file except in compliance with the License.
   6  * You may obtain a copy of the License at
   7  *
   8  * http://www.apache.org/licenses/LICENSE-2.0
   9  *
  10  * Unless required by applicable law or agreed to in writing, software
  11  * distributed under the License is distributed on an "AS IS" BASIS,
  12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13  * See the License for the specific language governing permissions and
  14  * limitations under the License.
  15  */
  16 using System;
  17 using Document = Lucene.Net.Documents.Document;
  18 using Field = Lucene.Net.Documents.Field;
  19 using Directory = Lucene.Net.Store.Directory;
  20 namespace Lucene.Net.Index
  21 {
  22
  23         /// <summary>An IndexReader which reads multiple indexes, appending their content.
  24         ///
  25         /// </summary>
  26         /// <version>  $Id: MultiReader.cs,v 1.2 2005/10/06 19:29:55 dsd Exp $
  27         /// </version>
  28         public class MultiReader : IndexReader
  29         {
  30                 private IndexReader[] subReaders;
  31                 private int[] starts; // 1st docno for each segment
  32                 private System.Collections.Hashtable normsCache = System.Collections.Hashtable.Synchronized(new System.Collections.Hashtable());
  33                 private int maxDoc = 0;
  34                 private int numDocs = -1;
  35                 private bool hasDeletions = false;
  36
  37                 /// <summary> <p>Construct a MultiReader aggregating the named set of (sub)readers.
  38                 /// Directory locking for delete, undeleteAll, and setNorm operations is
  39                 /// left to the subreaders. </p>
  40                 /// <p>Note that all subreaders are closed if this Multireader is closed.</p>
  41                 /// </summary>
  42                 /// <param name="subReaders">set of (sub)readers
  43                 /// </param>
  44                 /// <throws>  IOException </throws>
  45                 public MultiReader(IndexReader[] subReaders):base(subReaders.Length == 0?null:subReaders[0].Directory())
  46                 {
  47                         Initialize(subReaders);
  48                 }
  49
  50                 /// <summary>Construct reading the named set of readers. </summary>
  51         public /*internal*/ MultiReader(Directory directory, SegmentInfos sis, bool closeDirectory, IndexReader[] subReaders) : base(directory, sis, closeDirectory)
  52                 {
  53                         Initialize(subReaders);
  54                 }
  55
  56                 private void  Initialize(IndexReader[] subReaders)
  57                 {
  58                         this.subReaders = subReaders;
  59                         starts = new int[subReaders.Length + 1]; // build starts array
  60                         for (int i = 0; i < subReaders.Length; i++)
  61                         {
  62                                 starts[i] = maxDoc;
  63                                 maxDoc += subReaders[i].MaxDoc(); // compute maxDocs
  64
  65                                 if (subReaders[i].HasDeletions())
  66                                         hasDeletions = true;
  67                         }
  68                         starts[subReaders.Length] = maxDoc;
  69                 }
  70
  71
  72                 /// <summary>Return an array of term frequency vectors for the specified document.
  73                 /// The array contains a vector for each vectorized Field in the document.
  74                 /// Each vector vector contains term numbers and frequencies for all terms
  75                 /// in a given vectorized Field.
  76                 /// If no such fields existed, the method returns null.
  77                 /// </summary>
  78                 public override TermFreqVector[] GetTermFreqVectors(int n)
  79                 {
  80                         int i = ReaderIndex(n); // find segment num
  81                         return subReaders[i].GetTermFreqVectors(n - starts[i]); // dispatch to segment
  82                 }
  83
  84                 public override TermFreqVector GetTermFreqVector(int n, System.String field)
  85                 {
  86                         int i = ReaderIndex(n); // find segment num
  87                         return subReaders[i].GetTermFreqVector(n - starts[i], field);
  88                 }
  89
  90                 public override int NumDocs()
  91                 {
  92                         lock (this)
  93                         {
  94                                 if (numDocs == - 1)
  95                                 {
  96                                         // check cache
  97                                         int n = 0; // cache miss--recompute
  98                                         for (int i = 0; i < subReaders.Length; i++)
  99                                                 n += subReaders[i].NumDocs(); // sum from readers
 100                                         numDocs = n;
 101                                 }
 102                                 return numDocs;
 103                         }
 104                 }
 105
 106                 public override int MaxDoc()
 107                 {
 108                         return maxDoc;
 109                 }
 110
 111                 public override Document Document(int n)
 112                 {
 113                         int i = ReaderIndex(n); // find segment num
 114                         return subReaders[i].Document(n - starts[i]); // dispatch to segment reader
 115                 }
 116
 117                 public override bool IsDeleted(int n)
 118                 {
 119                         int i = ReaderIndex(n); // find segment num
 120                         return subReaders[i].IsDeleted(n - starts[i]); // dispatch to segment reader
 121                 }
 122
 123                 public override bool HasDeletions()
 124                 {
 125                         return hasDeletions;
 126                 }
 127
 128                 protected internal override void  DoDelete(int n)
 129                 {
 130                         numDocs = - 1; // invalidate cache
 131                         int i = ReaderIndex(n); // find segment num
 132                         subReaders[i].Delete(n - starts[i]); // dispatch to segment reader
 133                         hasDeletions = true;
 134                 }
 135
 136                 protected internal override void  DoUndeleteAll()
 137                 {
 138                         for (int i = 0; i < subReaders.Length; i++)
 139                                 subReaders[i].UndeleteAll();
 140                         hasDeletions = false;
 141                 }
 142
 143                 private int ReaderIndex(int n)
 144                 {
 145                         // find reader for doc n:
 146                         int lo = 0; // search starts array
 147                         int hi = subReaders.Length - 1; // for first element less
 148
 149                         while (hi >= lo)
 150                         {
 151                                 int mid = (lo + hi) >> 1;
 152                                 int midValue = starts[mid];
 153                                 if (n < midValue)
 154                                         hi = mid - 1;
 155                                 else if (n > midValue)
 156                                         lo = mid + 1;
 157                                 else
 158                                 {
 159                                         // found a match
 160                                         while (mid + 1 < subReaders.Length && starts[mid + 1] == midValue)
 161                                         {
 162                                                 mid++; // scan to last match
 163                                         }
 164                                         return mid;
 165                                 }
 166                         }
 167                         return hi;
 168                 }
 169
 170                 public override byte[] Norms(System.String field)
 171                 {
 172                         lock (this)
 173                         {
 174                                 byte[] bytes = (byte[]) normsCache[field];
 175                                 if (bytes != null)
 176                                         return bytes; // cache hit
 177
 178                                 bytes = new byte[MaxDoc()];
 179                                 for (int i = 0; i < subReaders.Length; i++)
 180                                         subReaders[i].Norms(field, bytes, starts[i]);
 181                                 normsCache[field] = bytes; // update cache
 182                                 return bytes;
 183                         }
 184                 }
 185
 186                 public override void  Norms(System.String field, byte[] result, int offset)
 187                 {
 188                         lock (this)
 189                         {
 190                                 byte[] bytes = (byte[]) normsCache[field];
 191                                 if (bytes != null)
 192                                 // cache hit
 193                                         Array.Copy(bytes, 0, result, offset, MaxDoc());
 194
 195                                 for (int i = 0; i < subReaders.Length; i++)
 196                                 // read from segments
 197                                         subReaders[i].Norms(field, result, offset + starts[i]);
 198                         }
 199                 }
 200
 201                 protected internal override void  DoSetNorm(int n, System.String field, byte value_Renamed)
 202                 {
 203                         normsCache.Remove(field); // clear cache
 204                         int i = ReaderIndex(n); // find segment num
 205                         subReaders[i].SetNorm(n - starts[i], field, value_Renamed); // dispatch
 206                 }
 207
 208                 public override TermEnum Terms()
 209                 {
 210                         return new MultiTermEnum(subReaders, starts, null);
 211                 }
 212
 213                 public override TermEnum Terms(Term term)
 214                 {
 215                         return new MultiTermEnum(subReaders, starts, term);
 216                 }
 217
 218                 public override int DocFreq(Term t)
 219                 {
 220                         int total = 0; // sum freqs in segments
 221                         for (int i = 0; i < subReaders.Length; i++)
 222                                 total += subReaders[i].DocFreq(t);
 223                         return total;
 224                 }
 225
 226                 public override TermDocs TermDocs()
 227                 {
 228                         return new MultiTermDocs(subReaders, starts);
 229                 }
 230
 231                 public override TermPositions TermPositions()
 232                 {
 233                         return new MultiTermPositions(subReaders, starts);
 234                 }
 235
 236                 protected internal override void  DoCommit()
 237                 {
 238                         for (int i = 0; i < subReaders.Length; i++)
 239                                 subReaders[i].Commit();
 240                 }
 241
 242                 protected internal override void  DoClose()
 243                 {
 244                         lock (this)
 245                         {
 246                                 for (int i = 0; i < subReaders.Length; i++)
 247                                         subReaders[i].Close();
 248                         }
 249                 }
 250
 251                 /// <seealso cref="IndexReader#GetFieldNames()">
 252                 /// </seealso>
 253                 public override System.Collections.ICollection GetFieldNames()
 254                 {
 255                         // maintain a unique set of Field names
 256                         System.Collections.Hashtable fieldSet = new System.Collections.Hashtable();
 257                         for (int i = 0; i < subReaders.Length; i++)
 258                         {
 259                                 IndexReader reader = subReaders[i];
 260                                 System.Collections.ICollection names = reader.GetFieldNames();
 261                                 // iterate through the Field names and add them to the set
 262                                 for (System.Collections.IEnumerator iterator = names.GetEnumerator(); iterator.MoveNext(); )
 263                                 {
 264                     System.Collections.DictionaryEntry fi = (System.Collections.DictionaryEntry) iterator.Current;
 265                                         System.String s = fi.Key.ToString();
 266                     if (fieldSet.ContainsKey(s) == false)
 267                     {
 268                         fieldSet.Add(s, s);
 269                     }
 270                                 }
 271                         }
 272                         return fieldSet;
 273                 }
 274
 275                 /// <seealso cref="IndexReader#GetFieldNames(boolean)">
 276                 /// </seealso>
 277                 public override System.Collections.ICollection GetFieldNames(bool indexed)
 278                 {
 279                         // maintain a unique set of Field names
 280                         System.Collections.Hashtable fieldSet = new System.Collections.Hashtable();
 281                         for (int i = 0; i < subReaders.Length; i++)
 282                         {
 283                                 IndexReader reader = subReaders[i];
 284                                 System.Collections.ICollection names = reader.GetFieldNames(indexed);
 285                 for (System.Collections.IEnumerator iterator = names.GetEnumerator(); iterator.MoveNext(); )
 286                 {
 287                     System.Collections.DictionaryEntry fi = (System.Collections.DictionaryEntry) iterator.Current;
 288                     System.String s = fi.Key.ToString();
 289                     if (fieldSet.ContainsKey(s) == false)
 290                     {
 291                         fieldSet.Add(s, s);
 292                     }
 293                 }
 294             }
 295                         return fieldSet;
 296                 }
 297
 298                 public override System.Collections.ICollection GetIndexedFieldNames(Field.TermVector tvSpec)
 299                 {
 300                         // maintain a unique set of Field names
 301                         System.Collections.Hashtable fieldSet = new System.Collections.Hashtable();
 302                         for (int i = 0; i < subReaders.Length; i++)
 303                         {
 304                                 IndexReader reader = subReaders[i];
 305                                 System.Collections.ICollection names = reader.GetIndexedFieldNames(tvSpec);
 306                 foreach (object item in names)
 307                 {
 308                     if (fieldSet.ContainsKey(item) == false)
 309                     {
 310                         fieldSet.Add(item, item);
 311                     }
 312                 }
 313             }
 314                         return fieldSet;
 315                 }
 316
 317         /// <seealso cref="IndexReader#GetFieldNames(IndexReader.FieldOption)">
 318         /// </seealso>
 319         public override System.Collections.ICollection GetFieldNames(IndexReader.FieldOption fieldNames)
 320         {
 321             // maintain a unique set of field names
 322             System.Collections.Hashtable fieldSet = new System.Collections.Hashtable();
 323             for (int i = 0; i < subReaders.Length; i++)
 324             {
 325                 IndexReader reader = subReaders[i];
 326                 System.Collections.ICollection names = reader.GetFieldNames(fieldNames);
 327                 foreach (object item in names)
 328                 {
 329                     if (fieldSet.ContainsKey(item) == false)
 330                     {
 331                         fieldSet.Add(item, item);
 332                     }
 333                 }
 334             }
 335             return fieldSet;
 336         }
 337     }
 338
 339         class MultiTermEnum : TermEnum
 340         {
 341                 private SegmentMergeQueue queue;
 342
 343                 private Term term;
 344                 private int docFreq;
 345
 346                 public MultiTermEnum(IndexReader[] readers, int[] starts, Term t)
 347                 {
 348                         queue = new SegmentMergeQueue(readers.Length);
 349                         for (int i = 0; i < readers.Length; i++)
 350                         {
 351                                 IndexReader reader = readers[i];
 352                                 TermEnum termEnum;
 353
 354                                 if (t != null)
 355                                 {
 356                                         termEnum = reader.Terms(t);
 357                                 }
 358                                 else
 359                                         termEnum = reader.Terms();
 360
 361                                 SegmentMergeInfo smi = new SegmentMergeInfo(starts[i], termEnum, reader);
 362                                 if (t == null?smi.Next():termEnum.Term() != null)
 363                                         queue.Put(smi);
 364                                 // initialize queue
 365                                 else
 366                                         smi.Close();
 367                         }
 368
 369                         if (t != null && queue.Size() > 0)
 370                         {
 371                                 Next();
 372                         }
 373                 }
 374
 375                 public override bool Next()
 376                 {
 377                         SegmentMergeInfo top = (SegmentMergeInfo) queue.Top();
 378                         if (top == null)
 379                         {
 380                                 term = null;
 381                                 return false;
 382                         }
 383
 384                         term = top.term;
 385                         docFreq = 0;
 386
 387                         while (top != null && term.CompareTo(top.term) == 0)
 388                         {
 389                                 queue.Pop();
 390                                 docFreq += top.termEnum.DocFreq(); // increment freq
 391                                 if (top.Next())
 392                                         queue.Put(top);
 393                                 // restore queue
 394                                 else
 395                                         top.Close(); // done with a segment
 396                                 top = (SegmentMergeInfo) queue.Top();
 397                         }
 398                         return true;
 399                 }
 400
 401                 public override Term Term()
 402                 {
 403                         return term;
 404                 }
 405
 406                 public override int DocFreq()
 407                 {
 408                         return docFreq;
 409                 }
 410
 411                 public override void  Close()
 412                 {
 413                         queue.Close();
 414                 }
 415         }
 416
 417         class MultiTermDocs : TermDocs
 418         {
 419                 protected internal IndexReader[] readers;
 420                 protected internal int[] starts;
 421                 protected internal Term term;
 422
 423                 protected internal int base_Renamed = 0;
 424                 protected internal int pointer = 0;
 425
 426                 private TermDocs[] readerTermDocs;
 427                 protected internal TermDocs current; // == readerTermDocs[pointer]
 428
 429                 public MultiTermDocs(IndexReader[] r, int[] s)
 430                 {
 431                         readers = r;
 432                         starts = s;
 433
 434                         readerTermDocs = new TermDocs[r.Length];
 435                 }
 436
 437                 public virtual int Doc()
 438                 {
 439                         return base_Renamed + current.Doc();
 440                 }
 441                 public virtual int Freq()
 442                 {
 443                         return current.Freq();
 444                 }
 445
 446                 public virtual void  Seek(Term term)
 447                 {
 448                         this.term = term;
 449                         this.base_Renamed = 0;
 450                         this.pointer = 0;
 451                         this.current = null;
 452                 }
 453
 454                 public virtual void  Seek(TermEnum termEnum)
 455                 {
 456                         Seek(termEnum.Term());
 457                 }
 458
 459                 public virtual bool Next()
 460                 {
 461                         if (current != null && current.Next())
 462                         {
 463                                 return true;
 464                         }
 465                         else if (pointer < readers.Length)
 466                         {
 467                                 base_Renamed = starts[pointer];
 468                                 current = TermDocs(pointer++);
 469                                 return Next();
 470                         }
 471                         else
 472                                 return false;
 473                 }
 474
 475                 /// <summary>Optimized implementation. </summary>
 476                 public virtual int Read(int[] docs, int[] freqs)
 477                 {
 478                         while (true)
 479                         {
 480                                 while (current == null)
 481                                 {
 482                                         if (pointer < readers.Length)
 483                                         {
 484                                                 // try next segment
 485                                                 base_Renamed = starts[pointer];
 486                                                 current = TermDocs(pointer++);
 487                                         }
 488                                         else
 489                                         {
 490                                                 return 0;
 491                                         }
 492                                 }
 493                                 int end = current.Read(docs, freqs);
 494                                 if (end == 0)
 495                                 {
 496                                         // none left in segment
 497                                         current = null;
 498                                 }
 499                                 else
 500                                 {
 501                                         // got some
 502                                         int b = base_Renamed; // adjust doc numbers
 503                                         for (int i = 0; i < end; i++)
 504                                                 docs[i] += b;
 505                                         return end;
 506                                 }
 507                         }
 508                 }
 509
 510                 /// <summary>As yet unoptimized implementation. </summary>
 511                 public virtual bool SkipTo(int target)
 512                 {
 513                         do
 514                         {
 515                                 if (!Next())
 516                                         return false;
 517                         }
 518                         while (target > Doc());
 519                         return true;
 520                 }
 521
 522                 private TermDocs TermDocs(int i)
 523                 {
 524                         if (term == null)
 525                                 return null;
 526                         TermDocs result = readerTermDocs[i];
 527                         if (result == null)
 528                                 result = readerTermDocs[i] = TermDocs(readers[i]);
 529                         result.Seek(term);
 530                         return result;
 531                 }
 532
 533                 protected internal virtual TermDocs TermDocs(IndexReader reader)
 534                 {
 535                         return reader.TermDocs();
 536                 }
 537
 538                 public virtual void  Close()
 539                 {
 540                         for (int i = 0; i < readerTermDocs.Length; i++)
 541                         {
 542                                 if (readerTermDocs[i] != null)
 543                                         readerTermDocs[i].Close();
 544                         }
 545                 }
 546         }
 547
 548         class MultiTermPositions : MultiTermDocs, TermPositions
 549         {
 550                 public MultiTermPositions(IndexReader[] r, int[] s):base(r, s)
 551                 {
 552                 }
 553
 554                 protected internal override TermDocs TermDocs(IndexReader reader)
 555                 {
 556                         return (TermDocs) reader.TermPositions();
 557                 }
 558
 559                 public virtual int NextPosition()
 560                 {
 561                         return ((TermPositions) current).NextPosition();
 562                 }
 563         }
 564 }