Thumbnail file hits. Based on a patch from D Bera

[beagle.git] / beagled / Lucene.Net / Index / SegmentMerger.cs

/*
 * Copyright 2004 The Apache Software Foundation
 * 
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 * 
 * http://www.apache.org/licenses/LICENSE-2.0
 * 
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
using System;
using Directory = Lucene.Net.Store.Directory;
using OutputStream = Lucene.Net.Store.OutputStream;
using RAMOutputStream = Lucene.Net.Store.RAMOutputStream;
namespace Lucene.Net.Index
{
        
        /// <summary> The SegmentMerger class combines two or more Segments, represented by an IndexReader ({@link #add},
        /// into a single Segment.  After adding the appropriate readers, call the merge method to combine the 
        /// segments.
        /// <P> 
        /// If the compoundFile flag is set, then the segments will be merged into a compound file.
        /// 
        /// 
        /// </summary>
        /// <seealso cref="#merge">
        /// </seealso>
        /// <seealso cref="#add">
        /// </seealso>
        sealed public class SegmentMerger
        {
                private bool useCompoundFile;
                private Directory directory;
                private System.String segment;
                
                private System.Collections.ArrayList readers = System.Collections.ArrayList.Synchronized(new System.Collections.ArrayList(10));
                private FieldInfos fieldInfos;
                
                // File extensions of old-style index files
                private static readonly System.String[] COMPOUND_EXTENSIONS = new System.String[]{"fnm", "frq", "prx", "fdx", "fdt", "tii", "tis"};
                private static readonly System.String[] VECTOR_EXTENSIONS = new System.String[]{"tvx", "tvd", "tvf"};
                
                /// <summary> </summary>
                /// <param name="dir">The Directory to merge the other segments into
                /// </param>
                /// <param name="name">The name of the new segment
                /// </param>
                /// <param name="compoundFile">true if the new segment should use a compoundFile
                /// </param>
                public /*internal*/ SegmentMerger(Directory dir, System.String name, bool compoundFile)
                {
                        directory = dir;
                        segment = name;
                        useCompoundFile = compoundFile;
                }
                
                /// <summary> Add an IndexReader to the collection of readers that are to be merged</summary>
                /// <param name="">reader
                /// </param>
                public /*internal*/ void  Add(IndexReader reader)
                {
                        readers.Add(reader);
                }
                
                /// <summary> </summary>
                /// <param name="i">The index of the reader to return
                /// </param>
                /// <returns> The ith reader to be merged
                /// </returns>
                internal IndexReader SegmentReader(int i)
                {
                        return (IndexReader) readers[i];
                }
                
                /// <summary> Merges the readers specified by the {@link #add} method into the directory passed to the constructor</summary>
                /// <returns> The number of documents that were merged
                /// </returns>
                /// <throws>  IOException </throws>
                public /*internal*/ int Merge()
                {
                        int value_Renamed;
                        
                        value_Renamed = MergeFields();
                        MergeTerms();
                        MergeNorms();
                        
                        if (fieldInfos.HasVectors())
                                MergeVectors();
                        
                        if (useCompoundFile)
                                CreateCompoundFile();
                        
                        return value_Renamed;
                }
                
                /// <summary> close all IndexReaders that have been added.
                /// Should not be called before merge().
                /// </summary>
                /// <throws>  IOException </throws>
                public /*internal*/ void  CloseReaders()
                {
                        for (int i = 0; i < readers.Count; i++)
                        {
                                // close readers
                                IndexReader reader = (IndexReader) readers[i];
                                reader.Close();
                        }
                }
                
                private void  CreateCompoundFile()
                {
                        CompoundFileWriter cfsWriter = new CompoundFileWriter(directory, segment + ".cfs");
                        
                        System.Collections.ArrayList files = new System.Collections.ArrayList(COMPOUND_EXTENSIONS.Length + fieldInfos.Size());
                        
                        // Basic files
                        for (int i = 0; i < COMPOUND_EXTENSIONS.Length; i++)
                        {
                                files.Add(segment + "." + COMPOUND_EXTENSIONS[i]);
                        }
                        
                        // Field norm files
                        for (int i = 0; i < fieldInfos.Size(); i++)
                        {
                                FieldInfo fi = fieldInfos.FieldInfo(i);
                                if (fi.isIndexed)
                                {
                                        files.Add(segment + ".f" + i);
                                }
                        }
                        
                        // Vector files
                        if (fieldInfos.HasVectors())
                        {
                                for (int i = 0; i < VECTOR_EXTENSIONS.Length; i++)
                                {
                                        files.Add(segment + "." + VECTOR_EXTENSIONS[i]);
                                }
                        }
                        
                        // Now merge all added files
                        System.Collections.IEnumerator it = files.GetEnumerator();
                        while (it.MoveNext())
                        {
                                cfsWriter.AddFile((System.String) it.Current);
                        }
                        
                        // Perform the merge
                        cfsWriter.Close();
                        
                        // Now delete the source files
                        it = files.GetEnumerator();
                        while (it.MoveNext())
                        {
                                directory.DeleteFile((System.String) it.Current);
                        }
                }
                
                /// <summary> </summary>
                /// <returns> The number of documents in all of the readers
                /// </returns>
                /// <throws>  IOException </throws>
                private int MergeFields()
                {
                        fieldInfos = new FieldInfos(); // merge Field names
                        int docCount = 0;
                        for (int i = 0; i < readers.Count; i++)
                        {
                                IndexReader reader = (IndexReader) readers[i];
                                fieldInfos.AddIndexed(reader.GetIndexedFieldNames(true), true);
                                fieldInfos.AddIndexed(reader.GetIndexedFieldNames(false), false);
                                fieldInfos.Add(reader.GetFieldNames(false), false);
                        }
                        fieldInfos.Write(directory, segment + ".fnm");
                        
                        FieldsWriter fieldsWriter = new FieldsWriter(directory, segment, fieldInfos);
                        try
                        {
                                for (int i = 0; i < readers.Count; i++)
                                {
                                        IndexReader reader = (IndexReader) readers[i];
                                        int maxDoc = reader.MaxDoc();
                                        for (int j = 0; j < maxDoc; j++)
                                                if (!reader.IsDeleted(j))
                                                {
                                                        // skip deleted docs
                                                        fieldsWriter.AddDocument(reader.Document(j));
                                                        docCount++;
                                                }
                                }
                        }
                        finally
                        {
                                fieldsWriter.Close();
                        }
                        return docCount;
                }
                
                /// <summary> Merge the TermVectors from each of the segments into the new one.</summary>
                /// <throws>  IOException </throws>
                private void  MergeVectors()
                {
                        TermVectorsWriter termVectorsWriter = new TermVectorsWriter(directory, segment, fieldInfos);
                        
                        try
                        {
                                for (int r = 0; r < readers.Count; r++)
                                {
                                        IndexReader reader = (IndexReader) readers[r];
                                        int maxDoc = reader.MaxDoc();
                                        for (int docNum = 0; docNum < maxDoc; docNum++)
                                        {
                                                // skip deleted docs
                                                if (reader.IsDeleted(docNum))
                                                {
                                                        continue;
                                                }
                                                termVectorsWriter.OpenDocument();
                                                
                                                // get all term vectors
                                                TermFreqVector[] sourceTermVector = reader.GetTermFreqVectors(docNum);
                                                
                                                if (sourceTermVector != null)
                                                {
                                                        for (int f = 0; f < sourceTermVector.Length; f++)
                                                        {
                                                                // translate Field numbers
                                                                TermFreqVector termVector = sourceTermVector[f];
                                                                termVectorsWriter.OpenField(termVector.GetField());
                                                                System.String[] terms = termVector.GetTerms();
                                                                int[] freqs = termVector.GetTermFrequencies();
                                                                
                                                                for (int t = 0; t < terms.Length; t++)
                                                                {
                                                                        termVectorsWriter.AddTerm(terms[t], freqs[t]);
                                                                }
                                                        }
                                                        termVectorsWriter.CloseDocument();
                                                }
                                        }
                                }
                        }
                        finally
                        {
                                termVectorsWriter.Close();
                        }
                }
                
                private OutputStream freqOutput = null;
                private OutputStream proxOutput = null;
                private TermInfosWriter termInfosWriter = null;
                private int skipInterval;
                private SegmentMergeQueue queue = null;
                
                private void  MergeTerms()
                {
                        try
                        {
                                freqOutput = directory.CreateFile(segment + ".frq");
                                proxOutput = directory.CreateFile(segment + ".prx");
                                termInfosWriter = new TermInfosWriter(directory, segment, fieldInfos);
                                skipInterval = termInfosWriter.skipInterval;
                                queue = new SegmentMergeQueue(readers.Count);
                                
                                MergeTermInfos();
                        }
                        finally
                        {
                                if (freqOutput != null)
                                        freqOutput.Close();
                                if (proxOutput != null)
                                        proxOutput.Close();
                                if (termInfosWriter != null)
                                        termInfosWriter.Close();
                                if (queue != null)
                                        queue.Close();
                        }
                }
                
                private void  MergeTermInfos()
                {
                        int base_Renamed = 0;
                        for (int i = 0; i < readers.Count; i++)
                        {
                                IndexReader reader = (IndexReader) readers[i];
                                TermEnum termEnum = reader.Terms();
                                SegmentMergeInfo smi = new SegmentMergeInfo(base_Renamed, termEnum, reader);
                                base_Renamed += reader.NumDocs();
                                if (smi.Next())
                                        queue.Put(smi);
                                // initialize queue
                                else
                                        smi.Close();
                        }
                        
                        SegmentMergeInfo[] match = new SegmentMergeInfo[readers.Count];
                        
                        while (queue.Size() > 0)
                        {
                                int matchSize = 0; // pop matching terms
                                match[matchSize++] = (SegmentMergeInfo) queue.Pop();
                                Term term = match[0].term;
                                SegmentMergeInfo top = (SegmentMergeInfo) queue.Top();
                                
                                while (top != null && term.CompareTo(top.term) == 0)
                                {
                                        match[matchSize++] = (SegmentMergeInfo) queue.Pop();
                                        top = (SegmentMergeInfo) queue.Top();
                                }
                                
                                MergeTermInfo(match, matchSize); // add new TermInfo
                                
                                while (matchSize > 0)
                                {
                                        SegmentMergeInfo smi = match[--matchSize];
                                        if (smi.Next())
                                                queue.Put(smi);
                                        // restore queue
                                        else
                                                smi.Close(); // done with a segment
                                }
                        }
                }
                
                private TermInfo termInfo = new TermInfo(); // minimize consing
                
                /// <summary>Merge one term found in one or more segments. The array <code>smis</code>
                /// contains segments that are positioned at the same term. <code>N</code>
                /// is the number of cells in the array actually occupied.
                /// 
                /// </summary>
                /// <param name="smis">array of segments
                /// </param>
                /// <param name="n">number of cells in the array actually occupied
                /// </param>
                private void  MergeTermInfo(SegmentMergeInfo[] smis, int n)
                {
                        long freqPointer = freqOutput.GetFilePointer();
                        long proxPointer = proxOutput.GetFilePointer();
                        
                        int df = AppendPostings(smis, n); // append posting data
                        
                        long skipPointer = WriteSkip();
                        
                        if (df > 0)
                        {
                                // add an entry to the dictionary with pointers to prox and freq files
                                termInfo.Set(df, freqPointer, proxPointer, (int) (skipPointer - freqPointer));
                                termInfosWriter.Add(smis[0].term, termInfo);
                        }
                }
                
                /// <summary>Process postings from multiple segments all positioned on the
                /// same term. Writes out merged entries into freqOutput and
                /// the proxOutput streams.
                /// 
                /// </summary>
                /// <param name="smis">array of segments
                /// </param>
                /// <param name="n">number of cells in the array actually occupied
                /// </param>
                /// <returns> number of documents across all segments where this term was found
                /// </returns>
                private int AppendPostings(SegmentMergeInfo[] smis, int n)
                {
                        int lastDoc = 0;
                        int df = 0; // number of docs w/ term
                        ResetSkip();
                        for (int i = 0; i < n; i++)
                        {
                                SegmentMergeInfo smi = smis[i];
                                TermPositions postings = smi.postings;
                                int base_Renamed = smi.base_Renamed;
                                int[] docMap = smi.docMap;
                                postings.Seek(smi.termEnum);
                                while (postings.Next())
                                {
                                        int doc = postings.Doc();
                                        if (docMap != null)
                                                doc = docMap[doc]; // map around deletions
                                        doc += base_Renamed; // convert to merged space
                                        
                                        if (doc < lastDoc)
                                                throw new System.SystemException("docs out of order");
                                        
                                        df++;
                                        
                                        if ((df % skipInterval) == 0)
                                        {
                                                BufferSkip(lastDoc);
                                        }
                                        
                                        int docCode = (doc - lastDoc) << 1; // use low bit to flag freq=1
                                        lastDoc = doc;
                                        
                                        int freq = postings.Freq();
                                        if (freq == 1)
                                        {
                                                freqOutput.WriteVInt(docCode | 1); // write doc & freq=1
                                        }
                                        else
                                        {
                                                freqOutput.WriteVInt(docCode); // write doc
                                                freqOutput.WriteVInt(freq); // write frequency in doc
                                        }
                                        
                                        int lastPosition = 0; // write position deltas
                                        for (int j = 0; j < freq; j++)
                                        {
                                                int position = postings.NextPosition();
                                                proxOutput.WriteVInt(position - lastPosition);
                                                lastPosition = position;
                                        }
                                }
                        }
                        return df;
                }
                
                private RAMOutputStream skipBuffer = new RAMOutputStream();
                private int lastSkipDoc;
                private long lastSkipFreqPointer;
                private long lastSkipProxPointer;
                
                private void  ResetSkip()
                {
                        skipBuffer.Leset();
                        lastSkipDoc = 0;
                        lastSkipFreqPointer = freqOutput.GetFilePointer();
                        lastSkipProxPointer = proxOutput.GetFilePointer();
                }
                
                private void  BufferSkip(int doc)
                {
                        long freqPointer = freqOutput.GetFilePointer();
                        long proxPointer = proxOutput.GetFilePointer();
                        
                        skipBuffer.WriteVInt(doc - lastSkipDoc);
                        skipBuffer.WriteVInt((int) (freqPointer - lastSkipFreqPointer));
                        skipBuffer.WriteVInt((int) (proxPointer - lastSkipProxPointer));
                        
                        lastSkipDoc = doc;
                        lastSkipFreqPointer = freqPointer;
                        lastSkipProxPointer = proxPointer;
                }
                
                private long WriteSkip()
                {
                        long skipPointer = freqOutput.GetFilePointer();
                        skipBuffer.WriteTo(freqOutput);
                        return skipPointer;
                }
                
                private void  MergeNorms()
                {
                        for (int i = 0; i < fieldInfos.Size(); i++)
                        {
                                FieldInfo fi = fieldInfos.FieldInfo(i);
                                if (fi.isIndexed)
                                {
                                        OutputStream output = directory.CreateFile(segment + ".f" + i);
                                        try
                                        {
                                                for (int j = 0; j < readers.Count; j++)
                                                {
                                                        IndexReader reader = (IndexReader) readers[j];
                                                        byte[] input = reader.Norms(fi.name);
                                                        int maxDoc = reader.MaxDoc();
                                                        for (int k = 0; k < maxDoc; k++)
                                                        {
                                                                byte norm = input != null?input[k]:(byte) 0;
                                                                if (!reader.IsDeleted(k))
                                                                {
                                                                        output.WriteByte(norm);
                                                                }
                                                        }
                                                }
                                        }
                                        finally
                                        {
                                                output.Close();
                                        }
                                }
                        }
                }
        }
}