delay a few things on startup, such as setting the visibility mode, which ensures...

[personal-kdebase.git] / runtime / nepomuk / services / storage / clucenetokenizer.cpp

/*
 * Modified version of StandardTokenizer.cpp for Nepomuk mostly to optimize for filename indexing
 * Copyright (C) 2008 Sebastian Trueg <trueg@kde.org>
 *
 * Based on StandardTokenizer.cpp from the CLucene package.
 * Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Library General Public
 * License as published by the Free Software Foundation; either
 * version 2 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Library General Public License for more details.
 *
 * You should have received a copy of the GNU Library General Public License
 * along with this library; see the file COPYING.LIB.  If not, write to
 * the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
 * Boston, MA 02110-1301, USA.
 */

/*
 * Modified version for Nepomuk mostly to optimize for filename indexing
 * Changes:
 * - underscore is treated as space, ie. tokens are always divided at underscores
 * - try to always divide tokens at dots.
 *   FIXME: we should still allow the acronym detection
 *   FIXME: is it possible to extract both? for example email addresses. Here it would
 *          be nice to have both the full address as a token and the separated portions.
 */

#include <CLucene/StdHeader.h>
#include "clucenetokenizer.h"

CL_NS_USE(analysis)
CL_NS_USE(util)

namespace Nepomuk {
    const TCHAR* tokenImageArray[] = {
        _T("<EOF>"),
        _T("<UNKNOWN>"),
        _T("<ALPHANUM>"),
        _T("<APOSTROPHE>"),
        _T("<ACRONYM>"),
        _T("<COMPANY>"),
        _T("<EMAIL>"),
        _T("<HOST>"),
        _T("<NUM>"),
        _T("<CJK>")
    };
    const TCHAR** tokenImage = tokenImageArray;
}


/* A bunch of shortcut macros, many of which make assumptions about variable
** names.  These macros enhance readability, not just convenience! */
#define EOS           (ch==-1 || rd->Eos())
#define SPACE         (_istspace((TCHAR)ch) != 0)
#define ALPHA         (_istalpha((TCHAR)ch) != 0)
#define ALNUM         (_istalnum(ch) != 0)
#define DIGIT         (_istdigit(ch) != 0)
#define UNDERSCORE    (ch == '_')

#define _CJK                    (  (ch>=0x3040 && ch<=0x318f) ||            \
                                                   (ch>=0x3300 && ch<=0x337f) ||            \
                                                   (ch>=0x3400 && ch<=0x3d2d) ||            \
                                                   (ch>=0x4e00 && ch<=0x9fff) ||            \
                                                   (ch>=0xf900 && ch<=0xfaff) ||            \
                                                   (ch>=0xac00 && ch<=0xd7af) ) //korean


#define DASH          (ch == '-')
#define NEGATIVE_SIGN_ DASH
//#define POSITIVE_SIGN_ (ch == '+')
//#define SIGN          (NEGATIVE_SIGN_ || POSITIVE_SIGN_)

#define DOT             (ch == '.')
#define DECIMAL         DOT


//freebsd seems to have a problem with defines over multiple lines, so this has to be one long line
#define _CONSUME_AS_LONG_AS(conditionFails) while (true) { ch = readChar(); if (ch==-1 || (!(conditionFails) || str.len >= LUCENE_MAX_WORD_LEN)) { break; } str.appendChar(ch);}

#define CONSUME_ALPHAS _CONSUME_AS_LONG_AS(ALPHA)

#define CONSUME_DIGITS _CONSUME_AS_LONG_AS(DIGIT)

/* otherMatches is a condition (possibly compound) under which a character
** that's not an ALNUM can be considered not to break the
** span.  Callers should pass false if only ALNUM are acceptable. */
#define CONSUME_WORD                  _CONSUME_AS_LONG_AS(ALNUM)

/*
** Consume CJK characters
*/
#define CONSUME_CJK                   _CONSUME_AS_LONG_AS(_CJK)


/* It is considered that "nothing of value" has been read if:
** a) The "read head" hasn't moved since specialCharPos was established.
** or
** b) The "read head" has moved by one character, but that character was
**    either whitespace or not among the characters found in the body of
**    a token (deliberately doesn't include the likes of '@'/'&'). */
#define CONSUMED_NOTHING_OF_VALUE (rdPos == specialCharPos || (rdPos == specialCharPos+1 && ( SPACE || !(ALNUM || DOT || DASH || UNDERSCORE) )))

#define RIGHTMOST(sb) (sb.getBuffer()[sb.len-1])
#define RIGHTMOST_IS(sb, c) (RIGHTMOST(sb) == c)
/* To discard the last character in a StringBuffer, we decrement the buffer's
** length indicator and move the terminator back by one character. */
#define SHAVE_RIGHTMOST(sb) (sb.getBuffer()[--sb.len] = '\0')

//#define REMOVE_TRAILING_CHARS(sb, charMatchesCondition) { TCHAR* sbBuf = sb.getBuffer(); for (int32_t i = sb.len-1; i >= 0; i--) { TCHAR c = sbBuf[i]; if (charMatchesCondition) { sbBuf[--sb.len] = '\0'; } else {break;}}}

/* Does StringBuffer sb contain any of the characters in string ofThese? */
#define CONTAINS_ANY(sb, ofThese) (_tcscspn(sb.getBuffer(), _T(ofThese)) != static_cast<size_t>(sb.len))


namespace Nepomuk {

    CLuceneTokenizer::CLuceneTokenizer(Reader* reader):
        rd(_CLNEW FastCharStream(reader)),
        /* rdPos is zero-based.  It starts at -1, and will advance to the first
        ** position when readChar() is first called. */
        rdPos(-1),
        tokenStart(-1)
    {
    }

    CLuceneTokenizer::~CLuceneTokenizer() {
        _CLDELETE(rd);
    }

    int CLuceneTokenizer::readChar() {
        /* Increment by 1 because we're speaking in terms of characters, not
        ** necessarily bytes: */
        rdPos++;
        return rd->GetNext();
    }

    void CLuceneTokenizer::unReadChar() {
        rd->UnGet();
        rdPos--;
    }

    inline bool CLuceneTokenizer::setToken(CL_NS( analysis )::Token* t, StringBuffer* sb, TokenTypes tokenCode) {
        t->setStartOffset(tokenStart);
        t->setEndOffset(tokenStart+sb->length());
        t->setType(tokenImage[tokenCode]);
        sb->getBuffer(); //null terminates the buffer
        t->resetTermTextLen();
        return true;
    }

    bool CLuceneTokenizer::next(Token* t) {
        int ch=0;
        while (!EOS) {
            ch = readChar();

            if ( ch == 0 || ch == -1 ){
                continue;
            } else if (SPACE || UNDERSCORE) {
                continue;
            } else if (ALPHA) {
                tokenStart = rdPos;
                return ReadAlphaNum(ch,t);
            } else if (DIGIT || NEGATIVE_SIGN_ || DECIMAL) {
                tokenStart = rdPos;
                /* ReadNumber returns NULL if it fails to extract a valid number; in
                ** that case, we just continue. */
                if (ReadNumber(NULL, ch,t))
                    return true;
            } else if ( _CJK ){
                if ( ReadCJK(ch,t) )
                    return true;
            }
        }
        return false;
    }

    bool CLuceneTokenizer::ReadNumber(const TCHAR* previousNumber, const TCHAR prev,Token* t) {
        /* previousNumber is only non-NULL if this function already read a complete
        ** number in a previous recursion, yet has been asked to read additional
        ** numeric segments.  For example, in the HOST "192.168.1.3", "192.168" is
        ** a complete number, but this function will recurse to read the "1.3",
        ** generating a single HOST token "192.168.1.3". */
        t->growBuffer(LUCENE_MAX_WORD_LEN+1);//make sure token can hold the next word
        StringBuffer str(t->_termText,t->bufferLength(),true); //use stringbuffer to read data onto the termText
        TokenTypes tokenType;
        bool decExhausted;
        if (previousNumber != NULL) {
            str.prepend(previousNumber);
            tokenType = HOST;
            decExhausted = false;
        } else {
            tokenType = NUM;
            decExhausted = (prev == '.');
        }
        if (  str.len >= LUCENE_MAX_WORD_LEN ){
            //if a number is too long, i would say there is no point
            //storing it, because its going to be the wrong number anyway?
            //what do people think?
            return false;
        }
        str.appendChar(prev);

        const bool signExhausted = (prev == '-');
        int ch = prev;

        CONSUME_DIGITS;

        if (str.len < 2 /* CONSUME_DIGITS didn't find any digits. */
            && (
                (signExhausted && !DECIMAL)
                || (decExhausted /* && !DIGIT is implied, since CONSUME_DIGITS stopped on a non-digit. */)
                )
            )
        {
            /* We have either:
            **   a) a negative sign that's not followed by either digit(s) or a decimal
            **   b) a decimal that's not followed by digit(s)
            ** so this is not a valid number. */
            if (!EOS) {
                /* Unread the character that stopped CONSUME_DIGITS: */
                unReadChar();
            }
            return false;
        }

        /* We just read a group of digits.  Is it followed by a decimal symbol,
        ** implying that there might be another group of digits available? */
        if (!EOS) {
            if (DECIMAL) {
                if (  str.len >= LUCENE_MAX_WORD_LEN )
                    return false; //read above for rationale
                str.appendChar(ch);
            } else {
                unReadChar();
                goto SUCCESSFULLY_EXTRACTED_NUMBER;
            }

            CONSUME_DIGITS;
            if (!DIGIT && !DECIMAL) {
                unReadChar();
            } else if (!EOS && DECIMAL && _istdigit(rd->Peek())) {
                /* We just read the fractional digit group, but it's also followed by
                ** a decimal symbol and at least one more digit, so this must be a
                ** HOST rather than a real number. */
                return ReadNumber(str.getBuffer(), '.',t);
            }
        }

    SUCCESSFULLY_EXTRACTED_NUMBER:
        TCHAR rightmost = RIGHTMOST(str);
        /* Don't including a trailing decimal point. */
        if (rightmost == '.') {
            SHAVE_RIGHTMOST(str);
            unReadChar();
            rightmost = RIGHTMOST(str);
        }
        /* If all we have left is a negative sign, it's not a valid number. */
        if (rightmost == '-') {
            CND_PRECONDITION (str.len == 1, "Number is invalid");
            return false;
        }

        return setToken(t,&str,tokenType);
    }

    bool CLuceneTokenizer::ReadAlphaNum(const TCHAR prev, Token* t) {
        t->growBuffer(LUCENE_MAX_WORD_LEN+1);//make sure token can hold the next word
        StringBuffer str(t->_termText,t->bufferLength(),true); //use stringbuffer to read data onto the termText
        if (  str.len < LUCENE_MAX_WORD_LEN ){
            str.appendChar(prev);
            int ch = prev;

            CONSUME_WORD;
            if (!EOS && str.len < LUCENE_MAX_WORD_LEN-1 ) { //still have space for 1 more character?
                switch(ch) { /* What follows the first alphanum segment? */
#if 0
                case '.':
                                        str.appendChar('.');
                                        return ReadDotted(&str, UNKNOWN,t);
#endif
                                case '\'':
                                        str.appendChar('\'');
                                        return ReadApostrophe(&str,t);
                                case '@':
                                        str.appendChar('@');
                                        return ReadAt(&str,t);
                                case '&':
                                        str.appendChar('&');
                                        return ReadCompany(&str,t);
                    /* default: fall through to end of this function. */
                }
            }
        }
        return setToken(t,&str,ALPHANUM);
    }

    bool CLuceneTokenizer::ReadCJK(const TCHAR prev, Token* t) {
        t->growBuffer(LUCENE_MAX_WORD_LEN+1);//make sure token can hold the next word
        StringBuffer str(t->_termText,t->bufferLength(),true); //use stringbuffer to read data onto the termText
        if ( str.len < LUCENE_MAX_WORD_LEN ){
            str.appendChar(prev);
            int ch = prev;

            CONSUME_CJK;
        }
        return setToken(t,&str,CJK);
    }


    bool CLuceneTokenizer::ReadDotted(StringBuffer* _str, TokenTypes forcedType, CL_NS(analysis)::Token* t) {
        const int32_t specialCharPos = rdPos;
        StringBuffer& str=*_str;

        /* A segment of a "dotted" is not allowed to begin with another dot or a dash.
        ** Even though hosts, e-mail addresses, etc., could have a dotted-segment
        ** that begins with a dot or a dash, it's far more common in source text
        ** for a pattern like "abc.--def" to be intended as two tokens. */
        int ch = rd->Peek();
        if (!(DOT || DASH)) {
            bool prevWasDot;
            bool prevWasDash;
            if (str.len == 0) {
                prevWasDot = false;
                prevWasDash = false;
            } else {
                prevWasDot = RIGHTMOST(str) == '.';
                prevWasDash = RIGHTMOST(str) == '-';
            }
            while (!EOS && str.len < LUCENE_MAX_WORD_LEN-1 ) {
                ch = readChar();
                const bool dot = ch == '.';
                const bool dash = ch == '-';

                if (!(ALNUM || UNDERSCORE || dot || dash)) {
                    break;
                }
                /* Multiple dots or dashes in succession end the token.
                ** Consider the following inputs:
                **   "Visit windowsupdate.microsoft.com--update today!"
                **   "In the U.S.A.--yes, even there!"                 */
                if ((dot || dash) && (prevWasDot || prevWasDash)) {
                    /* We're not going to append the character we just read, in any case.
                    ** As to the character before it (which is currently RIGHTMOST(str)):
                    ** Unless RIGHTMOST(str) is a dot, in which we need to save it so the
                    ** acronym-versus-host detection can work, we want to get rid of it. */
                    if (!prevWasDot) {
                        SHAVE_RIGHTMOST(str);
                    }
                    break;
                }

                str.appendChar(ch);

                prevWasDot = dot;
                prevWasDash = dash;
            }
        }

        /* There's a potential StringBuffer.append call in the code above, which
        ** could cause str to reallocate its internal buffer.  We must wait to
        ** obtain the optimization-oriented strBuf pointer until after the initial
        ** potentially realloc-triggering operations on str.
        ** Because there can be other such ops much later in this function, strBuf
        ** is guarded within a block to prevent its use during or after the calls
        ** that would potentially invalidate it. */
        { /* Begin block-guard of strBuf */
            TCHAR* strBuf = str.getBuffer();

            bool rightmostIsDot = RIGHTMOST_IS(str, '.');
            if (CONSUMED_NOTHING_OF_VALUE) {
                /* No more alphanums available for this token; shave trailing dot, if any. */
                if (rightmostIsDot) {
                    SHAVE_RIGHTMOST(str);
                }
                /* If there are no dots remaining, this is a generic ALPHANUM. */
                if (_tcschr(strBuf, '.') == NULL) {
                    forcedType = ALPHANUM;
                }

                /* Check the token to see if it's an acronym.  An acronym must have a
                ** letter in every even slot and a dot in every odd slot, including the
                ** last slot (for example, "U.S.A."). */
            } else if (rightmostIsDot) {
                bool isAcronym = true;
                const int32_t upperCheckLimit = str.len - 1; /* -1 b/c we already checked the last slot. */

                for (int32_t i = 0; i < upperCheckLimit; i++) {
                    const bool even = (i % 2 == 0);
                    ch = strBuf[i];
                    if ( (even && !ALPHA) || (!even && !DOT) ) {
                        isAcronym = false;
                        break;
                    }
                }
                if (isAcronym) {
                    forcedType = ACRONYM;
                } else {
                    /* If it's not an acronym, we don't want the trailing dot. */
                    SHAVE_RIGHTMOST(str);
                    /* If there are no dots remaining, this is a generic ALPHANUM. */
                    if (_tcschr(strBuf, '.') == NULL) {
                        forcedType = ALPHANUM;
                    }
                }
            }
        } /* End block-guard of strBuf */

        if (!EOS) {
            if (ch == '@' && str.len < LUCENE_MAX_WORD_LEN-1) {
                str.appendChar('@');
                return ReadAt(&str,t);
            } else {
                unReadChar();
            }
        }

        return setToken(t,&str,UNKNOWN
                        ? forcedType : HOST);
    }

    bool CLuceneTokenizer::ReadApostrophe(StringBuffer* _str, Token* t) {
        StringBuffer& str=*_str;

        TokenTypes tokenType = APOSTROPHE;
        const int32_t specialCharPos = rdPos;
        int ch=0;

        CONSUME_ALPHAS;
        if (RIGHTMOST_IS(str, '\'') || CONSUMED_NOTHING_OF_VALUE) {
            /* After the apostrophe, no more alphanums were available within this
            ** token; shave trailing apostrophe and revert to generic ALPHANUM. */
            SHAVE_RIGHTMOST(str);
            tokenType = ALPHANUM;
        }
        if (!EOS) {
            unReadChar();
        }

        return setToken(t,&str,tokenType);
    }

    bool CLuceneTokenizer::ReadAt(StringBuffer* str, Token* t) {
        ReadDotted(str, EMAIL,t);
        /* JLucene grammar indicates dots/digits not allowed in company name: */
        if (!CONTAINS_ANY((*str), ".0123456789")) {
            setToken(t,str,COMPANY);
        }
        return true;
    }

    bool CLuceneTokenizer::ReadCompany(StringBuffer* _str, Token* t) {
        StringBuffer& str = *_str;
        const int32_t specialCharPos = rdPos;
        int ch=0;

        CONSUME_WORD;
        if (CONSUMED_NOTHING_OF_VALUE) {
            /* After the ampersand, no more alphanums were available within this
            ** token; shave trailing ampersand and revert to ALPHANUM. */
            CND_PRECONDITION(RIGHTMOST_IS(str, '&'),"ReadCompany failed");
            SHAVE_RIGHTMOST(str);


            return setToken(t,&str,ALPHANUM);
        }
        if (!EOS) {
            unReadChar();
        }

        return setToken(t,&str,COMPANY);
    }
}