/*
 * $Id: SearchAnalyzer.java,v 1.3 2006/09/18 10:45:09 lawern Exp $
 *
 * Copyright (c) Siemens AG 2008 
 *           (c) Universität Paderborn 2008 
 * Licensed under EUPL 1.1. 
 *
 * $Log: SearchAnalyzer.java,v $
 * Revision 1.3  2006/09/18 10:45:09  lawern
 * ";:" zu punctuationMarks hinzugefügt.
 *
 * Revision 1.2  2006/09/14 12:27:55  lawern
 * Satzzeichen am Ende eines Tokens werden entfernt.
 *
 * Revision 1.1  2006/08/02 19:15:06  marbobo
 * + Initiale Version
 *
 */
package vkc.analyse;

import java.io.IOException;
import java.io.Reader;

import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.Token;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.WhitespaceTokenizer;

/**
 * Analyzer, welcher dem WhitespaceAnalyzer ähnlich ist, aber allen Tokens
 * lowercased liefert.
 * @author lawern
 */
public final class SearchAnalyzer extends Analyzer {
    
    public TokenStream tokenStream(String fieldName, Reader reader) {
        return new SearchTokenizer(reader);
    }
}


/**
 * Tokenizer, welcher dem WhitespaceTokenizer ähnlich ist, aber allen Tokens
 * lowercased liefert.
 * @author lawern
 */
final class SearchTokenizer extends TokenStream {
    WhitespaceTokenizer tokenizer = null;
    String punctuationMarks = ",.?!;:";

    public SearchTokenizer(Reader reader) {
        tokenizer = new WhitespaceTokenizer(reader);
    }

    public Token next() throws IOException {
        Token token = tokenizer.next();

        if (token != null) {
            String termText = token.termText();
            
            //Satzzeichen nicht indexieren
            int termTextLength = termText.length();
            if ((termTextLength > 1) && (punctuationMarks.indexOf(termText.charAt(termTextLength-1)) >= 0)) {
                termText = termText.substring(0, termTextLength-1) ;
            }
            
            Token newToken = new Token(termText.toLowerCase(),
                    token.startOffset(), token.endOffset(), token.type());
            newToken.setPositionIncrement(token.getPositionIncrement());

            return newToken;
        }

        return null;
    }
}