/*
 * $Id: NNClassifier.java,v 1.21 2007/12/05 08:19:56 marbobo Exp $
 *
 * Copyright (c) Siemens AG 2008 
 *           (c) Universität Paderborn 2008 
 * Licensed under EUPL 1.1. 
 *
 * $Log: NNClassifier.java,v $
 * Revision 1.21  2007/12/05 08:19:56  marbobo
 * + Bugfix
 *
 * Revision 1.20  2007/11/28 09:37:38  marbobo
 * + Bugfix
 *
 * Revision 1.19  2007/11/27 12:12:56  marbobo
 * + Bugfix
 *
 * Revision 1.18  2007/10/30 13:25:23  marbobo
 * + removed unnecessary statements
 *
 * Revision 1.17  2007/01/02 11:08:43  jonyp
 * delete unnecessary ";"
 *
 * Revision 1.16  2006/12/21 12:08:41  lawern
 * Benachrichtigung zu registrierten Suchanfragen überarbeitet.
 *
 * Revision 1.15  2006/12/18 15:50:23  lawern
 * Korowas Änderungen überarbeitet.
 *
 * Revision 1.14  2006/12/05 14:59:23  lawern
 * ArrayIndexOutOfBoundsException vermieden.
 *
 * Revision 1.13  2006/11/27 13:02:40  lawern
 * Der Startzeitpunkt der kNN-Parameteroptimierung kann jetzt mit einem Wochentag versehen werden.
 *
 * Revision 1.12  2006/11/25 21:27:31  vkc
 * + Bugfix
 *
 * Revision 1.11  2006/10/30 13:00:47  lawern
 * Synchronized wieder einkommentiert.
 *
 * Revision 1.10  2006/10/27 08:50:49  lawern
 * WRITABLE-Views eingefügt
 *
 * Revision 1.9  2006/10/05 20:51:02  marbobo
 * + Synchronisierung der Classifizierungsmethode
 *
 * Revision 1.8  2006/10/05 14:54:10  marbobo
 * + Parameteroptimierung wird jetzt auf WebShare.classifier ausgeführt
 *
 * Revision 1.7  2006/10/04 11:31:49  lawern
 * Keine Indexoptimierung während der Optimierung der Klassifikationsparameter.
 *
 * Revision 1.6  2006/09/13 12:30:10  lawern
 * Danieals Änderungen rückgängig gemacht.
 *
 * Revision 1.5  2006/08/31 19:59:09  marbobo
 * + Refactoring
 *
 * Revision 1.4  2006/07/19 15:19:37  lawern
 * Bugfix
 *
 * Revision 1.3  2006/07/18 11:42:11  lawern
 * Stoppwörter aus den Testvektoren für die Optimierung rausgeschmissen.
 *
 * Revision 1.2  2006/07/17 14:16:09  lawern
 * Umstellung der Schlagwort- und Summary-Extraktion auf TFIDF-Gewichtung.
 *
 * Revision 1.1  2006/07/17 13:04:02  lawern
 * Neuer typografiegewichteter Index und k-NN-Klassifikation aus meiner Diss.
 *
 */
package vkc.search;

import cern.colt.list.DoubleArrayList;
import cern.colt.list.IntArrayList;

import cern.colt.map.OpenIntDoubleHashMap;
import cern.colt.map.OpenIntIntHashMap;
import cern.colt.map.OpenIntObjectHashMap;

import cern.colt.matrix.impl.SparseDoubleMatrix2D;

import org.apache.lucene.index.Term;
import org.apache.lucene.index.TermFreqVector;
import org.apache.lucene.search.BooleanClause;
import org.apache.lucene.search.BooleanQuery;
import org.apache.lucene.search.DefaultSimilarity;
import org.apache.lucene.search.Hits;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.store.FSDirectory;

import org.quartz.Job;
import org.quartz.JobDetail;
import org.quartz.JobExecutionContext;
import org.quartz.JobExecutionException;
import org.quartz.Scheduler;
import org.quartz.SchedulerException;
import org.quartz.SimpleTrigger;

import vkc.analyse.SummaryEntry;

import vkc.db.entities.VKCCategory;
import vkc.db.entities.VKCDocument;
import vkc.db.entities.VKCUser;

import vkc.db.serializer.VKCCategorySerializer;
import vkc.db.serializer.VKCDocumentSerializer;
import vkc.db.serializer.VKCSysParameterSerializer;

import vkc.exception.DBException;

import vkc.misc.VKCLogManager;

import vkc.util.CategoryUtils;
import vkc.util.DateUtils;
import vkc.util.DocumentUtils;

import vkc.view.WebShare;

import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.ObjectInputStream;
import java.io.ObjectOutputStream;
import java.io.PrintStream;

import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Date;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;

/**
 * Klasse für den Aufbau und Pflege des Klassifikationsindex von Lucene
 * 
 * @author lawern
 */
public final class NNClassifier implements Classifier, Job {
    private static final int LOCAL_MUTUAL_INFORMATION = 0;
    private static final int LOCAL_CHI_SQUARE = 1;
    private static final int TF = 0;
    private static final int TFIDF = 1;
    private static final int BINARY = 2;
    private static final boolean debug = false;
    private static int featureSelector = LOCAL_MUTUAL_INFORMATION;
    private static final int termWeighting = TFIDF;
    private static int useKHits = 4; // -1 alle; (k beim kNN)
    private static double relFeatureSumTreshold = 0.09;
    private static double meanAccuracy;
    private static double meanKNNScore;
    private static final int minTestKeys = 10;
    private static final int minTestCatSize = 2;
    static final double log2 = Math.log(2);
    private static final int BE = 1;
    private static final int TR = 2;
    private static final int F1 = 3;
    private static final int BE_F1 = 4;
    private static final int TR_BE = 5;
    private static final int TR_F1 = 6;
    private static final int TR_BE_F1 = 7;
    private static final int perfMeasure = BE; // TR kann nicht empfohlen werden! Siehe Diss. Abschnitt 6.3.1

    // Verzeichnis, in dem der Suchindex liegt
    private static File indexDir = null;
    private static int hitSum = 0;

    // catFeatureRating
    private static OpenIntObjectHashMap catFeatureRating;
    private static OpenIntIntHashMap testCategorySize;
    private static boolean catBaseModified = false;
    private static Object catBaseMonitor = new Object();
    private long classifyStartTime;
    private int optimizationPeriod;
    private String optimizationTime;
    private boolean optimizationRunning = false;

    /**
     * Konstruktor
     */
    public NNClassifier() {
    }

    /**
     * Diese Methode liefert zu dem übergebenen Schlagwörtern eines Dokuments einer Liste aller ähnlichen Kategorien.
     * 
     * @param TestVector
     *            Zu klassifizierendes Testdokument
     * @param dateRangeMode
     *            gibt das Alter an das die Dokumente haben dürfen, die gesucht werden sollen
     * @param user
     * @param category
     *            falls !=null, dann soll ab dieser Kategorie klassifiziert werden
     * @return Liste mit Objekten vom Typ ClassifyResultEntry der ähnlichen Kategorien oder <code>null</code> falls etwas schief gegangen ist
     */
    public Collection classify(TestVector testDoc, int maxHits, int k, boolean[] featureSelection) {
        classifyStartTime = System.currentTimeMillis();

        if (testDoc.weightsNotNull == 0) {
            return null;
        }

        // normierte Suchanfrage aufbauen
        BooleanQuery query = new BooleanQuery();
        float weight;
        int currTermId;

        for (int i = testDoc.weights.length - 1; i >= 0; i--) {
            currTermId = testDoc.termIds[i];

            if ((currTermId != 0) && (featureSelection[currTermId])) {
                weight = (float) testDoc.weights[i];

                // System.out.println(weight);

                // Nur die besten Terme suchen
                if (weight >= 0.01) {
                    TermQuery termQuery = new TermQuery(new Term("contents", testDoc.terms[i]));
                    termQuery.setBoost(weight);
                    query.add(termQuery, BooleanClause.Occur.SHOULD);
                }
            }
        }

        // if (debug) {System.out.println("Classification-Query: "+query);}
        return searchCategory(query, testDoc.documentId, null, maxHits, k, false);
    }

    /**
     * Initialisiert das Classify-Modul
     * 
     * @param classifyIndexDir -
     *            Wurzelverzeichnis der VKC-Struktur
     * @param create -
     *            falls true, wird der Index neu aufgebaut
     */
    public void init(String classifyIndexDir, boolean create) {
        indexDir = new File(classifyIndexDir);

        try {
            File classifyIndex = new File(indexDir, "Index");

            if (!classifyIndex.exists() || create) {
                // Falls kein Index existiert, wird er automatisch
                // aufgebaut
                rebuildIndex();
            } else {
                loadCatBase();
            }
        } catch (Exception e) {
            VKCLogManager.errorLog(this, "8", e.getMessage(), e);
        }

        // Parameter-Optimierungsjob erzeugen
        try {
            // Nachschauen obs den Job schon gibt.
            if (WebShare.scheduler.getJobDetail("PARAMETER_OPTIMIZATION", Scheduler.DEFAULT_GROUP) == null) {
                JobDetail optimizationJob = new JobDetail("PARAMETER_OPTIMIZATION", Scheduler.DEFAULT_GROUP, NNClassifier.class);
                optimizationPeriod = VKCSysParameterSerializer.retrieveIntValue("INDEX", "ParameterOptimizationPeriod", 24);

                optimizationTime = VKCSysParameterSerializer.retrieveValue("INDEX", "ParameterOptimizationTime", "01:00:00");
                Date startTime = DateUtils.parseNextTime(optimizationTime);
                SimpleTrigger optimizationTrigger = new SimpleTrigger("PARAMETER_OPTIMIZATION_TRIGGER", Scheduler.DEFAULT_GROUP, startTime, null, SimpleTrigger.REPEAT_INDEFINITELY,
                        optimizationPeriod * 60 * 60 * 1000L);

                WebShare.scheduler.scheduleJob(optimizationJob, optimizationTrigger);

                VKCLogManager.infoLog(WebShare.class, optimizationJob.getName() + " scheduled with repeat interval: " + DateUtils.formatDuration(optimizationTrigger.getRepeatInterval())
                        + " starting at: " + startTime);
            }

            useKHits = VKCSysParameterSerializer.retrieveIntValue("INDEX", "NumberOfNeighbors", -1);

            if (useKHits == -1) {
                SimpleTrigger promptOptimizationTrigger = new SimpleTrigger("PARAMETER_OPTIMIZATION_PROMPT_TRIGGER", Scheduler.DEFAULT_GROUP, "PARAMETER_OPTIMIZATION", Scheduler.DEFAULT_GROUP,
                        new Date(System.currentTimeMillis() + 500), null, 0, 0);
                WebShare.scheduler.scheduleJob(promptOptimizationTrigger);
                useKHits = 1;
            }

            // optimale Parameter aus der Datenbank laden
            relFeatureSumTreshold = new Double(VKCSysParameterSerializer.retrieveValue("INDEX", "RelFeatureSumTreshold", "1")).doubleValue();
            meanAccuracy = new Double(VKCSysParameterSerializer.retrieveValue("INDEX", "MeanAccuracy", "1")).doubleValue();
            meanKNNScore = new Double(VKCSysParameterSerializer.retrieveValue("INDEX", "MeanScore", "1")).doubleValue();
        } catch (Exception e) {
            VKCLogManager.errorLog(this, "8", "Can't schedule parameter optimization!", e);
        }
    }

    /**
     * Sorgt dafür, dass der Index auf die Platte geschrieben wird.
     */
    public void saveCatBase() {
        if (!catBaseModified) {
            return;
        }

        synchronized (catBaseMonitor) {
            File file = null;
            FileOutputStream fos = null;
            ObjectOutputStream o = null;

            catBaseModified = false;

            try {
                // File file = new File(indexDir.getCanonicalPath() + "/Index");
                // file.delete();
                file = new File(indexDir.getCanonicalPath() + "/Index");

                fos = new FileOutputStream(file);

                o = new ObjectOutputStream(fos);
                o.writeObject(catFeatureRating);
                o.writeObject(testCategorySize);
                o.flush();
                o.close();
            } catch (Exception e) {
                catBaseModified = true;
                VKCLogManager.errorLog(this, "2", "Can't save categorization base!", e);
            }
        }
    }

    /**
     * Sorgt dafür, dass der Index von Platte geladen wird.
     */
    private void loadCatBase() {
        synchronized (catBaseMonitor) {
            try {
                FileInputStream file = new FileInputStream(new File(indexDir.getCanonicalPath() + "/Index"));
                ObjectInputStream o = new ObjectInputStream(file);
                catFeatureRating = (OpenIntObjectHashMap) o.readObject();
                testCategorySize = (OpenIntIntHashMap) o.readObject();
                catBaseModified = false;
                o.close();
            } catch (Exception e) {
                VKCLogManager.errorLog(this, "4", "Can't load categorization base!", e);
            }
        }
    }

    private static void catBaseModified() {
        if (catBaseModified) {
            // Zeitpunkt der Index-Synchronisation verschieben
            try {
                SimpleTrigger synchronizationTrigger = new SimpleTrigger("CATINDEX_SYNCHRONIZATION_TRIGGER", Scheduler.DEFAULT_GROUP, "CATINDEX_SYNCHRONIZATION", Scheduler.DEFAULT_GROUP, new Date(
                        System.currentTimeMillis() + 15000), null, 0, 0);

                WebShare.scheduler.rescheduleJob("CATINDEX_SYNCHRONIZATION_TRIGGER", Scheduler.DEFAULT_GROUP, synchronizationTrigger);
            } catch (SchedulerException e) {
                VKCLogManager.errorLog(NNClassifier.class, "7", "Can't reschedule synchronization job!", e);
            }

            return;
        }

        catBaseModified = true;

        try {
            File file = new File(indexDir.getCanonicalPath() + "/Index");
            file.delete();
        } catch (Exception e) {
            VKCLogManager.errorLog(NNClassifier.class, "5", "Can't delete categorization base!", e);
        }

        // Index-Synchronisation in 15 sec veranlassen
        try {
            JobDetail synchronizationJob = WebShare.scheduler.getJobDetail("CATINDEX_SYNCHRONIZATION", Scheduler.DEFAULT_GROUP);

            if (synchronizationJob == null) {
                synchronizationJob = new JobDetail("CATINDEX_SYNCHRONIZATION", Scheduler.DEFAULT_GROUP, NNClassifier.class);
                SimpleTrigger synchronizationTrigger = new SimpleTrigger("CATINDEX_SYNCHRONIZATION_TRIGGER", Scheduler.DEFAULT_GROUP, new Date(System.currentTimeMillis() + 15000), null, 0, 0);

                WebShare.scheduler.scheduleJob(synchronizationJob, synchronizationTrigger);
            } else {
                SimpleTrigger synchronizationTrigger = new SimpleTrigger("CATINDEX_SYNCHRONIZATION_TRIGGER", Scheduler.DEFAULT_GROUP, "CATINDEX_SYNCHRONIZATION", Scheduler.DEFAULT_GROUP, new Date(
                        System.currentTimeMillis() + 15000), null, 0, 0);

                WebShare.scheduler.rescheduleJob("CATINDEX_SYNCHRONIZATION_TRIGGER", Scheduler.DEFAULT_GROUP, synchronizationTrigger);
            }
        } catch (SchedulerException e) {
            VKCLogManager.errorLog(NNClassifier.class, "6", "Can't create synchronization job!", e);
        }
    }

    private void optimizeParameters() {
        ArrayList testVectors = buildTestVectors();

        if (testVectors.size() == 0) {
            return;
        }

        Collection allCatIds;

        try {
            allCatIds = VKCCategorySerializer.getAllCategoryIds();
        } catch (DBException e) {
            return;
        }

        optimizationRunning = true;

        File optimizationLogFile = new File(indexDir, "Optimization.log");

        try {
            PrintStream log;

            if (optimizationLogFile.exists()) {
                log = new PrintStream(new FileOutputStream(optimizationLogFile, true));
            } else {
                log = new PrintStream(new FileOutputStream(optimizationLogFile));
            }

            log.println();
            log.println("Parameter optimization protocoll from " + new Date() + ".");

            try {
                if (VKCSysParameterSerializer.retrieveIntValue("INDEX", "FullCrossValidation", 0) == 0) {
                    log.println("Optimization method: crossvalidation with newest document of every category");
                } else {
                    log.println("Optimization method: full crossvalidation");
                }
            } catch (DBException e) {
            }

            log.println("Number of test documents: " + testVectors.size());
            log.println();

            boolean[] featureSelection = null;
            double bestFeatures = 1.0;

            try {
                if (VKCSysParameterSerializer.retrieveIntValue("INDEX", "NumberOfNeighbors", -1) == -1) {
                    // Feature-Selektion mit allen Features
                    log.println("Optimization of the parameter k (kNN algorithm) with all features:");
                    bestFeatures = 1.0;
                    featureSelection = new boolean[Search.keyBaseSize + 1];

                    for (int i = Search.keyBaseSize; i >= 0; i--) {
                        featureSelection[i] = true;
                    }
                } else {
                    // Feature-Selektion mit dem letzten relFeatureSumTreshold
                    log.println("Optimization of the parameter k (kNN algorithm) with a feature rate of " + (int) ((relFeatureSumTreshold * 100) + 0.5) + "%");
                    bestFeatures = relFeatureSumTreshold;
                    featureSelection = calculateFeatureSelection(relFeatureSumTreshold, allCatIds);
                }
            } catch (DBException e) {
            }

            // Bestimmung des optimalen k mit allen Features
            double[] qualities;
            double[] maxQualities = { -1.0, -1.0, -1.0 };
            int bestK = -1;

            for (int k = 1; k < 20; k++) {
                if (k < 10) {
                    log.print("k= " + k);
                } else {
                    log.print("k=" + k);
                }

                qualities = runClassificationTest(testVectors, featureSelection, k);

                if (qualities == null) {
                    continue;
                }

                log.println(", quality: " + qualities[0]);

                if (qualities[0] > maxQualities[0]) {
                    maxQualities = qualities;
                    bestK = k;
                }
            }

            log.println();
            log.println("Optimal k: " + bestK + " quality: " + maxQualities[0]);
            log.println();
            log.println();

            log.println("Optimization of the feature rate for feature selection (k=" + bestK + "):");

            for (double features = 0.01; features < 1.0; features += 0.01) {
                if (features > 0.09) {
                    log.print("Feature rate=" + (int) ((features * 100) + 0.5) + "%");
                } else {
                    log.print("Feature rate= " + (int) ((features * 100) + 0.5) + "%");
                }

                featureSelection = calculateFeatureSelection(features, allCatIds);

                qualities = runClassificationTest(testVectors, featureSelection, bestK);

                if (qualities == null) {
                    continue;
                }

                log.println(", quality: " + qualities[0]);

                if (qualities[0] > maxQualities[0]) {
                    maxQualities = qualities;
                    bestFeatures = features;
                }
            }

            log.println();
            log.println("Optimal feature rate: " + (int) ((bestFeatures * 100) + 0.5) + "%" + " quality: " + maxQualities[0]);

            // Beste Parameter global setzen
            useKHits = bestK;
            relFeatureSumTreshold = bestFeatures;
            meanAccuracy = maxQualities[1];
            meanKNNScore = maxQualities[2];

            // Beste Parameter in der Datenbank setzen
            try {
                VKCSysParameterSerializer.saveReadOnlyValue("INDEX", "NumberOfNeighbors", "" + useKHits);
                VKCSysParameterSerializer.saveReadOnlyValue("INDEX", "RelFeatureSumTreshold", "" + relFeatureSumTreshold);
                VKCSysParameterSerializer.saveReadOnlyValue("INDEX", "MeanAccuracy", "" + meanAccuracy);
                VKCSysParameterSerializer.saveReadOnlyValue("INDEX", "MeanScore", "" + meanKNNScore);
                VKCSysParameterSerializer.saveReadOnlyValue("INDEX", "LastParameterOptimization", (new Date()).toString());
            } catch (DBException e) {
            }

            optimizationRunning = false;
            log.println();
            log.println("Parameter optimization successfully finished on " + new Date() + ".");
            log.flush();
            log.close();
        } catch (FileNotFoundException e1) {
        }
    }

    /**
     * Holt aus der DB aus jeder Kategorie mit mehr als 2 Dokumenten das jüngste und erzeugt einen Testvektor für die Klassifikation.
     * 
     * @return Testvektoren.
     */
    private ArrayList buildTestVectors() {
        ArrayList testDocs = null;

        try {
            if (VKCSysParameterSerializer.retrieveIntValue("INDEX", "FullCrossValidation", 0) == 0) {
                testDocs = (ArrayList) DocumentUtils.getNewestDocFromAllCategories();
            } else {
                testDocs = (ArrayList) VKCDocumentSerializer.getAllDocuments();
            }
        } catch (DBException e) {
            VKCLogManager.errorLog(this, "3", "Can't retrieve test documents!", e);

            return null;
        }

        // Testvektoren aufbauen
        ArrayList testVectors = new ArrayList(testDocs.size());
        TestVector currTestVect = null;
        VKCDocument currTestDoc = null;
        TermFreqVector currTermFreqVec = null;
        double probSize = Search.getNumDocs();

        for (int i = testDocs.size() - 1; i >= 0; i--) {
            currTestDoc = (VKCDocument) testDocs.get(i);
            currTestVect = new TestVector();
            currTestVect.targetCatId = currTestDoc.getCategoryId().intValue();

            // Nur Testdokumente verwenden die aus Kategorien mit mehr als minTestCatSize dokumenten stammen.
            if (testCategorySize.get(currTestVect.targetCatId) < minTestCatSize) {
                if (debug) {
                    System.out.println("Document mit ID " + currTestDoc.getDocumentId() + " für Test ignoriert.");
                }

                continue;
            }

            currTestVect.documentId = currTestDoc.getDocumentId();
            currTestVect.langPackId = currTestDoc.getLangPackId().intValue();
            currTermFreqVec = Search.getTermFreqVector(currTestDoc.getDocumentId());

            if (debug) {
                System.out.println("DocumentId: " + currTestVect.documentId + " langPackId: " + currTestVect.langPackId);
            }

            if (currTermFreqVec != null) {
                int[] freq = currTermFreqVec.getTermFrequencies();
                String[] terms = currTermFreqVec.getTerms();
                currTestVect.terms = terms;
                currTestVect.termIds = new int[terms.length];
                currTestVect.weights = new double[terms.length];

                // Summe der Term-Frequenzen berechnen
                double weightSum = 0.0;
                double df;
                double currWeight = -1;
                int currTermId;

                for (int j = freq.length - 1; j >= 0; j--) {
                    currTermId = Search.getKeyIndex(terms[j], currTestVect.langPackId, false, 0);

                    if (currTermId == 0) {
                        // Stoppwort oder Zahl, welche von der Methode Search.stemmKey entfernt wurde
                        continue;
                    }

                    currTestVect.termIds[j] = currTermId;

                    switch (termWeighting) {
                        case TF:
                            currWeight = freq[j];

                            break;

                        case TFIDF:
                            df = Search.idFreqBase.get(currTermId);

                            if (df != 0) {
                                currWeight = freq[j] * (Math.log(probSize / df) / log2); // * termMI[currKeyId];
                            } else {
                                currWeight = 0.0;
                            }

                            break;

                        case BINARY:

                            if (freq[j] > 0) {
                                currWeight = 1;
                            }

                            break;
                    }

                    currTestVect.weights[j] = currWeight;
                    weightSum += (currWeight * currWeight);
                }

                // Termgewichte in Dokument-Vektor eintragen
                double sqrtWeightSum = Math.sqrt(weightSum);

                for (int j = currTestVect.weights.length - 1; j >= 0; j--) {
                    currTestVect.weights[j] = currTestVect.weights[j] / sqrtWeightSum;

                    if (currTestVect.weights[j] > 0.01) {
                        currTestVect.weightsNotNull++;
                    }
                }

                testVectors.add(currTestVect);
            }
            
            Search.searchFinished();
        }

        return testVectors;
    }

    /**
     * Testet die Klassifikation mit den im übergebenen File enthaltenen Dateien.
     * 
     * @param testVectors
     * @param featureSelection
     * @return Breakeven-Punkt
     */
    private double[] runClassificationTest(ArrayList testVectors, boolean[] featureSelection, int k) {
        ArrayList testEntries = new ArrayList(testVectors.size());
        int treffer = 0;

        int targetId;
        int maxTargetID = -1;
        int docs = 0;
        double scoreSum = 0.0;
        ClassificationTestEntry testEntry;
        ClassifyResultEntry result;
        TestVector currDoc;
        hitSum = 0;

        long classifyTime = 0;

        for (int i = testVectors.size() - 1; i >= 0; i--) {
            currDoc = (TestVector) testVectors.get(i);

            targetId = currDoc.targetCatId;

            // Nur Dokumente berücksichtigen, die auch Keywords haben
            if (currDoc.weightsNotNull < minTestKeys) {
                continue;
            }

            ArrayList results = (ArrayList) classify(currDoc, 1, k, featureSelection);

            if (results != null) {
                classifyTime += (System.currentTimeMillis() - classifyStartTime);

                result = (ClassifyResultEntry) results.get(0);

                if (result != null) {
                    try {
                        // int mainCatId = getMainCatId(targetId.intValue());
                        if (targetId > maxTargetID) {
                            maxTargetID = targetId;
                        }

                        if (result.getCategoryId().equals("" + targetId)) {
                            treffer++;
                            docs++;

                            testEntry = new ClassificationTestEntry("" + targetId, // "" + mainCatId,
                                    result.getCategoryId(), result.getScore(), true);

                            // System.out.println("Treffer!");
                        } else {
                            docs++;
                            testEntry = new ClassificationTestEntry("" + targetId, // "" + mainCatId,
                                    result.getCategoryId(), result.getScore(), false);
                        }

                        scoreSum += testEntry.relevance;
                        testEntries.add(testEntry);

                        // System.out.println("Trefferrate: " +
                        // ((double) treffer / (double) docs * 100) + "%");
                    } catch (java.lang.Exception e) {
                        System.out.println("Can't test document with docId: " + currDoc.documentId + ", " + e);
                        e.printStackTrace();
                    }
                } else {
                    docs++;
                }
            }
        }

        if (debug) {
            System.out.println("Dauer einer Klassifikation: " + ((double) classifyTime / (double) docs) + " ms");
            System.out.println("durchschnittlicher Score: " + (scoreSum / docs));
        }

        if (testEntries.size() == 0) {
            return null;
        }

        // Ergebnisse nach Relevanz sortieren
        ClassificationTestEntry[] testEntriesArray = new ClassificationTestEntry[testEntries.size()];
        testEntriesArray = (ClassificationTestEntry[]) testEntries.toArray(testEntriesArray);
        Arrays.sort(testEntriesArray);

        // Precision-Recall-Diagramm berechnen
        double recall = 0.0;
        double precision = 0.0;
        double fOneMeasure = 0.0;
        double tmpFOneMeasure;
        double lastRecall;
        double lastPrecision;
        boolean breakEvenFound = false;
        double breakEven = 0.0;

        for (int targetA = 1; targetA <= treffer; targetA++) {
            lastRecall = recall;
            lastPrecision = precision;

            recall = (double) targetA / (double) treffer;

            int a = 0;
            int i = 0;

            while (a != targetA) {
                testEntry = testEntriesArray[i++];

                if (testEntry.correctClassified) {
                    a++;
                }
            }

            precision = (double) a / (double) (i);

            // maximales F1-Maß berechnen
            tmpFOneMeasure = (2 * precision * recall) / (precision + recall);

            if (tmpFOneMeasure > fOneMeasure) {
                fOneMeasure = tmpFOneMeasure;
            }

            // System.out.println("(" + recall + "," + precision + ")");
            if (!breakEvenFound) {
                if (precision < recall) {
                    breakEvenFound = true;
                    breakEven = ((recall * lastPrecision) - (lastRecall * precision)) / ((recall - precision + lastPrecision) - lastRecall);
                } else if (precision == recall) {
                    breakEvenFound = true;
                    breakEven = precision;
                }
            }
        }

        // Trefferraten der Kategorien berechnen
        SparseDoubleMatrix2D categories = (SparseDoubleMatrix2D) new SparseDoubleMatrix2D(maxTargetID + 1, 2).assign(0.0);
        int intTargetId;

        for (int i = testEntriesArray.length - 1; i >= 0; i--) {
            testEntry = testEntriesArray[i];
            intTargetId = new Integer(testEntry.targetId).intValue();

            if (testEntry.correctClassified) {
                categories.setQuick(intTargetId, 1, categories.getQuick(intTargetId, 1) + 1.0);
            } else {
                categories.setQuick(intTargetId, 0, categories.getQuick(intTargetId, 0) + 1.0);
            }
        }

        IntArrayList catIDs = new IntArrayList();
        categories.getNonZeros(catIDs, new IntArrayList(), new DoubleArrayList());

        if (debug) {
            System.out.println(((double) treffer / (double) docs) + "," + breakEven + "," + fOneMeasure + "\n");
        }

        double[] scores = new double[3];

        // Rückgabe für Bewertung
        switch (perfMeasure) {
            case BE:
                scores[0] = breakEven;

                break;

            case TR:
                scores[0] = (double) treffer / (double) docs;

                break;

            case F1:
                scores[0] = fOneMeasure;

                break;

            case BE_F1:
                scores[0] = (fOneMeasure + breakEven) / 2;

                break;

            case TR_BE:
                scores[0] = (((double) treffer / (double) docs) + breakEven) / 2;

                break;

            case TR_F1:
                scores[0] = (((double) treffer / (double) docs) + fOneMeasure) / 2;

                break;

            case TR_BE_F1:
                scores[0] = (((double) treffer / (double) docs) + fOneMeasure + breakEven) / 3;

                break;
        }

        scores[1] = (double) treffer / (double) docs; // durchschnittliche Trefferrate
        scores[2] = scoreSum / docs; // durchschnittliche kNN-Bewertung

        return scores; // Rückgabe für Bewertung
    }

    /**
     * Methode baut den Klassifikationsindex neu auf
     * 
     * @return
     */
    public void rebuildIndex() throws IOException {
        Collection catIDs;

        try {
            if (!indexDir.exists()) {
                FSDirectory.getDirectory(indexDir, true);
            }
        } catch (IOException e) {
        }

        try {
            catIDs = VKCCategorySerializer.getAllCategoryIds();
        } catch (DBException e) {
            e.printStackTrace();

            return;
        }

        synchronized (catBaseMonitor) {
            catFeatureRating = new OpenIntObjectHashMap(catIDs.size());
            testCategorySize = new OpenIntIntHashMap(catIDs.size());
        }

        Integer currCatID;

        long start = System.currentTimeMillis();

        // Auswahl der Besten Terme lokal für jede Kategorie
        for (Iterator it = catIDs.iterator(); it.hasNext();) {
            currCatID = (Integer) it.next();

            try {
                addCategory(VKCCategorySerializer.retrieveCategory(currCatID));
            } catch (DBException e) {
                VKCLogManager.errorLog(this, "1", "Can't retrieve documents for categoryId: !" + currCatID, e);

                return;
            }
        }

        catFeatureRating.trimToSize();
        testCategorySize.trimToSize();

        catBaseModified();

        VKCLogManager.infoLog(this, "Indexing time (ms): " + (System.currentTimeMillis() - start));
    }

    /**
     * Bewertet die Features(Terme) einer Kategorie bezüglich ihrer Klassifikationseigenschaften.
     * 
     * @param catTermVector
     *            Term-Vektor mit den summierten Termgewichten der Kategoriedokumente.
     * @param catDocs
     *            Anzahl der Dokumente der Kategorie.
     * @param ratingMethod
     *            Methode nach welcher die Terme bewertet werden sollen.
     * @return Array mit den Feature-Bewertungen.
     */
    private OpenIntDoubleHashMap localFeatureRating(OpenIntDoubleHashMap catTermVector, int catDocs, int ratingMethod) {
        OpenIntDoubleHashMap featureRating = null;
        int docSize = Search.getNumDocs();

        switch (ratingMethod) {
            case LOCAL_MUTUAL_INFORMATION:
                featureRating = localMIFeatureRating(catTermVector, catDocs, docSize);

                break;

            case LOCAL_CHI_SQUARE:
                featureRating = localChiSquareFeatureRating(catTermVector, catDocs, docSize);

                break;
        }

        return featureRating;
    }

    /**
     * Bewertet Terme nach dem lokalen Mutual-Information-Verfahren MI(x,c).
     * 
     * @param catTermVector
     *            Gibt an welcher Term wie oft in der betrachteten Kategorie vorkommt.
     * @param catDocs
     *            Anzahl der Dokumente in der Kategorie.
     * @param docSize
     *            Anzahl aller Dokumente.
     * @return Array mit den Feature-Bewertungen.
     */
    private OpenIntDoubleHashMap localMIFeatureRating(OpenIntDoubleHashMap catTermVector, int catDocs, int docSize) {
        OpenIntDoubleHashMap termMI = new OpenIntDoubleHashMap(catTermVector.size());

        int currTermId;
        double currTermFrequency;
        double miValue;
        double part1;
        double part2;
        double part3;
        double part4;

        IntArrayList termIDs = catTermVector.keys();
        DoubleArrayList catTermValues = catTermVector.values();

        for (int i = termIDs.size() - 1; i >= 0; i--) {
            currTermId = termIDs.getQuick(i);
            currTermFrequency = Search.idFreqBase.get(currTermId);

            // TODO Partielle Berechnung noch entfernen
            part1 = (double) catDocs / (double) docSize;
            part2 = catTermValues.getQuick(i) / docSize;
            part3 = (currTermFrequency / docSize) * part1;
            part4 = Math.log(part2 / part3) / log2;
            miValue = part2 * part4;

            termMI.put(currTermId, miValue);
        }

        termMI.trimToSize();

        return termMI;
    }

    /**
     * Bewertet Terme nach dem lokalen Chi-Square-Verfahren.
     * 
     * @param catTermVector
     *            Gibt an welcher Term wie oft in der betrachteten Kategorie vorkommt.
     * @param catDocs
     *            Anzahl der Dokumente in der Kategorie.
     * @param docSize
     *            Anzahl aller Dokumente.
     * @return Array mit den Feature-Bewertungen.
     */
    private OpenIntDoubleHashMap localChiSquareFeatureRating(OpenIntDoubleHashMap catTermVector, int catDocs, int docSize) {
        OpenIntDoubleHashMap termChi = new OpenIntDoubleHashMap(catTermVector.size());

        IntArrayList termIDs = Search.idFreqBase.keys();
        DoubleArrayList termFrequencies = Search.idFreqBase.values();

        int currTermId;
        double currTermFrequency;

        for (int i = termIDs.size() - 1; i >= 0; i--) {
            currTermId = termIDs.getQuick(i);
            currTermFrequency = termFrequencies.getQuick(i);

            // CHI bestimmen
            double a = catTermVector.get(currTermId);

            if (a > 0.0) {
                double b = currTermFrequency - a;
                double c = catDocs - a;
                double d = docSize - c;

                double adcb = (a * d) - (c * b);
                double chi = (docSize * adcb * adcb) / ((a + c) * (b + d) * (a + b) * (c + d));

                if (chi == chi) { // Test ob chi kein NaN ist.
                    termChi.put(currTermId, chi);
                }
            }
        }

        termChi.trimToSize();

        return termChi;
    }

    /**
     * Einfügemethode für einzelne VKCCategory-Objekte. Nach dem Aufruf muss das übergebene Objekt in der Datenbank aktualisiert werden, da die Keywords ergänzt werden.
     * 
     * @param categoryId
     *            Id der Kategorie welche dem Index hinzugefügt werden soll.
     */
    public void addCategory(VKCCategory data) {
        List documents;

        Integer categoryId = data.getCategoryId();

        documents = data.getDocuments();

        int catSize = documents.size();
        OpenIntDoubleHashMap catTermVector = new OpenIntDoubleHashMap(catSize);

        synchronized (catBaseMonitor) {
            for (int i = catSize - 1; i >= 0; i--) {
                VKCDocument currDoc = (VKCDocument) documents.get(i);

                TermFreqVector termVector = Search.getTermFreqVector(currDoc.getDocumentId());

                if (termVector != null) {
                    // catTermVector-Array aufbauen und füllen
                    int[] freq = termVector.getTermFrequencies();
                    String[] terms = termVector.getTerms();

                    double termWeightSum = 0;

                    for (int j = freq.length - 1; j >= 0; j--) {
                        termWeightSum += freq[j];
                    }

                    int docKeys = 0;

                    for (int j = freq.length - 1; j >= 0; j--) {
                        int termId = Search.getKeyIndex(terms[j], currDoc.getLangPackId().intValue(), false, 0);

                        if (termId != 0) {
                            docKeys++;
                            catTermVector.put(termId, catTermVector.get(termId) + (freq[j] / termWeightSum));
                        }
                    }

                    if (docKeys >= minTestKeys) {
                        testCategorySize.put(categoryId.intValue(), testCategorySize.get(categoryId.intValue()) + 1);
                    }
                }
                
                Search.searchFinished();

                catTermVector.trimToSize();

                // Feature-Bewertung
                OpenIntDoubleHashMap ratedFeatures = localFeatureRating(catTermVector, catSize, featureSelector);

                catFeatureRating.put(categoryId.intValue(), ratedFeatures);
            }

            catBaseModified();
        }
    }

    /**
     * Bestimmt auf Basis der globalen Feature-Bewertungsmatrix die Feature-Selektion.
     * 
     * @param relFeatSumTresh
     * @return
     */
    private boolean[] calculateFeatureSelection(double relFeatSumTresh, Collection validCategories) {
        boolean[] goodFeatures = new boolean[Search.keyBaseSize + 1];

        for (int i = Search.keyBaseSize; i >= 0; i--) {
            goodFeatures[i] = false;
        }

        int currCatID;

        // Auswahl der Besten Terme lokal für jede Kategorie
        for (Iterator it = validCategories.iterator(); it.hasNext();) {
            currCatID = ((Integer) it.next()).intValue();

            OpenIntDoubleHashMap currRow = (OpenIntDoubleHashMap) catFeatureRating.get(currCatID);

            if (currRow == null) {
                continue;
            }

            IntArrayList termIDs = currRow.keys();
            DoubleArrayList termValues = currRow.values();

            if (termValues.size() == 0) {
                // Kategorien ohne Terme überspringen
                if (debug) {
                    System.out.println("Kategorie " + currCatID + " hat keine Terme!");
                }

                continue;
            }

            // Schwelle für die zu verwerfenden Terme berechnen
            DoubleArrayList sortedTermValues = termValues.copy();
            sortedTermValues.sort();

            double treshold;
            double sum = 0.0;

            // Summe aller Gewichte berechnen
            for (int i = termValues.size() - 1; i >= 0; i--) {
                sum += sortedTermValues.getQuick(i);
            }

            // absoluten Schwellenwert berechnen
            double featureSumTreshold = relFeatSumTresh * sum;

            double featureSum = 0.0;
            int k = termValues.size();

            while ((featureSum <= featureSumTreshold) && (k > 0)) {
                featureSum += sortedTermValues.getQuick(--k);
            }

            if (k != termValues.size()) {
                treshold = sortedTermValues.getQuick(k);
            } else {
                treshold = 0.0;
            }

            // Markieren der guten Terme
            for (int i = termIDs.size() - 1; i >= 0; i--) {
                if (termValues.getQuick(i) >= treshold) {
                    int featureId = termIDs.getQuick(i);
                    
                    if (featureId < goodFeatures.length) {
                        goodFeatures[featureId] = true;
                    }
                }
            }
        }

        return goodFeatures;
    }

    /**
     * Sollte immer dann aufgerufen werden, wenn einer Kategorie ein neues Dokument hinzugefügt wird. Nach dem Aufruf muss das übergebene Objekt in der Datenbank aktualisiert werden, da die Keywords
     * ergänzt werden.
     * 
     * @param data
     *            VKCCategory-Objekt, welches im Index aktualisiert werden soll
     */
    public void updateCategory(VKCCategory data) throws IOException {
        addCategory(data);

        if (debug) {
            VKCLogManager.infoLog(this, "updateCategory: " + data.getCategoryId());
        }
    }

    /**
     * Methode zur Entfernung eines Dokuments aus dem Suchindex
     * 
     * @param documentId
     *            Eindeutiger Bezeichner für einen Eintrag
     */
    public void removeCategory(Integer categoryId) throws IOException {
        if (debug) {
            VKCLogManager.infoLog(this, "Classify.removeCategory: " + categoryId);
        }

        synchronized (catBaseMonitor) {
            catFeatureRating.removeKey(categoryId.intValue());
            catFeatureRating.trimToSize();
            testCategorySize.removeKey(categoryId.intValue());
            testCategorySize.trimToSize();
        }

        catBaseModified();
    }

    /**
     * Methode zum Suchen über die VKC-Kategorien
     * 
     * @param query
     *            Suchanfrage
     * @param maxHits -
     *            maximal Anzahl zurückgelieferter Treffer
     * @param dateRangeMode
     *            gibt das Alter an das die Dokumente haben dürfen, die gesucht werden sollen
     * @return Liste mit Objekten vom Typ ResultEntry der gefundenen Dokumente oder <code>null</code> falls etwas schief gegangen ist
     */
    private static final Collection searchCategory(BooleanQuery query, Integer documentId, Collection validCategories, int maxHits, int k, boolean normalize) {
        HashMap result = new HashMap(k);

        // Filter für die Suche initialisieren
        try {
            Hits hits = null;

            ClassifyResultFilter filter = new ClassifyResultFilter(validCategories, documentId);

            // Suche durchführen
            Search.startSearch();
            
            if (Search.classificationSearcher == null) {
                // Suchobjekt initialisieren
                try {
                    Search.classificationSearcher = new IndexSearcher(Search.indexDir.getCanonicalPath());
                    Search.classificationSearcher.setSimilarity(new NNSimilarity());
                    Search.classificationSearcherCreated = new Date(System.currentTimeMillis());
                } catch (IOException e) {
                    VKCLogManager.errorLog(NNClassifier.class, "7", "Can't create IndexSearcher: " + query, e);
                }
            }

            hits = Search.classificationSearcher.search(query, filter);
            // Ergebnisliste aufbauen´
            int hitsLength;
            double hitsTreshold = Double.MAX_VALUE;

            hitsLength = Math.min(hits.length(), k);

            String currCategoryId = null;
            ClassifyResultEntry currCategory = null;
            double scoreSum = 0.0;

            for (int i = 0; i < hitsLength; i++) {
                scoreSum += hits.score(i);
                hitSum++;

                currCategoryId = hits.doc(i).get("categoryId");
                currCategory = (ClassifyResultEntry) result.get(currCategoryId);

                if (currCategory != null) {
                    currCategory.incScore(hits.score(i));

                    // if (currCategory.getScore() < hits.score(i)) {
                    // currCategory.setScore(hits.score(i));
                    // }
                } else {
                    if (normalize) {
                        VKCCategory newCategory = VKCCategorySerializer.retrieveCategory(new Integer(currCategoryId));
                        result.put(currCategoryId, new ClassifyResultEntry(currCategoryId, newCategory.getName(), // hits.doc(i).get("name"),
                                newCategory.getKeywordsAsString(), // hits.doc(i).get("keywords"),
                                newCategory.getDescription(), // hits.doc(i).get("description"),
                                hits.score(i)));
                    } else {
                        // Beim internen Optimierungstest werden keine
                        // Kategorieinformationen benötigt!
                        result.put(currCategoryId, new ClassifyResultEntry(currCategoryId, null, null, null, hits.score(i)));
                    }
                }

                if (scoreSum >= hitsTreshold) {
                    break;
                }
            }
        } catch (Exception e) {
            VKCLogManager.errorLog(NNClassifier.class, "2", "Error in search call!", e);
        }

        Search.searchFinished();

        // Invers sortierte Liste erzeugen
        ClassifyResultEntry[] resultArray = (ClassifyResultEntry[]) result.values().toArray(new ClassifyResultEntry[1]);
        // Normierung auf die Kategoriegröße
        // for (int i=resultArray.length-1; i>=0; i--) {
        // resultArray[i].setScore(resultArray[i].getScore()/(float) (testCategorySize.get((new Integer(resultArray[i].categoryId)).intValue())));
        // }
        Arrays.sort(resultArray);

        ArrayList resultList = new ArrayList(resultArray.length);

        int stopI = resultArray.length - Math.min(resultArray.length, maxHits);
        ClassifyResultEntry currClassifyResultEntry;

        for (int i = resultArray.length - 1; i >= stopI; i--) {
            currClassifyResultEntry = resultArray[i];

            if (normalize) {
                currClassifyResultEntry.setScore((float) (currClassifyResultEntry.getScore() / meanKNNScore * meanAccuracy));
            }

            resultList.add(currClassifyResultEntry);
        }

        return resultList;
    }

    /**
     * Diese Methode liefert zu dem übergebenen Schlagwörtern eines Dokuments einer Liste aller ähnlichen Kategorien.
     * 
     * @param keywords
     *            Keywords die von der Methode vkc.analyse.Analyser.getKeywords(...) erzeugt wurden
     * @param dateRangeMode
     *            gibt das Alter an das die Dokumente haben dürfen, die gesucht werden sollen
     * @param user
     * @param category
     *            falls !=null, dann soll ab dieser Kategorie klassifiziert werden
     * @return Liste mit Objekten vom Typ ClassifyResultEntry der ähnlichen Kategorien oder <code>null</code> falls etwas schief gegangen ist
     */
    public Collection classify(SummaryEntry[] keywords, Integer documentId, int currDocLangId, int dateRangeMode, VKCUser user, VKCCategory category, int maxHits) {
        if (indexDir == null) {
            // Klassifikationsmodul noch nicht Initialisiert.
            return null;
        }

        SummaryEntry currKey;
        double weightSum = 0;
        ArrayList cleanKeys = new ArrayList(keywords.length);

        Collection validCategories = null;

        try {
            if (user != null) {
                // Kategorien holen in denen der User publizieren darf
                if (category != null) {
                    validCategories = CategoryUtils.getWriteableCategoryIds(category.getCategoryId(), user);
                } else {
                    validCategories = CategoryUtils.getWritableCategoryIds(user);
                }
            } else {
                validCategories = VKCCategorySerializer.getAllCategoryIds();
            }
        } catch (DBException e) {
        }

        boolean[] goodFeatures = calculateFeatureSelection(relFeatureSumTreshold, validCategories);

        // Keyword-TF*IDFs und deren Summe berechnen
        int currKeyId;
        double currWeight = -1;
        double df;
        int probSize = Search.getNumDocs();

        for (int i = keywords.length - 1; i >= 0; i--) {
            // Keywords rausschmeissen, die keine ID haben (Stoppwörter & Zahlen)
            currKeyId = Search.getKeyIndex(keywords[i].getText(), currDocLangId, false, 0);

            if ((currKeyId != 0) && (goodFeatures[currKeyId])) {
                switch (termWeighting) {
                    case TF:
                        currWeight = keywords[i].getWeight();

                        break;

                    case TFIDF:
                        df = Search.idFreqBase.get(currKeyId);
                        currWeight = keywords[i].getWeight() * (Math.log(probSize / df) / log2); // * termMI[currKeyId];

                        break;

                    case BINARY:

                        if (keywords[i].getWeight() > 0) {
                            currWeight = 1;
                        }

                        break;
                }

                keywords[i].setWeight(currWeight);

                if (currWeight > 0.0) {
                    cleanKeys.add(keywords[i]);
                    weightSum += (currWeight * currWeight);
                }
            }
        }

        classifyStartTime = System.currentTimeMillis();

        // normierte Suchanfrage aufbauen
        BooleanQuery query = new BooleanQuery();
        double sqrtWeightSum = Math.sqrt(weightSum);
        float weight;

        if (cleanKeys.size() > 0) {
            for (int i = cleanKeys.size() - 1; i >= 0; i--) {
                currKey = (SummaryEntry) cleanKeys.get(i);
                weight = (float) currKey.getWeight() / (float) sqrtWeightSum;

                // Nur die besten Terme suchen
                if (weight >= 0.01) {
                    TermQuery termQuery = new TermQuery(new Term("contents", currKey.getText().toLowerCase()));
                    termQuery.setBoost(weight);
                    query.add(termQuery, BooleanClause.Occur.SHOULD);
                }
            }

            return searchCategory(query, documentId, validCategories, maxHits, useKHits, true);
        }

        return null;
    }

    /**
     * Main-Methode für Tests
     */
    public static void main(java.lang.String[] args) {
        Classifier classifier = new LuceneClassifier();
        classifier.init("E:\\work\\VKC3\\defaultroot\\classifyindex", false);

        // search("data", -1, -1);
        // whatsRelated(createDocVector("46", -1), -1, -1);
        // int[] test = { 1024, 888, 666, 333, 220 };
        // System.out.println("ggT: " + ggT(test));
    }

    /*
     * (non-Javadoc)
     * 
     * @see vkc.search.Classifier#classify(vkc.analyse.SummaryEntry[], int, vkc.db.entities.VKCUser, vkc.db.entities.VKCCategory, int)
     */
    public Collection classify(SummaryEntry[] keywords, int currDocLangId, int dateRangeMode, VKCUser user, VKCCategory category, int maxHits) {
        return classify(keywords, null, currDocLangId, dateRangeMode, user, category, maxHits);
    }

    /**
     * Wird zyklisch vom Scheduler gestartet.
     */
    public void execute(JobExecutionContext context) throws JobExecutionException {
        JobDetail job = context.getJobDetail();
        NNClassifier classifier = (NNClassifier) WebShare.classifier;

        VKCLogManager.infoLog(this, "Executing Job " + job.getFullName());

        // DB-Parameter auf Veränderung prüfen und eventuell Optimierungs-Job reschedulen
        try {
            int newOptimizationPeriod = VKCSysParameterSerializer.retrieveIntValue("INDEX", "ParameterOptimizationPeriod", 24);

            String newOptimizationTime = VKCSysParameterSerializer.retrieveValue("INDEX", "ParameterOptimizationTime", "01:00:00");

            if ((classifier.optimizationPeriod != newOptimizationPeriod) || (!classifier.optimizationTime.equals(newOptimizationTime))) {
                Date startTime = DateUtils.parseNextTime(newOptimizationTime);
                SimpleTrigger optimizationTrigger = new SimpleTrigger("PARAMETER_OPTIMIZATION_TRIGGER", Scheduler.DEFAULT_GROUP, "PARAMETER_OPTIMIZATION", Scheduler.DEFAULT_GROUP, startTime, null,
                        SimpleTrigger.REPEAT_INDEFINITELY, newOptimizationPeriod * 60 * 60 * 1000L);

                WebShare.scheduler.rescheduleJob("PARAMETER_OPTIMIZATION_TRIGGER", Scheduler.DEFAULT_GROUP, optimizationTrigger);

                classifier.optimizationPeriod = newOptimizationPeriod;
                classifier.optimizationTime = newOptimizationTime;

                VKCLogManager.infoLog(WebShare.class, "PARAMETER_OPTIMIZATION_TRIGGER" + " scheduled with repeat interval: " + DateUtils.formatDuration(optimizationTrigger.getRepeatInterval())
                        + " starting at: " + startTime);
            }
        } catch (Exception e1) {
        }

        if (job.getName().equals("CATINDEX_SYNCHRONIZATION")) {
            saveCatBase();
        } else {
            // "PARAMETER_OPTIMIZATION"
            try {
                if ((VKCSysParameterSerializer.retrieveIntValue("INDEX", "ParameterOptimization", 1) == 1) && !classifier.optimizationRunning) {
                    classifier.optimizeParameters();
                }
            } catch (DBException e) {
            }
        }
    }

    final class ClassificationTestEntry implements Comparable {
        public String targetId;
        public String resultId;
        public float relevance;
        public boolean correctClassified;

        ClassificationTestEntry(String targetId, String resultId, float relevance, boolean correctClassified) {
            this.targetId = targetId;
            this.resultId = resultId;
            this.relevance = relevance;
            this.correctClassified = correctClassified;
        }

        /*
         * (non-Javadoc)
         * 
         * @see java.lang.Comparable#compareTo(java.lang.Object)
         */
        public int compareTo(Object anotherClassificationTestEntry) {
            // invers sortieren
            return Float.compare(((ClassificationTestEntry) anotherClassificationTestEntry).relevance, relevance);
        }
    }

    /**
     * Testvektor mit allen Daten eines Dokuments die zur Parameteroptimierung benötigt werden.
     * 
     * @author lawern
     */
    final class TestVector {
        public int langPackId;
        public double[] weights;
        public String[] terms;
        public int[] termIds;
        public int targetCatId;
        public Integer documentId;
        int weightsNotNull = 0;
    }

    /**
     * Liefert true, wenn die Optimierung der Klassifikationsparameter läuft.
     * 
     * @return True, wenn die Optimierung der Klassifikationsparameter läuft.
     */
    public boolean optimizationIsRunning() {
        return optimizationRunning;
    }
}

final class NNSimilarity extends DefaultSimilarity {
    // Die Lucene-Implementierung ist etwas besser als das reine Kosinusmaß!
    // public float coord(int overlap, int maxOverlap) {
    // return 1f;
    // }
}

final class InitialGene {
    public Double relFeatureSumTreshold;
    public Double useKHits;
    public double fitness;

    public InitialGene(String relFeatureSumTreshold, String useKHits, String fitness) {
        this.relFeatureSumTreshold = new Double(relFeatureSumTreshold);
        this.useKHits = new Double(useKHits);
        this.fitness = new Double(fitness).doubleValue();
    }
}