/*
 * $Id: RobotsTXT.java,v 1.6 2006/03/23 11:44:30 marbobo Exp $
 *
 * Copyright (c) Siemens AG 2008 
 *           (c) Universität Paderborn 2008 
 * Licensed under EUPL 1.1. 
 *
 * $Log: RobotsTXT.java,v $
 * Revision 1.6  2006/03/23 11:44:30  marbobo
 * + Mögliche NPE abgefangen
 *
 * Revision 1.5  2004/11/10 15:02:46  koppy
 * Projektweite Überarbeitung von Error-Logs, Stacktraces,..
 *
 * Revision 1.4  2004/07/20 08:46:27  jonyp
 * added copyright info + cvs log
 *
 *
 */

package vkc.util;

import java.io.BufferedReader;
import java.io.FileReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.Reader;
import java.io.StringReader;
import java.text.ParseException;
import java.util.Vector;

import vkc.misc.VKCLogManager;

/**
 * Parst eine robots.txt, die folgender Spezifikation entspricht
 * http://info.webcrawler.com/mak/projects/robots/norobots-rfc.html
 * Benutzung:
 *
 * InputStream in = url.openStream();
 * RobotsTXT robot = RobotsTXT.parse(in);
 * boolean isAllowed = robot.allowed(UserAgent, url.getPath());
 */

public class RobotsTXT
{
    public static class Entry
    {
        public String[] userAgents;
        boolean hasGlobal_;
        public boolean[] allow;
        public String[] paths;

        public Entry(String[] userAgents, boolean[] allow, String[] paths)
        {
            if (allow.length != paths.length)
                throw new IllegalArgumentException(this.getClass().getName() +
                                                   " constructor: length of <allow> and <paths> argument must match");

            this.userAgents = userAgents;
            this.hasGlobal_ = hasGlobal(userAgents);
            this.allow = allow;
            this.paths = paths;
        }

        protected static boolean hasGlobal(String[] s)
        {
            for (int i = 0; i < s.length; i++)
            {
                if (s[i].equals("*") || s[i].length() == 0)
                    return true;
            }

            return false;
        }

        public boolean hasGlobal()
        {
            return hasGlobal_;
        }

        public boolean applies(String agent)
        {
            if (hasGlobal())
                return true;

            for (int i = 0; i < userAgents.length; i++)
            {
                if (agent.equals(userAgents[i]))
                    return true;
            }

            return false;
        }

        public boolean allowed(String path)
        {
            for (int i = 0; i < paths.length; i++)
            {
                if (path.startsWith(paths[i]))
                    return allow[i];
            }

            return true;
        }

        public String toString()
        {
            StringBuffer rval = new StringBuffer();
            int i;

            for (i = 0; i < userAgents.length; i++)
            {
                rval.append("User-agent: ");
                rval.append(userAgents[i]);
                rval.append("\n");
            }

            for (i = 0; i < allow.length; i++)
            {
                rval.append(allow[i] ? "Allow" : "Disallow");
                rval.append(": ");
                rval.append(paths[i]);
                rval.append("\n");
            }

            return rval.toString();
        }
    }


    protected Entry[] entries;

    public RobotsTXT(Entry[] entries)
    {
        this.entries = entries;
    }

    public RobotsTXT(Vector v)
    {
        final int n = v.size();
        Entry[] e = new Entry[n];

        for (int i = 0; i < n; i++)
            e[i] = (Entry)v.elementAt(i);

        this.entries = e;
    }

    /**
     * Überprüft, ob der UserAgent den Pfad benutzen darf
     * <p>
     * @param agent User-Agent-String
     * @param path Pfad, der überprüft werden soll
     */

    public boolean allowed(String agent, String path)
    {
        int match = 0;
        int match_;
        Entry x;
        boolean rval = true;

        for (int i = 0; i < entries.length; i++)
        {
            //System.out.println("allowed["+i+"]? ("+agent+", "+path+")");
            x = entries[i];

            if (x.hasGlobal())
                match_ = 1;
            else if (x.applies(agent))
                match_ = 2;
            else
                continue;

            if (match_ < match)
                continue;

            rval = x.allowed(path);

            //System.out.println("= "+rval+"("+match_+")");
            match = match_;
        }

        return rval;
    }

    /**
     * Liefert die robots.txt als String zurück
     * <p>
     * @return String
     */

    public String toString()
    {
        StringBuffer rval = new StringBuffer();

        for (int i = 0; i < entries.length; i++)
        {
            rval.append(entries[i].toString());

            if (i + 1 < entries.length)
                rval.append("\n");
        }

        return rval.toString();
    }

    //////////////////////////////////////////////////////////////////////
    protected static String nukeComment(String str)
    {
        int off = str.indexOf('#');

        if (off < 0)
            return str;

        return str.substring(0, off);
    }

    protected static String[] toStringArray(Vector v)
    {
        final int n = v.size();
        String[] rval = new String[n];

        for (int i = 0; i < n; i++)
            rval[i] = (String)v.elementAt(i);

        return rval;
    }

    protected static boolean[] tobooleanArray(Vector v)
    {
        final int n = v.size();
        boolean[] rval = new boolean[n];

        for (int i = 0; i < n; i++)
            rval[i] = ((Boolean)v.elementAt(i)).booleanValue();

        return rval;
    }

    protected static Entry[] toEntryArray(Vector v)
    {
        final int n = v.size();
        Entry[] rval = new Entry[n];

        for (int i = 0; i < n; i++)
            rval[i] = (Entry)v.elementAt(i);

        return rval;
    }

    public static RobotsTXT parse(InputStream r) //, HTTPClient.ParseException
                           throws ParseException, IOException
    {
        return parse(new InputStreamReader(r));
    }

    public static RobotsTXT parse(Reader r) //, HTTPClient.ParseException
                           throws ParseException, IOException
    {
        return parse(new BufferedReader(r));
    }

    public static RobotsTXT parse(String txt) throws ParseException, IOException
    {
      return parse(new BufferedReader(new StringReader(txt)));
    }

    public static RobotsTXT parse(BufferedReader r) //, HTTPClient.ParseException
                           throws ParseException, IOException
    {
        Vector v;
        String line;
        int state = 0;
        Vector agents = null;
        Vector allows = null;
        Vector paths = null;
        final String ua = "user-agent:";
        final String al = "allow:";
        final String da = "disallow:";
        int lineno = 0;
        v = new Vector();

        while (null != (line = r.readLine()))
        {
            lineno++;

            //System.err.println(lineno+"/"+state+": "+line);
            if (line.length() > 0)
            {
                line = nukeComment(line);

                if (line.length() <= 0)
                    continue; // discard pure comments
            }

            //System.err.println("["+line.trim().length()+"]");
            if (line.toLowerCase().startsWith(ua))
            {
                if (agents == null)
                {
                    agents = new Vector();
                    allows = new Vector();
                    paths = new Vector();
                }

                if (state == 2)
                    throw new ParseException(RobotsTXT.class.getName() +
                                             ": user-agent in the middle of rules",
                                             lineno);

                agents.addElement(line.substring(ua.length()).trim());
                state = 1;
            }
            else if (line.toLowerCase().startsWith(al))
            {
                if (state == 0)
                    throw new java.text.ParseException(RobotsTXT.class.getName() +
                                                       ": allow without any user-agents",
                                                       lineno);
                
                if (allows != null) {
                    allows.addElement(Boolean.TRUE);
                }
                
                if (paths != null) {
                    paths.addElement(line.substring(al.length()).trim());
                }

                //System.out.println(line.substring(al.length()).trim());
                state = 2;
            }
            else if (line.toLowerCase().startsWith(da))
            {
                if (state == 0)
                    throw new ParseException(RobotsTXT.class.getName() +
                                             ": disallow without any user-agents",
                                             lineno);

                if (allows != null) {
                    allows.addElement(Boolean.FALSE);
                }

                if (paths != null) {
                    paths.addElement(line.substring(da.length()).trim());
                }

                //System.out.println(line.substring(da.length()).trim());
                state = 2;
            }
            else if (line.trim().length() == 0)
            {
                if (state == 1)
                    throw new ParseException(RobotsTXT.class.getName() +
                                             ": no paths in stanza", lineno);

                // blank line, end of record.
                if (agents != null && agents.size() > 0)
                {
                    v.addElement(new Entry(toStringArray(agents),
                                           tobooleanArray(allows),
                                           toStringArray(paths)));
                    agents = null;
                }

                state = 0;
            }
            else
            {
                int colon = line.indexOf(':');

                if (colon < 0)
                    throw new ParseException(RobotsTXT.class.getName() +
                                             ": bad line in robots.txt '" +
                                             line + "'", lineno);

                if (line.indexOf(' ') < colon)
                    throw new ParseException(RobotsTXT.class.getName() +
                                             ": space in directive name in robots.txt '" +
                                             line + "'", lineno);

                //unrecognized directve, ignore it
            }
        }

        if (agents != null && agents.size() > 0)
        {
            v.addElement(new Entry(toStringArray(agents),
                                   tobooleanArray(allows),
                                   toStringArray(paths)));
            agents = null;
        }

        return new RobotsTXT(toEntryArray(v));
    }

    public static void main(String[] argv) //, HTTPClient.ParseException
                     throws ParseException, IOException
    {
        VKCLogManager.init( "c:/temp/test_", ".log" );
        VKCLogManager.setEnableLogging(true);
        VKCLogManager.setLogToConsole(true);
        
        RobotsTXT robot = parse(new FileReader(argv[0]));

        BufferedReader r = new BufferedReader(new InputStreamReader(System.in));
        String agent = r.readLine().trim();
        String line;

        while (null != (line = r.readLine()))
        {
            line = line.trim();
            System.out.println(robot.allowed(agent, line));
        }
    }
}