import java.io.FileWriter;
import java.io.IOException;
import java.io.InputStream;
import java.io.PrintWriter;
import java.net.MalformedURLException;
import java.net.URL;
import java.net.URLConnection;
import java.util.LinkedList;
import java.util.List;

import org.apache.commons.lang3.StringEscapeUtils;

import edu.stanford.nlp.tagger.maxent.MaxentTagger;

/**
 * Utility for tagging text (in the form of a String or a URL for a webpage),
 * and tagging it with tokens indicating part-of-speech.
 *
 * @author chrchan
 */
public class PageTagger {

    private static final String DEFAULT_MODEL =
            "Taggers/english-left3words-distsim.tagger";

    private static final String DEFAULT_USER_AGENT =
            "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/38.0.2125.104 Safari/537.36";

    private MaxentTagger tagger = null;

    // the default chunk size in characters
    private static final int DEFAULT_CHUNK_SIZE = 2000;

    private static final int BUF_SIZE = 2048;

    /**
     * Main method that takes a single command-line argument of a URL whose
     * content is to be tagged.
     *
     * @param argv
     *            URL whose content we want to tag
     */
    public static void main(final String[] argv) {

        URL url = null;
        if (argv.length < 1) {
            System.err.println("You need to give me a URL.");
            System.exit(1);
        }
        try {
            url = new URL(argv[0]);
        } catch (MalformedURLException e) {
            // bail, since there is nothing we can do at this point
            e.printStackTrace();
            System.exit(1);
        }

        PageTagger pageTagger = new PageTagger();
        String text = null;
        try {
            text = pageTagger.getText(url);
        } catch (Exception e) {
            // bail, since there is nothing we can do at this point
            e.printStackTrace();
            System.exit(1);
        }
        String taggedText = pageTagger.tagText(text);
        //System.out.println(taggedText);
        PrintWriter out = null;
		try {
			out = new PrintWriter(new FileWriter("result.txt"));
			out.println(taggedText);
		} catch (IOException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}finally{			
			 out.close();
		}
		 
    }

    public PageTagger() {
        tagger = new MaxentTagger(DEFAULT_MODEL);
    }

    /**
     * Given a String, returns a new String where the text is tagged with tokens
     * indicating part-of-speech.
     *
     * @param text
     *            the text that we want to tag
     * @return a new String with tags
     */
    public String tagText(final String text) {

        // MaxentTagger does not like Unicode Replacement Character so strip it.
        String cleanText = text.replaceAll("\uFFFD","");

        // If text is too large, chunk it.
        List<String> chunks = chunkText(cleanText);
        StringBuilder sb = new StringBuilder();
        for (String chunk : chunks) {
            sb.append(tagger.tagString(chunk));
            sb.append(" ");
        }

        return sb.toString();
    }

    /**
     * Returns a String containing all the text in the body of a web page
     * excluding HTML tags and JavaScript.
     *
     * @param url
     *            the location of the the web page to fetch the text of.
     * @return String containing all the text in the body of a web page
     *         excluding HTML tags and JavaScript.
     * @throws IOException
     */
    public String getText(final URL url) throws IOException {

        URLConnection conn = url.openConnection(); 
        conn.setRequestProperty("User-Agent", DEFAULT_USER_AGENT);
        InputStream is = conn.getInputStream();
        StringBuilder sb = new StringBuilder();
        int bytesRead = 0;
        try {
            do {
                byte[] b = new byte[BUF_SIZE];
                bytesRead = is.read(b, 0, BUF_SIZE);
                
                if (bytesRead != -1) {
                    sb.append(new String(b, 0, bytesRead));
                }
            } while (bytesRead != -1);
        } catch (IOException ioe) {
            throw ioe;
        } finally {
            is.close();
        }

        String ret = sb.toString();
        ret = cleanText(ret);

        return ret;
    }

    /**
     * Cleans a String by removing thing that we do not consider text - scripts,
     * tags, comments.
     *
     * @param s input String to clean
     * @return a new String that is the old String with stuff removed
     */
    private String cleanText(final String s) {
        String ret = s;
        
        // Remove scripts. The "(?s)" enables single-line mode so that newlines
        // are included in the ".*" part of the pattern.
        ret = ret.replaceAll("(?is)<script.*?/script>", " ");

        // Remove anything in the head tag
        ret = ret.replaceAll("(?is)<head.*?/head>"," ");

        // Remove HTML comments.
        ret = ret.replaceAll("(?s)<!--.*?-->", " ");

        // Remove all tags.
        ret = ret.replaceAll("<.*?>", " ");

        // Unescape any escaped HTML entities.
        ret = StringEscapeUtils.unescapeHtml4(ret);
        return ret;
    }

    /**
     * Breaks a String into chunks of at most DEFAULT_CHUNK_SIZE.
     *
     * @param largeText the String we want to break up
     * @return a List of Strings
     */
    private List<String> chunkText(final String largeText) {
        List<String> chunks = new LinkedList<String>();
        int totalSize = largeText.length();
        int currentInd = 0;
        while (currentInd < totalSize) {
            int nextInd = currentInd + DEFAULT_CHUNK_SIZE - 1;
            if (nextInd > totalSize) {
                // we have reached the end
                nextInd = totalSize - 1;
            } else {
                // We are somewhere in the middle of the document, so find a
                // good place to separate. Look for a sentence boundary first.
                int periodIndex = largeText.lastIndexOf('.', nextInd);
                if ((periodIndex != -1) && (periodIndex > currentInd)) {
                    nextInd = periodIndex + 1;
                } else {
                    // no sentence boundary, so try a word boundary
                    int wordIndex = largeText.lastIndexOf(' ', nextInd);
                    if ((wordIndex != -1) && (wordIndex > currentInd)) {
                        nextInd = wordIndex + 1;
                    }
                }
                // If no sentence or word boundary could be found, just break
                // it at the chunk-size boundary.
            }
            String chunk = largeText.substring(currentInd, nextInd);
            chunks.add(chunk);
            currentInd = nextInd + 1;
        }

        return chunks;
    }

}