// Copyright
//  2016   Yclept Nemo <pscjtwjdjtAhnbjm/dpn>
// License
//  Apache 2.0 or GPL (compatible with Boilerpipe and Recoll)


// Internal:
import java.lang.reflect.Field;
import java.util.List;
import java.util.Map;
import java.util.HashMap;
import java.util.Collections;
import java.util.regex.Pattern;
import java.util.regex.Matcher;
import java.util.Locale;
import java.io.*;
import java.nio.ByteBuffer;
import java.nio.charset.Charset;
import java.nio.charset.IllegalCharsetNameException;
import java.nio.charset.UnsupportedCharsetException;
// External:
import org.apache.commons.cli.*;
import de.l3s.boilerpipe.BoilerpipeProcessingException;
import de.l3s.boilerpipe.extractors.*;
import org.jsoup.Jsoup;
import org.jsoup.nodes.*;
import org.mozilla.universalchardet.UniversalDetector;


// CheckedInputStream:
//  author:
//      rinde
//  source:
//      http://stackoverflow.com/a/35139613/2626865
//  notes:
//      Solves "Invalidate Stream without Closing"
final class CheckedInputStream extends InputStream {
  final InputStream delegate;
  boolean validated;

  public CheckedInputStream(InputStream stream) {
    delegate = stream;
    validated = true;
  }

  public void invalidate() {
    validated = false;
  }

  void checkValidated() {
    if (!validated) {
      throw new IllegalStateException("Stream is invalidated.");
    }
  }

  @Override
  public int read() throws IOException {
    checkValidated();
    return delegate.read();
  }

  @Override
  public int read(byte b[]) throws IOException {
    checkValidated();
    return delegate.read(b, 0, b.length);
  }

  @Override
  public int read(byte b[], int off, int len) throws IOException {
    checkValidated();
    return delegate.read(b, off, len);
  }

  @Override
  public long skip(long n) throws IOException {
    checkValidated();
    return delegate.skip(n);
  }

  @Override
  public int available() throws IOException {
    checkValidated();
    return delegate.available();
  }

  @Override
  public void close() throws IOException {
    checkValidated();
    delegate.close();
  }

  @Override
  public synchronized void mark(int readlimit) {
    checkValidated();
    delegate.mark(readlimit);
  }

  @Override
  public synchronized void reset() throws IOException {
    checkValidated();
    delegate.reset();
  }

  @Override
  public boolean markSupported() {
    checkValidated();
    return delegate.markSupported();
  }
}

class UnrecognizedArgumentException extends ParseException {
    private Option option;
    private String argument;

    public UnrecognizedArgumentException(String message) {
        super(message);
    }

    public UnrecognizedArgumentException(Option option, String argument) {
        this(String.format
                ( "Unrecognized argument \"%s\" for option \"%s\""
                , argument
                , option.getLongOpt() == null
                    ? option.getOpt()
                    : option.getLongOpt()
                )
        );
        this.option = option;
        this.argument = argument;
    }

    public Option getOption() {
        return option;
    }

    public String getArgument() {
        return argument;
    }
}

class BoilerpipeArguments {
    // from org.jsoup.helper.DataUtil, as we are re-implementing jsoup's
    // encoding detection. See below for more details.
    private static final Pattern charset_pattern = Pattern.compile("(?i)\\bcharset=\\s*(?:\"|')?([^\\s,;\"']+)");

    private static final Map<byte[], String> map_bom;
    static {
        Map<byte[], String> map_bom_t = new HashMap<byte[], String>();
        map_bom_t.put(new byte[] {(byte)0xEF, (byte)0xBB, (byte)0xBF},              "UTF-8");
        map_bom_t.put(new byte[] {(byte)0xFE, (byte)0xFF},                          "UTF-16BE");
        map_bom_t.put(new byte[] {(byte)0xFF, (byte)0xFE},                          "UTF-16LE");
        map_bom_t.put(new byte[] {(byte)0x00, (byte)0x00, (byte)0xFE, (byte)0xFF},  "UTF-32BE");
        map_bom_t.put(new byte[] {(byte)0xFF, (byte)0xFE, (byte)0x00, (byte)0x00},  "UTF-32LE");
        // UTF-7: 7-bit schemes: the last two bits of the fourth byte are not
        // used, so all possible combinations are listed. As the UTF-7 BOM is
        // insufficiently unique (standard ASCII/UTF-8 characters) and UTF-7 is
        // intended to be base64 encoded, heuristic algorithms (ie, character
        // frequency) should supplement or replace detection via the BOM.
        // Therefore, UTF-7 BOM detection is disabled.
        //map_bom_t.put(new byte[] {(byte)0x2B, (byte)0x2F, (byte)0x76, (byte)0x38},  "UTF-7");
        //map_bom_t.put(new byte[] {(byte)0x2B, (byte)0x2F, (byte)0x76, (byte)0x39},  "UTF-7");
        //map_bom_t.put(new byte[] {(byte)0x2B, (byte)0x2F, (byte)0x76, (byte)0x2B},  "UTF-7");
        //map_bom_t.put(new byte[] {(byte)0x2B, (byte)0x2F, (byte)0x76, (byte)0x38},  "UTF-7");
        map_bom_t.put(new byte[] {(byte)0xF7, (byte)0x64, (byte)0x4c},              "UTF-1");
        map_bom_t.put(new byte[] {(byte)0xDD, (byte)0x73, (byte)0x66, (byte)0x73},  "UTF-EBCDIC");
        // SCSU: SCSU uses U+FEFF (the UTF-16/32 BOM) as the "Signature Byte
        // Sequence", however U+FEFF has multiple encodings in SCSU. The
        // preferred encoding of U+FEFF for the "Signature Byte Sequence" is
        // listed and used below, as per UTR#6.
        map_bom_t.put(new byte[] {(byte)0x0E, (byte)0xFE, (byte)0xFF},              "SCSU");
        map_bom_t.put(new byte[] {(byte)0xFB, (byte)0xEE, (byte)0x28},              "BOCU-1");
        map_bom_t.put(new byte[] {(byte)0x84, (byte)0x31, (byte)0x95, (byte)0x33},  "GB-18030");
        map_bom = Collections.unmodifiableMap(map_bom_t);
    }

    private CommandLine         commandline;
    private Boolean             help            = null;
    private Boolean             html            = null;
    private BufferedWriter      output          = null;
    private RandomAccessFile    input           = null;
    private ExtractorBase       extractor       = null;
    private Charset             output_charset  = Charset.forName("UTF-8");
    private Charset             input_charset   = null;

    // Constructor

    public BoilerpipeArguments(CommandLine commandline) {
        this.commandline = commandline;
    }

    // Getters

    public boolean help() {
        if (this.help == null) {
            set_help();
        }
        return this.help.booleanValue();
    }

    public boolean html() {
        if (this.html == null) {
            set_html();
        }
        return this.html.booleanValue();
    }

    public BufferedWriter output() throws IOException {
        if (this.output == null) {
            set_output();
        }
        return this.output;
    }

    public RandomAccessFile input() throws ParseException, IOException {
        if (this.input == null) {
            set_input();
        }
        return this.input;
    }

    public ExtractorBase extractor() throws ParseException, IOException {
        if (this.extractor == null) {
            set_extractor();
        }
        return this.extractor;
    }

    public Charset output_charset() {
        assert (this.output_charset != null);
        return this.output_charset;
    }

    public Charset input_charset()
        throws ParseException,
               IOException,
               IllegalCharsetNameException,
               UnsupportedCharsetException {
        if (this.input_charset == null) {
            set_input_charset();
        }
        return this.input_charset;
    }

    public CheckedInputStream input_stream() throws ParseException, IOException {
        return new CheckedInputStream(new FileInputStream(this.input().getFD()));
    }

    // Setters

    private void set_help() {
        if (this.commandline.hasOption("h")) {
            this.help = Boolean.TRUE;
        } else {
            this.help = Boolean.FALSE;
        }
    }

    private void set_html() {
        if (this.commandline.hasOption("r")) {
            this.html = Boolean.TRUE;
        } else {
            this.html = Boolean.FALSE;
        }
    }

    private void set_output() throws IOException {
        String output_path = "";
        boolean output_path_use = false;

        if (this.commandline.hasOption("o")) {
            output_path = this.commandline.getOptionValue("o");
        }

        if (output_path.length() > 0 && !output_path.equals("-")) {
            output_path_use = true;
        }

        if (output_path_use) {
            this.output = new BufferedWriter
                ( new OutputStreamWriter
                    ( new FileOutputStream(output_path)
                    , output_charset()
                    )
                );
        } else {
            this.output = new BufferedWriter
                ( new OutputStreamWriter
                    ( System.out
                    )
                );
        }
    }

    private void set_input() throws ParseException, IOException {
        List<String> parameters = this.commandline.getArgList();

        if (parameters.size() != 1) {
            throw new MissingArgumentException("Missing parameter: input-path");
        }

        this.input = new RandomAccessFile(parameters.get(0), "r");
    }

    private void set_extractor() throws ParseException, IOException {
        ExtractorBase extractor = ArticleExtractor.INSTANCE;

        String exception_message = "Invalid Boilerpipe Extractor \"%s\": %s";

        if (this.commandline.hasOption("e")) {
            String extractor_string = this.commandline.getOptionValue("e");

            if (!is_valid_identifier(extractor_string)) {
                throw new UnrecognizedArgumentException(String.format(exception_message, "invalid identifier"));
            }

            String extractor_package = "de.l3s.boilerpipe.extractors.";

            // Not all derived classes provide getInstance()
            //try {
            //    Class<?> extractor_class = Class.forName(extractor_package + extractor_string);
            //    Method extractor_method = extractor_class.getDeclaredMethod("getInstance", ExtractorBase.class);
            //    extractor = extractor_method.invoke(null);
            //}
            //catch (ClassNotFoundException|IllegalAccessException|NoSuchMethodException|InvocationTargetException e) {
            //    System.exit(10);
            //}

            try {
                Class<?> extractor_class = Class.forName(extractor_package + extractor_string);
                Field extractor_field = extractor_class.getDeclaredField("INSTANCE");
                Object extractor_object = extractor_field.get(ExtractorBase.class);
                extractor = ExtractorBase.class.cast(extractor_object);
            }
            catch (ClassNotFoundException|NoSuchFieldException|IllegalAccessException e) {
                throw new UnrecognizedArgumentException(String.format(exception_message, "no such extractor"));
            }
        }

        this.extractor = extractor;
    }

    private void set_input_charset()
        throws ParseException,
               IOException,
               IllegalCharsetNameException,
               UnsupportedCharsetException {
        CheckedInputStream stream1 = input_stream();
        CheckedInputStream stream2 = input_stream();
        CheckedInputStream stream3 = input_stream();

        String encoding = null;

        if (encoding == null) {
            encoding = encoding_from_bom(stream1);
        }

        this.input().seek(0);

        if (encoding == null) {
            encoding = encoding_from_html(stream2);
        }

        this.input().seek(0);

        if (encoding == null) {
            encoding = encoding_detect(stream3);
        }

        this.input().seek(0);

        if (encoding == null) {
            encoding = Charset.defaultCharset().name();
        }

        this.input().seek(0);

        assert !stream1.validated;
        assert !stream2.validated;
        assert !stream3.validated;

        assert (encoding != null);

        this.input_charset = charset_from_string(encoding);
    }

    // Miscellaneous functions

    public void close() throws IOException {
        if (this.input != null) {
            this.input.close();
        }
        if (this.output != null) {
            this.output.close();
        }
    }

    // Support Methods (static)

    static boolean is_valid_identifier(String s) {
        if (s == null || s.length() == 0) {
            return false;
        }
        if (!Character.isJavaIdentifierStart(s.charAt(0))) {
            return false;
        }
        for (int i=1; i<s.length(); i++) {
            if (!Character.isJavaIdentifierPart(s.charAt(i))) {
                return false;
            }
        }
        return true;
    }

    static String encoding_from_bom(CheckedInputStream stream) throws IOException {
        byte[]  bom;
        int     bom_read;
        int     bom_length = 0;
        int     bom_offset = 0;

        for (byte[] key : map_bom.keySet()) {
            if (key.length > bom_length) {
                bom_length = key.length;
            }
        }

        bom = new byte[bom_length];

        while ((bom_read = stream.read(bom, bom_offset, bom_length)) > 0) {
            bom_length -= bom_read;
            bom_offset += bom_read;
        }

        stream.invalidate();

        for (Map.Entry<byte[], String> entry : map_bom.entrySet()) {
            if (equals(bom, entry.getKey(), 0)) {
                return entry.getValue();
            }
        }

        return null;
    }

    static String encoding_detect(CheckedInputStream stream) throws IOException {
        UniversalDetector detector = new UniversalDetector(null);

        byte[] buffer = new byte[4096];

        int bytes_read;

        while ((bytes_read = stream.read(buffer)) > 0 && !detector.isDone()) {
            detector.handleData(buffer, 0, bytes_read);
        }

        detector.dataEnd();

        String encoding = detector.getDetectedCharset();

        detector.reset();

        stream.invalidate();

        return encoding;
    }

    // Since org.jsoup.nodes.Document.OutputSettings:charset defaults to
    // UTF-8 if no charset is specified in the document, reimplement
    // jsoup's charset detection (org.jsoup.helper.DataUtil:parseByteData)
    static String encoding_from_html(CheckedInputStream stream) throws IOException {
        Document document = Jsoup.parse(stream, null, "");

        Element meta = document.select("meta[http-equiv=content-type], meta[charset]").first();

        String encoding = null;

        if (meta != null) {
            if (encoding == null && meta.hasAttr("http-equiv")) {
                encoding = encoding_from_content_type(meta.attr("content"));
            }
            if (encoding != null && !encoding_is_legal(encoding)) {
                encoding = null;
            }
            if (encoding == null && meta.hasAttr("charset")) {
                encoding = meta.attr("charset");
            }
            if (encoding != null && !encoding_is_legal(encoding)) {
                encoding = null;
            }
        }

        stream.invalidate();

        return encoding;
    }

    static String encoding_from_content_type(String s) {
        Matcher charset_matcher = charset_pattern.matcher(s);

        String charset = null;

        if (charset_matcher.matches()) {
            charset = charset_matcher.group(1);
            assert !charset.isEmpty();
        }

        return charset;
    }

    static boolean encoding_is_legal(String s) {
        boolean legal = true;

        try {
            Charset.isSupported(s);
        }
        catch (IllegalCharsetNameException e) {
            legal = false;
        }

        return legal;
    }

    static Charset charset_from_string(String s)
        throws IllegalCharsetNameException,
               UnsupportedCharsetException {
        Charset charset = null;

        try {
            try {
                charset = Charset.forName(s);
            }
            catch (IllegalCharsetNameException|UnsupportedCharsetException e) {
                charset = Charset.forName(s.toUpperCase(Locale.ENGLISH));
            }
        }
        catch (IllegalCharsetNameException e) {
            throw new IllegalCharsetNameException("Illegal charset name: " + e.getMessage());
        }
        catch (UnsupportedCharsetException e) {
            throw new UnsupportedCharsetException("Unsupported charset: " + e.getMessage());
        }

        return charset;
    }

    static boolean equals(byte[] a, byte[] b, int n) {
        if (a == null) {
            return (b == null);
        }

        int n_a = n <= 0 ? a.length : Math.min(n, a.length);
        int n_b = n <= 0 ? b.length : Math.min(n, b.length);

        ByteBuffer bb_a = ByteBuffer.wrap(a, 0, n_a);
        ByteBuffer bb_b = ByteBuffer.wrap(b, 0, n_b);

        return bb_a.equals(bb_b);
    }
}

class BoilerpipeHandler {
    private static final String html_metadata;
    static {
        String[] metadata_tags =
            { "head > title"
            , "head > base"
            , "head > meta[name=application-name]"
            , "head > meta[name=author]"
            , "head > meta[name=description]"
            , "head > meta[name=keywords]"
            , "head > link[rel=alternate]"
            , "head > link[rel=author]"
            , "head > link[rel=icon]"
            , "head > link[rel=license]"
            , "head > link[rel=next]"
            , "head > link[rel=pref]"
            , "head > link[rel=search]"
            , "head > link[rel=canonical]"
            };
        html_metadata = String.join(", ", metadata_tags);
    }

    static String error_string(String component, String error) {
        return String.join(": ", "Error", component, error);
    }

    static void error_print(String component, String error) {
        System.err.println(error_string(component, error));
    }

    static void help(Options options, String error) {
        StringWriter sw = new StringWriter();
        StringBuffer sb = sw.getBuffer();
        PrintWriter  pw = new PrintWriter(sw, true);

        HelpFormatter formatter = new HelpFormatter();

        formatter.printUsage(pw, 80, "BoilerpipeHandler", options);

        sb.deleteCharAt(sb.length() - 1);
        sb.append(" input-path");
        sb.append("\n\n");

        sb.append("Boilerpipe command-line wrapper");
        sb.append("\n\n");

        sb.append(String.join("\n"
                    , "positional arguments:"
                    , "    input-path"));
        sb.append("\n\n");

        sb.append("optional arguments:\n");

        formatter.printOptions(pw, 80, options, 4, 4);

        if (error != null && error.length() > 0) {
            sb.append("\n" + error_string("Command-line", error) + "\n");
        }

        System.err.print(sw.toString());
    }

    static Options args_build() {
        Options options = new Options();

        Option option_help = Option.builder("h")
            .longOpt("help")
            .desc("display help")
            .required(false)
            .build();

        Option option_output = Option.builder("o")
            .longOpt("output")
            .desc("output path")
            .required(false)
            .hasArg()
            .argName("output-path")
            .build();

        Option option_extractor = Option.builder("e")
            .longOpt("extractor")
            .desc("extractor, default 'ArticleExtractor'")
            .required(false)
            .hasArg()
            .argName("extractor")
            .build();

        Option option_html = Option.builder("r")
            .longOpt("html")
            .desc("html output suitable for recoll")
            .required(false)
            .build();

        options.addOption(option_help);
        options.addOption(option_output);
        options.addOption(option_extractor);
        options.addOption(option_html);

        return options;
    }

    static CommandLine args_parse(Options options, String[] args) throws ParseException {
        CommandLineParser parser = new DefaultParser();

        CommandLine commandline = parser.parse(options, args);

        return commandline;
    }

    static void process_help(BoilerpipeArguments arguments, Options options) {
        if (arguments.help()) {
            help(options, null);
            System.exit(0);
        }
    }

    static String process_boilerpipe(BoilerpipeArguments arguments)
        throws ParseException,
               IOException,
               BoilerpipeProcessingException {
        Charset charset = arguments.input_charset();

        CheckedInputStream stream = arguments.input_stream();

        BufferedReader reader = new BufferedReader
            ( new InputStreamReader
                ( stream
                , charset
                )
            );

        String text = arguments.extractor().getText(reader);

        stream.invalidate();

        arguments.input().seek(0);

        return text;
    }

    static String process_html(BoilerpipeArguments arguments)
        throws ParseException,
               IOException,
               BoilerpipeProcessingException {
        Charset charset = arguments.input_charset();

        CheckedInputStream stream = arguments.input_stream();

        Document document_source = Jsoup.parse(stream, charset.name(), "");

        stream.invalidate();

        arguments.input().seek(0);

        Document document_target = Document.createShell("");

        document_target.charset(arguments.output_charset());

        Element head_source = document_source.getElementsByTag("head").first();

        if (head_source != null) {
            Element head_target = document_target.getElementsByTag("head").first();

            assert (head_target != null);

            for(Element elem : head_source.select(html_metadata)) {
                if (elem.hasAttr("http-equiv") || elem.hasAttr("charset")) {
                    continue;
                }

                head_target.appendChild(elem);
            }
        }

        String text = process_boilerpipe(arguments);

        Element body = document_target.getElementsByTag("body").first();

        assert (body != null);

        body.text(text);

        return document_target.toString();
    }

    static String ensure_eol_terminated(String s) {
        if (!s.endsWith(System.lineSeparator())) {
            s = s + System.lineSeparator();
        }
        return s;
    }

    public static void main(String[] args) {
        Options options = args_build();

        try {
            CommandLine commandline = args_parse(options, args);

            BoilerpipeArguments arguments = new BoilerpipeArguments(commandline);

            process_help(arguments, options);

            String text;

            if (arguments.html()) {
                text = process_html(arguments);
            } else {
                text = process_boilerpipe(arguments);
            }

            text = ensure_eol_terminated(text);

            arguments.output().write(text);

            arguments.close();
        }
        catch (ParseException e) {
            help(options, e.getMessage());
            System.exit(1);
        }
        catch (IOException e) {
            error_print("IO", e.getMessage());
            System.exit(2);
        }
        catch (IllegalCharsetNameException|UnsupportedCharsetException e) {
            error_print("Charset", e.getMessage());
            System.exit(3);
        }
        catch (BoilerpipeProcessingException e) {
            error_print("Boilerpipe", e.getMessage());
            System.exit(4);
        }
    }
}

Manifest-version: 1.0
Main-Class: BoilerpipeHandler
Class-Path: /usr/share/java/apache-commons-cli.jar /usr/share/java/boilerpipe.jar /usr/share/java/nekohtml.jar /usr/share/java/xerces-j2.jar /usr/share/java/jsoup/jsoup.jar /usr/share/java/juniversalchardet.jar

all: BoilerpipeHandler.jar

BoilerpipeHandler.jar: BoilerpipeHandler.manifest BoilerpipeHandler.class BoilerpipeArguments.class CheckedInputStream.class UnrecognizedArgumentException.class
	jar -cfm $@ $^

BoilerpipeHandler.class: BoilerpipeHandler.java
	javac -cp "/usr/share/java/apache-commons-cli.jar:/usr/share/java/boilerpipe.jar:/usr/share/java/jsoup/jsoup.jar:/usr/share/java/juniversalchardet.jar:." BoilerpipeHandler.java

clean:
	rm *.jar *.class