QueryUtil

/*===========================================================================
  Copyright (C) 2009-2013 by the Okapi Framework contributors
-----------------------------------------------------------------------------
  Licensed under the Apache License, Version 2.0 (the "License");
  you may not use this file except in compliance with the License.
  You may obtain a copy of the License at

  http://www.apache.org/licenses/LICENSE-2.0

  Unless required by applicable law or agreed to in writing, software
  distributed under the License is distributed on an "AS IS" BASIS,
  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  See the License for the specific language governing permissions and
  limitations under the License.
===========================================================================*/

package net.sf.okapi.lib.translation;

import java.util.ArrayList;
import java.util.Collections;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Stack;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.w3c.dom.Element;
import org.w3c.dom.NamedNodeMap;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;

import net.sf.okapi.common.HTMLCharacterEntities;
import net.sf.okapi.common.Util;
import net.sf.okapi.common.filterwriter.XLIFFContent;
import net.sf.okapi.common.query.QueryResult;
import net.sf.okapi.common.resource.Code;
import net.sf.okapi.common.resource.TextFragment;
import net.sf.okapi.common.resource.TextFragment.TagType;

/**
 * Collection of helper method for preparing and querying translation resources.
 *
 *  @version 1.47.0
 *  SV: changed the exception in translateCodedHTMLToFragment() to a warning
 */
public class QueryUtil {
	private static final Pattern TAG = Pattern.compile("(<(br|u)(\\s+)id=['\"](.*?)['\"](\\s*?)/?>)|(</u>)", Pattern.CASE_INSENSITIVE);
	// Check also <span> as some engines add their own in the results
	private static final Pattern HTML_SPAN = Pattern.compile("\\<span\\s*?(.*?)>|\\</span>",
			Pattern.CASE_INSENSITIVE | Pattern.DOTALL);

	private static final Pattern CR = Pattern.compile("&(?:#(\\S+?)|\\w*?);");
	private final Logger logger = LoggerFactory.getLogger(getClass());

	private final StringBuilder codesMarkers;
	private final XLIFFContent fmt;
	private List<Code> codes;
	private HTMLCharacterEntities entities;

	public QueryUtil () {
		codesMarkers = new StringBuilder();
		fmt = new XLIFFContent();
	}

	/**
	 * Indicates if the last text fragment passed to {@link #separateCodesFromText(TextFragment)} has codes or not.
	 *
	 * @return true if the fragment has one or more code, false if it does not.
	 */
	public boolean hasCode () {
		if ( codes == null ) return false;
		return !codes.isEmpty();
	}

	/**
	 * Separate and store codes of a given text fragment.
	 *
	 * @param frag
	 *            the fragment to process. Use {@link #createNewFragmentWithCodes(String)} to
	 *            reconstruct the text back with its codes at the end.
	 * @return the fragment content stripped of its codes.
	 */
	public String separateCodesFromText (TextFragment frag) {
		// Reset
		codesMarkers.setLength(0);
		codes = frag.getCodes();
		// Get coded text
		String text = frag.getCodedText();
		if (!frag.hasCode()) {
			return text; // No codes
		}

		// If there are codes: store them apart
		StringBuilder tmp = new StringBuilder(text.length() - (codes.size() * 2));
		for (int i = 0; i < text.length(); i++) {
			switch (text.charAt(i)) {
			case TextFragment.MARKER_OPENING:
			case TextFragment.MARKER_CLOSING:
			case TextFragment.MARKER_ISOLATED:
				codesMarkers.append(text.charAt(i));
				codesMarkers.append(text.charAt(++i));
				break;
			default:
				tmp.append(text.charAt(i));
			}
		}
		// Return text without codes
		return tmp.toString();
	}

	/**
	 * Appends the codes stored apart using {@link #separateCodesFromText(TextFragment)} at the end of a given plain
	 * text. The text fragment provided must be the same and without code modifications, as the one used for the
	 * splitting.
	 *
	 * @param plainText
	 *            new text to use (must be plain)
	 * @return the provided fragment, but with the new text and the original codes appended at the end.
	 */
	public TextFragment createNewFragmentWithCodes (String plainText) {
		return new TextFragment(plainText + codesMarkers, codes);
	}

    /**
     * Converts from coded texts to coded HTML.
     * The resulting strings are also valid XML.
     * @param frags the fragments to convert.
     * @return The resulting HTML string.
     */
    public List<String> toCodedHTML(List<TextFragment> frags) {
        List<String> html = new ArrayList<>(frags.size());
        for (TextFragment frag : frags) {
            html.add(toCodedHTML(frag));
        }
        return html;
    }

	/**
	 * Converts from coded text to coded HTML.
	 * The resulting string is also valid XML.
	 * @param fragment
	 *            the fragment to convert.
	 * @return The resulting HTML string.
	 */
	public String toCodedHTML (TextFragment fragment) {
		if ( fragment == null ) {
			return "";
		}
		Code code;
		String text = fragment.getCodedText();
		StringBuilder sb = new StringBuilder();
		for ( int i = 0; i < text.length(); i++ ) {
			switch (text.charAt(i)) {
			case TextFragment.MARKER_OPENING:
				code = fragment.getCode(text.charAt(++i));
				sb.append("<u id='");
				sb.append(code.getId());
				sb.append("'>");
				break;
			case TextFragment.MARKER_CLOSING:
				i++;
				sb.append("</u>");
				break;
			case TextFragment.MARKER_ISOLATED:
				code = fragment.getCode(text.charAt(++i));
				switch ( code.getTagType() ) {
				case OPENING:
					sb.append("<br id='b");
					sb.append(code.getId());
					sb.append("'/>");
					break;
				case CLOSING:
					sb.append("<br id='e");
					sb.append(code.getId());
					sb.append("'/>");
					break;
				case PLACEHOLDER:
					sb.append("<br id='p");
					sb.append(code.getId());
					sb.append("'/>");
					break;
				}
				break;
			case '&':
				sb.append("&amp;");
				break;
			case '<':
				sb.append("&lt;");
				break;
			default:
				sb.append(text.charAt(i));
			}
		}
		return sb.toString();
	}

	/**
	 * Converts an HTML string created with {@link #toCodedHTML(TextFragment)} back into a text fragment,
	 * but with empty inline codes.
	 * @param text the HTML string to convert.
	 * @param fragment an optional text fragment where to place the converted content. Any existing codes will be
	 * replaced or removed by the codes coming from the HTML string. Use null to create a new fragment.
	 * @return the modified or created text fragment.
	 */
	public TextFragment fromCodedHTMLToFragment (String text,
		TextFragment fragment)
	{
		if ( Util.isEmpty(text) ) {
			if ( fragment != null ) {
				fragment.setCodedText("", true);
				return fragment;
			}
			else {
				return new TextFragment();
			}
		}

		text = unescapeCharacterReferences(text);

		ArrayList<Code> codes = new ArrayList<>();
		text = this.translateCodedHTMLToFragment(text, codes);

		// Remove any span elements that may have been added
		// (some MT engines mark up their output with extra info)
		StringBuilder sb = new StringBuilder(text);
		removeSpans(sb);

		// Create the fragment or update the existing one
		if ( fragment != null ) {
			fragment.setCodedText(sb.toString(), codes, true);
			return fragment;
		}
		else {
			return new TextFragment(sb.toString(), codes);
		}
	}

	private String unescapeCharacterReferences(String text) {
		initHTMLCharacterEntities();

		Matcher crMatch = CR.matcher(text);
		StringBuilder result = new StringBuilder(text.length());

		int textIndex = 0;
		String numeric;
		int character;
		while (crMatch.find()) {
			if (crMatch.start() > 0) {
				result.append(text, textIndex, crMatch.start());
			}
			textIndex = crMatch.end();

			// numeric character references
			numeric = crMatch.group(1);
			if (numeric != null) {
				try {
					if (numeric.charAt(0) == 'x') {
						// Hexadecimal
						character = Integer.parseInt(numeric.substring(1), 16);
					} else {
						// Decimal
						character = Integer.parseInt(numeric);
					}
				} catch (NumberFormatException e) {
					character = '?';
				}
				result.append((char) character);
				continue;
			}

			// character entity references
			character = entities.lookupReference(crMatch.group(0));
			if ( character == -1 ) {
				// Unknown entity
				// TODO: replace by something meaningful
				continue;
			}
			result.append((char) character);
		}
		result.append(text.substring(textIndex));

		return result.toString();
	}

	private String translateCodedHTMLToFragment(String text, ArrayList<Code> codes) {
		Matcher tagMatches = TAG.matcher(text);
		StringBuilder result = new StringBuilder(text.length());

		Code code;
		int codeId;
		int textIndex = 0;
		while (tagMatches.find()) {
			if (tagMatches.start() > 0) {
				result.append(text, textIndex, tagMatches.start());
			}
			textIndex = tagMatches.end();

			// Closing Tag
			if (tagMatches.group(1) == null) {
				// Code ID should be resolved automatically
				code = new Code(TagType.CLOSING, "Xpt", null);
				codes.add(code);

				// Write code marker
				result.append((char) TextFragment.MARKER_CLOSING);
				result.append(TextFragment.toChar(codes.size() - 1));
				continue;
			}

			// Opening Tag
			if ("u".equals(tagMatches.group(2))) {
				// Get code id from attribute
				codeId = Util.strToInt(tagMatches.group(4), -1);

				// Create the code
				code = new Code(TagType.OPENING, "Xpt", null);
				code.setId(codeId);
				codes.add(code);

				// Write code marker
				result.append((char) TextFragment.MARKER_OPENING);
				result.append(TextFragment.toChar(codes.size() - 1));
				continue;
			}

			// Get code id from attribute
			codeId = Util.strToInt(tagMatches.group(4).substring(1), -1);

			// Isolated Tag
			TagType tagType = TagType.PLACEHOLDER;
			char c = tagMatches.group(4).charAt(0);
			switch (c) {
				case 'b':
					tagType = TagType.OPENING;
					break;
				case 'e':
					tagType = TagType.CLOSING;
					break;
				case 'p':
					tagType = TagType.PLACEHOLDER;
					break;
				default:
					logger.warn(String.format("ID ('%s') of isolated code modified.", c));
			}
			code = new Code(tagType, "Xph", null);
			code.setId(codeId);
			codes.add(code);

			// Write code marker
			result.append((char) TextFragment.MARKER_ISOLATED);
			result.append(TextFragment.toChar(codes.size() - 1));
		}
		result.append(text.substring(textIndex));

		return result.toString();
	}

	/**
	 * Converts back a coded HTML to a coded text.
	 * (extra span elements are removed).
	 *
	 * @param htmlText the coded HTML to convert back.
	 * @param fragment the original text fragment.
	 * @param addMissingCodes true to added codes that are in the original fragment but not in the HTML string.
	 * @return the coded text with its code markers.
	 */
	public String fromCodedHTML (String htmlText,
		TextFragment fragment,
		boolean addMissingCodes)
	{
		return fromCodedHTML(htmlText, fragment, addMissingCodes, true);
	}

	/**
	 * Converts back a coded HTML to a coded text.
	 *
	 * @param htmlText the coded HTML to convert back.
	 * @param fragment the original text fragment.
	 * @param addMissingCodes true to added codes that are in the original fragment but not in the HTML string.
	 * @param removeSpans true to remove extra span HTML codes.
	 * @return the coded text with its code markers.
	 */
	public String fromCodedHTML (String htmlText,
		TextFragment fragment,
		boolean addMissingCodes,
		boolean removeSpans)
	{
		if ( Util.isEmpty(htmlText) ) {
			return "";
		}

		htmlText = unescapeCharacterReferences(htmlText);

		// Create a lists to verify the codes
		ArrayList<String> newCodes = new ArrayList<>(fragment.getCodes().size());
		String codedText = translateCodedHTML(fragment, htmlText, newCodes);

		StringBuilder sb = new StringBuilder(codedText);

		// Remove any span elements that may have been added
		// (some MT engines mark up their output with extra info)
		if ( removeSpans ) {
			removeSpans(sb);
		}

		// Try to correct missing codes
		if ( addMissingCodes) {
			ArrayList<String> oriCodes = getCodeIdAttrValues(fragment);
			if ( newCodes.size() < oriCodes.size() ) {
				for (String tmp : oriCodes) {
					if (!newCodes.contains(tmp)) {
						switch (tmp.charAt(0)) {
							case 'o':
								sb.append((char) TextFragment.MARKER_OPENING);
								sb.append(TextFragment.toChar(fragment.getIndex(Integer.parseInt(tmp.substring(1)))));
								break;
							case 'c':
								sb.append((char) TextFragment.MARKER_CLOSING);
								sb.append(TextFragment.toChar(fragment.getIndexForClosing(Integer.parseInt(tmp.substring(1)))));
								break;
							case 'b':
							case 'e':
							case 'p':
								sb.append((char) TextFragment.MARKER_ISOLATED);
								sb.append(TextFragment.toChar(fragment.getIndex(Integer.parseInt(tmp.substring(1)))));
								break;
						}
					}
				}
			}
		}

		return sb.toString();
	}

	private static void removeSpans(StringBuilder sb) {
		Matcher m = HTML_SPAN.matcher(sb.toString());
		int offset = 0;
		while ( m.find() ) {
			sb.delete(m.start() - offset, m.end() - offset);
			offset += m.end() - m.start();
		}
	}

	private void initHTMLCharacterEntities() {
		if ( entities == null ) {
			entities = new HTMLCharacterEntities();
			entities.ensureInitialization(true);
		}
	}

	private static String translateCodedHTML(TextFragment fragment, String text, ArrayList<String> newCodes) {
		Stack<Integer> stack = new Stack<>();

		Matcher tagMatches = TAG.matcher(text);
		StringBuilder result = new StringBuilder(text.length());

		int codeId;
		int textIndex = 0;
		char isoType;
		while (tagMatches.find()) {
			if (tagMatches.start() > 0) {
				result.append(text, textIndex, tagMatches.start());
			}
			textIndex = tagMatches.end();

			// Closing Tag
			if (tagMatches.group(1) == null) {
				if ( stack.isEmpty() ) {
					// If the stack is empty it means the string is not well-formed, or a start tag is missing.
					// The two codes will be added automatically at the end of the entry
					continue;
				}

				codeId = stack.pop();
				newCodes.add("c" + codeId);

				result.append((char) TextFragment.MARKER_CLOSING);
				result.append(TextFragment.toChar(fragment.getIndexForClosing(codeId)));
				continue;
			}

			// Opening Tag
			if ("u".equals(tagMatches.group(2))) {
				// It's an opening tag
				codeId = Util.strToInt(tagMatches.group(4), -1);
				stack.push(codeId);
				newCodes.add("o" + codeId);

				result.append((char) TextFragment.MARKER_OPENING);
				result.append(TextFragment.toChar(fragment.getIndex(codeId)));
				continue;
			}

			// Isolated Tag
			isoType = tagMatches.group(4).charAt(0);
			codeId = Util.strToInt(tagMatches.group(4).substring(1), -1);
			newCodes.add(isoType + Integer.toString(codeId));

			result.append((char) TextFragment.MARKER_ISOLATED);
			switch (isoType) {
				case 'b':
					result.append(TextFragment.toChar(fragment.getIndexForOpening(codeId)));
					break;
				case 'e':
					result.append(TextFragment.toChar(fragment.getIndexForClosing(codeId)));
					break;
				default:
					result.append(TextFragment.toChar(fragment.getIndex(codeId)));
					break;
			}
		}
		result.append(text.substring(textIndex));

		return result.toString();
	}

	private static ArrayList<String> getCodeIdAttrValues(TextFragment fragment) {
		ArrayList<String> oriCodes = new ArrayList<>(fragment.getCodes().size());
		for ( Code code : fragment.getCodes() ) {
			switch ( code.getTagType() ) {
			case OPENING:
				oriCodes.add("o" + code.getId());
				break;
			case CLOSING:
				oriCodes.add("c" + code.getId());
				break;
			case PLACEHOLDER:
				switch ( code.getTagType() ) {
				case OPENING:
					oriCodes.add("b" + code.getId());
					break;
				case CLOSING:
					oriCodes.add("e" + code.getId());
					break;
				case PLACEHOLDER:
					oriCodes.add("p" + code.getId());
					break;
				}
				break;
			}
		}
		return oriCodes;
	}

	/**
	 * Converts from coded text to XLIFF.
	 *
	 * @param fragment
	 *            the fragment to convert.
	 * @return The resulting XLIFF string.
	 * @see #fromXLIFF(Element, TextFragment)
	 */
	public String toXLIFF (TextFragment fragment) {
		if ( fragment == null ) {
			return "";
		}
		fmt.setContent(fragment);
		return fmt.toString();
	}

	// /**
	// * Converts back an XLIFF text to a coded text.
	// * @param text the XLIFF text to convert back.
	// * @param fragment the original text fragment.
	// * @return the coded text with its code markers.
	// */
	// public String fromXLIFF (String text,
	// TextFragment fragment)
	// {
	// if ( Util.isEmpty(text) ) return "";
	// // Un-escape first layer
	// text = text.replace("&apos;", "'");
	// text = text.replace("&lt;", "<");
	// text = text.replace("&gt;", ">");
	// text = text.replace("&quot;", "\"");
	// text = text.replace("&amp;", "&");
	// // Now we have XLIFF valid content
	//
	// // Read it to XML parser
	// // Un-escape XML
	//
	// //TODO: code conversion
	// return text;
	// }

	/**
	 * Converts back an XLIFF text contained in a given element into a TextFragment.
	 *
	 * @param elem
	 *            The element containing the XLIFF data.
	 * @param original
	 *            the original TextFragment (cannot be null).
	 * @see #toXLIFF(TextFragment)
	 * @return the newly created text fragment.
	 */
	public TextFragment fromXLIFF (Element elem,
		TextFragment original)
	{
		NodeList list = elem.getChildNodes();
		int lastId = -1;
		int id = -1;
		Node node;
		Stack<Integer> stack = new Stack<>();
		StringBuilder buffer = new StringBuilder();

		// Note that this parsing assumes non-overlapping codes.
		for ( int i = 0; i < list.getLength(); i++ ) {
			node = list.item(i);
			switch ( node.getNodeType() ) {
			case Node.TEXT_NODE:
				buffer.append(node.getNodeValue());
				break;
			case Node.ELEMENT_NODE:
				NamedNodeMap map = node.getAttributes();
				switch (node.getNodeName()) {
					case "bpt":
						id = getRawIndex(lastId, map.getNamedItem("id"));
						stack.push(id);
						buffer.append(new String(new char[]{TextFragment.MARKER_OPENING,
								TextFragment.toChar(original.getIndex(id))}));
						break;
					case "ept":
						buffer.append(new String(new char[]{ TextFragment.MARKER_CLOSING,
								TextFragment.toChar(original.getIndexForClosing(stack.pop()))}));
						break;
					case "ph":
						id = getRawIndex(lastId, map.getNamedItem("id"));
						buffer.append(new String(new char[]{ TextFragment.MARKER_ISOLATED,
								TextFragment.toChar(original.getIndex(id))}));
						break;
					case "it":
						Node pos = map.getNamedItem("pos");
						if (pos == null) { // Error, but just treat it as a placeholder
							id = getRawIndex(lastId, map.getNamedItem("id"));
							buffer.append(new String(new char[]{ TextFragment.MARKER_ISOLATED,
									TextFragment.toChar(original.getIndex(id))}));
						} else if (pos.getNodeValue().equals("begin")) {
							id = getRawIndex(lastId, map.getNamedItem("id"));
							buffer.append(new String(new char[]{ TextFragment.MARKER_OPENING,
									TextFragment.toChar(original.getIndex(id))}));
						} else { // Assumes 'end'
							id = getRawIndex(lastId, map.getNamedItem("id"));
							buffer.append(new String(new char[]{ TextFragment.MARKER_CLOSING,
									TextFragment.toChar(original.getIndexForClosing(id))}));
						}
						break;
				}
				break;
			}
		}

		return new TextFragment(buffer.toString(), original.getCodes());
	}

	/**
	 * Removes duplicates based on the Equals method of {@link QueryResult}.
	 * Preserve the highest ranked query results.
	 * For example, duplicate results with the newest creation date are always retained.<br>
	 * <b>WARNING:</b> order is not preserved!
	 *
	 * @param queryResults
	 *            the list of QueryResults to process.
	 * @return a new list of the QueryResults without duplicates.
	 */
	public static ArrayList<QueryResult> removeDuplicates (List<QueryResult> queryResults) {
		// Add QueryResults to linked hash in ranked order
		// to make sure we keep the highest ranked duplicates
		LinkedHashSet<QueryResult> dupRemove = new LinkedHashSet<>(queryResults.size());
		Collections.sort(queryResults);
		dupRemove.addAll(queryResults);
		return new ArrayList<>(dupRemove);
	}

	private int getRawIndex (int lastIndex, Node attr)
	{
		if ( attr == null ) {
			return ++lastIndex;
		}
		return Integer.parseInt(attr.getNodeValue());
	}
}