package werti.server;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.PrintWriter;
import java.io.OutputStreamWriter;
import java.io.FileOutputStream;
import java.io.FileWriter;
import java.net.URL;
import java.util.Arrays;
import java.util.Enumeration;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.Locale;
import javax.servlet.RequestDispatcher;
import javax.servlet.ServletConfig;
import javax.servlet.ServletException;
import javax.servlet.http.HttpServlet;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import org.apache.commons.lang.StringEscapeUtils;
import org.apache.logging.log4j.Logger;
import org.apache.logging.log4j.LogManager;
import org.apache.uima.jcas.JCas;
import org.jsoup.Connection;
import org.jsoup.parser.Parser;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Node;
import org.jsoup.nodes.Document;
import org.openid4java.OpenIDException;
import org.openid4java.consumer.ConsumerException;
import org.openid4java.consumer.ConsumerManager;
import org.openid4java.discovery.DiscoveryException;
import org.openid4java.discovery.DiscoveryInformation;
import org.openid4java.discovery.Identifier;
import org.openid4java.message.AuthRequest;
import org.openid4java.message.MessageException;
import werti.WERTiContext;
import werti.WERTiContext.WERTiContextException;
import werti.util.ActivitiesSessionLoader;
import werti.util.HTMLEnhancer;
import werti.util.HTMLUtils;
import werti.util.JSONEnhancer;
import werti.util.PageHandler;
import werti.util.PostRequest;
import werti.util.PracticeHandler;
import com.google.gson.Gson;
import weka.core.Instances;
import weka.filters.Filter;
import werti.util.Constants;
import java.io.File;
import java.io.FileInputStream;
import java.io.InputStream;
import java.io.OutputStream;
import java.util.Iterator;
import javax.servlet.ServletContext;
import javax.servlet.ServletOutputStream;
//import javax.servlet.annotation.WebServlet;
import javax.servlet.http.HttpSession;
/**
* The server side implementation of the WERTi service.
*
* This is where the work is coordinated. The rough outline of the procedure is as follows:
*
*
* - We take a request via the doGet() (web form) or doPost() (add-on) methods.
* - For web form requests, the HTML source of the URL is fetched and spans of text are
* identified.
* - The document is processed in UIMA, invoking the pre- and postprocessors for the current
* topic.
* - Afterwards, we take the resulting CAS and insert enhancement annotations
* (WERTi-<span>s) according to the target annotations from the
* postprocessor.
*
* @author Aleksandar Dimitrov
* @author Adriane Boyd
*/
public class WERTiServlet extends HttpServlet {
private static final Logger log =
LogManager.getLogger(WERTiServlet.class);
public static final String outputfileLoc = "InputLog.txt";
public static WERTiContext context;
// maximum amount of of ms to wait for a web-page to load
private static final int MAX_WAIT = 1000 * 20; // 20 seconds, was: 10
public static final long serialVersionUID = 10;
public static final Set supportedVersions = new HashSet(Arrays.asList("0.10"));
private Processors processors;
public static OpenIDConsumer openidConsumer = null;
public static String enhancement_type; // colorize, click, mc or cloze
public void init(ServletConfig config) throws ServletException {
super.init(config);
log.warn("Initializing servlet.");
// initialise servletcontext
try {
WERTiContext.init(config);
} catch (WERTiContextException wce) {
log.fatal("Context failed to initialize.");
log.fatal(wce);
}
}
public void destroy() {
// no-op
}
/* (non-Javadoc)
* @see javax.servlet.http.HttpServlet#doGet(javax.servlet.http.HttpServletRequest, javax.servlet.http.HttpServletResponse)
*/
@Override
protected void doGet(HttpServletRequest req, HttpServletResponse resp)
throws ServletException, IOException {
req.setCharacterEncoding("UTF-8");
resp.setCharacterEncoding("UTF-8");
resp.setContentType( "text/html" );
HttpSession session = req.getSession( true );
if ( session.getValue( "waitPage" ) == null ) {
session.putValue( "waitPage", Boolean.TRUE );
try {
PrintWriter out = resp.getWriter();
out.println( "" );
out.println( "Vuorddes..." );
out.println( "" );
out.println( "" );
out.println( "
" );
out.println( "Prográmma lea bargame.
" );
out.println( "Vuorddes...
" );
out.println( "
" );
out.close();
} catch (IOException ioe) {
log.error("Failed to write to temporary wait file");
throw new ServletException("", ioe);
}
} else {
session.removeValue( "waitPage" );
long startTime = System.currentTimeMillis();
log.debug("received GET request");
//req.setCharacterEncoding("UTF-8");
//resp.setCharacterEncoding("UTF-8");
// OpenID verification request
if ("true".equals(req.getParameter("openid_return"))) {
if (openidConsumer == null) {
String openidReturnToUrl = getOpenIDReturnToUrl(req);
openidConsumer = new OpenIDConsumer(openidReturnToUrl, this);
}
Identifier verified = openidConsumer.verifyResponse(req);
if (verified != null) {
resp.sendRedirect(getServletBaseUrl(req) + "/openid/return.jsp?openid.identity=" + verified.getIdentifier());
}
else {
resp.sendRedirect(getServletBaseUrl(req) + "/openid/verification-failed.jsp");
}
return;
}
String url = req.getParameter("url");
// accept url-s without http://
log.info("url:{}", url);
if (!url.startsWith("file:/")) {
if (!url.contains("http")) {
url = "http://" + url;
}
}
String activity = req.getParameter("activity");
String enhancement = req.getParameter("client.enhancement");
//log.info("enhancement type {}", enhancement);
enhancement_type = enhancement;
String lang = req.getParameter("language");
if (lang == null) {
lang = "en";
}
ActivityConfiguration config = loadActivitiesAndProcessors(req, activity); //track this
log.info("config:{}", config);
// merge config with request parameters
mergeConfigParams(config, req);
URL u = new URL(url);
log.info("URL again:{}", u);
Document htmlDoc;
try {
if (!url.startsWith("file:/")) {
/*
String myurl = "https://avvir.no/oddasat/2017/03/oddasit-lihcco-boine";
Document mydocument = Jsoup.connect(myurl).data("query", "Java").userAgent("Mozilla").cookie("auth", "token").timeout(3000).get();
log.info("mydocument={}", mydocument);*/
htmlDoc = Jsoup.parse(u, MAX_WAIT);
} else {
File myinput = new File(url.substring(7,url.length()));
htmlDoc = Jsoup.parse(myinput, "UTF-8"); //Document
}
//Commenting next ln to reduce output in catalina.out
//log.info("page source:{}", htmlDoc);
} catch (IOException ioe) {
throw new ServletException("Webpage retrieval failed.");
}
HTMLUtils.markTextNodes(htmlDoc, htmlDoc.body());
// TODO: potentially modify jsoup to return unescaped text so that this hack
// can be removed
String htmlString = spansToETags(htmlDoc, HTMLUtils.className, false);
String anl_dir_path = getServletContext().getInitParameter("files_anl_dir");
PageHandler ph = new PageHandler(processors, activity, url.replace("/","-"), anl_dir_path, htmlString, lang);
JCas cas;
cas = ph.process();
if (cas != null) {
HTMLEnhancer ge = new HTMLEnhancer(cas);
String result = ge.enhance(activity, u.toString(), req, config, getServletContext().getServletContextName());
log.info("Web ({}): {}, {}, {}, {}, {}", (System.currentTimeMillis() - startTime), req.getParameter("language"), activity, req.getParameter("client.enhancement"), url, cas.getDocumentLanguage());
// Write the url, topic and enhancement type into the file as well:
FileWriter outputfile = new FileWriter(outputfileLoc,true); //the true will append the new data
try {
outputfile.write("Topic: " + activity + ", exercise type: " + enhancement + ", URL: " + url + "\n");
}
finally {
outputfile.close();
}
try {
PrintWriter out = resp.getWriter();
out.write(result);
out.close();
} catch (IOException ioe) {
log.error("Failed to write to temporary result file");
throw new ServletException("", ioe);
}
} else {
throw new ServletException("The selected language/topic/activity combination is not currently available.");
}
}
}
/**
* Annotate according to the topic/activity/text provided in a JSON PostRequestObject.
*
* @param req the servlet request
* @param resp the servlet response
*/
@Override
protected void doPost(HttpServletRequest req, HttpServletResponse resp)
throws ServletException, IOException {
if (req.getParameter("word") != null) {
log.info("LogServlet received POST request");
String message = "", correct = "";
OutputStreamWriter outputfile = new OutputStreamWriter(new FileOutputStream(outputfileLoc, true), "UTF-8");
//true -> the new data will be appended to the end of the file, instead of overwriting the file
String extype = req.getParameter("extype");
String word = req.getParameter("word");
String facit = req.getParameter("facit");
if (req.getParameter("correct").matches("1"))
correct = "yes";
else
correct = "no";
String targetCorrect = req.getParameter("correctly_clicked");
String targetTotal = req.getParameter("total_clicked");
//String targetRatio1 = req.getParameter("ratio1");
//String targetRatio2 = req.getParameter("ratio2");
if (extype.matches("click"))
message = "The clicked word: " + word + ". Correct: " + correct + ". The user has clicked correctly " + targetCorrect + " words out of " + targetTotal + ".";
else
message = "User's answer: " + word + ". Facit: " + facit + ". Correct: " + correct + ". The user has written/chosen correctly " + targetCorrect + " words out of " + targetTotal + ".";
try {
outputfile.write(message + "\n");
}
finally {
outputfile.close();
}
}
else if (req.getParameter("nr_of_exercises") != null) {
log.info("LogServlet received POST request");
OutputStreamWriter outputfile = new OutputStreamWriter(new FileOutputStream(outputfileLoc, true), "UTF-8");
String nr_of_exercises = req.getParameter("nr_of_exercises");
String message = "Number of exercises on the page: " + nr_of_exercises + ".";
try {
outputfile.write(message + "\n");
}
finally {
outputfile.close();
}
}
else {
long startTime = System.currentTimeMillis();
log.debug("received POST request");
// read request in as string
// (gson.fromJson() seems unhappy with req.getReader() as its first argument, don't know why)
String line;
String requestString = "";
BufferedReader reader = req.getReader();
while((line = reader.readLine()) != null) {
requestString += line;
}
// parse this string into an object
Gson gson = new Gson();
PostRequest requestInfo = gson.fromJson(requestString, PostRequest.class);
// check if this version is supported
if (!supportedVersions.contains(requestInfo.version)) {
resp.sendError(490);
log.info("Add-on, version conflict ({}): {}, {}, {}", (System.currentTimeMillis() - startTime), requestInfo.topic, requestInfo.activity, requestInfo.url);
return;
}
// if this is an OpenID authentication request, handle it separately
if (requestInfo.type.matches("openid-authentication")) {
String userSuppliedIdentifier = requestInfo.url;
// TODO do something with this info
log.debug("requestInfo.document: {}" requestInfo.document);
if (openidConsumer == null) {
String openidReturnToUrl = getOpenIDReturnToUrl(req);
openidConsumer = new OpenIDConsumer(openidReturnToUrl, this);
}
openidConsumer.authRequest(userSuppliedIdentifier, req, resp);
return;
}
String lang = requestInfo.language;
if (lang == null) {
lang = "en";
}
ActivityConfiguration config = loadActivitiesAndProcessors(req, requestInfo.topic);
// check if the requested topic exists
if (config == null) {
resp.sendError(491);
log.info("Add-on, topic doesn't exist ({}): {}, {}, {}, {}", (System.currentTimeMillis() - startTime), lang, requestInfo.topic, requestInfo.activity, requestInfo.url);
return;
}
// check if the language-topic combination exists
if (config.getPreDesc(lang) == null || config.getPostDesc(lang) == null) {
resp.sendError(492);
log.info("Add-on, topic doesn't exist for language ({}): {}, {}, {}, {}", (System.currentTimeMillis() - startTime), lang, requestInfo.topic, requestInfo.activity, requestInfo.url);
return;
}
/* // disallow passives
if (requestInfo.topic.equals("Passives")) {
resp.sendError(491);
return;
}*/
// set enhancement type
config.setClientValue(lang, "enhancement", requestInfo.activity);
// extract the wertiview spans from the document
Document doc = Jsoup.parse(requestInfo.document);
String htmlString = spansToETags(doc, "wertiview", true);
String result = "";
// handling each type of request
if (requestInfo.type.matches("practice")) {
PracticeHandler ph = new PracticeHandler(requestInfo);
result = ph.process();
} else { // should be a "page" request
String anl_dir_path = getServletContext().getInitParameter("files_anl_dir");
PageHandler ph = new PageHandler(processors, requestInfo.topic, requestInfo.url.replace("/","-"), anl_dir_path, htmlString, lang);
JCas cas;
cas = ph.process();
JSONEnhancer pe = new JSONEnhancer(cas, requestInfo.activity);
result = pe.enhance();
log.info("Add-on ({}): {}, {}, {}, {}, {}", (System.currentTimeMillis() - startTime), requestInfo.language, requestInfo.topic, requestInfo.activity, requestInfo.url, cas.getDocumentLanguage());
}
try { // to write to the response stream
//Locale locale = new Locale("se-NO", "");
resp.setContentType("text/plain"); // charset=UTF-8");
final PrintWriter out = resp.getWriter();
//final PrintWriter out = new PrintWriter(new OutputStreamWriter(resp.getOutputStream(), "UTF8"), true);
out.write(result);
out.close();
} catch (IOException ioe) {
log.error("Error writing to response stream");
throw new ServletException("", ioe);
}
} // else
}
/**
* load the activity.xml files for all topics. Initialize the pre- and
* postprocessors for each.
* @param req the HTTP request
* @param topicName the topic chosen by the user
* @return the configuration of the activity chosen by the user
*/
private ActivityConfiguration loadActivitiesAndProcessors(HttpServletRequest req,
String topicName) throws IOException, ServletException {
// load activities from/into session
Activities acts = ActivitiesSessionLoader.createActivitiesInSession(req);
ActivityConfiguration config = acts.getActivity(topicName);
// load processors if necessary
loadProcessors(acts); // track this
return config;
}
/**
* replace all tags with tags. Copy the
* wertiview IDs if there are any. Turn the HTML character entities inside
* the spans/e-tags into unicode characters.
* @param doc the result of a Jsoup parse
* @param className the name of the class of the relevant spans
* @param haveIds whether the wertiview spans have IDs in the wertiviewid attribute
* @return the node as a string
*/
private String spansToETags(Document doc, String className, boolean haveIds) {
// find all added spans using the class name and replace everything inside
// the tokens with unescaped unicode characters
String htmlString = doc.html();
Pattern enhancePatt = Pattern.compile("(.*?)", Pattern.DOTALL);
Matcher enhanceMatcher = enhancePatt.matcher(htmlString);
while (enhanceMatcher.find()) {
htmlString = htmlString.replace(enhanceMatcher.group(3), StringEscapeUtils.unescapeHtml(enhanceMatcher.group(3)));
}
// replace these spans with tags for use in normal pipeline
if (haveIds) {
htmlString = htmlString.replaceAll(enhancePatt.toString(), "$3");
}
else {
htmlString = htmlString.replaceAll(enhancePatt.toString(), "$3");
}
return htmlString;
}
@SuppressWarnings("unchecked")
private void mergeConfigParams(ActivityConfiguration config, HttpServletRequest req) {
Enumeration paramNames = req.getParameterNames();
String lang = req.getParameter("language");
while (paramNames.hasMoreElements()) {
String key = paramNames.nextElement();
boolean worked = false;
boolean isConfigParam = false;
String value = req.getParameter(key);
if (key.startsWith(ActivityConfiguration.CLIENT_PREFIX)) {
worked = config.setClientValue(lang, key.substring(ActivityConfiguration.CLIENT_PREFIX.length()+1), value);
isConfigParam = true;
} else if (key.startsWith(ActivityConfiguration.PRE_PREFIX)) {
worked = config.setServerPreValue(lang, key.substring(ActivityConfiguration.PRE_PREFIX.length()+1), value);
isConfigParam = true;
} else if (key.startsWith(ActivityConfiguration.POST_PREFIX)) {
worked = config.setServerPostValue(lang, key.substring(ActivityConfiguration.POST_PREFIX.length()+1), value);
isConfigParam = true;
}
if (isConfigParam) {
if (worked) {
log.debug("Successfully set config param: {} to: {}", key, value);
} else {
log.debug("Access denied for config param: {}", key);
}
}
}
}
/**
* If the processors haven't been loaded, load them.
*
* @param acts the list of activities
* @throws IOException
* @throws ServletException
*/
private void loadProcessors(Activities acts) throws IOException, ServletException {
if (processors == null) {
long startTime = System.currentTimeMillis();
processors = new Processors(acts);
log.info("Loaded all UIMA processors ({})", (System.currentTimeMillis() - startTime));
}
}
private static String getServletBaseUrl(HttpServletRequest req) {
String baseUrl = req.getScheme() + "://" + req.getServerName() + ":" + req.getServerPort() + req.getContextPath();
return baseUrl;
}
private static String getOpenIDReturnToUrl(HttpServletRequest req) {
String baseUrl = getServletBaseUrl(req);
String openidReturnToUrl = baseUrl + "/VIEW?openid_return=true";
return openidReturnToUrl;
}
}