Merge pull request #13 from uts-cic/develop

Merge Develop for Initial 0.1 release

.travis.yml

@@ -1,5 +1,8 @@
 language: scala
 scala:
-   - 2.12.0
+   - 2.12.2
 jdk:
-  - oraclejdk8
+  - oraclejdk8
+
+install:
+  sudo apt-get install libc6-i386

README.md

@@ -1,5 +1,7 @@
 [![https://img.shields.io/badge/license-GPLv3-blue.svg](https://img.shields.io/badge/license-GPLv3-blue.svg)](https://github.com/uts-cic/athanor-server/blob/master/LICENSE)
-[![Build Status](https://travis-ci.org/uts-cic/athanor-server.svg?branch=master)](https://travis-ci.org/uts-cic/athanor-server.svg?branch=master)
+
+[![Build Status](https://travis-ci.org/uts-cic/athanor-server.svg?branch=master)](https://travis-ci.org/uts-cic/athanor-server)
+[![Coverage Status](https://coveralls.io/repos/github/uts-cic/athanor-server/badge.svg?branch=master)](https://coveralls.io/github/uts-cic/athanor-server?branch=master)
 
 **IMPORTANT: OPEN SOURCE APPROVAL PENDING. ALL RIGHTS RESERVED BY ORIGINAL OWNERS. PLEASE OPEN AN ISSUE BEFORE FORKING THIS CODE OR CONTACT [@andrewresearch](http://GitHub.com/andrewresearch)**
 

build.sbt

@@ -9,6 +9,9 @@ organization := "au.edu.utscic"
 // ensure that JavaAppPackaging is enabled - disable for Travis CI
 //enablePlugins(JavaAppPackaging)
 
+coverageEnabled := true
+
+
 //Scala library versions
 val akkaVersion = "2.5.3"
 val akkaStreamVersion = "2.5.3"
@@ -22,6 +25,7 @@ val nlytxCommonsVersion = "0.1.1"
 
 //Java library versions
 val coreNlpVersion = "3.8.0"
+val jsonassertVersion = "1.5.0"
 
 //Akka
 libraryDependencies ++= Seq(
@@ -43,11 +47,22 @@ libraryDependencies ++= Seq(
   //  "com.typesafe" % "config" % "1.3.1",
   "org.json4s" %% "json4s-jackson" % json4sVersion,
   "de.heikoseeberger" %% "akka-http-json4s" % akkaHttpJson4sVersion,
+  "org.skyscreamer" % "jsonassert" % jsonassertVersion,
   "org.scalatest" %% "scalatest" % scalatestVersion % "test",
   "org.slf4j" % "jcl-over-slf4j" % slf4jVersion,
   "ch.qos.logback" % "logback-classic" % logbackVersion
 )
 
 scalacOptions in (Compile, doc) ++= Seq("-doc-root-content", baseDirectory.value+"/src/main/scala/root-doc.md")
 
-resolvers += Resolver.bintrayRepo("nlytx", "nlytx_commons")
+resolvers += Resolver.bintrayRepo("nlytx", "nlytx_commons")
+
+coverageMinimum := 70
+
+coverageFailOnMinimum := false
+
+coverageHighlighting := true
+
+publishArtifact in Test := false
+
+parallelExecution in Test := false

project/build.properties

@@ -0,0 +1 @@
+sbt.version = 0.13.15

project/plugins.sbt

@@ -0,0 +1,9 @@
+addSbtPlugin("com.timushev.sbt" % "sbt-updates" % "0.3.0")
+addSbtPlugin("net.virtual-void" % "sbt-dependency-graph" % "0.8.2")
+//addSbtPlugin("org.scalariform" % "sbt-scalariform" % "1.6.0")
+//addSbtPlugin("org.scalastyle" %% "scalastyle-sbt-plugin" % "0.8.0")
+addSbtPlugin("org.scoverage" % "sbt-scoverage" % "1.5.0")
+addSbtPlugin("org.scoverage" % "sbt-coveralls" % "1.1.0")
+addSbtPlugin("com.typesafe.sbt" % "sbt-native-packager" % "1.2.0-M9")
+addSbtPlugin("com.typesafe.sbt" % "sbt-site" % "1.2.0")
+addSbtPlugin("com.typesafe.sbt" % "sbt-ghpages" % "0.5.4")

scripts/apply.kif

@@ -170,7 +170,7 @@ function traverse(vector v, treemapi nodes, vector deps,self surf,synode r) {
 function Apply(string js) {
     lock("apply");
     js=js.trim();
-    printlnerr(_threadhandle(),js[:80]+"...");
+    //printlnerr(_threadhandle(),js[:80]+"...");
     vector  v=js;
 
     //the tokens...
@@ -237,7 +237,7 @@ function Apply(string js) {
     //we return the result as a JSON map structure
     svector vres;
     vres=m.keys();
-    printlnerr("Within athanor:",m,vres);
+    //printlnerr("Within athanor:",m,vres);
     return(vres);
 }
 

src/main/resources/logback.xml

@@ -0,0 +1,14 @@
+<?xml version="1.0" encoding="ISO-8859-1"?>
+<configuration debug="false">
+    <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
+        <encoder>
+            <pattern>%green(%date{yyyyMMdd_HH:mm:ss.SSS, Australia/Sydney}) %highlight(%-5level) %cyan(%logger{36}) %yellow(%X{akkaSource}) - %msg%n</pattern>
+        </encoder>
+    </appender>
+    <logger name="com.zaxxer.hikari.pool.HikariPool" level="INFO" />
+    <logger name="slick.compiler" level="INFO" />
+
+    <root level="DEBUG">
+        <appender-ref ref="STDOUT" />
+    </root>
+</configuration>

src/main/scala/au/edu/utscic/athanorserver/athanor/Athanor.scala

@@ -1,17 +1,17 @@
 package au.edu.utscic.athanorserver.athanor
 
-import au.edu.utscic.athanorserver.data.RhetoricalTypes.{ConstituentTree, Dependencies, LexicalNodes, ParsedSentence}
+import au.edu.utscic.athanorserver.data.RhetoricalImplicits
+import au.edu.utscic.athanorserver.data.RhetoricalTypes._
 import com.typesafe.config.{Config, ConfigFactory}
 import com.xerox.jatanor.JAtanor
 import org.json4s.JsonAST.JValue
+import org.json4s.NoTypeHints
 import org.json4s.jackson.JsonMethods.parse
 import org.json4s.jackson.Serialization
 import org.json4s.jackson.Serialization.write
-import org.json4s.{DefaultFormats, NoTypeHints}
 
 import scala.io.Source
 
-
 /**
   * Created by [email protected] on 28/6/17.
   */
@@ -21,41 +21,41 @@ object Athanor {
   lazy val path: String = config.getString("app.path")
   lazy val athanor = new JAtanor
   lazy val handler = athanor.LoadProgram(program,"")
+  lazy val program: String = fullPath("apply.kif")
+  lazy val testSentence: String = fullPath("sentence.json")
+  lazy val demoFile:String = Source.fromFile(testSentence).getLines.mkString
 
   def fullPath(file:String): String = {
     s"$path/scripts/$file"
   }
 
-  def program: String = fullPath("apply.kif")
-
 
-  def testSentence: String = fullPath("sentence.json")
+  def parseJsonSentence(sent:String):ParsedSentence = {
 
-  def demoFile:String = {
-    Source.fromFile(testSentence).getLines.mkString
-  }
+    import RhetoricalImplicits._
 
-  def demoParsed:ParsedSentence = {
-    implicit val formats = DefaultFormats
-    val sent = demoFile
     val json:JValue = parse(sent)
-    val lexNodes:LexicalNodes = json(0).extract[LexicalNodes]
+    val lexNodes:LexicalNodes = json(0)
     val constTree:ConstituentTree = json(1).extract[ConstituentTree]
     val deps:Dependencies = json(2).extract[Dependencies]
     (lexNodes,constTree,deps)
   }
 
-  def process(json:String):List[String] = {
-    athanor.ExecuteFunctionArray(handler,"Apply",List(json).toArray).toList
+  def parsedSentenceToJsonString(parsedSent:ParsedSentence):String = {
+    implicit val formats = Serialization.formats(NoTypeHints)
+    val l = write(parsedSent._1)
+    val c = write(parsedSent._2)
+    val d = write(parsedSent._3)
+    s"[$l,$c,$d]"
   }
 
-  def process(parsed:ParsedSentence):List[String] = {
-    implicit val formats = Serialization.formats(NoTypeHints)
-    val l = write(parsed._1)
-    val c = write(parsed._2)
-    val d = write(parsed._3)
-    val output:String = s"[$l,$c,$d]"
-    this.process(output)
+  def analyseParsedSentence(parsed:ParsedSentence):List[String] = {
+    val jsonStr:String = parsedSentenceToJsonString(parsed)
+    this.analyseJson(jsonStr)
+  }
+
+  def analyseJson(json:String):List[String] = {
+    athanor.ExecuteFunctionArray(handler,"Apply",List(json).toArray).toList
   }
 
 }

src/main/scala/au/edu/utscic/athanorserver/corenlp/ConstituentTreeParser.scala

@@ -20,7 +20,6 @@ object ConstituentTreeParser {
     import scala.collection.JavaConverters._
     if(tree.numChildren()==0) {
       tree.yieldWords().asScala.map(_.value()).mkString(",")
-      //tree.taggedLabeledYield().asScala.toList.map(_.word())
     }
     else {
       tree.label().toString +:

src/main/scala/au/edu/utscic/athanorserver/corenlp/SentenceParser.scala

@@ -2,13 +2,13 @@ package au.edu.utscic.athanorserver.corenlp
 
 import au.edu.utscic.athanorserver.data.RhetoricalTypes
 import au.edu.utscic.athanorserver.data.RhetoricalTypes._
-import edu.stanford.nlp.ling.CoreAnnotations.{LemmaAnnotation, PartOfSpeechAnnotation, TextAnnotation}
+import edu.stanford.nlp.ling.CoreAnnotations._
 import edu.stanford.nlp.ling.CoreLabel
 import edu.stanford.nlp.semgraph.SemanticGraph
 import edu.stanford.nlp.trees.Tree
 
 import scala.collection.JavaConverters._
-import scala.collection.immutable.ListMap
+import scala.collection.immutable.SortedMap
 
 /**
   * Created by [email protected] on 12/7/17.
@@ -22,18 +22,27 @@ object SentenceParser {
     (ln,ct,dp)
   }
 
+  private def getOpt[T](a:Any):Option[T] = a match {
+    case s:T => Some(s)
+    case null => None
+    case _ => None
+  }
+
   def getNodes(tokens:List[CoreLabel]):LexicalNodes = {
 
       val nodes = tokens.zipWithIndex.map { case(token,i) =>
         val index = i+1
         val word = token.get(classOf[TextAnnotation])
         val lemma = token.get(classOf[LemmaAnnotation])
         val pos = token.get(classOf[PartOfSpeechAnnotation])
-        //val ne = token.get(classOf[NamedEntityTagAnnotation])
-        (index,RhetoricalTypes.Node(index,pos,Some(word),Some(lemma)))
+        val ne = token.get(classOf[NamedEntityTagAnnotation])
+        val spk = token.get(classOf[SpeakerAnnotation])
+        val left = token.beginPosition()
+        val right = token.endPosition()
+        (index,RhetoricalTypes.Node(index,pos,getOpt[String](word),getOpt[String](lemma),getOpt[String](ne),getOpt[String](spk),getOpt[Int](left),getOpt[Int](right)))
       }
 
-    ListMap() ++ nodes
+    SortedMap[Int,Node](0 -> Node(0,"ROOT",None,None,None,None,None,None)) ++ nodes
   }
 
   def getTree(constituentTree: Tree):ConstituentTree = {

src/main/scala/au/edu/utscic/athanorserver/corenlp/TextParser.scala

@@ -10,6 +10,7 @@ import edu.stanford.nlp.semgraph.SemanticGraph
 import edu.stanford.nlp.semgraph.SemanticGraphCoreAnnotations.EnhancedPlusPlusDependenciesAnnotation
 import edu.stanford.nlp.trees.Tree
 import edu.stanford.nlp.trees.TreeCoreAnnotations.TreeAnnotation
+import edu.stanford.nlp.util.CoreMap
 
 import scala.collection.JavaConverters._
 
@@ -19,24 +20,34 @@ import scala.collection.JavaConverters._
 //noinspection SpellCheckingInspection,SpellCheckingInspection,SpellCheckingInspection,SpellCheckingInspection
 object TextParser {
 
-  def parse(text:String):List[ParsedSentence] = {
+  val pipeline:StanfordCoreNLP = {
     val props = new Properties
-    props.setProperty("annotators", "tokenize, ssplit, pos, lemma, parse") // ner, dcoref
-    val pipeline = new StanfordCoreNLP(props)
+    props.setProperty("annotators", "tokenize, ssplit, pos, lemma, parse") //ner dcoref
+    new StanfordCoreNLP(props)
+  }
+
+  def parse(text:String):List[ParsedSentence] = {
+    val sentences = annotateSentences(text)
+    parseSentences(sentences)
+  }
+
+  def annotateSentences(text:String):List[CoreMap] = {
     val document = new Annotation(text)
     pipeline.annotate(document)
+    document.get(classOf[SentencesAnnotation]).asScala.toList
+  }
 
-    val sentences = document.get(classOf[SentencesAnnotation]).asScala.toList
-
+  def parseSentences(sentences:List[CoreMap]):List[ParsedSentence] = {
     sentences.map { sentence =>
-      val tokens:List[CoreLabel] = sentence.get(classOf[TokensAnnotation]).asScala.toList
+      val tokens = getTokens(sentence)
       val tree:Tree = sentence.get(classOf[TreeAnnotation])
       val dependencies:SemanticGraph = sentence.get(classOf[EnhancedPlusPlusDependenciesAnnotation])
       SentenceParser.parse(tokens,tree,dependencies)
     }
-
   }
 
+  def getTokens(sentence:CoreMap):List[CoreLabel] = sentence.get(classOf[TokensAnnotation]).asScala.toList
+
   // This is the coreference link graph
   // Each chain stores a set of mentions that link to each other,
   // along with a method for getting the most representative mention

src/main/scala/au/edu/utscic/athanorserver/data/RhetoricalImplicits.scala

@@ -0,0 +1,17 @@
+package au.edu.utscic.athanorserver.data
+
+import au.edu.utscic.athanorserver.data.RhetoricalTypes._
+import org.json4s.DefaultFormats
+import org.json4s.JsonAST.JValue
+
+import scala.collection.immutable.SortedMap
+
+/**
+  * Created by [email protected] on 19/7/17.
+  */
+object RhetoricalImplicits {
+
+  implicit val formats = DefaultFormats
+  implicit def jvalueToLexicalNodes(jvalue:JValue):LexicalNodes = SortedMap[Int,Node]() ++ jvalue.extract[Map[Int,Node]]
+
+}

src/main/scala/au/edu/utscic/athanorserver/data/RhetoricalTypes.scala

@@ -1,22 +1,28 @@
 package au.edu.utscic.athanorserver.data
 
-import scala.collection.immutable.ListMap
+import scala.collection.immutable.SortedMap
 
 /**
   * Created by [email protected] on 29/6/17.
   */
 
 object RhetoricalTypes {
-  type ConstituentTree = List[Serializable]
-  type LexicalNodes = ListMap[Int,Node]
+
+  type LexicalNodes = SortedMap[Int,Node]
+  type ConstituentTree = List[Any]
   type Dependencies = List[Dependency]
+
   type ParsedSentence = (LexicalNodes,ConstituentTree,Dependencies)
 
   case class Node(
                    id:Int,
                    POS:String,
                    surface:Option[String],
-                   lemma:Option[String]
+                   lemma:Option[String],
+                   NER:Option[String] = None,
+                   Speaker:Option[String] = None,
+                   left:Option[Int] = None,
+                   right:Option[Int] = None
                  )
 
   case class Dependency(