Skip to content

Commit

Permalink
Minor updates to separate NER and Parse from other pipelines
Browse files Browse the repository at this point in the history
  • Loading branch information
andrewresearch committed Nov 2, 2017
1 parent 560854f commit 1212abf
Show file tree
Hide file tree
Showing 3 changed files with 12 additions and 9 deletions.
4 changes: 2 additions & 2 deletions .travis.yml
Original file line number Diff line number Diff line change
Expand Up @@ -3,10 +3,10 @@ language: scala
jdk: oraclejdk8

scala:
- 2.12.4
- 2.12.3

script:
- sbt ++$TRAVIS_SCALA_VERSION -J-Xmx6G test
- sbt ++$TRAVIS_SCALA_VERSION -J-Xmx2000m test

# Use container-based infrastructure
sudo: false
Expand Down
2 changes: 1 addition & 1 deletion build.sbt
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
name := "factorie-nlp-api"

version := "0.5.0"
version := "0.5.1"

scalaVersion := "2.12.3"

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ object AnnotatorPipelines {
val logger = Logging(system.eventStream, "factorie-nlp-api")

type Pipeline = String => RunnableGraph[Future[Document]]
type DocPipeline = Document => RunnableGraph[Future[Document]]

//Make Document
private lazy val doc = Flow[String].map(new Document(_))
Expand Down Expand Up @@ -74,21 +75,23 @@ object AnnotatorPipelines {

val defaultPipeline = lemmaPipeline

val parsePipeline = (s:String) =>
Source.single(s)
.via(doc.map(tokeniser).map(segmenter).map(normaliser).map(postagger).map(lemmatiser).map(parser))
val parsePipeline = (d:Document) =>
Source.single(d)
.map(parser)
.toMat(Sink.head[Document])(Keep.right)

val nerPipeline = (s:String) =>
Source.single(s)
.via(doc.map(tokeniser).map(segmenter).map(normaliser).map(postagger).map(lemmatiser).map(parser).mapAsync(2)(nerTagger))
val nerPipeline = (d:Document) =>
Source.single(d)
.mapAsync(2)(nerTagger)
.toMat(Sink.head[Document])(Keep.right)

val completePipeline = nerPipeline

/* The main method for running a pipeline */
def process(text:String,pipeline:Pipeline=defaultPipeline):Future[Document] = pipeline(text).run

def processDoc(doc:Document,pipeline:DocPipeline):Future[Document] = pipeline(doc).run

def profile(text:String,pipeline:Pipeline=defaultPipeline,wait:Int=180):Document = {
logger.info(s"Profiling pipeline...")
val start = System.currentTimeMillis()
Expand Down

0 comments on commit 1212abf

Please sign in to comment.