Remove text extraction

nlnwa · Apr 27, 2020 · e82645c · e82645c
1 parent 6640320
commit e82645c
Show file tree

Hide file tree

Showing 7 changed files with 7 additions and 301 deletions.
diff --git a/pom.xml b/pom.xml
@@ -29,7 +29,6 @@
         <veidemann.api.version>1.0.0-beta14</veidemann.api.version>
         <veidemann.commons.version>0.4.1</veidemann.commons.version>
         <veidemann.rethinkdbadapter.version>0.4.5</veidemann.rethinkdbadapter.version>
-        <org.apache.tika.version>1.22</org.apache.tika.version>
         <org.jwat.version>1.1.1</org.jwat.version>
         <log4j.version>2.7</log4j.version>
         <rethinkdb.version>2.4.0</rethinkdb.version>
@@ -88,22 +87,6 @@
             <version>${org.jwat.version}</version>
         </dependency>
 
-        <dependency>
-            <groupId>org.apache.tika</groupId>
-            <artifactId>tika-core</artifactId>
-            <version>${org.apache.tika.version}</version>
-        </dependency>
-        <dependency>
-            <groupId>org.apache.tika</groupId>
-            <artifactId>tika-parsers</artifactId>
-            <version>${org.apache.tika.version}</version>
-        </dependency>
-        <dependency>
-            <groupId>org.apache.tika</groupId>
-            <artifactId>tika-langdetect</artifactId>
-            <version>${org.apache.tika.version}</version>
-        </dependency>
-
         <!-- Configuration framework -->
         <dependency>
             <groupId>com.typesafe</groupId>

diff --git a/src/main/java/no/nb/nna/veidemann/contentwriter/ApiServer.java b/src/main/java/no/nb/nna/veidemann/contentwriter/ApiServer.java
@@ -20,7 +20,6 @@
 import io.grpc.ServerBuilder;
 import io.opentracing.contrib.ServerTracingInterceptor;
 import io.opentracing.util.GlobalTracer;
-import no.nb.nna.veidemann.contentwriter.text.TextExtractor;
 import no.nb.nna.veidemann.contentwriter.warc.WarcCollectionRegistry;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -44,12 +43,12 @@ public class ApiServer implements AutoCloseable {
     /**
      * Construct a new REST API server.
      */
-    public ApiServer(int port, int shutdownTimeoutSeconds, WarcCollectionRegistry warcCollectionRegistry, TextExtractor textExtractor) {
-        this(ServerBuilder.forPort(port), warcCollectionRegistry, textExtractor);
+    public ApiServer(int port, int shutdownTimeoutSeconds, WarcCollectionRegistry warcCollectionRegistry) {
+        this(ServerBuilder.forPort(port), warcCollectionRegistry);
         this.shutdownTimeoutSeconds = shutdownTimeoutSeconds;
     }
 
-    public ApiServer(ServerBuilder<?> serverBuilder, WarcCollectionRegistry warcCollectionRegistry, TextExtractor textExtractor) {
+    public ApiServer(ServerBuilder<?> serverBuilder, WarcCollectionRegistry warcCollectionRegistry) {
 
         ServerTracingInterceptor tracingInterceptor = new ServerTracingInterceptor.Builder(GlobalTracer.get())
                 .withTracedAttributes(ServerTracingInterceptor.ServerRequestAttribute.CALL_ATTRIBUTES,
@@ -61,7 +60,7 @@ public ApiServer(ServerBuilder<?> serverBuilder, WarcCollectionRegistry warcColl
         threadPool = Executors.newCachedThreadPool();
         serverBuilder.executor(threadPool);
 
-        server = serverBuilder.addService(new ContentWriterService(warcCollectionRegistry, textExtractor)).build();
+        server = serverBuilder.addService(new ContentWriterService(warcCollectionRegistry)).build();
     }
 
     public ApiServer start() {

diff --git a/src/main/java/no/nb/nna/veidemann/contentwriter/ContentWriter.java b/src/main/java/no/nb/nna/veidemann/contentwriter/ContentWriter.java
@@ -23,7 +23,6 @@
 import no.nb.nna.veidemann.commons.db.DbService;
 import no.nb.nna.veidemann.commons.opentracing.TracerFactory;
 import no.nb.nna.veidemann.contentwriter.settings.Settings;
-import no.nb.nna.veidemann.contentwriter.text.TextExtractor;
 import no.nb.nna.veidemann.contentwriter.warc.WarcCollectionRegistry;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -60,8 +59,7 @@ public ContentWriter() {
     public ContentWriter start() {
         try (DbService db = DbService.configure(SETTINGS);
              WarcCollectionRegistry warcCollectionRegistry = new WarcCollectionRegistry();
-             TextExtractor textExtractor = new TextExtractor();
-             ApiServer apiServer = new ApiServer(SETTINGS.getApiPort(), SETTINGS.getTerminationGracePeriodSeconds(), warcCollectionRegistry, textExtractor);) {
+             ApiServer apiServer = new ApiServer(SETTINGS.getApiPort(), SETTINGS.getTerminationGracePeriodSeconds(), warcCollectionRegistry)) {
 
             registerShutdownHook();
 
@@ -89,15 +87,7 @@ private void registerShutdownHook() {
         Runtime.getRuntime().addShutdownHook(new Thread(() -> {
             // Use stderr here since the logger may have been reset by its JVM shutdown hook.
             System.err.println("*** shutting down since JVM is shutting down");
-
             mainThread.interrupt();
-            try {
-                mainThread.join();
-            } catch (InterruptedException e) {
-                //
-            }
-            System.err.println("*** gracefully shut down");
-
         }));
     }
 

diff --git a/src/main/java/no/nb/nna/veidemann/contentwriter/ContentWriterService.java b/src/main/java/no/nb/nna/veidemann/contentwriter/ContentWriterService.java
@@ -23,7 +23,6 @@
 import no.nb.nna.veidemann.api.contentwriter.v1.WriteRequest;
 import no.nb.nna.veidemann.api.contentwriter.v1.WriteResponseMeta;
 import no.nb.nna.veidemann.contentwriter.WriteSessionContext.RecordData;
-import no.nb.nna.veidemann.contentwriter.text.TextExtractor;
 import no.nb.nna.veidemann.contentwriter.warc.SingleWarcWriter;
 import no.nb.nna.veidemann.contentwriter.warc.WarcCollection;
 import no.nb.nna.veidemann.contentwriter.warc.WarcCollection.Instance;
@@ -43,11 +42,8 @@ public class ContentWriterService extends ContentWriterGrpc.ContentWriterImplBas
 
     private final WarcCollectionRegistry warcCollectionRegistry;
 
-    private final TextExtractor textExtractor;
-
-    public ContentWriterService(WarcCollectionRegistry warcCollectionRegistry, TextExtractor textExtractor) {
+    public ContentWriterService(WarcCollectionRegistry warcCollectionRegistry) {
         this.warcCollectionRegistry = warcCollectionRegistry;
-        this.textExtractor = textExtractor;
     }
 
     @Override
@@ -140,10 +136,6 @@ public void onCompleted() {
 
                             URI ref = warcWriters.getWarcWriter(recordData.getSubCollectionType()).writeRecord(recordData);
 
-                            if (context.getCollectionConfig().getCrawlConfig().getExtra().getExtractText()) {
-                                textExtractor.extractText(recordData);
-                            }
-
                             WriteResponseMeta.RecordMeta.Builder responseMeta = WriteResponseMeta.RecordMeta.newBuilder()
                                     .setRecordNum(recordNum)
                                     .setType(recordData.getRecordType())

diff --git a/src/main/java/no/nb/nna/veidemann/contentwriter/text/SkipSpaceContentHandler.java b/src/main/java/no/nb/nna/veidemann/contentwriter/text/SkipSpaceContentHandler.java
diff --git a/src/main/java/no/nb/nna/veidemann/contentwriter/text/TextExtractor.java b/src/main/java/no/nb/nna/veidemann/contentwriter/text/TextExtractor.java