add robots.txt, introduce first search draft

querwurzel · Nov 12, 2023 · 339f48e · 339f48e
1 parent 6a102e5
commit 339f48e
Show file tree

Hide file tree

Showing 24 changed files with 393 additions and 73 deletions.
diff --git a/.mvn/wrapper/maven-wrapper.properties b/.mvn/wrapper/maven-wrapper.properties
@@ -14,5 +14,5 @@
 # KIND, either express or implied.  See the License for the
 # specific language governing permissions and limitations
 # under the License.
-distributionUrl=https://repo.maven.apache.org/maven2/org/apache/maven/apache-maven/3.9.4/apache-maven-3.9.4-bin.zip
+distributionUrl=https://repo.maven.apache.org/maven2/org/apache/maven/apache-maven/3.9.5/apache-maven-3.9.5-bin.zip
 wrapperUrl=https://repo.maven.apache.org/maven2/org/apache/maven/wrapper/maven-wrapper/3.2.0/maven-wrapper-3.2.0.jar
diff --git a/README.md b/README.md
@@ -24,7 +24,7 @@ Apache License 2.0 ([Apache-2.0](https://www.apache.org/licenses/LICENSE-2.0))
 * JDK 17+
 * MySQL 8+
 
-Just put the [JDK](https://adoptium.net/temurin/releases/) somewhere on your file system.
+Just put the [JDK](https://adoptium.net/temurin/releases/?os=any&arch=any&package=jdk&version=17) somewhere on your file system.
 The `bin` folder contains the `java` binary.
 
 ### How to configure

diff --git a/backend/src/main/java/com/github/binpastes/config/WebServerConfig.java b/backend/src/main/java/com/github/binpastes/config/WebServerConfig.java
@@ -49,6 +49,8 @@ public RouterFunctionMapping indexRoute(@Value("static/index.html") final ClassP
 
         var route = route(RequestPredicates
                         .method(HttpMethod.GET)
+                        .and(path("/robots.txt").negate())
+                        .and(path("/favicon.png").negate())
                         .and(path("/assets/**").negate())
                         .and(path("/api/**").negate()),
                 request -> ok().contentType(MediaType.TEXT_HTML).bodyValue(indexHtml));

diff --git a/backend/src/main/java/com/github/binpastes/paste/api/PasteController.java b/backend/src/main/java/com/github/binpastes/paste/api/PasteController.java
@@ -29,8 +29,8 @@
 
 import static com.github.binpastes.paste.api.model.ListView.ListItemView;
 
-@RestController
 @Validated
+@RestController
 @RequestMapping("/api/v1/paste")
 class PasteController {
 
@@ -83,7 +83,7 @@ public Mono<SearchView> searchPastes(
             final String term,
             final ServerHttpResponse response
     ) {
-        response.getHeaders().add(HttpHeaders.CACHE_CONTROL, "max-age=300");
+        response.getHeaders().add(HttpHeaders.CACHE_CONTROL, "max-age=60");
         return pasteService
                 .findByFullText(term)
                 .map(paste -> SearchItemView.of(paste, term))

diff --git a/backend/src/main/java/com/github/binpastes/paste/api/model/SearchView.java b/backend/src/main/java/com/github/binpastes/paste/api/model/SearchView.java
@@ -22,7 +22,7 @@ public record SearchItemView(
             LocalDateTime dateOfExpiry
     ) {
 
-        private static final short HIGHLIGHT_RANGE = 25;
+        private static final short HIGHLIGHT_RANGE = 30;
 
         public static SearchItemView of(final Paste reference, final String term) {
             return new SearchItemView(

diff --git a/backend/src/main/java/com/github/binpastes/paste/business/fulltext/Indexer.java b/backend/src/main/java/com/github/binpastes/paste/business/fulltext/Indexer.java
@@ -0,0 +1,104 @@
+package com.github.binpastes.paste.business.fulltext;
+
+import com.github.binpastes.paste.domain.Paste;
+import com.github.binpastes.paste.domain.PasteRepository;
+import com.github.binpastes.util.IdGenerator;
+import jakarta.annotation.PostConstruct;
+import org.apache.lucene.analysis.standard.StandardAnalyzer;
+import org.apache.lucene.document.*;
+import org.apache.lucene.index.*;
+import org.apache.lucene.queryparser.classic.ParseException;
+import org.apache.lucene.queryparser.classic.QueryParser;
+import org.apache.lucene.search.*;
+import org.apache.lucene.store.ByteBuffersDirectory;
+import org.apache.lucene.store.NIOFSDirectory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Component;
+import reactor.core.publisher.Flux;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.util.ArrayList;
+import java.util.List;
+
+@Component
+public class Indexer {
+
+    private final ByteBuffersDirectory index = new ByteBuffersDirectory();
+
+    IndexWriter writer;
+
+    private final PasteRepository pasteRepository;
+
+    @Autowired
+    public Indexer(PasteRepository pasteRepository) {
+        this.pasteRepository = pasteRepository;
+    }
+
+    @PostConstruct
+    private void postConstruct() {
+        try {
+            var analyzer = new StandardAnalyzer();
+            var config = new IndexWriterConfig(analyzer);
+            writer = new IndexWriter(index, config);
+        } catch (IOException e) {
+            throw new UncheckedIOException(e);
+        }
+
+    }
+
+    public void index(Paste paste) {
+        try {
+            if (paste.getTitle() == null && paste.isEncrypted()) {
+                return;
+            }
+
+            Document document = new Document();
+            document.add(new StringField("id", paste.getId(), Field.Store.YES));
+
+            if (paste.getTitle() != null) {
+                document.add(new TextField("title", paste.getTitle(), Field.Store.NO));
+            }
+
+            if (!paste.isEncrypted()) {
+                document.add(new TextField("content", paste.getContent(), Field.Store.NO));
+            }
+
+            writer.addDocument(document);
+            writer.flush();
+            writer.commit();
+
+        } catch (IOException e) {
+            throw new UncheckedIOException(e);
+        }
+    }
+
+    public Flux<Paste> query(String term) {
+        try {
+            var indexSearcher = new IndexSearcher(DirectoryReader.open(index));
+
+            BooleanQuery booleanQuery
+                    = new BooleanQuery.Builder()
+                    .add(new FuzzyQuery(new Term("title", term)), BooleanClause.Occur.SHOULD)
+                    .add(new FuzzyQuery(new Term("content", term)), BooleanClause.Occur.SHOULD)
+                    .build();
+
+            TopDocs hits = indexSearcher.search(booleanQuery, 10);
+            StoredFields storedFields = indexSearcher.storedFields();
+
+            List<Document> documents = new ArrayList<>();
+            for (ScoreDoc hit : hits.scoreDocs) {
+                Document doc = storedFields.document(hit.doc);
+                documents.add(doc);
+            }
+
+            System.out.println(hits.totalHits.value + " / " + indexSearcher.getIndexReader().maxDoc());
+
+            return Flux.fromIterable(documents)
+                    .map(doc -> doc.get("id"))
+                    .flatMap(pasteRepository::findOneLegitById);
+        } catch (Exception e) {
+            throw new RuntimeException(e);
+        }
+    }
+}
diff --git a/backend/src/main/java/com/github/binpastes/paste/domain/MySqlFullTextSupportImpl.java b/backend/src/main/java/com/github/binpastes/paste/domain/MySqlFullTextSupportImpl.java
@@ -27,40 +27,27 @@ public MySqlFullTextSupportImpl(final R2dbcEntityTemplate entityTemplate) {
 
     @Override
     public Flux<Paste> searchByFullText(final String text) {
-        /*
-         * Seems not to be supported by dev.miku:r2dbc-mysql
-         * java.lang.IllegalArgumentException: Cannot encode value of type 'class io.r2dbc.spi.Parameters$InParameter'
-         **/
-/*
-        entityTemplate
-                .getDatabaseClient()
-                .sql("SELECT * FROM pastes WHERE (date_of_expiry IS NULL OR date_of_expiry > CURRENT_TIMESTAMP) AND MATCH(title, content) AGAINST(?text IN BOOLEAN MODE)")
-                .bind("text", text + '*'))
-*/
-
-        var connectionFactory = entityTemplate.getDatabaseClient().getConnectionFactory();
-
-        var query = String.format("SELECT * FROM %s WHERE %s = ? AND (%s IS NULL OR %s > ?) AND (MATCH(%s) AGAINST(? IN BOOLEAN MODE) OR (MATCH(%s) AGAINST(? IN BOOLEAN MODE) AND %s IS FALSE)) ORDER BY %s DESC",
+        var query = String.format("SELECT * FROM %s WHERE %s = ? AND (%s IS NULL OR %s > ?) AND (MATCH(%s) AGAINST(?) OR (%s IS FALSE AND MATCH(%s) AGAINST(?)))",
                 PasteSchema.TABLE_NAME,
                 PasteSchema.EXPOSURE,
                 PasteSchema.DATE_OF_EXPIRY,
                 PasteSchema.DATE_OF_EXPIRY,
                 PasteSchema.TITLE,
-                PasteSchema.CONTENT,
                 PasteSchema.IS_ENCRYPTED,
-                PasteSchema.DATE_CREATED
+                PasteSchema.CONTENT
         );
 
+        var connectionFactory = entityTemplate.getDatabaseClient().getConnectionFactory();
         return Mono.from(connectionFactory.create())
                 .flatMap(mySqlConnection -> Mono.from(mySqlConnection
-                        .createStatement(query)
-                        .bind(0, PasteExposure.PUBLIC.name())
-                        .bind(1, LocalDateTime.now())
-                        .bind(2, text + '*')
-                        .bind(3, text + '*')
-                        .execute()
+                            .createStatement(query)
+                            .bind(0, PasteExposure.PUBLIC)
+                            .bind(1, LocalDateTime.now())
+                            .bind(2, text)
+                            .bind(3, text)
+                            .execute()
                 ))
-                .flatMapMany(mySqlResult -> Flux.from(mySqlResult.map((row, rowMetadata) -> {
+                .flatMapMany(mySqlResult -> Flux.from(mySqlResult.map((row) -> {
                     var paste = new Paste();
                     paste.setId(row.get(PasteSchema.ID, String.class));
                     paste.setVersion(row.get(PasteSchema.VERSION, Long.class));

diff --git a/backend/src/main/java/com/github/binpastes/paste/domain/SimpleFullTextSupportImpl.java b/backend/src/main/java/com/github/binpastes/paste/domain/SimpleFullTextSupportImpl.java
@@ -26,7 +26,7 @@ public SimpleFullTextSupportImpl(final R2dbcEntityTemplate entityTemplate) {
     @Override
     public Flux<Paste> searchByFullText(final String text) {
         var criteria = Criteria
-                .where(PasteSchema.EXPOSURE).is(PasteExposure.PUBLIC.name())
+                .where(PasteSchema.EXPOSURE).is(PasteExposure.PUBLIC)
                 .and(Criteria
                         .where(PasteSchema.DATE_OF_EXPIRY).isNull()
                         .or(PasteSchema.DATE_OF_EXPIRY).greaterThan(LocalDateTime.now())
@@ -47,6 +47,5 @@ public Flux<Paste> searchByFullText(final String text) {
                                 .sort(Sort.by(Sort.Direction.DESC, PasteSchema.DATE_CREATED))
                 )
                 .all();
-
     }
 }
diff --git a/backend/src/main/resources/application-dev.properties b/backend/src/main/resources/application-dev.properties
@@ -4,8 +4,6 @@
 #logging.level.org.flywaydb.core.internal.license.VersionPrinter=INFO
 logging.level.com.github.binpastes.paste.business.tracking=DEBUG
 
-spring.jpa.database-platform=org.hibernate.dialect.H2Dialect
-
 spring.flyway.enabled=true
 spring.flyway.driver-class-name=org.h2.Driver
 spring.flyway.fail-on-missing-locations=true

diff --git a/backend/src/main/resources/application-mysql.properties b/backend/src/main/resources/application-mysql.properties
@@ -1,5 +1,3 @@
-spring.jpa.database-platform=org.hibernate.dialect.MySqlDialect
-
 spring.flyway.enabled=true
 spring.flyway.driver-class-name=com.mysql.cj.jdbc.Driver
 spring.flyway.fail-on-missing-locations=true

diff --git a/backend/src/main/resources/broker.xml b/backend/src/main/resources/broker.xml
@@ -0,0 +1,29 @@
+<configuration xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns="urn:activemq" xsi:schemaLocation="urn:activemq /schema/artemis-server.xsd">
+    <core xmlns="urn:activemq:core">
+
+        <persistence-enabled>true</persistence-enabled>
+
+        <security-enabled>false</security-enabled>
+
+        <acceptors>
+            <acceptor name="in-vm">vm://0</acceptor>
+        </acceptors>
+
+        <bindings-directory>./tracking</bindings-directory>
+
+        <journal-pool-files>3</journal-pool-files>
+
+        <addresses>
+            <address name="binpastes">
+                <anycast>
+                    <queue name="pasteTrackingQueue" max-consumers="1">
+                        <durable>true</durable>
+                    </queue>
+                </anycast>
+            </address>
+        </addresses>
+
+
+    </core>
+</configuration>
+
diff --git a/frontend/index.html b/frontend/index.html
@@ -8,6 +8,7 @@
   <meta http-equiv="Content-Type" content="text/html; charset=utf-8"/>
   <meta name="robots" content="noindex, nofollow"/>
   <meta name="viewport" content="width=device-width, initial-scale=1"/>
+  <meta name="color-scheme" content="dark light">
   <link rel="icon shortcut" type="image/png" href="/favicon.png"/>
 </head>
 <body id="root">

diff --git a/frontend/public/robots.txt b/frontend/public/robots.txt
@@ -0,0 +1,2 @@
+User-agent: *
+Disallow: /api/
diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx
@@ -23,8 +23,8 @@ const App: () => JSX.Element = () => {
           <div class={styles.leftContainer}>
           <Routes>
             <Route path="/" component={Create}  />
-            <Route path="/paste/search" component={Search} />
             <Route path="/paste/:id" component={Read} />
+            <Route path="/paste/search" component={Search} />
             <Route path="*" component={NotFound} />
           </Routes>
           </div>

diff --git a/frontend/src/api/client.ts b/frontend/src/api/client.ts
@@ -56,14 +56,15 @@ const findAll = (): Promise<Array<PasteListView>> => {
 
 const searchAll = (term: string): Promise<Array<PasteSearchView>> => {
   const params = new URLSearchParams([['term', term]]);
-  const url = new URL('/api/v1/paste/search?' + params.toString(), apiBaseUrl());
+  const url = new URL('/api/v1/paste/search?' + encodeURI(params.toString()), apiBaseUrl());
 
   return fetch(url)
     .then(value => value.json())
-    .then(value => value.pastes);
+    .then(value => value.pastes)
+    .catch(_ => [])
 }
 
-const deletePaste = (id: string) => {
+const deletePaste = (id: string): Promise<void> => {
   const url = new URL('/api/v1/paste/' + id, apiBaseUrl());
 
   return fetch(url, {

diff --git a/frontend/src/components/CreatePaste/CreatePaste.tsx b/frontend/src/components/CreatePaste/CreatePaste.tsx
@@ -36,6 +36,7 @@ const CreatePaste: Component<CreatePasteProps> = ({onCreatePaste, initialPaste})
   const [lastPasteUrl, setLastPasteUrl] = createSignal<string>();
 
   let creationForm: HTMLFormElement
+  let submitInput: HTMLInputElement
 
   const updateFormField = (fieldName: keyof FormModel) => (event: Event) => {
     const inputElement = event.currentTarget as HTMLInputElement;
@@ -81,6 +82,7 @@ const CreatePaste: Component<CreatePasteProps> = ({onCreatePaste, initialPaste})
         resetStore();
         setLastPasteUrl(url);
       })
+      .catch(e => submitInput.style.backgroundColor = 'red');
   }
 
   return (
@@ -140,8 +142,8 @@ const CreatePaste: Component<CreatePasteProps> = ({onCreatePaste, initialPaste})
         <div>
           <textarea minLength="5"
                     maxLength="4096"
-                    required={true}
-                    autofocus={true}
+                    required
+                    autofocus
                     rows="20"
                     cols="75"
                     placeholder="Paste here"
@@ -153,7 +155,7 @@ const CreatePaste: Component<CreatePasteProps> = ({onCreatePaste, initialPaste})
         <Show when={lastPasteUrl()}>
           <p class={styles.lastPaste}>{lastPasteUrl()}<Copy/></p>
         </Show>
-        <input type="submit" value="Paste"/>
+        <input ref={submitInput} type="submit" value="Paste"/>
         <input type="reset" value="Reset"/>
       </fieldset>
 

diff --git a/frontend/src/components/CreatePaste/createPaste.module.css b/frontend/src/components/CreatePaste/createPaste.module.css
@@ -65,7 +65,7 @@
 .createForm input[type=submit],
 .createForm input[type=reset] {
     display: inline-block;
-    width: initial;
+    color: var(--color-text);
     margin: .5rem;
 }
 

diff --git a/frontend/src/components/RecentPastes/RecentPastes.tsx b/frontend/src/components/RecentPastes/RecentPastes.tsx
@@ -76,6 +76,8 @@ const RecentPastes: () => JSX.Element = () => {
             <span class={styles.refetch} onClick={manualRefetch}>↻</span>
           </h3>
 
+          <A class={styles.searchLink} activeClass={styles.searchLinkActive} href={'/paste/search'}>Search all pastes</A>
+
           <ol>
             <For each={pastes()}>{item =>
             <li class={styles.item}>

diff --git a/frontend/src/components/RecentPastes/recentPastes.module.css b/frontend/src/components/RecentPastes/recentPastes.module.css
@@ -14,7 +14,6 @@
     list-style: none;
     margin: 0;
     padding: 0;
-
 }
 
 .recentPastes .item {
@@ -50,3 +49,12 @@
         border-bottom: 1px dotted #a5a5a5;
     }
 }
+
+.searchLink {
+    margin-left: 1rem;
+    margin-bottom: 1rem;
+}
+
+.searchLinkActive {
+    display: none;
+}