feat: Web UI for auto-indexing #438

src/mqueryfront/src/App.css

@@ -87,3 +87,27 @@
 .cursor-pointer {
     cursor: pointer;
 }
+
+.index-form-wrapper {
+    display: grid;
+    grid-auto-flow: row;
+    grid-row-gap: 10px;
+}
+
+.index-links-wrapper {
+    display: grid;
+    grid-auto-flow: row;
+    grid-row-gap: 3px;
+}
+
+.index-navlink {
+    border-color: black;
+    border-style: solid;
+    border-width: thin;
+    font-size: large;
+    display: grid;
+    grid-auto-flow: row;
+    grid-row-gap: 10px;
+    cursor: pointer;
+    width: fit-content;
+}

src/mqueryfront/src/App.js

@@ -9,6 +9,7 @@ import AboutPage from "./about/AboutPage";
 import AuthPage from "./auth/AuthPage";
 import api, { parseJWT } from "./api";
 import "./App.css";
+import IndexPage from "./indexFiles/IndexPage";
 
 function getCurrentTokenOrNull() {
     // This function handles missing and corrupted token in the same way.
@@ -69,6 +70,7 @@ function App() {
                     path="/auth"
                     element={<AuthPage config={config} login={login} />}
                 />
+                <Route path="/index-files/:ursa_id" element={<IndexPage />} />
             </Routes>
         </div>
     );

src/mqueryfront/src/indexFiles/IndexClearQueueButton.js

@@ -0,0 +1,28 @@
+import React, { Component } from "react";
+import api from "../api";
+
+class IndexClearQueueButton extends Component {
+    onClick() {
+        api.post(`/queue/${this.props.ursa_id}/clear`, {}).catch((_e) => {});
+    }
+
+    render() {
+        return (
+            <span
+                data-toggle="tooltip"
+                title={
+                    "This action will clear all unindexed files from this queue"
+                }
+            >
+                <button
+                    className="btn btn-secondary btn-sm btn-danger my-2"
+                    onClick={() => this.onClick()}
+                >
+                    Clear queue
+                </button>
+            </span>
+        );
+    }
+}
+
+export default IndexClearQueueButton;

src/mqueryfront/src/indexFiles/IndexMultiSelect.js

@@ -0,0 +1,27 @@
+import { Component } from "react";
+import Select from "react-select";
+
+class IndexMultiselect extends Component {
+    get optionsList() {
+        return this.props.options.map((obj) => ({
+            label: obj,
+            value: obj,
+        }));
+    }
+
+    render() {
+        return (
+            <Select
+                form="formName"
+                name={this.props.name}
+                options={this.optionsList}
+                placeholder={this.props.placeholder}
+                isMulti
+                isSearchable
+                onChange={this.props.onChange}
+            />
+        );
+    }
+}
+
+export default IndexMultiselect;

src/mqueryfront/src/indexFiles/IndexPage.js

@@ -0,0 +1,154 @@
+import { Component } from "react";
+import IndexMultiSelect from "./IndexMultiSelect";
+import { useParams } from "react-router-dom";
+import api from "../api";
+import ErrorBoundary from "../components/ErrorBoundary";
+import IndexProgressBar from "./IndexProgressBar";
+import IndexClearQueueButton from "./IndexClearQueueButton";
+
+// function getAvailableTaintsListFromDatasets(datasets) {
+//     var taintList = Object.values(datasets)
+//         .map((ds) => ds.taints)
+//         .flat();
+//     return [...new Set(taintList)];
+// }
+
+class IndexPageInner extends Component {
+    constructor(props) {
+        super(props);
+        this.state = {
+            filenames: [],
+            availableNGrams: ["gram3", "text4", "wide8", "hash4"],
+            selectedNGrams: [],
+            availableTaints: [],
+            selectedTaints: [],
+            allQueuedFilesLength: 0,
+            finishedQueuedFilesLength: 0,
+        };
+        this.handleSubmit = this.handleSubmit.bind(this);
+        this.handleTextareaInput = this.handleTextareaInput.bind(this);
+        this.handleNGramSelect = this.handleNGramSelect.bind(this);
+        this.handleTaintSelect = this.handleTaintSelect.bind(this);
+    }
+
+    ursa_id = this.props.params.ursa_id;
+
+    componentDidMount() {
+        api.get(`/queue/${this.props.params.ursa_id}`)
+            // .then((response) => {
+            //     this.setState({
+            //         allQueuedFilesLength: response.data.all_files, // NOTE: dunno whether these params will be provided at all
+            //         finishedQueuedFilesLength: response.data.finished_files
+            //     })
+            // })
+            // .catch((error) => {
+            //     this.setState({ error: error });
+            // });
+            .catch((_e) => {
+                this.setState({
+                    allQueuedFilesLength: 10000,
+                    finishedQueuedFilesLength: 2137,
+                });
+            }); // TODO: uncomment prev then-catch clause after API implementation
+
+        api.get("/backend/datasets")
+            .then((_response) => {
+                // this.setState({
+                //     availableTaints: getAvailableTaintsListFromDatasets(response.data.datasets),
+                // });
+                this.setState({
+                    availableTaints: ["test_taint", "some other taint"],
+                }); // TODO: uncomment prev set state after API implementation
+            })
+            .catch((error) => {
+                this.setState({ error: error });
+            });
+    }
+
+    handleTextareaInput(e) {
+        const splitFiles = e.target.value.split("\n").filter((file) => !!file);
+        this.setState({ filenames: splitFiles });
+    }
+
+    handleNGramSelect(selection) {
+        this.setState({ selectedNGrams: selection });
+    }
+
+    handleTaintSelect(selection) {
+        this.setState({ selectedTaints: selection });
+    }
+
+    handleSubmit() {
+        // TODO: process following data accordingly to new endpoint
+        // c_onsole.log(this.state.filenames); // list of strings
+        // c_onsole.log(this.state.selectedNGrams); // list of {value: nGram, label: nGram} objects
+        // c_onsole.log(this.state.selectedTaints); // list of {value: taint, label: taint} objects
+        api.post(`/queue/${this.ursa_id}`, {})
+            .then((_r) => {})
+            .catch((_e) => {});
+    }
+
+    render() {
+        const fileLen = this.state.filenames.length;
+        const percentage = this.state.allQueuedFilesLength
+            ? (this.state.finishedQueuedFilesLength * 100.0) /
+              this.state.allQueuedFilesLength
+            : 0;
+        return (
+            <ErrorBoundary error={this.state.error}>
+                <div className="container-fluid">
+                    <h1 className="text-center mq-bottom">{`Index ${this.props.params.ursa_id}`}</h1>
+                    <div className="index-form-wrapper">
+                        <textarea
+                            id="filenames-textarea"
+                            className="form-control"
+                            name="rows"
+                            placeholder="Input filenames here (each line representing one filename)"
+                            onChange={this.handleTextareaInput}
+                        />
+                        <IndexMultiSelect
+                            placeholder="Select nGrams"
+                            options={this.state.availableNGrams}
+                            onChange={this.handleNGramSelect}
+                        />
+                        {this.state.availableTaints.length > 0 && (
+                            <IndexMultiSelect
+                                placeholder="Select taints"
+                                options={this.state.availableTaints}
+                                onChange={this.handleTaintSelect}
+                            />
+                        )}
+                        <button
+                            className="btn btn-secondary btn-sm btn-success"
+                            onClick={this.handleSubmit}
+                        >
+                            {`Add to queue${
+                                fileLen > 0
+                                    ? ` (${fileLen} file${
+                                          fileLen > 1 ? "s" : ""
+                                      })`
+                                    : ""
+                            }`}
+                        </button>
+                    </div>
+                    {percentage ? (
+                        <>
+                            <IndexProgressBar percentage={percentage} />
+                            <IndexClearQueueButton
+                                ursa_id={this.props.params.ursa_id}
+                            />
+                        </>
+                    ) : (
+                        <></>
+                    )}
+                </div>
+            </ErrorBoundary>
+        );
+    }
+}
+
+function IndexPage() {
+    return <IndexPageInner params={useParams()} />;
+}
+
+export default IndexPage;

src/mqueryfront/src/indexFiles/IndexProgressBar.js

@@ -0,0 +1,25 @@
+import { Component } from "react";
+
+class IndexProgressBar extends Component {
+    render() {
+        return (
+            <div className="row my-2">
+                <h4 className="text-center mq-bottom">Indexing progress</h4>
+                <div className="progress my-2">
+                    <div
+                        className="progress-bar progress-bar-striped progress-bar-animated"
+                        role="progressbar"
+                        aria-valuenow={this.props.percentage}
+                        aria-valuemin="0"
+                        aria-valuemax="100"
+                        style={{ width: `${this.props.percentage}%` }}
+                    >
+                        {`${this.props.percentage}%`}
+                    </div>
+                </div>
+            </div>
+        );
+    }
+}
+
+export default IndexProgressBar;

src/mqueryfront/src/status/IndexLink.js

@@ -0,0 +1,21 @@
+import { FontAwesomeIcon } from "@fortawesome/react-fontawesome";
+import { faArrowRight } from "@fortawesome/free-solid-svg-icons";
+import { Link } from "react-router-dom";
+
+const IndexLink = (props) => {
+    return (
+        <div className="index-navlink">
+            <Link exact to={`/index-files/${props.ursaID}`}>
+                Index using UrsaDB id.:{props.ursaID}
+                <FontAwesomeIcon
+                    className="mx-2"
+                    icon={faArrowRight}
+                    size="xl"
+                    color="black"
+                />
+            </Link>
+        </div>
+    );
+};
+
+export default IndexLink;

src/mqueryfront/src/status/StatusPage.js

@@ -5,6 +5,7 @@ import DatabaseTopology from "./DatabaseTopology";
 import VersionStatus from "./VersionStatus";
 import api from "../api";
 import WarningPage from "../components/WarningPage";
+import IndexLink from "./IndexLink";
 
 class StatusPage extends Component {
     constructor(props) {
@@ -30,6 +31,8 @@ class StatusPage extends Component {
         this._ismounted = true;
     }
 
+    usraIDs = [1, 2, "asd"]; // TODO: collect from endpoint
+
     getAgentsUrsaURLDuplicatesWarning(agentgroups) {
         var ursaURLS = agentgroups.map((agent) => agent.spec.ursadb_url);
         var duplicateURLS = ursaURLS.filter(
@@ -82,6 +85,14 @@ class StatusPage extends Component {
                             <DatabaseTopology />
                         </div>
                     </div>
+                    <div className="row">
+                        <h1 className="text-center mq-bottom">Index files</h1>
+                        <div className="index-links-wrapper">
+                            {this.usraIDs.map((ursaID) => (
+                                <IndexLink ursaID={ursaID} />
+                            ))}
+                        </div>
+                    </div>
                 </div>
             </ErrorBoundary>
         );