Add CLI (#35)

* add * add * add * add * add * add * add * wip * wip * wip * add * add * add * add * add * add * add * add * wip * rename * add * mv * wip * mv * add * add * add * add * add * add * add * mv * add * mv * add * add * add * mv * fix * add * rm * add * wip * fix * wip * wip * add * add * add * renew * mv * wip * wip * add * fiz * wip * add * add * add * add * add * add
kampersanda · Oct 14, 2024 · 21349fc · 21349fc
1 parent f90a0ba
commit 21349fc
Show file tree

Hide file tree

Showing 40 changed files with 19,477 additions and 204 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -73,12 +73,12 @@ jobs:
         uses: actions/setup-python@v5
         with:
           python-version: 3.12
-      - name: Build elinor-evaluate
-        run: cargo build --release -p elinor-evaluate
+      - name: Build elinor-cli
+        run: cargo build --release -p elinor-cli
       - name: Run correctness test
         run: |
-          ./correctness-test/prepare_trec_eval.sh
-          ./correctness-test/compare_with_trec_eval.py trec_eval-9.0.8 target/release
+          ./scripts/compare_with_trec_eval.py target/release test-data/trec-eval/qrels.test.jsonl test-data/trec-eval/results.test.jsonl test-data/trec-eval/qrels.test.output.json
+          ./scripts/compare_with_trec_eval.py target/release test-data/trec-eval/qrels.rel_level.jsonl test-data/trec-eval/results.test.jsonl test-data/trec-eval/qrels.rel_level.output.json
 
   publish:
     name: Publish

diff --git a/Cargo.toml b/Cargo.toml
@@ -35,7 +35,7 @@ rstest = "0.22.0"
 serde_json = "1.0.128"
 
 [workspace]
-members = ["elinor-evaluate"]
+members = ["elinor-cli"]
 
 [package.metadata.docs.rs]
 rustdoc-args = ["--html-in-header", "katex.html"]

diff --git a/README.md b/README.md
@@ -34,7 +34,49 @@ Or, you can build and open the documentation locally
 by running the following command:
 
 ```sh
-RUSTDOCFLAGS="--html-in-header katex.html" cargo doc --no-deps --open
+RUSTDOCFLAGS="--html-in-header katex.html" cargo doc --no-deps --features serde --open
+```
+
+## Command-line tools
+
+[elinor-cli](./elinor-cli) provides command-line tools for evaluating and comparing IR systems.
+See the [README](./elinor-cli/README.md) for more details.
+
+For example, you can obtain various statistics from several statistical tests, as shown below:
+
+```
+# score
++----------+--------+---------+
+| System   | Mean   | 95% MOE |
++----------+--------+---------+
+| System 1 | 0.3450 | 0.0670  |
+| System 2 | 0.2700 | 0.0670  |
+| System 3 | 0.2450 | 0.0670  |
++----------+--------+---------+
+## Two-way ANOVA without replication
++-----------------+------------+----+----------+--------+---------+
+| Factor          | Variation  | DF | Variance | F Stat | P Value |
++-----------------+------------+----+----------+--------+---------+
+| Between-systems | 0.1083     | 2  | 0.0542   | 2.4749 | 0.0976  |
+| Between-topics  | 1.0293     | 19 | 0.0542   | 2.4754 | 0.0086  |
+| Residual        | 0.8317     | 38 | 0.0219   |        |         |
++-----------------+------------+----+----------+--------+---------+
+## Between-system effect sizes (ES) from Tukey Hsd test
++----------+----------+----------+----------+
+| ES       | System_1 | System_2 | System_3 |
++----------+----------+----------+----------+
+| System_1 | 0.0000   | 0.5070   | 0.6760   |
+| System_2 | -0.5070  | 0.0000   | 0.1690   |
+| System_3 | -0.6760  | -0.1690  | 0.0000   |
++----------+----------+----------+----------+
+## Between-system P values from randomized Tukey Hsd test (n_iters=10000)
++----------+----------+----------+----------+
+| P Value  | System_1 | System_2 | System_3 |
++----------+----------+----------+----------+
+| System_1 | 1.0000   | 0.2734   | 0.1060   |
+| System_2 | 0.2734   | 1.0000   | 0.8913   |
+| System_3 | 0.1060   | 0.8913   | 1.0000   |
++----------+----------+----------+----------+
 ```
 
 ## Licensing

diff --git a/correctness-test/compare_with_trec_eval.py b/correctness-test/compare_with_trec_eval.py
diff --git a/elinor-cli/.gitignore b/elinor-cli/.gitignore
diff --git a/elinor-cli/Cargo.toml b/elinor-cli/Cargo.toml
@@ -0,0 +1,39 @@
+[package]
+name = "elinor-cli"
+version = "0.1.0"
+edition = "2021"
+authors = ["Shunsuke Kanda (kampersanda) <[email protected]>"]
+description = "Command line tools for elinor."
+license = "MIT OR Apache-2.0"
+readme = "README.md"
+documentation = "https://docs.rs/elinor"
+repository = "https://github.com/kampersanda/elinor"
+homepage = "https://github.com/kampersanda/elinor"
+keywords = ["information", "retrieval", "evaluation"]
+categories = ["science"]
+rust-version = "1.65.0"
+
+[dependencies]
+anyhow = "1.0.89"
+clap = { version = "4.5.20", features = ["derive"] }
+polars = "0.43.1"
+polars-lazy = "0.43.1"
+prettytable-rs = "0.10.0"
+serde_json = "1.0.128"
+
+[dependencies.elinor]
+path = ".."
+features = ["serde"]
+default-features = false
+
+[[bin]]
+name = "elinor-convert"
+path = "src/bin/convert.rs"
+
+[[bin]]
+name = "elinor-evaluate"
+path = "src/bin/evaluate.rs"
+
+[[bin]]
+name = "elinor-compare"
+path = "src/bin/compare.rs"