From ed443a832bb51cff2c3186709f2e3b01ffff6599 Mon Sep 17 00:00:00 2001 From: Jimmy Lin Date: Sat, 2 Mar 2024 13:03:33 -0500 Subject: [PATCH] Rename dl20-passage to dl20 to be consistent with other topics (#2404) ref: castorini/anserini-tools#73 castorini/anserini-tools#75 --- ...20-passage-cohere-embed-english-v3.0-hnsw-int8.md | 12 ++++++------ ...ns-dl20-passage-cohere-embed-english-v3.0-hnsw.md | 12 ++++++------ .../regressions-dl20-passage-openai-ada2-int8.md | 12 ++++++------ .../regressions-dl20-passage-openai-ada2.md | 12 ++++++------ ...-passage-cohere-embed-english-v3.0-hnsw-int8.yaml | 2 +- .../dl20-passage-cohere-embed-english-v3.0-hnsw.yaml | 2 +- .../regression/dl20-passage-openai-ada2-int8.yaml | 2 +- .../regression/dl20-passage-openai-ada2.yaml | 2 +- tools | 2 +- 9 files changed, 29 insertions(+), 29 deletions(-) diff --git a/docs/regressions/regressions-dl20-passage-cohere-embed-english-v3.0-hnsw-int8.md b/docs/regressions/regressions-dl20-passage-cohere-embed-english-v3.0-hnsw-int8.md index 3622ad82ef..6e4e667853 100644 --- a/docs/regressions/regressions-dl20-passage-cohere-embed-english-v3.0-hnsw-int8.md +++ b/docs/regressions/regressions-dl20-passage-cohere-embed-english-v3.0-hnsw-int8.md @@ -73,19 +73,19 @@ After indexing has completed, you should be able to perform retrieval as follows ```bash target/appassembler/bin/SearchHnswDenseVectors \ -index indexes/lucene-hnsw.msmarco-passage-cohere-embed-english-v3.0-int8/ \ - -topics tools/topics-and-qrels/topics.dl20-passage.cohere-embed-english-v3.0.jsonl.gz \ + -topics tools/topics-and-qrels/topics.dl20.cohere-embed-english-v3.0.jsonl.gz \ -topicReader JsonIntVector \ - -output runs/run.msmarco-passage-cohere-embed-english-v3.0.cohere-embed-english-v3.0.topics.dl20-passage.cohere-embed-english-v3.0.jsonl.txt \ + -output runs/run.msmarco-passage-cohere-embed-english-v3.0.cohere-embed-english-v3.0.topics.dl20.cohere-embed-english-v3.0.jsonl.txt \ -generator VectorQueryGenerator -topicField vector -threads 16 -hits 1000 -efSearch 1000 & ``` Evaluation can be performed using `trec_eval`: ```bash -target/appassembler/bin/trec_eval -m map -c -l 2 tools/topics-and-qrels/qrels.dl20-passage.txt runs/run.msmarco-passage-cohere-embed-english-v3.0.cohere-embed-english-v3.0.topics.dl20-passage.cohere-embed-english-v3.0.jsonl.txt -target/appassembler/bin/trec_eval -m ndcg_cut.10 -c tools/topics-and-qrels/qrels.dl20-passage.txt runs/run.msmarco-passage-cohere-embed-english-v3.0.cohere-embed-english-v3.0.topics.dl20-passage.cohere-embed-english-v3.0.jsonl.txt -target/appassembler/bin/trec_eval -m recall.100 -c -l 2 tools/topics-and-qrels/qrels.dl20-passage.txt runs/run.msmarco-passage-cohere-embed-english-v3.0.cohere-embed-english-v3.0.topics.dl20-passage.cohere-embed-english-v3.0.jsonl.txt -target/appassembler/bin/trec_eval -m recall.1000 -c -l 2 tools/topics-and-qrels/qrels.dl20-passage.txt runs/run.msmarco-passage-cohere-embed-english-v3.0.cohere-embed-english-v3.0.topics.dl20-passage.cohere-embed-english-v3.0.jsonl.txt +target/appassembler/bin/trec_eval -m map -c -l 2 tools/topics-and-qrels/qrels.dl20-passage.txt runs/run.msmarco-passage-cohere-embed-english-v3.0.cohere-embed-english-v3.0.topics.dl20.cohere-embed-english-v3.0.jsonl.txt +target/appassembler/bin/trec_eval -m ndcg_cut.10 -c tools/topics-and-qrels/qrels.dl20-passage.txt runs/run.msmarco-passage-cohere-embed-english-v3.0.cohere-embed-english-v3.0.topics.dl20.cohere-embed-english-v3.0.jsonl.txt +target/appassembler/bin/trec_eval -m recall.100 -c -l 2 tools/topics-and-qrels/qrels.dl20-passage.txt runs/run.msmarco-passage-cohere-embed-english-v3.0.cohere-embed-english-v3.0.topics.dl20.cohere-embed-english-v3.0.jsonl.txt +target/appassembler/bin/trec_eval -m recall.1000 -c -l 2 tools/topics-and-qrels/qrels.dl20-passage.txt runs/run.msmarco-passage-cohere-embed-english-v3.0.cohere-embed-english-v3.0.topics.dl20.cohere-embed-english-v3.0.jsonl.txt ``` ## Effectiveness diff --git a/docs/regressions/regressions-dl20-passage-cohere-embed-english-v3.0-hnsw.md b/docs/regressions/regressions-dl20-passage-cohere-embed-english-v3.0-hnsw.md index 7930304da7..9c44b7eb14 100644 --- a/docs/regressions/regressions-dl20-passage-cohere-embed-english-v3.0-hnsw.md +++ b/docs/regressions/regressions-dl20-passage-cohere-embed-english-v3.0-hnsw.md @@ -73,19 +73,19 @@ After indexing has completed, you should be able to perform retrieval as follows ```bash target/appassembler/bin/SearchHnswDenseVectors \ -index indexes/lucene-hnsw.msmarco-passage-cohere-embed-english-v3.0/ \ - -topics tools/topics-and-qrels/topics.dl20-passage.cohere-embed-english-v3.0.jsonl.gz \ + -topics tools/topics-and-qrels/topics.dl20.cohere-embed-english-v3.0.jsonl.gz \ -topicReader JsonIntVector \ - -output runs/run.msmarco-passage-cohere-embed-english-v3.0.cohere-embed-english-v3.0.topics.dl20-passage.cohere-embed-english-v3.0.jsonl.txt \ + -output runs/run.msmarco-passage-cohere-embed-english-v3.0.cohere-embed-english-v3.0.topics.dl20.cohere-embed-english-v3.0.jsonl.txt \ -generator VectorQueryGenerator -topicField vector -threads 16 -hits 1000 -efSearch 1000 & ``` Evaluation can be performed using `trec_eval`: ```bash -target/appassembler/bin/trec_eval -m map -c -l 2 tools/topics-and-qrels/qrels.dl20-passage.txt runs/run.msmarco-passage-cohere-embed-english-v3.0.cohere-embed-english-v3.0.topics.dl20-passage.cohere-embed-english-v3.0.jsonl.txt -target/appassembler/bin/trec_eval -m ndcg_cut.10 -c tools/topics-and-qrels/qrels.dl20-passage.txt runs/run.msmarco-passage-cohere-embed-english-v3.0.cohere-embed-english-v3.0.topics.dl20-passage.cohere-embed-english-v3.0.jsonl.txt -target/appassembler/bin/trec_eval -m recall.100 -c -l 2 tools/topics-and-qrels/qrels.dl20-passage.txt runs/run.msmarco-passage-cohere-embed-english-v3.0.cohere-embed-english-v3.0.topics.dl20-passage.cohere-embed-english-v3.0.jsonl.txt -target/appassembler/bin/trec_eval -m recall.1000 -c -l 2 tools/topics-and-qrels/qrels.dl20-passage.txt runs/run.msmarco-passage-cohere-embed-english-v3.0.cohere-embed-english-v3.0.topics.dl20-passage.cohere-embed-english-v3.0.jsonl.txt +target/appassembler/bin/trec_eval -m map -c -l 2 tools/topics-and-qrels/qrels.dl20-passage.txt runs/run.msmarco-passage-cohere-embed-english-v3.0.cohere-embed-english-v3.0.topics.dl20.cohere-embed-english-v3.0.jsonl.txt +target/appassembler/bin/trec_eval -m ndcg_cut.10 -c tools/topics-and-qrels/qrels.dl20-passage.txt runs/run.msmarco-passage-cohere-embed-english-v3.0.cohere-embed-english-v3.0.topics.dl20.cohere-embed-english-v3.0.jsonl.txt +target/appassembler/bin/trec_eval -m recall.100 -c -l 2 tools/topics-and-qrels/qrels.dl20-passage.txt runs/run.msmarco-passage-cohere-embed-english-v3.0.cohere-embed-english-v3.0.topics.dl20.cohere-embed-english-v3.0.jsonl.txt +target/appassembler/bin/trec_eval -m recall.1000 -c -l 2 tools/topics-and-qrels/qrels.dl20-passage.txt runs/run.msmarco-passage-cohere-embed-english-v3.0.cohere-embed-english-v3.0.topics.dl20.cohere-embed-english-v3.0.jsonl.txt ``` ## Effectiveness diff --git a/docs/regressions/regressions-dl20-passage-openai-ada2-int8.md b/docs/regressions/regressions-dl20-passage-openai-ada2-int8.md index 37b74f67e8..eb8459b0ea 100644 --- a/docs/regressions/regressions-dl20-passage-openai-ada2-int8.md +++ b/docs/regressions/regressions-dl20-passage-openai-ada2-int8.md @@ -82,19 +82,19 @@ After indexing has completed, you should be able to perform retrieval as follows ```bash target/appassembler/bin/SearchHnswDenseVectors \ -index indexes/lucene-hnsw.msmarco-passage-openai-ada2-int8/ \ - -topics tools/topics-and-qrels/topics.dl20-passage.openai-ada2.jsonl.gz \ + -topics tools/topics-and-qrels/topics.dl20.openai-ada2.jsonl.gz \ -topicReader JsonIntVector \ - -output runs/run.msmarco-passage-openai-ada2.openai-ada2.topics.dl20-passage.openai-ada2.jsonl.txt \ + -output runs/run.msmarco-passage-openai-ada2.openai-ada2.topics.dl20.openai-ada2.jsonl.txt \ -generator VectorQueryGenerator -topicField vector -threads 16 -hits 1000 -efSearch 1000 & ``` Evaluation can be performed using `trec_eval`: ```bash -target/appassembler/bin/trec_eval -m map -c -l 2 tools/topics-and-qrels/qrels.dl20-passage.txt runs/run.msmarco-passage-openai-ada2.openai-ada2.topics.dl20-passage.openai-ada2.jsonl.txt -target/appassembler/bin/trec_eval -m ndcg_cut.10 -c tools/topics-and-qrels/qrels.dl20-passage.txt runs/run.msmarco-passage-openai-ada2.openai-ada2.topics.dl20-passage.openai-ada2.jsonl.txt -target/appassembler/bin/trec_eval -m recall.100 -c -l 2 tools/topics-and-qrels/qrels.dl20-passage.txt runs/run.msmarco-passage-openai-ada2.openai-ada2.topics.dl20-passage.openai-ada2.jsonl.txt -target/appassembler/bin/trec_eval -m recall.1000 -c -l 2 tools/topics-and-qrels/qrels.dl20-passage.txt runs/run.msmarco-passage-openai-ada2.openai-ada2.topics.dl20-passage.openai-ada2.jsonl.txt +target/appassembler/bin/trec_eval -m map -c -l 2 tools/topics-and-qrels/qrels.dl20-passage.txt runs/run.msmarco-passage-openai-ada2.openai-ada2.topics.dl20.openai-ada2.jsonl.txt +target/appassembler/bin/trec_eval -m ndcg_cut.10 -c tools/topics-and-qrels/qrels.dl20-passage.txt runs/run.msmarco-passage-openai-ada2.openai-ada2.topics.dl20.openai-ada2.jsonl.txt +target/appassembler/bin/trec_eval -m recall.100 -c -l 2 tools/topics-and-qrels/qrels.dl20-passage.txt runs/run.msmarco-passage-openai-ada2.openai-ada2.topics.dl20.openai-ada2.jsonl.txt +target/appassembler/bin/trec_eval -m recall.1000 -c -l 2 tools/topics-and-qrels/qrels.dl20-passage.txt runs/run.msmarco-passage-openai-ada2.openai-ada2.topics.dl20.openai-ada2.jsonl.txt ``` ## Effectiveness diff --git a/docs/regressions/regressions-dl20-passage-openai-ada2.md b/docs/regressions/regressions-dl20-passage-openai-ada2.md index a59d2d5b74..b77032cbe1 100644 --- a/docs/regressions/regressions-dl20-passage-openai-ada2.md +++ b/docs/regressions/regressions-dl20-passage-openai-ada2.md @@ -78,19 +78,19 @@ After indexing has completed, you should be able to perform retrieval as follows ```bash target/appassembler/bin/SearchHnswDenseVectors \ -index indexes/lucene-hnsw.msmarco-passage-openai-ada2/ \ - -topics tools/topics-and-qrels/topics.dl20-passage.openai-ada2.jsonl.gz \ + -topics tools/topics-and-qrels/topics.dl20.openai-ada2.jsonl.gz \ -topicReader JsonIntVector \ - -output runs/run.msmarco-passage-openai-ada2.openai-ada2.topics.dl20-passage.openai-ada2.jsonl.txt \ + -output runs/run.msmarco-passage-openai-ada2.openai-ada2.topics.dl20.openai-ada2.jsonl.txt \ -generator VectorQueryGenerator -topicField vector -threads 16 -hits 1000 -efSearch 1000 & ``` Evaluation can be performed using `trec_eval`: ```bash -target/appassembler/bin/trec_eval -m map -c -l 2 tools/topics-and-qrels/qrels.dl20-passage.txt runs/run.msmarco-passage-openai-ada2.openai-ada2.topics.dl20-passage.openai-ada2.jsonl.txt -target/appassembler/bin/trec_eval -m ndcg_cut.10 -c tools/topics-and-qrels/qrels.dl20-passage.txt runs/run.msmarco-passage-openai-ada2.openai-ada2.topics.dl20-passage.openai-ada2.jsonl.txt -target/appassembler/bin/trec_eval -m recall.100 -c -l 2 tools/topics-and-qrels/qrels.dl20-passage.txt runs/run.msmarco-passage-openai-ada2.openai-ada2.topics.dl20-passage.openai-ada2.jsonl.txt -target/appassembler/bin/trec_eval -m recall.1000 -c -l 2 tools/topics-and-qrels/qrels.dl20-passage.txt runs/run.msmarco-passage-openai-ada2.openai-ada2.topics.dl20-passage.openai-ada2.jsonl.txt +target/appassembler/bin/trec_eval -m map -c -l 2 tools/topics-and-qrels/qrels.dl20-passage.txt runs/run.msmarco-passage-openai-ada2.openai-ada2.topics.dl20.openai-ada2.jsonl.txt +target/appassembler/bin/trec_eval -m ndcg_cut.10 -c tools/topics-and-qrels/qrels.dl20-passage.txt runs/run.msmarco-passage-openai-ada2.openai-ada2.topics.dl20.openai-ada2.jsonl.txt +target/appassembler/bin/trec_eval -m recall.100 -c -l 2 tools/topics-and-qrels/qrels.dl20-passage.txt runs/run.msmarco-passage-openai-ada2.openai-ada2.topics.dl20.openai-ada2.jsonl.txt +target/appassembler/bin/trec_eval -m recall.1000 -c -l 2 tools/topics-and-qrels/qrels.dl20-passage.txt runs/run.msmarco-passage-openai-ada2.openai-ada2.topics.dl20.openai-ada2.jsonl.txt ``` ## Effectiveness diff --git a/src/main/resources/regression/dl20-passage-cohere-embed-english-v3.0-hnsw-int8.yaml b/src/main/resources/regression/dl20-passage-cohere-embed-english-v3.0-hnsw-int8.yaml index 16bbac17dc..ba4d644498 100644 --- a/src/main/resources/regression/dl20-passage-cohere-embed-english-v3.0-hnsw-int8.yaml +++ b/src/main/resources/regression/dl20-passage-cohere-embed-english-v3.0-hnsw-int8.yaml @@ -46,7 +46,7 @@ topic_reader: JsonIntVector topics: - name: "[DL20 (Passage)](https://trec.nist.gov/data/deep2020.html)" id: dl20 - path: topics.dl20-passage.cohere-embed-english-v3.0.jsonl.gz + path: topics.dl20.cohere-embed-english-v3.0.jsonl.gz qrel: qrels.dl20-passage.txt models: diff --git a/src/main/resources/regression/dl20-passage-cohere-embed-english-v3.0-hnsw.yaml b/src/main/resources/regression/dl20-passage-cohere-embed-english-v3.0-hnsw.yaml index c9ae9e91bd..e13dde9a2f 100644 --- a/src/main/resources/regression/dl20-passage-cohere-embed-english-v3.0-hnsw.yaml +++ b/src/main/resources/regression/dl20-passage-cohere-embed-english-v3.0-hnsw.yaml @@ -46,7 +46,7 @@ topic_reader: JsonIntVector topics: - name: "[DL20 (Passage)](https://trec.nist.gov/data/deep2020.html)" id: dl20 - path: topics.dl20-passage.cohere-embed-english-v3.0.jsonl.gz + path: topics.dl20.cohere-embed-english-v3.0.jsonl.gz qrel: qrels.dl20-passage.txt models: diff --git a/src/main/resources/regression/dl20-passage-openai-ada2-int8.yaml b/src/main/resources/regression/dl20-passage-openai-ada2-int8.yaml index b5c764e583..98bff7595f 100644 --- a/src/main/resources/regression/dl20-passage-openai-ada2-int8.yaml +++ b/src/main/resources/regression/dl20-passage-openai-ada2-int8.yaml @@ -46,7 +46,7 @@ topic_reader: JsonIntVector topics: - name: "[DL20 (Passage)](https://trec.nist.gov/data/deep2020.html)" id: dl20 - path: topics.dl20-passage.openai-ada2.jsonl.gz + path: topics.dl20.openai-ada2.jsonl.gz qrel: qrels.dl20-passage.txt models: diff --git a/src/main/resources/regression/dl20-passage-openai-ada2.yaml b/src/main/resources/regression/dl20-passage-openai-ada2.yaml index 4c8463db8d..1205dace7e 100644 --- a/src/main/resources/regression/dl20-passage-openai-ada2.yaml +++ b/src/main/resources/regression/dl20-passage-openai-ada2.yaml @@ -46,7 +46,7 @@ topic_reader: JsonIntVector topics: - name: "[DL20 (Passage)](https://trec.nist.gov/data/deep2020.html)" id: dl20 - path: topics.dl20-passage.openai-ada2.jsonl.gz + path: topics.dl20.openai-ada2.jsonl.gz qrel: qrels.dl20-passage.txt models: diff --git a/tools b/tools index 5a03340ff1..eac8940148 160000 --- a/tools +++ b/tools @@ -1 +1 @@ -Subproject commit 5a03340ff1fc7408061d2e6bd70687eec7547985 +Subproject commit eac89401480fb340f96678487e018dcb7b4b5d66