diff --git a/docs/script_examples/train_clanp.sh b/docs/script_examples/train_clanp.sh new file mode 100644 index 000000000..9feb0f0ce --- /dev/null +++ b/docs/script_examples/train_clanp.sh @@ -0,0 +1,62 @@ +#!/bin/bash +#SBATCH --partition=g40 +#SBATCH --account=laion +#SBATCH --job-name=CLANP +#SBATCH --nodes=4 +#SBATCH --gpus-per-node=8 +#SBATCH --cpus-per-gpu=10 +#SBATCH --output=logs/%x_%j.out +#SBATCH --comment=laion +#SBATCH --open-mode=append +#SBATCH --exclusive +source /etc/profile.d/modules.sh +module load openmpi +module load cuda/11.7 + +export PYTHONFAULTHANDLER=1 +export CUDA_LAUNCH_BLOCKING=0 +export HOSTNAMES=`scontrol show hostnames "$SLURM_JOB_NODELIST"` +export MASTER_ADDR=$(scontrol show hostnames "$SLURM_JOB_NODELIST" | head -n 1) +export MASTER_PORT=12802 +export COUNT_NODE=`scontrol show hostnames "$SLURM_JOB_NODELIST" | wc -l` + +echo go $COUNT_NODE +echo $HOSTNAMES + + +cd /admin/home-jianz/open_clip/src +export PYTHONPATH="$PYTHONPATH:/admin/home-jianz/open_clip/src" + +EXP_NAME="" + + + +srun --comment laion --cpu_bind=v --accel-bind=gn python3 -m training.main \ + --save-frequency 1 \ + --dataset-type="webdataset" \ + --text-a-key="txt" \ + --text-b-key="txt" \ + --report-to wandb \ + --wandb-project-name="CLANP" \ + --train-data="pipe:aws s3 --cli-connect-timeout 1000 --cli-read-timeout 1000 cp s3://s-laion/pile/pile_wds/pile-{00000..01159}.tar -" \ + --train-num-samples=1000000 \ + --warmup 5000 \ + --batch-size=256 \ + --lr=0.00001 \ + --wd=0.01 \ + --epochs=100 \ + --workers=10 \ + --model="Siamese-xlm-roberta-large" \ + --save-most-recent \ + --seed 42 \ + --local-loss \ + --gather-with-grad \ + --ddp-static-graph \ + --grad-checkpointing \ + --precision amp_bfloat16 \ + --model-type "SiameseCLANP" \ + --unsupervised-pretraining \ + --context-length=256 \ + --logs /fsx/home-jianz/logs \ + --resume /fsx/home-jianz/logs/2023_05_25-09_21_35-model_Siamese-xlm-roberta-large-lr_1e-05-b_256-j_10-p_amp_bfloat16/checkpoints/epoch_latest.pt + diff --git a/error_logs.txt b/error_logs.txt new file mode 100644 index 000000000..1bd406aa9 --- /dev/null +++ b/error_logs.txt @@ -0,0 +1,3116 @@ +2023-05-21 22:14:41.116492 >>> ArxivClusteringP2P +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 235, in run + results = task.evaluate(model, split, **kwargs) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/AbsTaskClustering.py", line 19, in evaluate + metrics = evaluator(model) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/evaluators/ClusteringEvaluator.py", line 25, in __call__ + corpus_embeddings = np.asarray(model.encode(self.sentences)) + File "src/training/mteb_eval.py", line 35, in encode + features = self.tokenize(sentences_batch) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/torch/nn/modules/module.py", line 1269, in __getattr__ + raise AttributeError("'{}' object has no attribute '{}'".format( +AttributeError: 'CLANP_For_MTEB' object has no attribute 'tokenize' + + +2023-05-21 22:14:43.063198 >>> ArxivClusteringS2S +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 235, in run + results = task.evaluate(model, split, **kwargs) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/AbsTaskClustering.py", line 19, in evaluate + metrics = evaluator(model) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/evaluators/ClusteringEvaluator.py", line 25, in __call__ + corpus_embeddings = np.asarray(model.encode(self.sentences)) + File "src/training/mteb_eval.py", line 35, in encode + features = self.tokenize(sentences_batch) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/torch/nn/modules/module.py", line 1269, in __getattr__ + raise AttributeError("'{}' object has no attribute '{}'".format( +AttributeError: 'CLANP_For_MTEB' object has no attribute 'tokenize' + + +2023-05-21 22:14:46.659623 >>> BiorxivClusteringP2P +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 235, in run + results = task.evaluate(model, split, **kwargs) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/AbsTaskClustering.py", line 19, in evaluate + metrics = evaluator(model) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/evaluators/ClusteringEvaluator.py", line 25, in __call__ + corpus_embeddings = np.asarray(model.encode(self.sentences)) + File "src/training/mteb_eval.py", line 35, in encode + features = self.tokenize(sentences_batch) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/torch/nn/modules/module.py", line 1269, in __getattr__ + raise AttributeError("'{}' object has no attribute '{}'".format( +AttributeError: 'CLANP_For_MTEB' object has no attribute 'tokenize' + + +2023-05-21 22:14:47.605287 >>> BiorxivClusteringS2S +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 235, in run + results = task.evaluate(model, split, **kwargs) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/AbsTaskClustering.py", line 19, in evaluate + metrics = evaluator(model) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/evaluators/ClusteringEvaluator.py", line 25, in __call__ + corpus_embeddings = np.asarray(model.encode(self.sentences)) + File "src/training/mteb_eval.py", line 35, in encode + features = self.tokenize(sentences_batch) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/torch/nn/modules/module.py", line 1269, in __getattr__ + raise AttributeError("'{}' object has no attribute '{}'".format( +AttributeError: 'CLANP_For_MTEB' object has no attribute 'tokenize' + + +2023-05-21 22:14:49.655345 >>> MedrxivClusteringP2P +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 235, in run + results = task.evaluate(model, split, **kwargs) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/AbsTaskClustering.py", line 19, in evaluate + metrics = evaluator(model) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/evaluators/ClusteringEvaluator.py", line 25, in __call__ + corpus_embeddings = np.asarray(model.encode(self.sentences)) + File "src/training/mteb_eval.py", line 35, in encode + features = self.tokenize(sentences_batch) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/torch/nn/modules/module.py", line 1269, in __getattr__ + raise AttributeError("'{}' object has no attribute '{}'".format( +AttributeError: 'CLANP_For_MTEB' object has no attribute 'tokenize' + + +2023-05-21 22:14:50.423953 >>> MedrxivClusteringS2S +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 235, in run + results = task.evaluate(model, split, **kwargs) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/AbsTaskClustering.py", line 19, in evaluate + metrics = evaluator(model) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/evaluators/ClusteringEvaluator.py", line 25, in __call__ + corpus_embeddings = np.asarray(model.encode(self.sentences)) + File "src/training/mteb_eval.py", line 35, in encode + features = self.tokenize(sentences_batch) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/torch/nn/modules/module.py", line 1269, in __getattr__ + raise AttributeError("'{}' object has no attribute '{}'".format( +AttributeError: 'CLANP_For_MTEB' object has no attribute 'tokenize' + + +2023-05-21 22:14:51.993315 >>> RedditClustering +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 235, in run + results = task.evaluate(model, split, **kwargs) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/AbsTaskClustering.py", line 19, in evaluate + metrics = evaluator(model) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/evaluators/ClusteringEvaluator.py", line 25, in __call__ + corpus_embeddings = np.asarray(model.encode(self.sentences)) + File "src/training/mteb_eval.py", line 35, in encode + features = self.tokenize(sentences_batch) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/torch/nn/modules/module.py", line 1269, in __getattr__ + raise AttributeError("'{}' object has no attribute '{}'".format( +AttributeError: 'CLANP_For_MTEB' object has no attribute 'tokenize' + + +2023-05-21 22:15:01.074955 >>> RedditClusteringP2P +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 235, in run + results = task.evaluate(model, split, **kwargs) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/AbsTaskClustering.py", line 19, in evaluate + metrics = evaluator(model) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/evaluators/ClusteringEvaluator.py", line 25, in __call__ + corpus_embeddings = np.asarray(model.encode(self.sentences)) + File "src/training/mteb_eval.py", line 35, in encode + features = self.tokenize(sentences_batch) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/torch/nn/modules/module.py", line 1269, in __getattr__ + raise AttributeError("'{}' object has no attribute '{}'".format( +AttributeError: 'CLANP_For_MTEB' object has no attribute 'tokenize' + + +2023-05-21 22:15:03.724835 >>> StackExchangeClustering +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 235, in run + results = task.evaluate(model, split, **kwargs) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/AbsTaskClustering.py", line 19, in evaluate + metrics = evaluator(model) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/evaluators/ClusteringEvaluator.py", line 25, in __call__ + corpus_embeddings = np.asarray(model.encode(self.sentences)) + File "src/training/mteb_eval.py", line 35, in encode + features = self.tokenize(sentences_batch) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/torch/nn/modules/module.py", line 1269, in __getattr__ + raise AttributeError("'{}' object has no attribute '{}'".format( +AttributeError: 'CLANP_For_MTEB' object has no attribute 'tokenize' + + +2023-05-21 22:15:06.890844 >>> StackExchangeClusteringP2P +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 235, in run + results = task.evaluate(model, split, **kwargs) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/AbsTaskClustering.py", line 19, in evaluate + metrics = evaluator(model) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/evaluators/ClusteringEvaluator.py", line 25, in __call__ + corpus_embeddings = np.asarray(model.encode(self.sentences)) + File "src/training/mteb_eval.py", line 35, in encode + features = self.tokenize(sentences_batch) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/torch/nn/modules/module.py", line 1269, in __getattr__ + raise AttributeError("'{}' object has no attribute '{}'".format( +AttributeError: 'CLANP_For_MTEB' object has no attribute 'tokenize' + + +2023-05-21 22:15:07.588629 >>> TwentyNewsgroupsClustering +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 235, in run + results = task.evaluate(model, split, **kwargs) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/AbsTaskClustering.py", line 19, in evaluate + metrics = evaluator(model) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/evaluators/ClusteringEvaluator.py", line 25, in __call__ + corpus_embeddings = np.asarray(model.encode(self.sentences)) + File "src/training/mteb_eval.py", line 35, in encode + features = self.tokenize(sentences_batch) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/torch/nn/modules/module.py", line 1269, in __getattr__ + raise AttributeError("'{}' object has no attribute '{}'".format( +AttributeError: 'CLANP_For_MTEB' object has no attribute 'tokenize' + + +2023-05-21 22:15:07.592022 >>> ArguAna +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:15:07.595497 >>> ClimateFEVER +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:15:07.597198 >>> CQADupstackAndroidRetrieval +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:15:07.598847 >>> CQADupstackEnglishRetrieval +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:15:07.600403 >>> CQADupstackGamingRetrieval +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:15:07.602096 >>> CQADupstackGisRetrieval +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:15:07.603802 >>> CQADupstackMathematicaRetrieval +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:15:07.605398 >>> CQADupstackPhysicsRetrieval +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:15:07.606962 >>> CQADupstackProgrammersRetrieval +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:15:07.608499 >>> CQADupstackStatsRetrieval +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:15:07.610021 >>> CQADupstackTexRetrieval +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:15:07.611630 >>> CQADupstackUnixRetrieval +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:15:07.613429 >>> CQADupstackWebmastersRetrieval +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:15:07.614980 >>> CQADupstackWordpressRetrieval +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:15:07.616518 >>> DBPedia +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:15:07.618017 >>> FEVER +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:15:07.619576 >>> FiQA2018 +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:15:07.621133 >>> HotpotQA +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:15:07.622719 >>> MSMARCO +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:15:07.624253 >>> MSMARCOv2 +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:15:07.625774 >>> NFCorpus +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:15:07.627625 >>> NQ +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:15:07.629457 >>> QuoraRetrieval +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:15:07.631024 >>> SCIDOCS +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:15:07.632535 >>> SciFact +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:15:07.634130 >>> Touche2020 +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:15:07.635689 >>> TRECCOVID +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:18:07.819906 >>> ArxivClusteringP2P +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 235, in run + results = task.evaluate(model, split, **kwargs) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/AbsTaskClustering.py", line 19, in evaluate + metrics = evaluator(model) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/evaluators/ClusteringEvaluator.py", line 25, in __call__ + corpus_embeddings = np.asarray(model.encode(self.sentences)) + File "src/training/mteb_eval.py", line 36, in encode + features = self.batch_to_device(features, self.device) + File "src/training/mteb_eval.py", line 53, in batch_to_device + if isinstance(batch[key], Tensor): +IndexError: index 1673 is out of bounds for dimension 0 with size 32 + + +2023-05-21 22:18:08.270324 >>> ArxivClusteringS2S +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 235, in run + results = task.evaluate(model, split, **kwargs) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/AbsTaskClustering.py", line 19, in evaluate + metrics = evaluator(model) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/evaluators/ClusteringEvaluator.py", line 25, in __call__ + corpus_embeddings = np.asarray(model.encode(self.sentences)) + File "src/training/mteb_eval.py", line 36, in encode + features = self.batch_to_device(features, self.device) + File "src/training/mteb_eval.py", line 53, in batch_to_device + if isinstance(batch[key], Tensor): +IndexError: index 25074 is out of bounds for dimension 0 with size 32 + + +2023-05-21 22:18:08.768140 >>> BiorxivClusteringP2P +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 235, in run + results = task.evaluate(model, split, **kwargs) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/AbsTaskClustering.py", line 19, in evaluate + metrics = evaluator(model) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/evaluators/ClusteringEvaluator.py", line 25, in __call__ + corpus_embeddings = np.asarray(model.encode(self.sentences)) + File "src/training/mteb_eval.py", line 36, in encode + features = self.batch_to_device(features, self.device) + File "src/training/mteb_eval.py", line 53, in batch_to_device + if isinstance(batch[key], Tensor): +IndexError: index 380 is out of bounds for dimension 0 with size 32 + + +2023-05-21 22:18:09.064860 >>> BiorxivClusteringS2S +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 235, in run + results = task.evaluate(model, split, **kwargs) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/AbsTaskClustering.py", line 19, in evaluate + metrics = evaluator(model) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/evaluators/ClusteringEvaluator.py", line 25, in __call__ + corpus_embeddings = np.asarray(model.encode(self.sentences)) + File "src/training/mteb_eval.py", line 36, in encode + features = self.batch_to_device(features, self.device) + File "src/training/mteb_eval.py", line 53, in batch_to_device + if isinstance(batch[key], Tensor): +IndexError: index 28521 is out of bounds for dimension 0 with size 32 + + +2023-05-21 22:18:09.547762 >>> MedrxivClusteringP2P +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 235, in run + results = task.evaluate(model, split, **kwargs) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/AbsTaskClustering.py", line 19, in evaluate + metrics = evaluator(model) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/evaluators/ClusteringEvaluator.py", line 25, in __call__ + corpus_embeddings = np.asarray(model.encode(self.sentences)) + File "src/training/mteb_eval.py", line 36, in encode + features = self.batch_to_device(features, self.device) + File "src/training/mteb_eval.py", line 53, in batch_to_device + if isinstance(batch[key], Tensor): +IndexError: index 998 is out of bounds for dimension 0 with size 32 + + +2023-05-21 22:18:09.830162 >>> MedrxivClusteringS2S +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 235, in run + results = task.evaluate(model, split, **kwargs) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/AbsTaskClustering.py", line 19, in evaluate + metrics = evaluator(model) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/evaluators/ClusteringEvaluator.py", line 25, in __call__ + corpus_embeddings = np.asarray(model.encode(self.sentences)) + File "src/training/mteb_eval.py", line 36, in encode + features = self.batch_to_device(features, self.device) + File "src/training/mteb_eval.py", line 53, in batch_to_device + if isinstance(batch[key], Tensor): +IndexError: index 998 is out of bounds for dimension 0 with size 32 + + +2023-05-21 22:18:10.178599 >>> RedditClustering +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 235, in run + results = task.evaluate(model, split, **kwargs) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/AbsTaskClustering.py", line 19, in evaluate + metrics = evaluator(model) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/evaluators/ClusteringEvaluator.py", line 25, in __call__ + corpus_embeddings = np.asarray(model.encode(self.sentences)) + File "src/training/mteb_eval.py", line 36, in encode + features = self.batch_to_device(features, self.device) + File "src/training/mteb_eval.py", line 53, in batch_to_device + if isinstance(batch[key], Tensor): +IndexError: index 35 is out of bounds for dimension 0 with size 32 + + +2023-05-21 22:18:10.718680 >>> RedditClusteringP2P +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 235, in run + results = task.evaluate(model, split, **kwargs) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/AbsTaskClustering.py", line 19, in evaluate + metrics = evaluator(model) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/evaluators/ClusteringEvaluator.py", line 25, in __call__ + corpus_embeddings = np.asarray(model.encode(self.sentences)) + File "src/training/mteb_eval.py", line 36, in encode + features = self.batch_to_device(features, self.device) + File "src/training/mteb_eval.py", line 53, in batch_to_device + if isinstance(batch[key], Tensor): +IndexError: index 56 is out of bounds for dimension 0 with size 32 + + +2023-05-21 22:18:11.112059 >>> StackExchangeClustering +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 235, in run + results = task.evaluate(model, split, **kwargs) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/AbsTaskClustering.py", line 19, in evaluate + metrics = evaluator(model) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/evaluators/ClusteringEvaluator.py", line 25, in __call__ + corpus_embeddings = np.asarray(model.encode(self.sentences)) + File "src/training/mteb_eval.py", line 36, in encode + features = self.batch_to_device(features, self.device) + File "src/training/mteb_eval.py", line 53, in batch_to_device + if isinstance(batch[key], Tensor): +IndexError: index 2513 is out of bounds for dimension 0 with size 32 + + +2023-05-21 22:18:11.606707 >>> StackExchangeClusteringP2P +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 235, in run + results = task.evaluate(model, split, **kwargs) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/AbsTaskClustering.py", line 19, in evaluate + metrics = evaluator(model) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/evaluators/ClusteringEvaluator.py", line 25, in __call__ + corpus_embeddings = np.asarray(model.encode(self.sentences)) + File "src/training/mteb_eval.py", line 36, in encode + features = self.batch_to_device(features, self.device) + File "src/training/mteb_eval.py", line 53, in batch_to_device + if isinstance(batch[key], Tensor): +IndexError: index 1694 is out of bounds for dimension 0 with size 32 + + +2023-05-21 22:18:12.282844 >>> TwentyNewsgroupsClustering +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 235, in run + results = task.evaluate(model, split, **kwargs) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/AbsTaskClustering.py", line 19, in evaluate + metrics = evaluator(model) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/evaluators/ClusteringEvaluator.py", line 25, in __call__ + corpus_embeddings = np.asarray(model.encode(self.sentences)) + File "src/training/mteb_eval.py", line 36, in encode + features = self.batch_to_device(features, self.device) + File "src/training/mteb_eval.py", line 53, in batch_to_device + if isinstance(batch[key], Tensor): +IndexError: index 46 is out of bounds for dimension 0 with size 32 + + +2023-05-21 22:18:12.285650 >>> ArguAna +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:18:12.288074 >>> ClimateFEVER +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:18:12.289633 >>> CQADupstackAndroidRetrieval +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:18:12.291188 >>> CQADupstackEnglishRetrieval +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:18:12.292730 >>> CQADupstackGamingRetrieval +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:18:12.294196 >>> CQADupstackGisRetrieval +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:18:12.295704 >>> CQADupstackMathematicaRetrieval +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:18:12.297226 >>> CQADupstackPhysicsRetrieval +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:18:12.298715 >>> CQADupstackProgrammersRetrieval +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:18:12.300210 >>> CQADupstackStatsRetrieval +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:18:12.301720 >>> CQADupstackTexRetrieval +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:18:12.303295 >>> CQADupstackUnixRetrieval +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:18:12.304907 >>> CQADupstackWebmastersRetrieval +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:18:12.306460 >>> CQADupstackWordpressRetrieval +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:18:12.307999 >>> DBPedia +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:18:12.309583 >>> FEVER +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:18:12.311148 >>> FiQA2018 +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:18:12.312693 >>> HotpotQA +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:18:12.314289 >>> MSMARCO +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:18:12.315852 >>> MSMARCOv2 +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:18:12.317538 >>> NFCorpus +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:18:12.319044 >>> NQ +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:18:12.320612 >>> QuoraRetrieval +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:18:12.322100 >>> SCIDOCS +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:18:12.323566 >>> SciFact +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:18:12.325068 >>> Touche2020 +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:18:12.326566 >>> TRECCOVID +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:26:13.233910 >>> ArxivClusteringP2P +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 235, in run + results = task.evaluate(model, split, **kwargs) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/AbsTaskClustering.py", line 19, in evaluate + metrics = evaluator(model) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/evaluators/ClusteringEvaluator.py", line 25, in __call__ + corpus_embeddings = np.asarray(model.encode(self.sentences)) + File "src/training/mteb_eval.py", line 34, in encode + print(sentences_batch.shape) +AttributeError: 'list' object has no attribute 'shape' + + +2023-05-21 22:26:13.672080 >>> ArxivClusteringS2S +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 235, in run + results = task.evaluate(model, split, **kwargs) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/AbsTaskClustering.py", line 19, in evaluate + metrics = evaluator(model) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/evaluators/ClusteringEvaluator.py", line 25, in __call__ + corpus_embeddings = np.asarray(model.encode(self.sentences)) + File "src/training/mteb_eval.py", line 34, in encode + print(sentences_batch.shape) +AttributeError: 'list' object has no attribute 'shape' + + +2023-05-21 22:26:14.092847 >>> BiorxivClusteringP2P +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 235, in run + results = task.evaluate(model, split, **kwargs) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/AbsTaskClustering.py", line 19, in evaluate + metrics = evaluator(model) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/evaluators/ClusteringEvaluator.py", line 25, in __call__ + corpus_embeddings = np.asarray(model.encode(self.sentences)) + File "src/training/mteb_eval.py", line 34, in encode + print(sentences_batch.shape) +AttributeError: 'list' object has no attribute 'shape' + + +2023-05-21 22:26:14.347831 >>> BiorxivClusteringS2S +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 235, in run + results = task.evaluate(model, split, **kwargs) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/AbsTaskClustering.py", line 19, in evaluate + metrics = evaluator(model) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/evaluators/ClusteringEvaluator.py", line 25, in __call__ + corpus_embeddings = np.asarray(model.encode(self.sentences)) + File "src/training/mteb_eval.py", line 34, in encode + print(sentences_batch.shape) +AttributeError: 'list' object has no attribute 'shape' + + +2023-05-21 22:26:14.747403 >>> MedrxivClusteringP2P +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 235, in run + results = task.evaluate(model, split, **kwargs) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/AbsTaskClustering.py", line 19, in evaluate + metrics = evaluator(model) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/evaluators/ClusteringEvaluator.py", line 25, in __call__ + corpus_embeddings = np.asarray(model.encode(self.sentences)) + File "src/training/mteb_eval.py", line 34, in encode + print(sentences_batch.shape) +AttributeError: 'list' object has no attribute 'shape' + + +2023-05-21 22:26:14.992638 >>> MedrxivClusteringS2S +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 235, in run + results = task.evaluate(model, split, **kwargs) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/AbsTaskClustering.py", line 19, in evaluate + metrics = evaluator(model) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/evaluators/ClusteringEvaluator.py", line 25, in __call__ + corpus_embeddings = np.asarray(model.encode(self.sentences)) + File "src/training/mteb_eval.py", line 34, in encode + print(sentences_batch.shape) +AttributeError: 'list' object has no attribute 'shape' + + +2023-05-21 22:26:15.311379 >>> RedditClustering +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 235, in run + results = task.evaluate(model, split, **kwargs) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/AbsTaskClustering.py", line 19, in evaluate + metrics = evaluator(model) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/evaluators/ClusteringEvaluator.py", line 25, in __call__ + corpus_embeddings = np.asarray(model.encode(self.sentences)) + File "src/training/mteb_eval.py", line 34, in encode + print(sentences_batch.shape) +AttributeError: 'list' object has no attribute 'shape' + + +2023-05-21 22:26:15.762837 >>> RedditClusteringP2P +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 235, in run + results = task.evaluate(model, split, **kwargs) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/AbsTaskClustering.py", line 19, in evaluate + metrics = evaluator(model) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/evaluators/ClusteringEvaluator.py", line 25, in __call__ + corpus_embeddings = np.asarray(model.encode(self.sentences)) + File "src/training/mteb_eval.py", line 34, in encode + print(sentences_batch.shape) +AttributeError: 'list' object has no attribute 'shape' + + +2023-05-21 22:26:16.138870 >>> StackExchangeClustering +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 235, in run + results = task.evaluate(model, split, **kwargs) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/AbsTaskClustering.py", line 19, in evaluate + metrics = evaluator(model) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/evaluators/ClusteringEvaluator.py", line 25, in __call__ + corpus_embeddings = np.asarray(model.encode(self.sentences)) + File "src/training/mteb_eval.py", line 34, in encode + print(sentences_batch.shape) +AttributeError: 'list' object has no attribute 'shape' + + +2023-05-21 22:26:16.554528 >>> StackExchangeClusteringP2P +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 235, in run + results = task.evaluate(model, split, **kwargs) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/AbsTaskClustering.py", line 19, in evaluate + metrics = evaluator(model) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/evaluators/ClusteringEvaluator.py", line 25, in __call__ + corpus_embeddings = np.asarray(model.encode(self.sentences)) + File "src/training/mteb_eval.py", line 34, in encode + print(sentences_batch.shape) +AttributeError: 'list' object has no attribute 'shape' + + +2023-05-21 22:26:16.769926 >>> TwentyNewsgroupsClustering +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 235, in run + results = task.evaluate(model, split, **kwargs) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/AbsTaskClustering.py", line 19, in evaluate + metrics = evaluator(model) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/evaluators/ClusteringEvaluator.py", line 25, in __call__ + corpus_embeddings = np.asarray(model.encode(self.sentences)) + File "src/training/mteb_eval.py", line 34, in encode + print(sentences_batch.shape) +AttributeError: 'list' object has no attribute 'shape' + + +2023-05-21 22:26:16.772275 >>> ArguAna +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:26:16.775636 >>> ClimateFEVER +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:26:16.777437 >>> CQADupstackAndroidRetrieval +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:26:16.778978 >>> CQADupstackEnglishRetrieval +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:26:16.780513 >>> CQADupstackGamingRetrieval +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:26:16.782036 >>> CQADupstackGisRetrieval +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:26:16.783554 >>> CQADupstackMathematicaRetrieval +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:26:16.785117 >>> CQADupstackPhysicsRetrieval +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:26:16.786643 >>> CQADupstackProgrammersRetrieval +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:26:16.788211 >>> CQADupstackStatsRetrieval +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:26:16.789983 >>> CQADupstackTexRetrieval +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:26:16.791514 >>> CQADupstackUnixRetrieval +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:26:16.793020 >>> CQADupstackWebmastersRetrieval +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:26:16.794537 >>> CQADupstackWordpressRetrieval +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:26:16.796097 >>> DBPedia +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:26:16.797619 >>> FEVER +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:26:16.799175 >>> FiQA2018 +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:26:16.800729 >>> HotpotQA +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:26:16.802244 >>> MSMARCO +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:26:16.803751 >>> MSMARCOv2 +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:26:16.805325 >>> NFCorpus +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:26:16.806855 >>> NQ +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:26:16.808415 >>> QuoraRetrieval +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:26:16.809923 >>> SCIDOCS +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:26:16.811601 >>> SciFact +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:26:16.813134 >>> Touche2020 +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:26:16.814639 >>> TRECCOVID +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:27:22.007826 >>> ArxivClusteringP2P +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 235, in run + results = task.evaluate(model, split, **kwargs) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/AbsTaskClustering.py", line 19, in evaluate + metrics = evaluator(model) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/evaluators/ClusteringEvaluator.py", line 25, in __call__ + corpus_embeddings = np.asarray(model.encode(self.sentences)) + File "src/training/mteb_eval.py", line 36, in encode + features = self.batch_to_device(features, self.device) + File "src/training/mteb_eval.py", line 53, in batch_to_device + if isinstance(batch[key], Tensor): +IndexError: index 1673 is out of bounds for dimension 0 with size 32 + + +2023-05-21 22:27:22.261201 >>> ArxivClusteringS2S +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 235, in run + results = task.evaluate(model, split, **kwargs) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/AbsTaskClustering.py", line 19, in evaluate + metrics = evaluator(model) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/evaluators/ClusteringEvaluator.py", line 25, in __call__ + corpus_embeddings = np.asarray(model.encode(self.sentences)) + File "src/training/mteb_eval.py", line 36, in encode + features = self.batch_to_device(features, self.device) + File "src/training/mteb_eval.py", line 53, in batch_to_device + if isinstance(batch[key], Tensor): +IndexError: index 25074 is out of bounds for dimension 0 with size 32 + + +2023-05-21 22:27:22.544696 >>> BiorxivClusteringP2P +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 235, in run + results = task.evaluate(model, split, **kwargs) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/AbsTaskClustering.py", line 19, in evaluate + metrics = evaluator(model) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/evaluators/ClusteringEvaluator.py", line 25, in __call__ + corpus_embeddings = np.asarray(model.encode(self.sentences)) + File "src/training/mteb_eval.py", line 36, in encode + features = self.batch_to_device(features, self.device) + File "src/training/mteb_eval.py", line 53, in batch_to_device + if isinstance(batch[key], Tensor): +IndexError: index 380 is out of bounds for dimension 0 with size 32 + + +2023-05-21 22:27:23.258645 >>> BiorxivClusteringS2S +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 235, in run + results = task.evaluate(model, split, **kwargs) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/AbsTaskClustering.py", line 19, in evaluate + metrics = evaluator(model) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/evaluators/ClusteringEvaluator.py", line 25, in __call__ + corpus_embeddings = np.asarray(model.encode(self.sentences)) + File "src/training/mteb_eval.py", line 36, in encode + features = self.batch_to_device(features, self.device) + File "src/training/mteb_eval.py", line 53, in batch_to_device + if isinstance(batch[key], Tensor): +IndexError: index 28521 is out of bounds for dimension 0 with size 32 + + +2023-05-21 22:27:23.632457 >>> MedrxivClusteringP2P +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 235, in run + results = task.evaluate(model, split, **kwargs) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/AbsTaskClustering.py", line 19, in evaluate + metrics = evaluator(model) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/evaluators/ClusteringEvaluator.py", line 25, in __call__ + corpus_embeddings = np.asarray(model.encode(self.sentences)) + File "src/training/mteb_eval.py", line 36, in encode + features = self.batch_to_device(features, self.device) + File "src/training/mteb_eval.py", line 53, in batch_to_device + if isinstance(batch[key], Tensor): +IndexError: index 998 is out of bounds for dimension 0 with size 32 + + +2023-05-21 22:27:23.983727 >>> MedrxivClusteringS2S +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 235, in run + results = task.evaluate(model, split, **kwargs) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/AbsTaskClustering.py", line 19, in evaluate + metrics = evaluator(model) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/evaluators/ClusteringEvaluator.py", line 25, in __call__ + corpus_embeddings = np.asarray(model.encode(self.sentences)) + File "src/training/mteb_eval.py", line 36, in encode + features = self.batch_to_device(features, self.device) + File "src/training/mteb_eval.py", line 53, in batch_to_device + if isinstance(batch[key], Tensor): +IndexError: index 998 is out of bounds for dimension 0 with size 32 + + +2023-05-21 22:27:24.197034 >>> RedditClustering +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 235, in run + results = task.evaluate(model, split, **kwargs) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/AbsTaskClustering.py", line 19, in evaluate + metrics = evaluator(model) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/evaluators/ClusteringEvaluator.py", line 25, in __call__ + corpus_embeddings = np.asarray(model.encode(self.sentences)) + File "src/training/mteb_eval.py", line 36, in encode + features = self.batch_to_device(features, self.device) + File "src/training/mteb_eval.py", line 53, in batch_to_device + if isinstance(batch[key], Tensor): +IndexError: index 35 is out of bounds for dimension 0 with size 32 + + +2023-05-21 22:27:24.545544 >>> RedditClusteringP2P +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 235, in run + results = task.evaluate(model, split, **kwargs) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/AbsTaskClustering.py", line 19, in evaluate + metrics = evaluator(model) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/evaluators/ClusteringEvaluator.py", line 25, in __call__ + corpus_embeddings = np.asarray(model.encode(self.sentences)) + File "src/training/mteb_eval.py", line 36, in encode + features = self.batch_to_device(features, self.device) + File "src/training/mteb_eval.py", line 53, in batch_to_device + if isinstance(batch[key], Tensor): +IndexError: index 1620 is out of bounds for dimension 0 with size 32 + + +2023-05-21 22:27:24.917339 >>> StackExchangeClustering +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 235, in run + results = task.evaluate(model, split, **kwargs) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/AbsTaskClustering.py", line 19, in evaluate + metrics = evaluator(model) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/evaluators/ClusteringEvaluator.py", line 25, in __call__ + corpus_embeddings = np.asarray(model.encode(self.sentences)) + File "src/training/mteb_eval.py", line 36, in encode + features = self.batch_to_device(features, self.device) + File "src/training/mteb_eval.py", line 53, in batch_to_device + if isinstance(batch[key], Tensor): +IndexError: index 2513 is out of bounds for dimension 0 with size 32 + + +2023-05-21 22:27:25.195054 >>> StackExchangeClusteringP2P +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 235, in run + results = task.evaluate(model, split, **kwargs) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/AbsTaskClustering.py", line 19, in evaluate + metrics = evaluator(model) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/evaluators/ClusteringEvaluator.py", line 25, in __call__ + corpus_embeddings = np.asarray(model.encode(self.sentences)) + File "src/training/mteb_eval.py", line 36, in encode + features = self.batch_to_device(features, self.device) + File "src/training/mteb_eval.py", line 53, in batch_to_device + if isinstance(batch[key], Tensor): +IndexError: index 1694 is out of bounds for dimension 0 with size 32 + + +2023-05-21 22:27:25.435361 >>> TwentyNewsgroupsClustering +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 235, in run + results = task.evaluate(model, split, **kwargs) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/AbsTaskClustering.py", line 19, in evaluate + metrics = evaluator(model) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/evaluators/ClusteringEvaluator.py", line 25, in __call__ + corpus_embeddings = np.asarray(model.encode(self.sentences)) + File "src/training/mteb_eval.py", line 36, in encode + features = self.batch_to_device(features, self.device) + File "src/training/mteb_eval.py", line 53, in batch_to_device + if isinstance(batch[key], Tensor): +IndexError: index 46 is out of bounds for dimension 0 with size 32 + + +2023-05-21 22:27:25.437428 >>> ArguAna +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:27:25.439691 >>> ClimateFEVER +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:27:25.440930 >>> CQADupstackAndroidRetrieval +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:27:25.442174 >>> CQADupstackEnglishRetrieval +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:27:25.443383 >>> CQADupstackGamingRetrieval +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:27:25.444590 >>> CQADupstackGisRetrieval +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:27:25.445818 >>> CQADupstackMathematicaRetrieval +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:27:25.447101 >>> CQADupstackPhysicsRetrieval +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:27:25.448322 >>> CQADupstackProgrammersRetrieval +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:27:25.449536 >>> CQADupstackStatsRetrieval +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:27:25.450682 >>> CQADupstackTexRetrieval +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:27:25.451912 >>> CQADupstackUnixRetrieval +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:27:25.453140 >>> CQADupstackWebmastersRetrieval +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:27:25.454311 >>> CQADupstackWordpressRetrieval +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:27:25.455513 >>> DBPedia +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:27:25.456750 >>> FEVER +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:27:25.457911 >>> FiQA2018 +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:27:25.459269 >>> HotpotQA +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:27:25.460568 >>> MSMARCO +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:27:25.461740 >>> MSMARCOv2 +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:27:25.462949 >>> NFCorpus +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:27:25.464165 >>> NQ +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:27:25.465347 >>> QuoraRetrieval +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:27:25.466498 >>> SCIDOCS +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:27:25.467773 >>> SciFact +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:27:25.469019 >>> Touche2020 +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:27:25.470211 >>> TRECCOVID +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:28:19.439473 >>> ArxivClusteringP2P +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 235, in run + results = task.evaluate(model, split, **kwargs) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/AbsTaskClustering.py", line 19, in evaluate + metrics = evaluator(model) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/evaluators/ClusteringEvaluator.py", line 25, in __call__ + corpus_embeddings = np.asarray(model.encode(self.sentences)) + File "src/training/mteb_eval.py", line 37, in encode + text_features = model.encode_text(features) +NameError: name 'model' is not defined + + +2023-05-21 22:28:19.700152 >>> ArxivClusteringS2S +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 235, in run + results = task.evaluate(model, split, **kwargs) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/AbsTaskClustering.py", line 19, in evaluate + metrics = evaluator(model) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/evaluators/ClusteringEvaluator.py", line 25, in __call__ + corpus_embeddings = np.asarray(model.encode(self.sentences)) + File "src/training/mteb_eval.py", line 37, in encode + text_features = model.encode_text(features) +NameError: name 'model' is not defined + + +2023-05-21 22:28:19.977665 >>> BiorxivClusteringP2P +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 235, in run + results = task.evaluate(model, split, **kwargs) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/AbsTaskClustering.py", line 19, in evaluate + metrics = evaluator(model) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/evaluators/ClusteringEvaluator.py", line 25, in __call__ + corpus_embeddings = np.asarray(model.encode(self.sentences)) + File "src/training/mteb_eval.py", line 37, in encode + text_features = model.encode_text(features) +NameError: name 'model' is not defined + + +2023-05-21 22:28:20.230267 >>> BiorxivClusteringS2S +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 235, in run + results = task.evaluate(model, split, **kwargs) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/AbsTaskClustering.py", line 19, in evaluate + metrics = evaluator(model) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/evaluators/ClusteringEvaluator.py", line 25, in __call__ + corpus_embeddings = np.asarray(model.encode(self.sentences)) + File "src/training/mteb_eval.py", line 37, in encode + text_features = model.encode_text(features) +NameError: name 'model' is not defined + + +2023-05-21 22:28:20.733546 >>> MedrxivClusteringP2P +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 235, in run + results = task.evaluate(model, split, **kwargs) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/AbsTaskClustering.py", line 19, in evaluate + metrics = evaluator(model) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/evaluators/ClusteringEvaluator.py", line 25, in __call__ + corpus_embeddings = np.asarray(model.encode(self.sentences)) + File "src/training/mteb_eval.py", line 37, in encode + text_features = model.encode_text(features) +NameError: name 'model' is not defined + + +2023-05-21 22:28:20.977127 >>> MedrxivClusteringS2S +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 235, in run + results = task.evaluate(model, split, **kwargs) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/AbsTaskClustering.py", line 19, in evaluate + metrics = evaluator(model) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/evaluators/ClusteringEvaluator.py", line 25, in __call__ + corpus_embeddings = np.asarray(model.encode(self.sentences)) + File "src/training/mteb_eval.py", line 37, in encode + text_features = model.encode_text(features) +NameError: name 'model' is not defined + + +2023-05-21 22:28:21.200248 >>> RedditClustering +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 235, in run + results = task.evaluate(model, split, **kwargs) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/AbsTaskClustering.py", line 19, in evaluate + metrics = evaluator(model) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/evaluators/ClusteringEvaluator.py", line 25, in __call__ + corpus_embeddings = np.asarray(model.encode(self.sentences)) + File "src/training/mteb_eval.py", line 37, in encode + text_features = model.encode_text(features) +NameError: name 'model' is not defined + + +2023-05-21 22:28:21.507616 >>> RedditClusteringP2P +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 235, in run + results = task.evaluate(model, split, **kwargs) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/AbsTaskClustering.py", line 19, in evaluate + metrics = evaluator(model) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/evaluators/ClusteringEvaluator.py", line 25, in __call__ + corpus_embeddings = np.asarray(model.encode(self.sentences)) + File "src/training/mteb_eval.py", line 37, in encode + text_features = model.encode_text(features) +NameError: name 'model' is not defined + + +2023-05-21 22:28:21.764115 >>> StackExchangeClustering +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 235, in run + results = task.evaluate(model, split, **kwargs) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/AbsTaskClustering.py", line 19, in evaluate + metrics = evaluator(model) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/evaluators/ClusteringEvaluator.py", line 25, in __call__ + corpus_embeddings = np.asarray(model.encode(self.sentences)) + File "src/training/mteb_eval.py", line 37, in encode + text_features = model.encode_text(features) +NameError: name 'model' is not defined + + +2023-05-21 22:28:22.154970 >>> StackExchangeClusteringP2P +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 235, in run + results = task.evaluate(model, split, **kwargs) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/AbsTaskClustering.py", line 19, in evaluate + metrics = evaluator(model) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/evaluators/ClusteringEvaluator.py", line 25, in __call__ + corpus_embeddings = np.asarray(model.encode(self.sentences)) + File "src/training/mteb_eval.py", line 37, in encode + text_features = model.encode_text(features) +NameError: name 'model' is not defined + + +2023-05-21 22:28:22.423249 >>> TwentyNewsgroupsClustering +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 235, in run + results = task.evaluate(model, split, **kwargs) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/AbsTaskClustering.py", line 19, in evaluate + metrics = evaluator(model) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/evaluators/ClusteringEvaluator.py", line 25, in __call__ + corpus_embeddings = np.asarray(model.encode(self.sentences)) + File "src/training/mteb_eval.py", line 37, in encode + text_features = model.encode_text(features) +NameError: name 'model' is not defined + + +2023-05-21 22:28:22.425438 >>> ArguAna +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:28:22.427412 >>> ClimateFEVER +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:28:22.428703 >>> CQADupstackAndroidRetrieval +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:28:22.430040 >>> CQADupstackEnglishRetrieval +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:28:22.431280 >>> CQADupstackGamingRetrieval +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:28:22.432470 >>> CQADupstackGisRetrieval +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:28:22.433670 >>> CQADupstackMathematicaRetrieval +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:28:22.434858 >>> CQADupstackPhysicsRetrieval +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:28:22.436104 >>> CQADupstackProgrammersRetrieval +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:28:22.437305 >>> CQADupstackStatsRetrieval +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:28:22.438490 >>> CQADupstackTexRetrieval +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:28:22.439876 >>> CQADupstackUnixRetrieval +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:28:22.441061 >>> CQADupstackWebmastersRetrieval +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:28:22.442262 >>> CQADupstackWordpressRetrieval +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:28:22.443481 >>> DBPedia +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:28:22.444719 >>> FEVER +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:28:22.445916 >>> FiQA2018 +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:28:22.447140 >>> HotpotQA +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:28:22.448406 >>> MSMARCO +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:28:22.449612 >>> MSMARCOv2 +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:28:22.450877 >>> NFCorpus +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:28:22.452045 >>> NQ +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:28:22.453390 >>> QuoraRetrieval +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:28:22.454588 >>> SCIDOCS +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:28:22.455874 >>> SciFact +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:28:22.457095 >>> Touche2020 +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:28:22.458297 >>> TRECCOVID +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:30:44.495512 >>> ArxivClusteringP2P +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 235, in run + results = task.evaluate(model, split, **kwargs) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/AbsTaskClustering.py", line 19, in evaluate + metrics = evaluator(model) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/evaluators/ClusteringEvaluator.py", line 25, in __call__ + corpus_embeddings = np.asarray(model.encode(self.sentences)) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/torch/_tensor.py", line 956, in __array__ + return self.numpy() +TypeError: can't convert cuda:0 device type tensor to numpy. Use Tensor.cpu() to copy the tensor to host memory first. + + +2023-05-21 22:33:15.493726 >>> ArxivClusteringP2P +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 235, in run + results = task.evaluate(model, split, **kwargs) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/AbsTaskClustering.py", line 19, in evaluate + metrics = evaluator(model) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/evaluators/ClusteringEvaluator.py", line 25, in __call__ + corpus_embeddings = np.asarray(model.encode(self.sentences)) +ValueError: setting an array element with a sequence. The requested array has an inhomogeneous shape after 1 dimensions. The detected shape was (8,) + inhomogeneous part. + + +2023-05-21 22:36:46.248030 >>> ArxivClusteringP2P +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 235, in run + results = task.evaluate(model, split, **kwargs) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/AbsTaskClustering.py", line 19, in evaluate + metrics = evaluator(model) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/evaluators/ClusteringEvaluator.py", line 25, in __call__ + corpus_embeddings = np.asarray(model.encode(self.sentences)) +ValueError: setting an array element with a sequence. The requested array has an inhomogeneous shape after 1 dimensions. The detected shape was (25000,) + inhomogeneous part. + + +2023-05-21 22:42:50.337546 >>> ArxivClusteringP2P +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 235, in run + results = task.evaluate(model, split, **kwargs) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/AbsTaskClustering.py", line 19, in evaluate + metrics = evaluator(model) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/evaluators/ClusteringEvaluator.py", line 25, in __call__ + corpus_embeddings = np.asarray(model.encode(self.sentences)) +ValueError: setting an array element with a sequence. The requested array has an inhomogeneous shape after 1 dimensions. The detected shape was (25000,) + inhomogeneous part. + + +2023-05-29 16:35:30.769925 >>> ClimateFEVER +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 235, in run + results = task.evaluate(model, split, **kwargs) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/AbsTaskRetrieval.py", line 95, in evaluate + results = retriever.retrieve(corpus, queries) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/beir/retrieval/evaluation.py", line 23, in retrieve + return self.retriever.search(corpus, queries, self.top_k, self.score_function, **kwargs) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/beir/retrieval/search/dense/exact_search.py", line 39, in search + query_embeddings = self.model.encode_queries( + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/AbsTaskRetrieval.py", line 148, in encode_queries + return self.model.encode(queries, batch_size=batch_size, **kwargs) + File "src/training/mteb_eval.py", line 42, in encode + return np.stack(all_embeddings,0) +NameError: name 'np' is not defined + + +2023-05-29 17:10:01.510309 >>> SciFact +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 235, in run + results = task.evaluate(model, split, **kwargs) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/AbsTaskRetrieval.py", line 95, in evaluate + results = retriever.retrieve(corpus, queries) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/beir/retrieval/evaluation.py", line 23, in retrieve + return self.retriever.search(corpus, queries, self.top_k, self.score_function, **kwargs) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/beir/retrieval/search/dense/exact_search.py", line 65, in search + cos_scores = self.score_functions[score_function](query_embeddings, sub_corpus_embeddings) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/beir/retrieval/search/dense/util.py", line 24, in cos_sim + return torch.mm(a_norm, b_norm.transpose(0, 1)) #TODO: this keeps allocating GPU memory +RuntimeError: mat1 and mat2 shapes cannot be multiplied (1x153600 and 2653696x1) + + +2023-05-29 17:15:31.249976 >>> SciFact +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 235, in run + results = task.evaluate(model, split, **kwargs) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/AbsTaskRetrieval.py", line 95, in evaluate + results = retriever.retrieve(corpus, queries) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/beir/retrieval/evaluation.py", line 23, in retrieve + return self.retriever.search(corpus, queries, self.top_k, self.score_function, **kwargs) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/beir/retrieval/search/dense/exact_search.py", line 39, in search + query_embeddings = self.model.encode_queries( + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/AbsTaskRetrieval.py", line 148, in encode_queries + return self.model.encode(queries, batch_size=batch_size, **kwargs) + File "src/training/mteb_eval.py", line 40, in encode + all_embeddings.extend(embeddings.detach()) +NameError: name 'embeddings' is not defined + + +2023-05-29 17:20:08.362515 >>> SciFact +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 235, in run + results = task.evaluate(model, split, **kwargs) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/AbsTaskRetrieval.py", line 95, in evaluate + results = retriever.retrieve(corpus, queries) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/beir/retrieval/evaluation.py", line 23, in retrieve + return self.retriever.search(corpus, queries, self.top_k, self.score_function, **kwargs) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/beir/retrieval/search/dense/exact_search.py", line 65, in search + cos_scores = self.score_functions[score_function](query_embeddings, sub_corpus_embeddings) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/beir/retrieval/search/dense/util.py", line 24, in cos_sim + return torch.mm(a_norm, b_norm.transpose(0, 1)) #TODO: this keeps allocating GPU memory +RuntimeError: mat1 and mat2 shapes cannot be multiplied (1x153600 and 2653696x1) + + +2023-05-29 17:28:59.689506 >>> SciFact +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 235, in run + results = task.evaluate(model, split, **kwargs) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/AbsTaskRetrieval.py", line 95, in evaluate + results = retriever.retrieve(corpus, queries) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/beir/retrieval/evaluation.py", line 23, in retrieve + return self.retriever.search(corpus, queries, self.top_k, self.score_function, **kwargs) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/beir/retrieval/search/dense/exact_search.py", line 65, in search + cos_scores = self.score_functions[score_function](query_embeddings, sub_corpus_embeddings) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/beir/retrieval/search/dense/util.py", line 11, in cos_sim + a = torch.tensor(a) +RuntimeError: Could not infer dtype of NoneType + + +2023-05-29 17:39:05.588611 >>> SciFact +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 235, in run + results = task.evaluate(model, split, **kwargs) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/AbsTaskRetrieval.py", line 95, in evaluate + results = retriever.retrieve(corpus, queries) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/beir/retrieval/evaluation.py", line 23, in retrieve + return self.retriever.search(corpus, queries, self.top_k, self.score_function, **kwargs) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/beir/retrieval/search/dense/exact_search.py", line 65, in search + cos_scores = self.score_functions[score_function](query_embeddings, sub_corpus_embeddings) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/beir/retrieval/search/dense/util.py", line 11, in cos_sim + a = torch.tensor(a) +RuntimeError: Could not infer dtype of NoneType + + +2023-05-29 17:48:00.556928 >>> SciFact +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 235, in run + results = task.evaluate(model, split, **kwargs) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/AbsTaskRetrieval.py", line 95, in evaluate + results = retriever.retrieve(corpus, queries) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/beir/retrieval/evaluation.py", line 23, in retrieve + return self.retriever.search(corpus, queries, self.top_k, self.score_function, **kwargs) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/beir/retrieval/search/dense/exact_search.py", line 39, in search + query_embeddings = self.model.encode_queries( + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/AbsTaskRetrieval.py", line 148, in encode_queries + return self.model.encode(queries, batch_size=batch_size, **kwargs) + File "src/training/mteb_eval.py", line 41, in encode + all_embeddings = np.stack(all_embeddings,0) + File "<__array_function__ internals>", line 200, in stack + File "/admin/home-jianz/.local/lib/python3.8/site-packages/numpy/core/shape_base.py", line 464, in stack + raise ValueError('all input arrays must have the same shape') +ValueError: all input arrays must have the same shape + + +2023-05-29 18:00:29.464698 >>> SciFact +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 235, in run + results = task.evaluate(model, split, **kwargs) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/AbsTaskRetrieval.py", line 95, in evaluate + results = retriever.retrieve(corpus, queries) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/beir/retrieval/evaluation.py", line 23, in retrieve + return self.retriever.search(corpus, queries, self.top_k, self.score_function, **kwargs) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/beir/retrieval/search/dense/exact_search.py", line 65, in search + cos_scores = self.score_functions[score_function](query_embeddings, sub_corpus_embeddings) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/beir/retrieval/search/dense/util.py", line 11, in cos_sim + a = torch.tensor(a) +RuntimeError: Could not infer dtype of NoneType + + diff --git a/models_gh_runner.txt b/models_gh_runner.txt new file mode 100644 index 000000000..e69de29bb diff --git a/requirements-training.txt b/requirements-training.txt index c44eb61d7..70f76fc29 100644 --- a/requirements-training.txt +++ b/requirements-training.txt @@ -8,5 +8,7 @@ pandas braceexpand huggingface_hub transformers +datasets timm fsspec +scipy diff --git a/src/open_clip/__init__.py b/src/open_clip/__init__.py index 088c86441..30f59fb80 100644 --- a/src/open_clip/__init__.py +++ b/src/open_clip/__init__.py @@ -2,8 +2,9 @@ from .constants import OPENAI_DATASET_MEAN, OPENAI_DATASET_STD from .factory import create_model, create_model_and_transforms, create_model_from_pretrained, get_tokenizer, create_loss from .factory import list_models, add_model_config, get_model_config, load_checkpoint + from .loss import ClipLoss, DistillClipLoss, CoCaLoss -from .model import CLIP, CustomTextCLIP, CLIPTextCfg, CLIPVisionCfg, \ +from .model import CLIP, CustomTextCLIP, SiameseCLANP, CLANP, CLIPTextCfg, CLIPVisionCfg,\ convert_weights_to_lp, convert_weights_to_fp16, trace_model, get_cast_dtype from .openai import load_openai_model, list_openai_models from .pretrained import list_pretrained, list_pretrained_models_by_tag, list_pretrained_tags_by_model, \ diff --git a/src/open_clip/factory.py b/src/open_clip/factory.py index 14011f934..92be625ff 100644 --- a/src/open_clip/factory.py +++ b/src/open_clip/factory.py @@ -10,7 +10,7 @@ import torch from .constants import OPENAI_DATASET_MEAN, OPENAI_DATASET_STD -from .model import CLIP, CustomTextCLIP, convert_weights_to_lp, convert_to_custom_text_state_dict,\ +from .model import CLIP, CustomTextCLIP, CLANP, SiameseCLANP, convert_weights_to_lp, convert_to_custom_text_state_dict,\ resize_pos_embed, get_cast_dtype from .coca_model import CoCa from .loss import ClipLoss, DistillClipLoss, CoCaLoss @@ -44,10 +44,16 @@ def _rescan_model_configs(): for cf in config_files: with open(cf, 'r') as f: model_cfg = json.load(f) - if all(a in model_cfg for a in ('embed_dim', 'vision_cfg', 'text_cfg')): + + is_clip = all(a in model_cfg for a in ('embed_dim', 'vision_cfg', 'text_cfg')) + is_text_only = all(a in model_cfg for a in ('embed_dim', 'tower_a_cfg', 'tower_b_cfg')) + is_siamese_text_only = all(a in model_cfg for a in ('embed_dim', 'text_cfg')) + is_multimodal = all(a in model_cfg for a in ('embed_dim', 'vision_cfg', 'text_cfg', 'multimodal_cfg')) + if is_clip or is_text_only or is_siamese_text_only or is_multimodal: _MODEL_CONFIGS[cf.stem] = model_cfg _MODEL_CONFIGS = {k: v for k, v in sorted(_MODEL_CONFIGS.items(), key=lambda x: _natural_key(x[0]))} + _rescan_model_configs() # initial populate of model config registry @@ -74,12 +80,24 @@ def get_model_config(model_name): def get_tokenizer(model_name): + if model_name.startswith(HF_HUB_PREFIX): tokenizer = HFTokenizer(model_name[len(HF_HUB_PREFIX):]) else: config = get_model_config(model_name) - tokenizer = HFTokenizer( - config['text_cfg']['hf_tokenizer_name']) if 'hf_tokenizer_name' in config['text_cfg'] else tokenize + + if 'text_cfg' in config.keys(): + key = 'text_cfg' + elif 'tower_a_cfg' in config.keys(): + key = 'tower_a_cfg' + if 'hf_tokenizer_name' in config[key]: + tokenizer = HFTokenizer(config[key]['hf_tokenizer_name']) + if "pythia" in config[key]['hf_tokenizer_name']: + tokenizer.tokenizer.pad_token_id = 1 + else: + tokenizer = tokenize + + return tokenizer @@ -117,8 +135,10 @@ def create_model( pretrained_image: bool = False, pretrained_hf: bool = True, cache_dir: Optional[str] = None, + model_type: Optional[str] = "CLIP", output_dict: Optional[bool] = None, require_pretrained: bool = False, + ): has_hf_hub_prefix = model_name.startswith(HF_HUB_PREFIX) if has_hf_hub_prefix: @@ -180,18 +200,35 @@ def create_model( assert False, 'pretrained image towers currently only supported for timm models' cast_dtype = get_cast_dtype(precision) + is_hf_model = 'hf_model_name' in model_cfg.get('text_cfg', {}) custom_text = model_cfg.pop('custom_text', False) or force_custom_text or is_hf_model - if custom_text: - if is_hf_model: - model_cfg['text_cfg']['hf_model_pretrained'] = pretrained_hf - if "coca" in model_name: - model = CoCa(**model_cfg, cast_dtype=cast_dtype) + # switch to CLANP + if model_type=="CLANP": + if 'hf_model_name' in model_cfg.get('tower_a_cfg', {}): + model_cfg['tower_a_cfg']['hf_model_pretrained'] = pretrained_hf + if 'hf_model_name' in model_cfg.get('tower_b_cfg', {}): + model_cfg['tower_b_cfg']['hf_model_pretrained'] = pretrained_hf + model = CLANP(**model_cfg, cast_dtype=cast_dtype) + + elif model_type=="SiameseCLANP": + if 'hf_model_name' in model_cfg.get('tower_a_cfg', {}): + model_cfg['tower_a_cfg']['hf_model_pretrained'] = pretrained_hf + model = SiameseCLANP(**model_cfg, cast_dtype=cast_dtype) + + elif model_type=="CLIP": + + if custom_text: + if is_hf_model: + model_cfg['text_cfg']['hf_model_pretrained'] = pretrained_hf + if "coca" in model_name: + model = CoCa(**model_cfg, cast_dtype=cast_dtype) + else: + model = CustomTextCLIP(**model_cfg, cast_dtype=cast_dtype) else: - model = CustomTextCLIP(**model_cfg, cast_dtype=cast_dtype) - else: - model = CLIP(**model_cfg, cast_dtype=cast_dtype) + model = CLIP(**model_cfg, cast_dtype=cast_dtype) + pretrained_loaded = False if pretrained: @@ -226,9 +263,10 @@ def create_model( if precision in ("fp16", "bf16"): convert_weights_to_lp(model, dtype=torch.bfloat16 if precision == 'bf16' else torch.float16) + if model_type=="CLIP": # set image / mean metadata from pretrained_cfg if available, or use default - model.visual.image_mean = pretrained_cfg.get('mean', None) or OPENAI_DATASET_MEAN - model.visual.image_std = pretrained_cfg.get('std', None) or OPENAI_DATASET_STD + model.visual.image_mean = pretrained_cfg.get('mean', None) or OPENAI_DATASET_MEAN + model.visual.image_std = pretrained_cfg.get('std', None) or OPENAI_DATASET_STD # to always output dict even if it is clip if output_dict and hasattr(model, "output_dict"): @@ -287,6 +325,7 @@ def create_model_and_transforms( image_std: Optional[Tuple[float, ...]] = None, aug_cfg: Optional[Union[Dict[str, Any], AugmentationCfg]] = None, cache_dir: Optional[str] = None, + model_type: Optional[str] = "CLIP", output_dict: Optional[bool] = None, ): model = create_model( @@ -302,24 +341,29 @@ def create_model_and_transforms( pretrained_image=pretrained_image, pretrained_hf=pretrained_hf, cache_dir=cache_dir, + model_type=model_type, output_dict=output_dict, ) - image_mean = image_mean or getattr(model.visual, 'image_mean', None) - image_std = image_std or getattr(model.visual, 'image_std', None) - preprocess_train = image_transform( - model.visual.image_size, - is_train=True, - mean=image_mean, - std=image_std, - aug_cfg=aug_cfg, - ) - preprocess_val = image_transform( - model.visual.image_size, - is_train=False, - mean=image_mean, - std=image_std, - ) + if model_type=="CLIP": + image_mean = image_mean or getattr(model.visual, 'image_mean', None) + image_std = image_std or getattr(model.visual, 'image_std', None) + preprocess_train = image_transform( + model.visual.image_size, + is_train=True, + mean=image_mean, + std=image_std + ) + preprocess_val = image_transform( + model.visual.image_size, + is_train=False, + mean=image_mean, + std=image_std + ) + else: + preprocess_val = None + preprocess_train = None + return model, preprocess_train, preprocess_val @@ -337,6 +381,7 @@ def create_model_from_pretrained( image_mean: Optional[Tuple[float, ...]] = None, image_std: Optional[Tuple[float, ...]] = None, cache_dir: Optional[str] = None, + model_type: Optional[str] = "CLIP", ): model = create_model( model_name, @@ -348,19 +393,23 @@ def create_model_from_pretrained( force_custom_text=force_custom_text, force_image_size=force_image_size, cache_dir=cache_dir, + model_type=model_type, require_pretrained=True, ) if not return_transform: return model - image_mean = image_mean or getattr(model.visual, 'image_mean', None) - image_std = image_std or getattr(model.visual, 'image_std', None) - preprocess = image_transform( - model.visual.image_size, - is_train=False, - mean=image_mean, - std=image_std, - ) + + if model_type=="CLIP": + image_mean = image_mean or getattr(model.visual, 'image_mean', None) + image_std = image_std or getattr(model.visual, 'image_std', None) + preprocess = image_transform( + model.visual.image_size, + is_train=False, + mean=image_mean, + std=image_std + ) + return model, preprocess diff --git a/src/open_clip/hf_configs.py b/src/open_clip/hf_configs.py index e236222ba..41fd5a68b 100644 --- a/src/open_clip/hf_configs.py +++ b/src/open_clip/hf_configs.py @@ -42,4 +42,18 @@ }, "pooler": "mean_pooler", }, + # https://huggingface.co/docs/transformers/main/en/model_doc/gpt_neox#transformers.GPTNeoXConfig + "gpt_neox": { + "config_names": { + # https://github.com/huggingface/transformers/blob/c612628045822f909020f7eb6784c79700813eda/src/transformers/models/gpt_neox/modeling_gpt_neox.py#L410 + "context_length": "max_position_embeddings", + "vocab_size": "vocab_size", + "width": "hidden_size", + "heads": "num_attention_heads", + "layers": "num_hidden_layers", + "layer_attr": "layers", + "token_embeddings_attr": "embed_in" + }, + "pooler": "weighted_mean_pooler", + }, } diff --git a/src/open_clip/hf_model.py b/src/open_clip/hf_model.py index fbccc8127..29af8b9ea 100644 --- a/src/open_clip/hf_model.py +++ b/src/open_clip/hf_model.py @@ -61,6 +61,29 @@ def forward(self, x: BaseModelOutput, attention_mask: TensorType): return masked_output.max(1).values +@register_pooler +class WeightedMeanPooler(nn.Module): + """Weighted mean pooling for autoregressive models""" + + def forward(self, x: BaseModelOutput, attention_mask: TensorType): + + input_mask_expanded = attention_mask.unsqueeze(-1).expand(x.last_hidden_state.size()).float() + weights = ( + torch.arange(start=1, end=x.last_hidden_state.shape[1] + 1) + .unsqueeze(0) + .unsqueeze(-1) + .expand(x.last_hidden_state.size()) + .float().to(x.last_hidden_state.device) + ) + assert weights.shape == x.last_hidden_state.shape == input_mask_expanded.shape + input_mask_expanded = input_mask_expanded * weights + + sum_embeddings = torch.sum(x.last_hidden_state * input_mask_expanded, 1) + sum_mask = input_mask_expanded.sum(1) + sum_mask = torch.clamp(sum_mask, min=1e-9) + return sum_embeddings / sum_mask + + @register_pooler class ClsPooler(nn.Module): """CLS token pooling""" @@ -78,6 +101,8 @@ def forward(self, x: BaseModelOutput, attention_mask: TensorType): return x.pooler_output return x.last_hidden_state[:, self.cls_token_position, :] + + class HFTextEncoder(nn.Module): @@ -111,6 +136,9 @@ def __init__( if hasattr(self.config, "is_encoder_decoder") and self.config.is_encoder_decoder: self.transformer = create_func(model_args) self.transformer = self.transformer.encoder + elif 'gpt' in self.config.model_type: + self.transformer = create_func(model_args) + self.config.pad_token_id = 1 # this is for GPT-NeoX. It might need to be changed if other models are needed. else: self.transformer = create_func(model_args, add_pooling_layer=uses_transformer_pooler) else: @@ -139,23 +167,28 @@ def forward(self, x: TensorType): out = self.transformer(input_ids=x, attention_mask=attn_mask) pooled_out = self.pooler(out, attn_mask) projected = self.proj(pooled_out) - - seq_len = out.last_hidden_state.shape[1] - tokens = ( - out.last_hidden_state[:, torch.arange(seq_len) != self.pooler.cls_token_position, :] - if type(self.pooler) == ClsPooler - else out.last_hidden_state - ) if self.output_tokens: + seq_len = out.last_hidden_state.shape[1] + tokens = ( + out.last_hidden_state[:, torch.arange(seq_len) != self.pooler.cls_token_position, :] + if type(self.pooler) == ClsPooler + else out.last_hidden_state + ) return projected, tokens + return projected - def lock(self, unlocked_layers: int = 0, freeze_layer_norm: bool = True): + def lock(self, unlocked_layers: int = 0, freeze_layer_norm: bool = True, unlocked_biases: bool = False): if not unlocked_layers: # full freezing for n, p in self.transformer.named_parameters(): - p.requires_grad = (not freeze_layer_norm) if "LayerNorm" in n.split(".") else False + p.requires_grad = False + if "LayerNorm" in n.split("."): + p.requires_grad = (not freeze_layer_norm) + if 'bias' in n.split("."): + p.requires_grad = unlocked_biases return + encoder = self.transformer.encoder if hasattr(self.transformer, 'encoder') else self.transformer layer_list = getattr(encoder, arch_dict[self.config.model_type]["config_names"]["layer_attr"]) diff --git a/src/open_clip/loss.py b/src/open_clip/loss.py index 4fbf61dac..8737d2346 100644 --- a/src/open_clip/loss.py +++ b/src/open_clip/loss.py @@ -17,8 +17,8 @@ def gather_features( - image_features, - text_features, + features_a, + features_b, local_loss=False, gather_with_grad=False, rank=0, @@ -29,38 +29,38 @@ def gather_features( if use_horovod: assert hvd is not None, 'Please install horovod' if gather_with_grad: - all_image_features = hvd.allgather(image_features) - all_text_features = hvd.allgather(text_features) + all_features_a = hvd.allgather(features_a) + all_features_b = hvd.allgather(features_b) else: with torch.no_grad(): - all_image_features = hvd.allgather(image_features) - all_text_features = hvd.allgather(text_features) + all_features_a = hvd.allgather(features_a) + all_features_b = hvd.allgather(features_b) if not local_loss: # ensure grads for local rank when all_* features don't have a gradient - gathered_image_features = list(all_image_features.chunk(world_size, dim=0)) - gathered_text_features = list(all_text_features.chunk(world_size, dim=0)) - gathered_image_features[rank] = image_features - gathered_text_features[rank] = text_features - all_image_features = torch.cat(gathered_image_features, dim=0) - all_text_features = torch.cat(gathered_text_features, dim=0) + gathered_features_a = list(all_features_a.chunk(world_size, dim=0)) + gathered_features_b = list(all_features_b.chunk(world_size, dim=0)) + gathered_features_a[rank] = features_a + gathered_features_b[rank] = features_b + all_features_a = torch.cat(gathered_features_a, dim=0) + all_features_b = torch.cat(gathered_features_b, dim=0) else: # We gather tensors from all gpus if gather_with_grad: - all_image_features = torch.cat(torch.distributed.nn.all_gather(image_features), dim=0) - all_text_features = torch.cat(torch.distributed.nn.all_gather(text_features), dim=0) + all_features_a = torch.cat(torch.distributed.nn.all_gather(features_a), dim=0) + all_features_b = torch.cat(torch.distributed.nn.all_gather(features_b), dim=0) else: - gathered_image_features = [torch.zeros_like(image_features) for _ in range(world_size)] - gathered_text_features = [torch.zeros_like(text_features) for _ in range(world_size)] - dist.all_gather(gathered_image_features, image_features) - dist.all_gather(gathered_text_features, text_features) + gathered_features_a = [torch.zeros_like(features_a) for _ in range(world_size)] + gathered_features_b = [torch.zeros_like(features_b) for _ in range(world_size)] + dist.all_gather(gathered_features_a, features_a) + dist.all_gather(gathered_features_b, features_b) if not local_loss: # ensure grads for local rank when all_* features don't have a gradient - gathered_image_features[rank] = image_features - gathered_text_features[rank] = text_features - all_image_features = torch.cat(gathered_image_features, dim=0) - all_text_features = torch.cat(gathered_text_features, dim=0) + gathered_features_a[rank] = features_a + gathered_features_b[rank] = features_b + all_features_a = torch.cat(gathered_features_a, dim=0) + all_features_b = torch.cat(gathered_features_b, dim=0) - return all_image_features, all_text_features + return all_features_a, all_features_b class ClipLoss(nn.Module): @@ -99,33 +99,44 @@ def get_ground_truth(self, device, num_logits) -> torch.Tensor: labels = self.labels[device] return labels - def get_logits(self, image_features, text_features, logit_scale): + def get_logits(self, features_a, features_b, logit_scale): + if self.world_size > 1: - all_image_features, all_text_features = gather_features( - image_features, text_features, + all_features_a, all_features_b = gather_features( + features_a, features_b, self.local_loss, self.gather_with_grad, self.rank, self.world_size, self.use_horovod) if self.local_loss: - logits_per_image = logit_scale * image_features @ all_text_features.T - logits_per_text = logit_scale * text_features @ all_image_features.T + logits_per_feature_a = logit_scale * features_a @ all_features_b.T + logits_per_feature_b = logit_scale * features_b @ all_features_a.T else: - logits_per_image = logit_scale * all_image_features @ all_text_features.T - logits_per_text = logits_per_image.T + logits_per_feature_a = logit_scale * all_features_a @ all_features_b.T + logits_per_feature_b = logits_per_feature_a.T else: - logits_per_image = logit_scale * image_features @ text_features.T - logits_per_text = logit_scale * text_features @ image_features.T + logits_per_feature_a = logit_scale * features_a @ features_b.T + logits_per_feature_b = logit_scale * features_b @ features_a.T + + + return logits_per_feature_a, logits_per_feature_b + + + def forward(self, image_features=None, text_features=None, logit_scale=None, text_a_features=None, text_b_features=None, output_dict=False): - return logits_per_image, logits_per_text + if image_features is not None and text_features is not None: + features_a = image_features + features_b = text_features + elif text_a_features is not None and text_b_features is not None: + features_a = text_a_features + features_b = text_b_features - def forward(self, image_features, text_features, logit_scale, output_dict=False): - device = image_features.device - logits_per_image, logits_per_text = self.get_logits(image_features, text_features, logit_scale) + device = features_a.device + logits_per_feature_a, logits_per_feature_b = self.get_logits(features_a, features_b, logit_scale) - labels = self.get_ground_truth(device, logits_per_image.shape[0]) + labels = self.get_ground_truth(device, logits_per_feature_a.shape[0]) total_loss = ( - F.cross_entropy(logits_per_image, labels) + - F.cross_entropy(logits_per_text, labels) + F.cross_entropy(logits_per_feature_a, labels) + + F.cross_entropy(logits_per_feature_b, labels) ) / 2 return {"contrastive_loss": total_loss} if output_dict else total_loss @@ -210,3 +221,4 @@ def forward( return {"contrastive_loss": contrastive_loss, "distill_loss": distill_loss} return contrastive_loss, distill_loss + diff --git a/src/open_clip/model.py b/src/open_clip/model.py index 4f5e775a7..0b0b97376 100644 --- a/src/open_clip/model.py +++ b/src/open_clip/model.py @@ -261,8 +261,8 @@ def lock_image_tower(self, unlocked_groups=0, freeze_bn_stats=False): # lock image tower as per LiT - https://arxiv.org/abs/2111.07991 self.visual.lock(unlocked_groups=unlocked_groups, freeze_bn_stats=freeze_bn_stats) - def lock_text_tower(self, unlocked_layers: int = 0, freeze_layer_norm: bool = True): - self.text.lock(unlocked_layers, freeze_layer_norm) + def lock_text_tower(self, unlocked_layers: int = 0, freeze_layer_norm: bool = True, unlocked_biases: bool = False): + self.text.lock(unlocked_layers, freeze_layer_norm, unlocked_biases) @torch.jit.ignore def set_grad_checkpointing(self, enable=True): @@ -289,6 +289,94 @@ def forward(self, image, text): return image_features, text_features, self.logit_scale.exp() + +class CLANP(nn.Module): + def __init__( + self, + embed_dim: int, + tower_a_cfg: CLIPTextCfg, + tower_b_cfg: CLIPTextCfg, + quick_gelu: bool = False, + cast_dtype: Optional[torch.dtype] = None, + output_dict: bool = False, + ): + super().__init__() + self.output_dict = output_dict + self.tower_a = _build_text_tower(embed_dim, tower_a_cfg, quick_gelu, cast_dtype) + self.tower_b = _build_text_tower(embed_dim, tower_b_cfg, quick_gelu, cast_dtype) + self.logit_scale = nn.Parameter(torch.ones([]) * np.log(1 / 0.07)) + + def lock_tower_a(self, unlocked_layers: int = 0, freeze_layer_norm: bool = True, unlocked_biases: bool = False): + self.tower_a.lock(unlocked_layers, freeze_layer_norm, unlocked_biases) + + def lock_tower_b(self, unlocked_layers: int = 0, freeze_layer_norm: bool = True, unlocked_biases: bool = False): + self.tower_b.lock(unlocked_layers, freeze_layer_norm, unlocked_biases) + + @torch.jit.ignore + def set_grad_checkpointing(self, enable=True): + self.tower_a.set_grad_checkpointing(enable) + self.tower_b.set_grad_checkpointing(enable) + + def encode_text_a(self, text, normalize: bool = False): + features = self.tower_a(text) + return F.normalize(features, dim=-1) if normalize else features + + def encode_text_b(self, text, normalize: bool = False): + features = self.tower_b(text) + return F.normalize(features, dim=-1) if normalize else features + + def forward(self, text_a, text_b): + features_a = self.encode_text_a(text_a, normalize=True) + features_b = self.encode_text_b(text_b, normalize=True) + if self.output_dict: + return { + "text_a_features": features_a, + "text_b_features": features_b, + "logit_scale": self.logit_scale.exp() + } + return features_a, features_b, self.logit_scale.exp() + + + + +class SiameseCLANP(nn.Module): + def __init__( + self, + embed_dim: int, + text_cfg: CLIPTextCfg, + quick_gelu: bool = False, + cast_dtype: Optional[torch.dtype] = None, + output_dict: bool = False, + ): + super().__init__() + self.output_dict = output_dict + self.text_tower = _build_text_tower(embed_dim, text_cfg, quick_gelu, cast_dtype) + self.logit_scale = nn.Parameter(torch.ones([]) * np.log(1 / 0.07)) + + def lock_text_tower(self, unlocked_layers: int = 0, freeze_layer_norm: bool = True, unlocked_biases: bool = False): + self.text_tower.lock(unlocked_layers, freeze_layer_norm, unlocked_biases) + + @torch.jit.ignore + def set_grad_checkpointing(self, enable=True): + self.text_tower.set_grad_checkpointing(enable) + + def encode_text(self, text, normalize: bool = False): + features = self.text_tower(text) + return F.normalize(features, dim=-1) if normalize else features + + def forward(self, text_a, text_b): + features_a = self.encode_text(text_a, normalize=True) + features_b = self.encode_text(text_b, normalize=True) + if self.output_dict: + return { + "text_a_features": features_a, + "text_b_features": features_b, + "logit_scale": self.logit_scale.exp() + } + return features_a, features_b, self.logit_scale.exp() + + + def convert_weights_to_lp(model: nn.Module, dtype=torch.float16): """Convert applicable model parameters to low-precision (bf16 or fp16)""" diff --git a/src/open_clip/model_configs/Siamese-mt5-xl.json b/src/open_clip/model_configs/Siamese-mt5-xl.json new file mode 100644 index 000000000..8066ae07b --- /dev/null +++ b/src/open_clip/model_configs/Siamese-mt5-xl.json @@ -0,0 +1,10 @@ +{ + "embed_dim": 1024, + "quick_gelu": true, + "text_cfg": { + "hf_model_name": "google/mt5-xl", + "hf_tokenizer_name": "google/mt5-xl", + "proj": "mlp", + "pooler_type": "mean_pooler" + } +} \ No newline at end of file diff --git a/src/open_clip/model_configs/Siamese-pythia-410m.json b/src/open_clip/model_configs/Siamese-pythia-410m.json new file mode 100644 index 000000000..1c86bfabb --- /dev/null +++ b/src/open_clip/model_configs/Siamese-pythia-410m.json @@ -0,0 +1,10 @@ +{ + "embed_dim": 512, + "quick_gelu": true, + "text_cfg": { + "hf_model_name": "EleutherAI/pythia-410m-deduped", + "hf_tokenizer_name": "EleutherAI/pythia-410m-deduped", + "proj": "mlp", + "pooler_type": "weighted_mean_pooler" + } +} \ No newline at end of file diff --git a/src/open_clip/model_configs/Siamese-xlm-roberta-large.json b/src/open_clip/model_configs/Siamese-xlm-roberta-large.json new file mode 100644 index 000000000..037a97d8a --- /dev/null +++ b/src/open_clip/model_configs/Siamese-xlm-roberta-large.json @@ -0,0 +1,10 @@ +{ + "embed_dim": 512, + "quick_gelu": true, + "text_cfg": { + "hf_model_name": "xlm-roberta-large", + "hf_tokenizer_name": "xlm-roberta-large", + "proj": "mlp", + "pooler_type": "mean_pooler" + } +} \ No newline at end of file diff --git a/src/open_clip/model_configs/pythia-410m-pythia-410m.json b/src/open_clip/model_configs/pythia-410m-pythia-410m.json new file mode 100644 index 000000000..e5931980f --- /dev/null +++ b/src/open_clip/model_configs/pythia-410m-pythia-410m.json @@ -0,0 +1,16 @@ +{ + "embed_dim": 512, + "quick_gelu": true, + "tower_a_cfg": { + "hf_model_name": "EleutherAI/pythia-410m-deduped", + "hf_tokenizer_name": "EleutherAI/pythia-410m-deduped", + "proj": "mlp", + "pooler_type": "weighted_mean_pooler" + }, + "tower_b_cfg": { + "hf_model_name": "EleutherAI/pythia-410m-deduped", + "hf_tokenizer_name": "EleutherAI/pythia-410m-deduped", + "proj": "mlp", + "pooler_type": "weighted_mean_pooler" + } +} \ No newline at end of file diff --git a/src/open_clip/model_configs/pythia-70m-pythia-70m.json b/src/open_clip/model_configs/pythia-70m-pythia-70m.json new file mode 100644 index 000000000..5c49ed5c5 --- /dev/null +++ b/src/open_clip/model_configs/pythia-70m-pythia-70m.json @@ -0,0 +1,16 @@ +{ + "embed_dim": 512, + "quick_gelu": true, + "tower_a_cfg": { + "hf_model_name": "EleutherAI/pythia-70m-deduped", + "hf_tokenizer_name": "EleutherAI/pythia-70m-deduped", + "proj": "mlp", + "pooler_type": "weighted_mean_pooler" + }, + "tower_b_cfg": { + "hf_model_name": "EleutherAI/pythia-70m-deduped", + "hf_tokenizer_name": "EleutherAI/pythia-70m-deduped", + "proj": "mlp", + "pooler_type": "weighted_mean_pooler" + } +} \ No newline at end of file diff --git a/src/open_clip/model_configs/roberta-roberta.json b/src/open_clip/model_configs/roberta-roberta.json new file mode 100644 index 000000000..fe21c14df --- /dev/null +++ b/src/open_clip/model_configs/roberta-roberta.json @@ -0,0 +1,16 @@ +{ + "embed_dim": 512, + "quick_gelu": true, + "tower_a_cfg": { + "hf_model_name": "roberta-base", + "hf_tokenizer_name": "roberta-base", + "proj": "mlp", + "pooler_type": "mean_pooler" + }, + "tower_b_cfg": { + "hf_model_name": "roberta-base", + "hf_tokenizer_name": "roberta-base", + "proj": "mlp", + "pooler_type": "mean_pooler" + } +} \ No newline at end of file diff --git a/src/open_clip/model_configs/xlm-roberta-large-xlm-roberta-large.json b/src/open_clip/model_configs/xlm-roberta-large-xlm-roberta-large.json new file mode 100644 index 000000000..766e71439 --- /dev/null +++ b/src/open_clip/model_configs/xlm-roberta-large-xlm-roberta-large.json @@ -0,0 +1,16 @@ +{ + "embed_dim": 512, + "quick_gelu": true, + "tower_a_cfg": { + "hf_model_name": "xlm-roberta-large", + "hf_tokenizer_name": "xlm-roberta-large", + "proj": "mlp", + "pooler_type": "mean_pooler" + }, + "tower_b_cfg": { + "hf_model_name": "xlm-roberta-large", + "hf_tokenizer_name": "xlm-roberta-large", + "proj": "mlp", + "pooler_type": "mean_pooler" + } +} \ No newline at end of file diff --git a/src/open_clip/text_augmentation.py b/src/open_clip/text_augmentation.py new file mode 100644 index 000000000..66bd8a568 --- /dev/null +++ b/src/open_clip/text_augmentation.py @@ -0,0 +1,101 @@ +import torch +import random +import numpy as np + + +''' +Data augmentation scripts from the original contriever implementation: https://github.com/facebookresearch/contriever/blob/main/src/data.py +''' + +def randomcrop(x, ratio_min, ratio_max): + + ratio = random.uniform(ratio_min, ratio_max) + length = int(len(x) * ratio) + start = random.randint(0, len(x) - length) + end = start + length + crop = x[start:end].clone() + return crop + + +def add_token(x, token): + x = torch.cat((torch.tensor([token]), x)) + return x + + +def deleteword(x, p=0.1): + mask = np.random.rand(len(x)) + x = [e for e, m in zip(x, mask) if m > p] + return x + + +def replaceword(x, min_random, max_random, p=0.1): + mask = np.random.rand(len(x)) + x = [e if m > p else random.randint(min_random, max_random) for e, m in zip(x, mask)] + return x + + +def maskword(x, mask_id, p=0.1): + mask = np.random.rand(len(x)) + x = [e if m > p else mask_id for e, m in zip(x, mask)] + return x + + +def shuffleword(x, p=0.1): + count = (np.random.rand(len(x)) < p).sum() + """Shuffles any n number of values in a list""" + indices_to_shuffle = random.sample(range(len(x)), k=count) + to_shuffle = [x[i] for i in indices_to_shuffle] + random.shuffle(to_shuffle) + for index, value in enumerate(to_shuffle): + old_index = indices_to_shuffle[index] + x[old_index] = value + return x + + +def apply_augmentation(x, tokenizer,args): + if args.text_augmentation == "mask": + return torch.tensor(maskword(x, mask_id=tokenizer.mask_token_id, p=args.text_prob_augmentation)) + elif args.text_augmentation == "replace": + return torch.tensor( + replaceword(x, min_random=args.start_id, max_random=len(tokenizer) - 1, p=args.text_prob_augmentation) + ) + elif args.text_augmentation == "delete": + return torch.tensor(deleteword(x, p=args.text_prob_augmentation)) + elif args.text_augmentation == "shuffle": + return torch.tensor(shuffleword(x, p=args.text_prob_augmentation)) + else: + if not isinstance(x, torch.Tensor): + x = torch.Tensor(x) + return x + + +def add_bos_eos(x, bos_token_id, eos_token_id): + if not isinstance(x, torch.Tensor): + x = torch.Tensor(x) + if bos_token_id is None and eos_token_id is not None: + x = torch.cat([x.clone().detach(), torch.tensor([eos_token_id])]) + elif bos_token_id is not None and eos_token_id is None: + x = torch.cat([torch.tensor([bos_token_id]), x.clone().detach()]) + elif bos_token_id is None and eos_token_id is None: + pass + else: + x = torch.cat([torch.tensor([bos_token_id]), x.clone().detach(), torch.tensor([eos_token_id])]) + return x + + +def create_unsupervised_sample(text,tokenizer,args): + # remove padding + text = text[text!=tokenizer.tokenizer.pad_token_id] + if tokenizer.tokenizer.bos_token_id is not None: + text = text[text!=tokenizer.tokenizer.bos_token_id] + if tokenizer.tokenizer.eos_token_id is not None: + text = text[text!=tokenizer.tokenizer.eos_token_id] + # perform data augmentation + text = randomcrop(text, args.text_aug_ratio_min, args.text_aug_ratio_max) + text = apply_augmentation(text, tokenizer.tokenizer, args) + # add bos/eos tokens back + text = add_bos_eos(text, tokenizer.tokenizer.bos_token_id, tokenizer.tokenizer.eos_token_id) + # padding back to max length + output = torch.zeros(args.context_length) + output[:len(text)] = text + return output.long() \ No newline at end of file diff --git a/src/training/data.py b/src/training/data.py index 8eee19f85..8bee63eb3 100644 --- a/src/training/data.py +++ b/src/training/data.py @@ -12,6 +12,7 @@ import numpy as np import pandas as pd +from functools import partial import torch import torchvision.datasets as datasets import webdataset as wds @@ -20,6 +21,7 @@ from torch.utils.data.distributed import DistributedSampler from webdataset.filters import _shuffle from webdataset.tariterators import base_plus_ext, url_opener, tar_file_expander, valid_sample +from open_clip.text_augmentation import create_unsupervised_sample try: import horovod.torch as hvd @@ -27,6 +29,67 @@ hvd = None +class STSDataset(Dataset): + def __init__(self, input_filename, tokenizer=None): + from datasets import load_dataset + logging.debug(f'Loading STS data from {input_filename}.') + self.dataset = load_dataset(input_filename,keep_in_memory=True) + if "train" in self.dataset.keys(): + self.dataset = self.dataset['train'] + + logging.debug('Done loading data.') + + self.tokenize = tokenizer + + def __len__(self): + return len(self.dataset) + + def __getitem__(self, idx): + entry = self.dataset[idx] + sent1 = self.tokenize([str(entry['sentence1'])])[0] + sent2 = self.tokenize([str(entry['sentence2'])])[0] + score = entry['score'] + return sent1,sent2,score + + + +def get_STS_dataset(args, preprocess_fn, is_train, epoch=0, floor=False, tokenizer=None): + input_filename = args.sts_val_data + assert input_filename + + + dataset = STSDataset(input_filename,tokenizer=tokenizer) + + sampler = DistributedSampler(dataset) if args.distributed and is_train else None + shuffle = is_train and sampler is None + + dataloader = DataLoader( + dataset, + batch_size=args.batch_size, + shuffle=shuffle, + num_workers=args.workers, + pin_memory=True, + sampler=sampler, + drop_last=is_train, + ) + + num_samples = args.train_num_samples + round_fn = math.floor if floor else math.ceil + global_batch_size = args.batch_size * args.world_size + num_batches = round_fn(num_samples / global_batch_size) + num_workers = max(1, args.workers) + num_worker_batches = round_fn(num_batches / num_workers) # per dataloader worker + num_batches = num_worker_batches * num_workers + + logging.debug("%s"%(num_samples)) + dataloader.num_samples = num_samples + dataloader.num_batches = num_batches + + return DataInfo(dataloader, sampler) + + + + class CsvDataset(Dataset): def __init__(self, input_filename, transforms, img_key, caption_key, sep="\t", tokenizer=None): logging.debug(f'Loading csv data from {input_filename}.') @@ -375,14 +438,30 @@ def get_wds_dataset(args, preprocess_img, is_train, epoch=0, floor=False, tokeni # at this point, we have an iterator over the shards assigned to each worker wds.tarfile_to_samples(handler=log_and_continue), ]) - pipeline.extend([ - wds.select(filter_no_caption_or_no_image), - wds.decode("pilrgb", handler=log_and_continue), - wds.rename(image="jpg;png;jpeg;webp", text="txt"), - wds.map_dict(image=preprocess_img, text=lambda text: tokenizer(text)[0]), - wds.to_tuple("image", "text"), - wds.batched(args.batch_size, partial=not is_train) - ]) + + if 'CLANP' in args.model_type and not args.unsupervised_pretraining: + pipeline.extend([ + wds.to_tuple(args.text_a_key, args.text_b_key), + wds.map_tuple(lambda text: tokenizer(text.decode('utf8'),context_length=args.context_length)[0], lambda text: tokenizer(text.decode('utf8'),context_length=args.context_length)[0]), + wds.batched(args.batch_size, partial=not is_train), + ]) + elif 'CLANP' in args.model_type and args.unsupervised_pretraining: + pipeline.extend([ + wds.to_tuple(args.text_a_key, args.text_b_key), + wds.map_tuple(lambda text: tokenizer(text.decode('utf8'),context_length=args.context_length)[0], lambda text: tokenizer(text.decode('utf8'),context_length=args.context_length)[0]), + wds.map_tuple(lambda text: create_unsupervised_sample(text,tokenizer,args), lambda text: create_unsupervised_sample(text,tokenizer,args)), + wds.batched(args.batch_size, partial=not is_train), + ]) + else: + pipeline.extend([ + wds.select(filter_no_caption_or_no_image), + wds.decode("pilrgb", handler=log_and_continue), + wds.rename(image="jpg;png;jpeg;webp", text="txt"), + wds.map_dict(image=preprocess_img, text=lambda text: tokenizer(text)[0]), + wds.to_tuple("image", "text"), + wds.batched(args.batch_size, partial=not is_train), + ]) + dataset = wds.DataPipeline(*pipeline) @@ -402,6 +481,7 @@ def get_wds_dataset(args, preprocess_img, is_train, epoch=0, floor=False, tokeni # last batches are partial, eval is done on single (master) node num_batches = math.ceil(num_samples / args.batch_size) + dataloader = wds.WebLoader( dataset, batch_size=None, @@ -505,6 +585,8 @@ def get_synthetic_dataset(args, preprocess_fn, is_train, epoch=0, tokenizer=None def get_dataset_fn(data_path, dataset_type): + if dataset_type == 'sts': + return get_STS_dataset if dataset_type == "webdataset": return get_wds_dataset elif dataset_type == "csv": @@ -541,5 +623,8 @@ def get_data(args, preprocess_fns, epoch=0, tokenizer=None): if args.imagenet_v2 is not None: data["imagenet-v2"] = get_imagenet(args, preprocess_fns, "v2") + + if args.sts_val_data is not None: + data['sts-val'] = get_dataset_fn(args,'sts')(args, preprocess_fns, is_train=False, epoch=epoch, tokenizer=tokenizer) return data diff --git a/src/training/main.py b/src/training/main.py index f70c9f953..54ff04072 100644 --- a/src/training/main.py +++ b/src/training/main.py @@ -228,6 +228,7 @@ def main(args): pretrained_image=args.pretrained_image, image_mean=args.image_mean, image_std=args.image_std, + model_type=args.model_type, aug_cfg=args.aug_cfg, output_dict=True, ) @@ -254,7 +255,18 @@ def main(args): if args.lock_text: model.lock_text_tower( unlocked_layers=args.lock_text_unlocked_layers, - freeze_layer_norm=args.lock_text_freeze_layer_norm) + freeze_layer_norm=args.lock_text_freeze_layer_norm, + unlocked_biases=args.text_unlocked_biases) + if args.lock_tower_a: + model.lock_tower_a( + unlocked_layers=args.lock_tower_a_unlocked_layers, + freeze_layer_norm=args.lock_tower_a_freeze_layer_norm, + unlocked_biases=args.text_a_unlocked_biases) + if args.lock_tower_b: + model.lock_tower_b( + unlocked_layers=args.lock_tower_b_unlocked_layers, + freeze_layer_norm=args.lock_tower_b_freeze_layer_norm, + unlocked_biases=args.text_b_unlocked_biases) if args.grad_checkpointing: model.set_grad_checkpointing() @@ -336,6 +348,7 @@ def main(args): # initialize datasets data = get_data(args, (preprocess_train, preprocess_val), epoch=start_epoch, tokenizer=get_tokenizer(args.model)) assert len(data), 'At least one train or eval dataset must be specified.' + print(len(data)) # create scheduler if train scheduler = None @@ -398,7 +411,8 @@ def main(args): train_one_epoch(model, data, loss, epoch, optimizer, scaler, scheduler, dist_model, args, tb_writer=writer) completed_epoch = epoch + 1 - if any(v in data for v in ('val', 'imagenet-val', 'imagenet-v2')): + if any(v in data for v in ('val', 'imagenet-val', 'imagenet-v2', 'sts-val')): + logging.info('Running evaluation.') evaluate(model, data, completed_epoch, args, writer) # Saving checkpoints. diff --git a/src/training/mteb_eval.py b/src/training/mteb_eval.py new file mode 100644 index 000000000..f4ca1028b --- /dev/null +++ b/src/training/mteb_eval.py @@ -0,0 +1,65 @@ +import sys +import numpy as np +import torch +import torch.nn as nn +from mteb import MTEB +from training.params import parse_args +from tqdm.autonotebook import trange +import open_clip +from open_clip.factory import get_tokenizer + +class CLANP_For_MTEB(nn.Module): + + def __init__(self,args): + super().__init__() + self.model, _, _ = open_clip.create_model_and_transforms(args.model, pretrained=args.pretrained, jit=False, model_type='SiameseCLANP') + self.tokenizer = get_tokenizer(args.model) + self.model.eval() + self.device = 'cuda' if torch.cuda.is_available() else 'cpu' + self.model.to(self.device) + + def encode(self, sentences, batch_size=128, show_progress_bar=True, **kwargs): + """ Returns a list of embeddings for the given sentences. + Args: + sentences (`List[str]`): List of sentences to encode + batch_size (`int`): Batch size for the encoding + + Returns: + `List[np.ndarray]` or `List[tensor]`: List of embeddings for the given sentences + """ + + + all_embeddings = [] + for start_index in trange(0, len(sentences), batch_size, desc="Batches", disable=not show_progress_bar): + sentences_batch = sentences[start_index:start_index+batch_size] + features = self.tokenizer(sentences_batch) + features = features.to(self.device) + with torch.no_grad(): + embeddings = self.model.encode_text(features) + + all_embeddings.append(embeddings.detach()) + + all_embeddings = torch.cat(all_embeddings,0) + + return all_embeddings + + + + +def run_mteb(args): + + model = CLANP_For_MTEB(args) + evaluation = MTEB(tasks=['MSMARCO']) #task_types=['Clustering','retrieval'] + results = evaluation.run(model, eval_splits=["test"], output_folder=args.mteb_output) + + +if __name__ == "__main__": + + args = parse_args(sys.argv[1:]) + run_mteb(args) + + # /fsx/home-jianz/logs/2023_05_11-16_04_50-model_Siamese-pythia-410m-lr_0.0001-b_1024-j_6-p_amp_bfloat16/checkpoints/epoch_1.pt + # srun --account=laion --partition=g40 nodes=1 --ntasks-per-node=1 --cpus-per-task=4 --job-name=MTEB --pty bash -i + # srun --account laion --partition=g40 --gpus=1 --cpus-per-gpu=4 --job-name=pty --pty bash -i + # python3 src/training/mteb_eval.py --model="Siamese-xlm-roberta-large" --pretrained /fsx/home-jianz/logs/2023_05_25-09_21_35-model_Siamese-xlm-roberta-large-lr_1e-05-b_256-j_10-p_amp_bfloat16/checkpoints/epoch_latest.pt --mteb-output /fsx/home-jianz/logs/mteb --context-length=128 + \ No newline at end of file diff --git a/src/training/params.py b/src/training/params.py index 36c693bc7..b4da2cdc3 100644 --- a/src/training/params.py +++ b/src/training/params.py @@ -61,7 +61,7 @@ def parse_args(args): ) parser.add_argument( "--dataset-type", - choices=["webdataset", "csv", "synthetic", "auto"], + choices=["webdataset", "csv", "synthetic", "auto", "textpair"], default="auto", help="Which type of dataset to process." ) @@ -89,6 +89,18 @@ def parse_args(args): default="title", help="For csv-like datasets, the name of the key for the captions." ) + parser.add_argument( + "--text-a-key", + type=str, + default="text_a", + help="For text pair datasets, the name of the key for the texts a." + ) + parser.add_argument( + "--text-b-key", + type=str, + default="text_b", + help="For text pair datasets, the name of the key for the texts b." + ) parser.add_argument( "--imagenet-val", type=str, @@ -101,6 +113,12 @@ def parse_args(args): default=None, help="Path to imagenet v2 for conducting zero shot evaluation.", ) + parser.add_argument( + "--sts-val-data", + type=str, + default=None, + help="Path to STS17 crosslingual evaluation data. The default path is: lingjzhu/sts17-crosslingual", + ) parser.add_argument( "--logs", type=str, @@ -364,20 +382,82 @@ def parse_args(args): "--lock-text-unlocked-layers", type=int, default=0, - help="Leave last n image tower layer groups unlocked.", + help="Leave last n text tower layer groups unlocked.", + ) + parser.add_argument( + "--text-unlocked-biases", + action='store_true', + default=False, + help="Only unlock model biases", ) parser.add_argument( "--lock-text-freeze-layer-norm", default=False, action='store_true', + help="Freeze BatchNorm running stats in text tower for any locked layers.", + ) + parser.add_argument( + "--lock-tower-b", + default=False, + action='store_true', + help="Lock full text tower by disabling gradients.", + ) + parser.add_argument( + "--lock-tower-b-unlocked-layers", + type=int, + default=0, + help="Leave last n tower layer groups unlocked.", + ) + parser.add_argument( + "--lock-tower-b-freeze-layer-norm", + default=False, + action='store_true', + help="Freeze BatchNorm running stats in image tower for any locked layers.", + ) + parser.add_argument( + "--lock-tower-a", + default=False, + action='store_true', + help="Lock full text tower by disabling gradients.", + ) + parser.add_argument( + "--lock-tower-a-unlocked-layers", + type=int, + default=0, + help="Leave last n tower layer groups unlocked.", + ) + parser.add_argument( + "--lock-tower-a-freeze-layer-norm", + default=False, + action='store_true', help="Freeze BatchNorm running stats in image tower for any locked layers.", ) + parser.add_argument( + "--text-b-unlocked-biases", + action='store_true', + default=False, + help="Only unlock model biases", + ) + parser.add_argument( + "--text-a-unlocked-biases", + action='store_true', + default=False, + help="Only unlock model biases", + ) parser.add_argument( "--log-every-n-steps", type=int, - default=100, + default=10, help="Log every n steps to tensorboard/console/wandb.", ) + parser.add_argument( + "--model-type", + type=str, + choices=["CLIP", "CLANP", "SiameseCLANP"], + default="CLIP", + help="Specify the model type", + ) + parser.add_argument( "--coca-caption-loss-weight", type=float, @@ -414,6 +494,7 @@ def parse_args(args): action="store_true", help="If true, delete previous checkpoint after storing a new one." ) + parser.add_argument( "--distill-model", default=None, @@ -424,6 +505,51 @@ def parse_args(args): default=None, help='Which pre-trained weights to distill from, if any.' ) + + ''' + Arguments related to training a contriever-like text-to-text model + ''' + parser.add_argument( + "--unsupervised-pretraining", + action="store_true", + help="Whether to run contriever-style unupservised contrastive pretraining" + ) + parser.add_argument( + "--text-augmentation", + default='delete', + choices=['delete','mask','replace','shuffle'], + help='Which text augmentation to create postive pairs for training unsupervised text retreival models' + ) + parser.add_argument( + "--text-aug-ratio-min", + default=0.1, + type=float, + help='min ratio for random cropping of texts (only applies to unsupervised text retreival models)' + ) + parser.add_argument( + "--text-aug-ratio-max", + default=0.5, + type=float, + help='max ratio for random cropping of texts (only applies to unsupervised text retreival models)' + ) + parser.add_argument( + "--text-prob-augmentation", + default=0.1, + type=float, + help='probability of applying augmentations to texts (only applies to unsupervised text retreival models)' + ) + parser.add_argument( + "--context-length", + default=77, + type=int, + help="max number of tokens in the input text" + ) + parser.add_argument( + "--mteb-output", + default=None, + type=str, + help="location to store MTEB evaluation results" + ) args = parser.parse_args(args) # If some params are not passed, we use the default values based on model name. diff --git a/src/training/sts_eval.py b/src/training/sts_eval.py new file mode 100644 index 000000000..ec677b370 --- /dev/null +++ b/src/training/sts_eval.py @@ -0,0 +1,56 @@ +import torch +import torch.nn.functional as F +import open_clip +from datasets import load_dataset +from scipy.stats import spearmanr, pearsonr +from tqdm import tqdm +import numpy as np + +import logging + +from open_clip import get_cast_dtype, get_tokenizer +from .precision import get_autocast + +def run(model,dataloader,args): + + preds = [] + gt_scores = [] + + for sent1,sent2,scores in tqdm(dataloader, unit_scale=args.batch_size): + + with torch.no_grad(), torch.cuda.amp.autocast(): + a_features = model.encode_text(sent1.to(args.device)) + b_features = model.encode_text(sent2.to(args.device)) + + pred = F.cosine_similarity(a_features,b_features,dim=-1).squeeze() + + preds.append(pred.cpu().detach().numpy()) + gt_scores.append(scores) + + preds = np.concatenate(preds,axis=0) + gt_scores = np.concatenate(gt_scores,axis=0) + + return spearmanr(preds,gt_scores)[0], pearsonr(preds,gt_scores)[0] + + +def sts_eval(model, data, epoch, args): + + if 'sts-val' not in data: + return {} + + logging.info('Starting STS-17 evaluation.') + + + results = {} + + spearman_corr, pearson_corr = run(model,data['sts-val'].dataloader, args) + results['sts-val-spearman-corr'] = spearman_corr + results['sts-val-pearson-corr'] = pearson_corr + + logging.info('Finished STS-17') + + return results + + + + diff --git a/src/training/train.py b/src/training/train.py index e0a140f9c..9edba64db 100644 --- a/src/training/train.py +++ b/src/training/train.py @@ -17,6 +17,7 @@ from open_clip import get_cast_dtype, CLIP, CustomTextCLIP from .distributed import is_master from .zero_shot import zero_shot_eval +from .sts_eval import sts_eval from .precision import get_autocast @@ -239,9 +240,13 @@ def evaluate(model, data, epoch, args, tb_writer=None): return metrics device = torch.device(args.device) model.eval() - - zero_shot_metrics = zero_shot_eval(model, data, epoch, args) - metrics.update(zero_shot_metrics) + + if hasattr(model,'visual'): + zero_shot_metrics = zero_shot_eval(model, data, epoch, args) + metrics.update(zero_shot_metrics) + elif hasattr(model,'text_tower'): + sts_metrics = sts_eval(model, data, epoch, args) + metrics.update(sts_metrics) autocast = get_autocast(args.precision) cast_dtype = get_cast_dtype(args.precision) diff --git a/tests/test_CLANP.py b/tests/test_CLANP.py new file mode 100644 index 000000000..b198d9e56 --- /dev/null +++ b/tests/test_CLANP.py @@ -0,0 +1,93 @@ +import torch +from open_clip.factory import get_tokenizer +import pytest +import open_clip +import os +import sys +import pandas as pd +from training.data import get_data +from training.params import parse_args +from training.main import main + +os.environ["CUDA_VISIBLE_DEVICES"] = "" + +@pytest.mark.parametrize("model_cfg", [("roberta-roberta")]) +def test_inference_simple(model_cfg, pretrained=None): + model, _, preprocess = open_clip.create_model_and_transforms(model_cfg, pretrained=pretrained, jit=False, model_type='CLANP') + tokenizer = get_tokenizer(model_cfg) + + text_a = tokenizer(['this', 'is', 'a', 'document']) + text_b = tokenizer(['this', 'is', 'a', 'summary']) + + with torch.no_grad(): + text_a_features = model.encode_text_a(text_a) + text_b_features = model.encode_text_b(text_b) + + print(text_a_features.shape) + print(text_b_features.shape) + + text_probs = (100.0 * text_a_features @ text_b_features.T).softmax(dim=-1) + print(text_probs) + + model.lock_tower_a(unlocked_layers=0,freeze_layer_norm=True,unlock_biases=True) + model.lock_tower_b(unlocked_layers=0,freeze_layer_norm=True,unlock_biases=True) + + + +@pytest.mark.parametrize("model_cfg", [("Siamese-xlm-roberta-large")]) +def test_inference_simple(model_cfg, pretrained=None): + model, _, preprocess = open_clip.create_model_and_transforms(model_cfg, pretrained=pretrained, jit=False, model_type='SiameseCLANP') + tokenizer = get_tokenizer(model_cfg) + + text_a = tokenizer(['this', 'is', 'a', 'document']) + text_b = tokenizer(['this', 'is', 'a', 'summary']) + + with torch.no_grad(): + text_a_features = model.encode_text(text_a) + text_b_features = model.encode_text(text_b) + + print(text_a_features.shape) + print(text_b_features.shape) + + text_probs = (100.0 * text_a_features @ text_b_features.T).softmax(dim=-1) + print(text_probs) + + model.lock_text_tower(unlocked_layers=0,freeze_layer_norm=True,unlock_biases=True) + + +@pytest.mark.parametrize("model_cfg", [("pythia-410m-pythia-410m")]) +def test_inference_simple(model_cfg, pretrained=None): + model, _, preprocess = open_clip.create_model_and_transforms(model_cfg, pretrained=pretrained, jit=False, model_type='CLANP') + tokenizer = get_tokenizer(model_cfg) + + text_a = tokenizer(['this', 'is', 'a', 'document']) + text_b = tokenizer(['this', 'is', 'a', 'summary']) + + with torch.no_grad(): + text_a_features = model.encode_text_a(text_a) + text_b_features = model.encode_text_b(text_b) + + print(text_a_features.shape) + print(text_b_features.shape) + + text_probs = (100.0 * text_a_features @ text_b_features.T).softmax(dim=-1) + print(text_probs) + + +@pytest.mark.parametrize("model_cfg", [("Siamese-pythia-410m")]) +def test_inference_simple(model_cfg, pretrained=None): + model, _, preprocess = open_clip.create_model_and_transforms(model_cfg, pretrained=pretrained, jit=False, model_type='SiameseCLANP') + tokenizer = get_tokenizer(model_cfg) + + text_a = tokenizer(['this', 'is', 'a', 'document']) + text_b = tokenizer(['this', 'is', 'a', 'summary']) + + with torch.no_grad(): + text_a_features = model.encode_text(text_a) + text_b_features = model.encode_text(text_b) + + print(text_a_features.shape) + print(text_b_features.shape) + + text_probs = (100.0 * text_a_features @ text_b_features.T).softmax(dim=-1) + print(text_probs) diff --git a/tests/test_inference.py b/tests/test_inference.py index dca8dc44c..2e4b93c0b 100644 --- a/tests/test_inference.py +++ b/tests/test_inference.py @@ -29,7 +29,9 @@ 'mt5-xl-ViT-H-14', 'coca_base', 'coca_ViT-B-32', - 'coca_roberta-ViT-B-32' + 'coca_roberta-ViT-B-32', + 'roberta-roberta', + 'xlm-roberta-large-xlm-roberta-large', }) if 'OPEN_CLIP_TEST_REG_MODELS' in os.environ: