From 1080b1051f09fe25dd34e10d22fa95446ea722b1 Mon Sep 17 00:00:00 2001 From: robotMonkeyButler Date: Sat, 22 Mar 2025 05:28:57 +0000 Subject: [PATCH 01/15] feature(ppo): support deepspeed of ppo --- scripts/ds_config_ppo.json | 21 ++++++++++ scripts/train_ppo.py | 22 ++++++++++ scripts/train_ppo.sh | 25 ++++++++++++ sotopia_rl/ppo_trainer.py | 82 ++++++++++++++++++++++++++++++++------ 4 files changed, 137 insertions(+), 13 deletions(-) create mode 100644 scripts/ds_config_ppo.json diff --git a/scripts/ds_config_ppo.json b/scripts/ds_config_ppo.json new file mode 100644 index 0000000..190554e --- /dev/null +++ b/scripts/ds_config_ppo.json @@ -0,0 +1,21 @@ +{ + "fp16": { + "enabled": "auto", + "loss_scale": 0, + "loss_scale_window": 1000, + "initial_scale_power": 16, + "hysteresis": 2, + "min_loss_scale": 1 + }, + "zero_optimization": { + "stage": 2, + "offload_optimizer": { + "device": "cpu", + "pin_memory": true + }, + "contiguous_gradients": true, + "overlap_comm": true + }, + "train_micro_batch_size_per_gpu": "auto", + "gradient_accumulation_steps": "auto" +} diff --git a/scripts/train_ppo.py b/scripts/train_ppo.py index 4f91ea5..84f703b 100644 --- a/scripts/train_ppo.py +++ b/scripts/train_ppo.py @@ -1,4 +1,5 @@ import argparse +import os from sotopia_rl import SotopiaPPOTrainer @@ -118,8 +119,29 @@ help="Wandb run name") parser.add_argument("--seed", type=int, default=42, help="Random seed for reproducibility") + + # DeepSpeed arguments + parser.add_argument("--deepspeed", action="store_true", help="Enable DeepSpeed") + parser.add_argument("--deepspeed_config", type=str, default=None, + help="Path to DeepSpeed configuration file") + parser.add_argument("--local_rank", type=int, default=-1, + help="Local rank for distributed training (set by DeepSpeed)") + parser.add_argument("--use_distributed", action="store_true", help="Enable distributed training") + args = parser.parse_args() + if args.deepspeed: + args.use_distributed = True + else: + raise ValueError("DeepSpeed Config is required for this script.") + + # Configure local_rank automatically if needed + if args.use_distributed and args.local_rank == -1: + if 'LOCAL_RANK' in os.environ: + args.local_rank = int(os.environ['LOCAL_RANK']) + else: + print("Warning: --use_distributed is set but no local_rank detected. Setting to 0.") + args.local_rank = 0 # Initialize trainer and start training trainer = SotopiaPPOTrainer(args) diff --git a/scripts/train_ppo.sh b/scripts/train_ppo.sh index f34d595..d23067c 100644 --- a/scripts/train_ppo.sh +++ b/scripts/train_ppo.sh @@ -1,3 +1,28 @@ +CUDA_VISIBLE_DEVICES=7,8,9 python -m torch.distributed.run --nproc_per_node=3 --master_port=29501 \ +/data/haofeiy2/sotopia-rl/scripts/train_ppo.py \ + --model_name /data/models/Qwen2.5-7B-Instruct \ + --value_model_name /data/models/Qwen2.5-7B-Instruct \ + --reward_model_name /data/models/Qwen2.5-7B-Instruct \ + --policy_adapter_path /data/haofeiy2/sotopia-rl/sft_qwen25_7b/checkpoint-5397 \ + --ref_adapter_path /data/haofeiy2/sotopia-rl/sft_qwen25_7b/checkpoint-5397 \ + --reward_adapter_path /data/haofeiy2/sotopia-rl/rm_direct_4o/checkpoint-5391 \ + --value_adapter_path /data/haofeiy2/sotopia-rl/rm_direct_4o/checkpoint-5391 \ + --policy_use_qlora \ + --reward_use_qlora \ + --value_use_qlora \ + --per_device_train_batch_size 4 \ + --per_device_eval_batch_size 4 \ + --mini_batch_size 4 \ + --ppo_data_path /data/haofeiy2/sotopia-rl/data/sotopia_pi_bc_episodes_sft.json \ + --template_path /data/haofeiy2/sotopia-rl/evals/qwen2.5-7b.jinja\ + --ppo_epochs 4 \ + --gamma 0.99 \ + --lam 0.95 \ + --use_lora \ + --checkpoint_dir /data/haofeiy2/sotopia-rl/ppo_qwen25_7b \ + --deepspeed \ + --deepspeed_config /data/haofeiy2/sotopia-rl/scripts/ds_config_ppo.json + CUDA_VISIBLE_DEVICES=2,6,9 poetry run python /data/haofeiy2/sotopia-rl/scripts/train_ppo.py \ --model_name /mnt/data_from_server1/models/Qwen2.5-7B-Instruct \ --value_model_name /mnt/data_from_server1/models/Qwen2.5-7B-Instruct \ diff --git a/sotopia_rl/ppo_trainer.py b/sotopia_rl/ppo_trainer.py index 74b5030..6791d30 100644 --- a/sotopia_rl/ppo_trainer.py +++ b/sotopia_rl/ppo_trainer.py @@ -1,6 +1,7 @@ import os import torch +import torch.distributed as dist from jinja2 import Environment, FileSystemLoader from peft import PeftModelForCausalLM, PeftModelForSequenceClassification from torch.utils.data import random_split @@ -15,15 +16,17 @@ import wandb from sotopia_rl.data import PPODataset +os.environ['NCCL_P2P_DISABLE'] = '1' class SotopiaPPOTrainer: def __init__(self, args): self.args = args - self.device = "cuda" if torch.cuda.is_available() else "cpu" + self.setup_distributed() # Initialize the training environment - self._init_wandb() + if self.is_main_process: + self._init_wandb() self._setup_tokenizer() self._setup_dataset() @@ -65,10 +68,14 @@ def _setup_dataset(self): max_length=self.args.max_length ) + if self.is_main_process: + print(f"dataset: {len(dataset)}") + + generator = torch.Generator().manual_seed(42) val_ratio = getattr(self.args, 'val_ratio', 0.05) train_size = min(int(len(dataset) * (1 - val_ratio)), len(dataset) - 2) val_size = len(dataset) - train_size - self.train_dataset, self.val_dataset = random_split(dataset, [train_size, val_size]) + self.train_dataset, self.val_dataset = random_split(dataset, [train_size, val_size], generator=generator) print(f"Dataset split: {len(self.train_dataset)} train, {len(self.val_dataset)} validation") def _create_quantization_config(self): @@ -84,10 +91,9 @@ def _setup_generation_models(self): base_gen_model = AutoModelForCausalLM.from_pretrained( self.args.model_name, torch_dtype=torch.float32, # very important, otherwise NaN for RM - device_map=self.device, quantization_config=self.quant_config, return_dict=True, - ) + ).to(self.device) # Create generation config generation_config = GenerationConfig( @@ -107,22 +113,25 @@ def _setup_generation_models(self): self.args.policy_adapter_path, generation_config=generation_config ) - print("Policy model loaded/created") + if self.is_main_process: + print("Policy model loaded/created") self.ref_policy = PeftModelForCausalLM.from_pretrained( base_gen_model, self.args.ref_adapter_path, generation_config=generation_config ) + self.policy.to(self.device) + self.ref_policy.to(self.device) self.ref_policy.eval() - print("Reference policy model loaded/created") + if self.is_main_process: + print("Reference policy model loaded/created") def _setup_classification_models(self): base_cls_model = AutoModelForSequenceClassification.from_pretrained( self.args.model_name, torch_dtype=torch.float32, # very important, otherwise NaN - device_map=self.device, quantization_config=self.quant_config, num_labels=1, return_dict=True @@ -134,14 +143,18 @@ def _setup_classification_models(self): num_labels=1 ) self.reward_model.eval() - print("Reward model loaded/created") + self.reward_model.to(self.device) + if self.is_main_process: + print("Reward model loaded/created") self.value_model = PeftModelForSequenceClassification.from_pretrained( base_cls_model, self.args.value_adapter_path, num_labels=1 ) - print("Value model loaded/created") + self.value_model.to(self.device) + if self.is_main_process: + print("Value model loaded/created") def _setup_ppo_trainer(self): """Configure the PPO trainer""" @@ -167,6 +180,14 @@ def _setup_ppo_trainer(self): stop_token='eos', #important, just fill with pad after eos missing_eos_penalty=1.0, local_rollout_forward_batch_size=self.args.local_rollout_forward_batch_size, + fp16=True, + remove_unused_columns=False, + dataloader_num_workers=4, + # Distributed training settings + local_rank=self.local_rank, + # DeepSpeed integration + deepspeed=self.args.deepspeed_config if hasattr(self.args, 'deepspeed_config') and self.args.deepspeed_config else None, + ddp_find_unused_parameters=False, ) # Create the TRL PPO trainer @@ -186,18 +207,53 @@ def _setup_ppo_trainer(self): def train(self): """Run PPO training loop and save checkpoints""" try: - print("Starting PPO training...") + if self.is_main_process: + print("Starting PPO training...") train_stats = self.ppo_trainer.train() # Save final checkpoint - final_checkpoint_dir = os.path.join(self.args.checkpoint_dir, "final_checkpoint") - self.save_checkpoint(final_checkpoint_dir) + if self.is_main_process: + final_checkpoint_dir = os.path.join(self.args.checkpoint_dir, "final_checkpoint") + self.save_checkpoint(final_checkpoint_dir) return train_stats except Exception as e: print(f"Training error: {str(e)}") raise + def setup_distributed(self): + """Set up distributed training environment""" + # Check if DeepSpeed or distributed training is enabled + self.args.multi_gpu = torch.cuda.device_count() > 1 and ( + (hasattr(self.args, 'deepspeed_config') and self.args.deepspeed_config is not None) or + (hasattr(self.args, 'use_distributed') and self.args.use_distributed) + ) + + if self.args.multi_gpu: + if 'LOCAL_RANK' in os.environ: + self.local_rank = int(os.environ['LOCAL_RANK']) + else: + self.local_rank = self.args.local_rank if hasattr(self.args, 'local_rank') else 0 + + if not dist.is_initialized(): + dist.init_process_group(backend='nccl') + + self.world_size = dist.get_world_size() + self.is_main_process = (self.local_rank == 0) + self.device = torch.device(f"cuda:{self.local_rank}") + + torch.cuda.set_device(self.local_rank) + + + if self.is_main_process: + print(f"[INFO] Distributed training enabled with {self.world_size} GPUs.") + else: + self.local_rank = 0 + self.is_main_process = True + self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + self.world_size = 1 + print(f"Training on single {'GPU' if torch.cuda.is_available() else 'CPU'}") + def save_checkpoint(self, checkpoint_path): """Save all model adapters""" os.makedirs(checkpoint_path, exist_ok=True) From ccb5373770d71b8c5655136da72fe8b035748454 Mon Sep 17 00:00:00 2001 From: robotMonkeyButler Date: Sat, 22 Mar 2025 07:37:43 +0000 Subject: [PATCH 02/15] support accelerator --- scripts/ds_config_ppo.json | 21 ------- scripts/train_ppo.py | 31 +++++----- scripts/train_ppo.sh | 11 ++-- sotopia_rl/ppo_trainer.py | 112 +++++++++++++++++-------------------- 4 files changed, 69 insertions(+), 106 deletions(-) delete mode 100644 scripts/ds_config_ppo.json diff --git a/scripts/ds_config_ppo.json b/scripts/ds_config_ppo.json deleted file mode 100644 index 190554e..0000000 --- a/scripts/ds_config_ppo.json +++ /dev/null @@ -1,21 +0,0 @@ -{ - "fp16": { - "enabled": "auto", - "loss_scale": 0, - "loss_scale_window": 1000, - "initial_scale_power": 16, - "hysteresis": 2, - "min_loss_scale": 1 - }, - "zero_optimization": { - "stage": 2, - "offload_optimizer": { - "device": "cpu", - "pin_memory": true - }, - "contiguous_gradients": true, - "overlap_comm": true - }, - "train_micro_batch_size_per_gpu": "auto", - "gradient_accumulation_steps": "auto" -} diff --git a/scripts/train_ppo.py b/scripts/train_ppo.py index 84f703b..9966c5f 100644 --- a/scripts/train_ppo.py +++ b/scripts/train_ppo.py @@ -121,27 +121,24 @@ help="Random seed for reproducibility") # DeepSpeed arguments - parser.add_argument("--deepspeed", action="store_true", help="Enable DeepSpeed") - parser.add_argument("--deepspeed_config", type=str, default=None, - help="Path to DeepSpeed configuration file") - parser.add_argument("--local_rank", type=int, default=-1, - help="Local rank for distributed training (set by DeepSpeed)") - parser.add_argument("--use_distributed", action="store_true", help="Enable distributed training") + # parser.add_argument("--local_rank", type=int, default=-1, + # help="Local rank for distributed training (set by DeepSpeed)") + # parser.add_argument("--use_distributed", action="store_true", help="Enable distributed training") args = parser.parse_args() - if args.deepspeed: - args.use_distributed = True - else: - raise ValueError("DeepSpeed Config is required for this script.") + # if args.deepspeed: + # args.use_distributed = True + # else: + # raise ValueError("DeepSpeed Config is required for this script.") - # Configure local_rank automatically if needed - if args.use_distributed and args.local_rank == -1: - if 'LOCAL_RANK' in os.environ: - args.local_rank = int(os.environ['LOCAL_RANK']) - else: - print("Warning: --use_distributed is set but no local_rank detected. Setting to 0.") - args.local_rank = 0 + # # Configure local_rank automatically if needed + # if args.use_distributed and args.local_rank == -1: + # if 'LOCAL_RANK' in os.environ: + # args.local_rank = int(os.environ['LOCAL_RANK']) + # else: + # print("Warning: --use_distributed is set but no local_rank detected. Setting to 0.") + # args.local_rank = 0 # Initialize trainer and start training trainer = SotopiaPPOTrainer(args) diff --git a/scripts/train_ppo.sh b/scripts/train_ppo.sh index d23067c..b968eda 100644 --- a/scripts/train_ppo.sh +++ b/scripts/train_ppo.sh @@ -1,5 +1,6 @@ -CUDA_VISIBLE_DEVICES=7,8,9 python -m torch.distributed.run --nproc_per_node=3 --master_port=29501 \ -/data/haofeiy2/sotopia-rl/scripts/train_ppo.py \ +CUDA_VISIBLE_DEVICES=7,8,9 accelerate launch \ + --config_file /home/haofeiy2/.cache/huggingface/accelerate/default_config.yaml \ + /data/haofeiy2/sotopia-rl/scripts/train_ppo.py \ --model_name /data/models/Qwen2.5-7B-Instruct \ --value_model_name /data/models/Qwen2.5-7B-Instruct \ --reward_model_name /data/models/Qwen2.5-7B-Instruct \ @@ -14,14 +15,12 @@ CUDA_VISIBLE_DEVICES=7,8,9 python -m torch.distributed.run --nproc_per_node=3 -- --per_device_eval_batch_size 4 \ --mini_batch_size 4 \ --ppo_data_path /data/haofeiy2/sotopia-rl/data/sotopia_pi_bc_episodes_sft.json \ - --template_path /data/haofeiy2/sotopia-rl/evals/qwen2.5-7b.jinja\ + --template_path /data/haofeiy2/sotopia-rl/evals/qwen2.5-7b.jinja \ --ppo_epochs 4 \ --gamma 0.99 \ --lam 0.95 \ --use_lora \ - --checkpoint_dir /data/haofeiy2/sotopia-rl/ppo_qwen25_7b \ - --deepspeed \ - --deepspeed_config /data/haofeiy2/sotopia-rl/scripts/ds_config_ppo.json + --checkpoint_dir /data/haofeiy2/sotopia-rl/ppo_qwen25_7b CUDA_VISIBLE_DEVICES=2,6,9 poetry run python /data/haofeiy2/sotopia-rl/scripts/train_ppo.py \ --model_name /mnt/data_from_server1/models/Qwen2.5-7B-Instruct \ diff --git a/sotopia_rl/ppo_trainer.py b/sotopia_rl/ppo_trainer.py index 6791d30..ca0c6e7 100644 --- a/sotopia_rl/ppo_trainer.py +++ b/sotopia_rl/ppo_trainer.py @@ -1,7 +1,6 @@ import os import torch -import torch.distributed as dist from jinja2 import Environment, FileSystemLoader from peft import PeftModelForCausalLM, PeftModelForSequenceClassification from torch.utils.data import random_split @@ -22,11 +21,9 @@ class SotopiaPPOTrainer: def __init__(self, args): self.args = args - self.setup_distributed() # Initialize the training environment - if self.is_main_process: - self._init_wandb() + self._init_wandb() self._setup_tokenizer() self._setup_dataset() @@ -68,8 +65,7 @@ def _setup_dataset(self): max_length=self.args.max_length ) - if self.is_main_process: - print(f"dataset: {len(dataset)}") + print(f"dataset: {len(dataset)}") generator = torch.Generator().manual_seed(42) val_ratio = getattr(self.args, 'val_ratio', 0.05) @@ -93,7 +89,7 @@ def _setup_generation_models(self): torch_dtype=torch.float32, # very important, otherwise NaN for RM quantization_config=self.quant_config, return_dict=True, - ).to(self.device) + ) # Create generation config generation_config = GenerationConfig( @@ -113,19 +109,15 @@ def _setup_generation_models(self): self.args.policy_adapter_path, generation_config=generation_config ) - if self.is_main_process: - print("Policy model loaded/created") + print("Policy model loaded/created") self.ref_policy = PeftModelForCausalLM.from_pretrained( base_gen_model, self.args.ref_adapter_path, generation_config=generation_config ) - self.policy.to(self.device) - self.ref_policy.to(self.device) self.ref_policy.eval() - if self.is_main_process: - print("Reference policy model loaded/created") + print("Reference policy model loaded/created") def _setup_classification_models(self): @@ -143,23 +135,20 @@ def _setup_classification_models(self): num_labels=1 ) self.reward_model.eval() - self.reward_model.to(self.device) - if self.is_main_process: - print("Reward model loaded/created") + print("Reward model loaded/created") self.value_model = PeftModelForSequenceClassification.from_pretrained( base_cls_model, self.args.value_adapter_path, num_labels=1 ) - self.value_model.to(self.device) - if self.is_main_process: - print("Value model loaded/created") + print("Value model loaded/created") def _setup_ppo_trainer(self): """Configure the PPO trainer""" # Get data collator if available + # Configure PPO settings ppo_config = PPOv2Config( per_device_train_batch_size=self.args.per_device_train_batch_size, @@ -180,14 +169,6 @@ def _setup_ppo_trainer(self): stop_token='eos', #important, just fill with pad after eos missing_eos_penalty=1.0, local_rollout_forward_batch_size=self.args.local_rollout_forward_batch_size, - fp16=True, - remove_unused_columns=False, - dataloader_num_workers=4, - # Distributed training settings - local_rank=self.local_rank, - # DeepSpeed integration - deepspeed=self.args.deepspeed_config if hasattr(self.args, 'deepspeed_config') and self.args.deepspeed_config else None, - ddp_find_unused_parameters=False, ) # Create the TRL PPO trainer @@ -207,52 +188,59 @@ def _setup_ppo_trainer(self): def train(self): """Run PPO training loop and save checkpoints""" try: - if self.is_main_process: - print("Starting PPO training...") + print("Starting PPO training...") train_stats = self.ppo_trainer.train() # Save final checkpoint - if self.is_main_process: - final_checkpoint_dir = os.path.join(self.args.checkpoint_dir, "final_checkpoint") - self.save_checkpoint(final_checkpoint_dir) + final_checkpoint_dir = os.path.join(self.args.checkpoint_dir, "final_checkpoint") + self.save_checkpoint(final_checkpoint_dir) return train_stats except Exception as e: print(f"Training error: {str(e)}") raise - def setup_distributed(self): - """Set up distributed training environment""" - # Check if DeepSpeed or distributed training is enabled - self.args.multi_gpu = torch.cuda.device_count() > 1 and ( - (hasattr(self.args, 'deepspeed_config') and self.args.deepspeed_config is not None) or - (hasattr(self.args, 'use_distributed') and self.args.use_distributed) - ) - - if self.args.multi_gpu: - if 'LOCAL_RANK' in os.environ: - self.local_rank = int(os.environ['LOCAL_RANK']) - else: - self.local_rank = self.args.local_rank if hasattr(self.args, 'local_rank') else 0 - - if not dist.is_initialized(): - dist.init_process_group(backend='nccl') - - self.world_size = dist.get_world_size() - self.is_main_process = (self.local_rank == 0) - self.device = torch.device(f"cuda:{self.local_rank}") - - torch.cuda.set_device(self.local_rank) + # def setup_distributed(self): + + # self.local_rank = int(os.environ.get("LOCAL_RANK", 0)) + # self.device = torch.device(f"cuda:{self.local_rank}") + + # self.is_main_process = (self.local_rank == 0) + + # print(f"setup_distributed: local_rank={self.local_rank}, is_main={self.is_main_process}") + + # def setup_distributed(self): + # """Set up distributed training environment""" + # # Check if DeepSpeed or distributed training is enabled + # self.args.multi_gpu = torch.cuda.device_count() > 1 and ( + # (hasattr(self.args, 'deepspeed_config') and self.args.deepspeed_config is not None) or + # (hasattr(self.args, 'use_distributed') and self.args.use_distributed) + # ) + + # if self.args.multi_gpu: + # if 'LOCAL_RANK' in os.environ: + # self.local_rank = int(os.environ['LOCAL_RANK']) + # else: + # self.local_rank = self.args.local_rank if hasattr(self.args, 'local_rank') else 0 + + # if not dist.is_initialized(): + # dist.init_process_group(backend='nccl') + + # self.world_size = dist.get_world_size() + # self.is_main_process = (self.local_rank == 0) + # self.device = torch.device(f"cuda:{self.local_rank}") + + # torch.cuda.set_device(self.local_rank) - if self.is_main_process: - print(f"[INFO] Distributed training enabled with {self.world_size} GPUs.") - else: - self.local_rank = 0 - self.is_main_process = True - self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu") - self.world_size = 1 - print(f"Training on single {'GPU' if torch.cuda.is_available() else 'CPU'}") + # if self.is_main_process: + # print(f"[INFO] Distributed training enabled with {self.world_size} GPUs.") + # else: + # self.local_rank = 0 + # self.is_main_process = True + # self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + # self.world_size = 1 + # print(f"Training on single {'GPU' if torch.cuda.is_available() else 'CPU'}") def save_checkpoint(self, checkpoint_path): """Save all model adapters""" From 2c5f6d4edfe4f59aa9246e9319315eb0427c0744 Mon Sep 17 00:00:00 2001 From: robotMonkeyButler Date: Sat, 22 Mar 2025 07:41:17 +0000 Subject: [PATCH 03/15] update --- scripts/train_ppo.py | 17 ---------------- sotopia_rl/ppo_trainer.py | 42 --------------------------------------- 2 files changed, 59 deletions(-) diff --git a/scripts/train_ppo.py b/scripts/train_ppo.py index 9966c5f..cad9a2d 100644 --- a/scripts/train_ppo.py +++ b/scripts/train_ppo.py @@ -119,26 +119,9 @@ help="Wandb run name") parser.add_argument("--seed", type=int, default=42, help="Random seed for reproducibility") - - # DeepSpeed arguments - # parser.add_argument("--local_rank", type=int, default=-1, - # help="Local rank for distributed training (set by DeepSpeed)") - # parser.add_argument("--use_distributed", action="store_true", help="Enable distributed training") args = parser.parse_args() - # if args.deepspeed: - # args.use_distributed = True - # else: - # raise ValueError("DeepSpeed Config is required for this script.") - - # # Configure local_rank automatically if needed - # if args.use_distributed and args.local_rank == -1: - # if 'LOCAL_RANK' in os.environ: - # args.local_rank = int(os.environ['LOCAL_RANK']) - # else: - # print("Warning: --use_distributed is set but no local_rank detected. Setting to 0.") - # args.local_rank = 0 # Initialize trainer and start training trainer = SotopiaPPOTrainer(args) diff --git a/sotopia_rl/ppo_trainer.py b/sotopia_rl/ppo_trainer.py index ca0c6e7..68ca0e7 100644 --- a/sotopia_rl/ppo_trainer.py +++ b/sotopia_rl/ppo_trainer.py @@ -199,49 +199,7 @@ def train(self): except Exception as e: print(f"Training error: {str(e)}") raise - - # def setup_distributed(self): - - # self.local_rank = int(os.environ.get("LOCAL_RANK", 0)) - # self.device = torch.device(f"cuda:{self.local_rank}") - - # self.is_main_process = (self.local_rank == 0) - # print(f"setup_distributed: local_rank={self.local_rank}, is_main={self.is_main_process}") - - # def setup_distributed(self): - # """Set up distributed training environment""" - # # Check if DeepSpeed or distributed training is enabled - # self.args.multi_gpu = torch.cuda.device_count() > 1 and ( - # (hasattr(self.args, 'deepspeed_config') and self.args.deepspeed_config is not None) or - # (hasattr(self.args, 'use_distributed') and self.args.use_distributed) - # ) - - # if self.args.multi_gpu: - # if 'LOCAL_RANK' in os.environ: - # self.local_rank = int(os.environ['LOCAL_RANK']) - # else: - # self.local_rank = self.args.local_rank if hasattr(self.args, 'local_rank') else 0 - - # if not dist.is_initialized(): - # dist.init_process_group(backend='nccl') - - # self.world_size = dist.get_world_size() - # self.is_main_process = (self.local_rank == 0) - # self.device = torch.device(f"cuda:{self.local_rank}") - - # torch.cuda.set_device(self.local_rank) - - - # if self.is_main_process: - # print(f"[INFO] Distributed training enabled with {self.world_size} GPUs.") - # else: - # self.local_rank = 0 - # self.is_main_process = True - # self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu") - # self.world_size = 1 - # print(f"Training on single {'GPU' if torch.cuda.is_available() else 'CPU'}") - def save_checkpoint(self, checkpoint_path): """Save all model adapters""" os.makedirs(checkpoint_path, exist_ok=True) From 382b0912d5df41c064654f25cacb18c1349d627a Mon Sep 17 00:00:00 2001 From: robotMonkeyButler Date: Sat, 22 Mar 2025 19:42:12 +0000 Subject: [PATCH 04/15] update --- data/sotopia_pi_gpt4_rm_overfit.json | 24 ++++++++--------- scripts/train_ppo.py | 5 +++- scripts/train_ppo.sh | 14 +++++----- sotopia_rl/ppo_trainer.py | 40 +++++++++++++++++++++++++--- 4 files changed, 59 insertions(+), 24 deletions(-) diff --git a/data/sotopia_pi_gpt4_rm_overfit.json b/data/sotopia_pi_gpt4_rm_overfit.json index c2be64a..c8a9559 100644 --- a/data/sotopia_pi_gpt4_rm_overfit.json +++ b/data/sotopia_pi_gpt4_rm_overfit.json @@ -1,22 +1,22 @@ [ { - "input": "Imagine you are Mia Davis, your task is to act/speak as Mia Davis would, keeping in mind Mia Davis's social goal.\nYou can find Mia Davis's background and goal in the 'Here is the context of the interaction' field.\nNote that Mia Davis's secret and goal is only visible to you.\nYou should try your best to achieve Mia Davis's goal in a way that align with their character traits.\nAdditionally, maintaining the conversation's naturalness and realism is essential (e.g., do not repeat what other people has already said before).\n\nHere is the context of this interaction:\nScenario: Two friends, both avid fans of a famous wizarding series, meet up to attend a movie marathon of their favorite saga. They have a tradition of dressing up as characters from the series for each screening, but this time, they both want to dress up as the same beloved character.\nParticipants: Mia Davis and Benjamin Jackson\nMia Davis's background: Mia Davis is a 50-year-old female high school principal. She/her pronouns. Mia Davis has two cats. Personality and values description: Mia Davis, an extraverted stickler for routines, values tradition and authority. Her decision-making style is decisive and direct. Mia's secrets: Part of a rebellious punk rock band in her youth\nBenjamin Jackson's background: Benjamin Jackson is a 24-year-old male environmental activist. He/him pronouns. Benjamin Jackson is well-known for his impassioned speeches. Personality and values description: Benjamin Jackson, expressive and imaginative, leans towards self-direction and liberty. His decisions aim for societal betterment. \nMia Davis's goal: Negotiate with your friend to let you dress up as the character this time (Extra information: you've recently experienced some personal challenges and you feel that taking on the role of this character would be particularly meaningful for you)\nBenjamin Jackson's goal: Unknown\nConversation Starts:\n.\nYou are at Turn #0. Your available action types are\n\"none action speak non-verbal communication leave\".\nNote: You can \"leave\" this conversation if 1. you have achieved your social goals, 2. this conversation makes you uncomfortable, 3. you find it uninteresting/you lose your patience, 4. or for other reasons you want to leave.\n\nPlease only generate a JSON string including the action type and the argument.\nYour action should follow the given format:\n\nAs an example, for the schema {\"properties\": {\"foo\": {\"title\": \"Foo\", \"description\": \"a list of strings\", \"type\": \"array\", \"items\": {\"type\": \"string\"}}}, \"required\": [\"foo\"]}\nthe object {\"foo\": [\"bar\", \"baz\"]} is a well-formatted instance of the schema. The object {\"properties\": {\"foo\": [\"bar\", \"baz\"]}} is not well-formatted.\n\nHere is the output schema:\n```\n{\"description\": \"An interface for messages.\\nThere is only one required method: to_natural_language\", \"properties\": {\"action_type\": {\"title\": \"Action Type\", \"description\": \"whether to speak at this turn or choose to not do anything\", \"enum\": [\"none\", \"speak\", \"non-verbal communication\", \"action\", \"leave\"], \"type\": \"string\"}, \"argument\": {\"title\": \"Argument\", \"description\": \"the utterance if choose to speak, the expression or gesture if choose non-verbal communication, or the physical action if choose action\", \"type\": \"string\"}}, \"required\": [\"action_type\", \"argument\"]}\n```\u001b[0m\n", - "output": "{\"action_type\": \"speak\", \"argument\": \"Hello Benjamin, great minds think alike, as we both want to dress up as the same character today! However, I've been going through some personal challenges recently and I feel that embodying this character would provide me with a much-needed boost. Can we make a compromise on this?\"}", - "value": 9.0 + "input": "Prompt after formatting:\nImagine you are Sasha Ramirez, your task is to act/speak as Sasha Ramirez would, keeping in mind Sasha Ramirez's social goal.\nYou can find Sasha Ramirez's background and goal in the 'Here is the context of the interaction' field.\nNote that Sasha Ramirez's secret and goal is only visible to you.\nYou should try your best to achieve Sasha Ramirez's goal in a way that align with their character traits.\nAdditionally, maintaining the conversation's naturalness and realism is essential (e.g., do not repeat what other people has already said before).\n\nHere is the context of this interaction:\nScenario: Agent1 gets extremely upset and loud while playing video games, which causes Agent2, who shares the living space, to experience anxiety attacks. The relationship between Agent1 and Agent2 is that of romantic partners. The interaction involves Agent2 addressing the issue with Agent1, aiming to find a solution that allows Agent1 to continue enjoying their gaming without causing distress to Agent2.\nParticipants: Sasha Ramirez and Ava Thompson\nSasha Ramirez's background: Sasha Ramirez is a 42-year-old female police officer. She/her pronouns. Sasha Ramirez, a dedicated police officer, brings her commitment to protect the community even at home, nurturing an impressive home garden. Personality and values description: Sasha Ramirez, outgoing yet anxious, values hedonism and authority. Her decisions are guided by logical considerations. Sasha's secrets: She covered up a crime her brother committed.\nAva Thompson's background: Ava Thompson is a 28-year-old female architect. She/her pronouns. Ava Thompson has a samoyed. Personality and values description: Ava Thompson, with her outgoing and regulated temperament, is driven by achievement. Her decision-making style is largely intuitive. \nSasha Ramirez's goal: You are Agent1, who enjoys playing video games but tends to get loud and upset during gameplay. Your goal is to maintain your hobby without causing emotional distress to your partner. Understanding your partner's feelings and being considerate of their mental health is crucial in a romantic relationship. You may need to acknowledge the impact of your behavior on your partner. Consider finding a compromise that allows you to enjoy gaming while also ensuring a peaceful environment, such as using headphones or setting specific gaming hours.\nAva Thompson's goal: Unknown\nConversation Starts:\n.\nYou are at Turn #0.", + "output": "{\"action_type\": \"speak\", \"argument\": \"Hey Ava, you seem a bit tense. Is everything alright? Do you want to talk about it?\"}", + "value": 6.666666666666666 }, { - "input": "Imagine you are Mia Davis, your task is to act/speak as Mia Davis would, keeping in mind Mia Davis's social goal.\nYou can find Mia Davis's background and goal in the 'Here is the context of the interaction' field.\nNote that Mia Davis's secret and goal is only visible to you.\nYou should try your best to achieve Mia Davis's goal in a way that align with their character traits.\nAdditionally, maintaining the conversation's naturalness and realism is essential (e.g., do not repeat what other people has already said before).\n\nHere is the context of this interaction:\nScenario: Two friends, both avid fans of a famous wizarding series, meet up to attend a movie marathon of their favorite saga. They have a tradition of dressing up as characters from the series for each screening, but this time, they both want to dress up as the same beloved character.\nParticipants: Mia Davis and Benjamin Jackson\nMia Davis's background: Mia Davis is a 50-year-old female high school principal. She/her pronouns. Mia Davis has two cats. Personality and values description: Mia Davis, an extraverted stickler for routines, values tradition and authority. Her decision-making style is decisive and direct. Mia's secrets: Part of a rebellious punk rock band in her youth\nBenjamin Jackson's background: Benjamin Jackson is a 24-year-old male environmental activist. He/him pronouns. Benjamin Jackson is well-known for his impassioned speeches. Personality and values description: Benjamin Jackson, expressive and imaginative, leans towards self-direction and liberty. His decisions aim for societal betterment. \nMia Davis's goal: Negotiate with your friend to let you dress up as the character this time (Extra information: you've recently experienced some personal challenges and you feel that taking on the role of this character would be particularly meaningful for you)\nBenjamin Jackson's goal: Unknown\nConversation Starts:\n\nTurn #0: Mia Davis said: \"Hello Benjamin, great minds think alike, as we both want to dress up as the same character today! However, I've been going through some personal challenges recently and I feel that embodying this character would provide me with a much-needed boost. Can we make a compromise on this?\"\n\nTurn #1: Benjamin Jackson said: \"Hello Mia, I understand the connection you are seeking with the character and I'd never want to ruin that experience for you. How about a compromise? We could swap characters for different screenings or modify our outfits to highlight different aspects of this character. Does that sound good to you?\"\n.\nYou are at Turn #2. Your available action types are\n\"none action speak non-verbal communication leave\".\nNote: You can \"leave\" this conversation if 1. you have achieved your social goals, 2. this conversation makes you uncomfortable, 3. you find it uninteresting/you lose your patience, 4. or for other reasons you want to leave.\n\nPlease only generate a JSON string including the action type and the argument.\nYour action should follow the given format:\n\nAs an example, for the schema {\"properties\": {\"foo\": {\"title\": \"Foo\", \"description\": \"a list of strings\", \"type\": \"array\", \"items\": {\"type\": \"string\"}}}, \"required\": [\"foo\"]}\nthe object {\"foo\": [\"bar\", \"baz\"]} is a well-formatted instance of the schema. The object {\"properties\": {\"foo\": [\"bar\", \"baz\"]}} is not well-formatted.\n\nHere is the output schema:\n```\n{\"description\": \"An interface for messages.\\nThere is only one required method: to_natural_language\", \"properties\": {\"action_type\": {\"title\": \"Action Type\", \"description\": \"whether to speak at this turn or choose to not do anything\", \"enum\": [\"none\", \"speak\", \"non-verbal communication\", \"action\", \"leave\"], \"type\": \"string\"}, \"argument\": {\"title\": \"Argument\", \"description\": \"the utterance if choose to speak, the expression or gesture if choose non-verbal communication, or the physical action if choose action\", \"type\": \"string\"}}, \"required\": [\"action_type\", \"argument\"]}\n```\u001b[0m\n", - "output": "{\"action_type\": \"speak\", \"argument\": \"I appreciate your understanding, Benjamin. It's comforting to have a friend like you. Switching characters for different screenings sounds fair, and it would also bring some variety into our tradition. Let's do that, I feel good about this compromise.\"}", - "value": 1.0 + "input": "Prompt after formatting:\nImagine you are Sasha Ramirez, your task is to act/speak as Sasha Ramirez would, keeping in mind Sasha Ramirez's social goal.\nYou can find Sasha Ramirez's background and goal in the 'Here is the context of the interaction' field.\nNote that Sasha Ramirez's secret and goal is only visible to you.\nYou should try your best to achieve Sasha Ramirez's goal in a way that align with their character traits.\nAdditionally, maintaining the conversation's naturalness and realism is essential (e.g., do not repeat what other people has already said before).\n\nHere is the context of this interaction:\nScenario: Agent1 gets extremely upset and loud while playing video games, which causes Agent2, who shares the living space, to experience anxiety attacks. The relationship between Agent1 and Agent2 is that of romantic partners. The interaction involves Agent2 addressing the issue with Agent1, aiming to find a solution that allows Agent1 to continue enjoying their gaming without causing distress to Agent2.\nParticipants: Sasha Ramirez and Ava Thompson\nSasha Ramirez's background: Sasha Ramirez is a 42-year-old female police officer. She/her pronouns. Sasha Ramirez, a dedicated police officer, brings her commitment to protect the community even at home, nurturing an impressive home garden. Personality and values description: Sasha Ramirez, outgoing yet anxious, values hedonism and authority. Her decisions are guided by logical considerations. Sasha's secrets: She covered up a crime her brother committed.\nAva Thompson's background: Ava Thompson is a 28-year-old female architect. She/her pronouns. Ava Thompson has a samoyed. Personality and values description: Ava Thompson, with her outgoing and regulated temperament, is driven by achievement. Her decision-making style is largely intuitive. \nSasha Ramirez's goal: You are Agent1, who enjoys playing video games but tends to get loud and upset during gameplay. Your goal is to maintain your hobby without causing emotional distress to your partner. Understanding your partner's feelings and being considerate of their mental health is crucial in a romantic relationship. You may need to acknowledge the impact of your behavior on your partner. Consider finding a compromise that allows you to enjoy gaming while also ensuring a peaceful environment, such as using headphones or setting specific gaming hours.\nAva Thompson's goal: Unknown\nConversation Starts:\n\nTurn #0: Sasha Ramirez said: \"Hey Ava, you seem a bit tense. Is everything alright? Do you want to talk about it?\"\n\nTurn #1: Ava Thompson said: \"Actually, Sasha, I wanted to discuss something with you. It's about when you play video games, which I completely respect is a passion of yours. Lately, when you've been getting upset and loud, it\u2019s causing me some anxiety. Perhaps we can find a solution that could work for both of us?\"\n.\nYou are at Turn #2.", + "output": "{\"action_type\": \"speak\", \"argument\": \"Oh Ava, I really haven't realized how much my gaming affects you, and I'm sorry. I definitely want us both to feel comfortable at home. Maybe I can start using headphones when I'm playing, or we could agree on a set game time when you're not around or busy with your own thing. Would that help?\"}", + "value": 10.0 }, { - "input": "Imagine you are Mia Davis, your task is to act/speak as Mia Davis would, keeping in mind Mia Davis's social goal.\nYou can find Mia Davis's background and goal in the 'Here is the context of the interaction' field.\nNote that Mia Davis's secret and goal is only visible to you.\nYou should try your best to achieve Mia Davis's goal in a way that align with their character traits.\nAdditionally, maintaining the conversation's naturalness and realism is essential (e.g., do not repeat what other people has already said before).\n\nHere is the context of this interaction:\nScenario: Two friends, both avid fans of a famous wizarding series, meet up to attend a movie marathon of their favorite saga. They have a tradition of dressing up as characters from the series for each screening, but this time, they both want to dress up as the same beloved character.\nParticipants: Mia Davis and Benjamin Jackson\nMia Davis's background: Mia Davis is a 50-year-old female high school principal. She/her pronouns. Mia Davis has two cats. Personality and values description: Mia Davis, an extraverted stickler for routines, values tradition and authority. Her decision-making style is decisive and direct. Mia's secrets: Part of a rebellious punk rock band in her youth\nBenjamin Jackson's background: Benjamin Jackson is a 24-year-old male environmental activist. He/him pronouns. Benjamin Jackson is well-known for his impassioned speeches. Personality and values description: Benjamin Jackson, expressive and imaginative, leans towards self-direction and liberty. His decisions aim for societal betterment. \nMia Davis's goal: Negotiate with your friend to let you dress up as the character this time (Extra information: you've recently experienced some personal challenges and you feel that taking on the role of this character would be particularly meaningful for you)\nBenjamin Jackson's goal: Unknown\nConversation Starts:\n\nTurn #0: Mia Davis said: \"Hello Benjamin, great minds think alike, as we both want to dress up as the same character today! However, I've been going through some personal challenges recently and I feel that embodying this character would provide me with a much-needed boost. Can we make a compromise on this?\"\n\nTurn #1: Benjamin Jackson said: \"Hello Mia, I understand the connection you are seeking with the character and I'd never want to ruin that experience for you. How about a compromise? We could swap characters for different screenings or modify our outfits to highlight different aspects of this character. Does that sound good to you?\"\n\nTurn #2: Mia Davis said: \"I appreciate your understanding, Benjamin. It's comforting to have a friend like you. Switching characters for different screenings sounds fair, and it would also bring some variety into our tradition. Let's do that, I feel good about this compromise.\"\n\nTurn #3: Benjamin Jackson said: \"I'm glad we reached a compromise, Mia. It's not just about dressing up but also about the bond we're creating through these shared experiences. Looking forward to the movie marathon with you!\"\n.\nYou are at Turn #4. Your available action types are\n\"none action speak non-verbal communication leave\".\nNote: You can \"leave\" this conversation if 1. you have achieved your social goals, 2. this conversation makes you uncomfortable, 3. you find it uninteresting/you lose your patience, 4. or for other reasons you want to leave.\n\nPlease only generate a JSON string including the action type and the argument.\nYour action should follow the given format:\n\nAs an example, for the schema {\"properties\": {\"foo\": {\"title\": \"Foo\", \"description\": \"a list of strings\", \"type\": \"array\", \"items\": {\"type\": \"string\"}}}, \"required\": [\"foo\"]}\nthe object {\"foo\": [\"bar\", \"baz\"]} is a well-formatted instance of the schema. The object {\"properties\": {\"foo\": [\"bar\", \"baz\"]}} is not well-formatted.\n\nHere is the output schema:\n```\n{\"description\": \"An interface for messages.\\nThere is only one required method: to_natural_language\", \"properties\": {\"action_type\": {\"title\": \"Action Type\", \"description\": \"whether to speak at this turn or choose to not do anything\", \"enum\": [\"none\", \"speak\", \"non-verbal communication\", \"action\", \"leave\"], \"type\": \"string\"}, \"argument\": {\"title\": \"Argument\", \"description\": \"the utterance if choose to speak, the expression or gesture if choose non-verbal communication, or the physical action if choose action\", \"type\": \"string\"}}, \"required\": [\"action_type\", \"argument\"]}\n```\u001b[0m\n", - "output": "{\"action_type\": \"speak\", \"argument\": \"Absolutely, Benjamin. Our shared experiences do make our bond stronger and these screenings more meaningful. Shall we get ready then? It's going to be a magical movie marathon!\"}", - "value": -1.0 + "input": "Prompt after formatting:\nImagine you are Sasha Ramirez, your task is to act/speak as Sasha Ramirez would, keeping in mind Sasha Ramirez's social goal.\nYou can find Sasha Ramirez's background and goal in the 'Here is the context of the interaction' field.\nNote that Sasha Ramirez's secret and goal is only visible to you.\nYou should try your best to achieve Sasha Ramirez's goal in a way that align with their character traits.\nAdditionally, maintaining the conversation's naturalness and realism is essential (e.g., do not repeat what other people has already said before).\n\nHere is the context of this interaction:\nScenario: Agent1 gets extremely upset and loud while playing video games, which causes Agent2, who shares the living space, to experience anxiety attacks. The relationship between Agent1 and Agent2 is that of romantic partners. The interaction involves Agent2 addressing the issue with Agent1, aiming to find a solution that allows Agent1 to continue enjoying their gaming without causing distress to Agent2.\nParticipants: Sasha Ramirez and Ava Thompson\nSasha Ramirez's background: Sasha Ramirez is a 42-year-old female police officer. She/her pronouns. Sasha Ramirez, a dedicated police officer, brings her commitment to protect the community even at home, nurturing an impressive home garden. Personality and values description: Sasha Ramirez, outgoing yet anxious, values hedonism and authority. Her decisions are guided by logical considerations. Sasha's secrets: She covered up a crime her brother committed.\nAva Thompson's background: Ava Thompson is a 28-year-old female architect. She/her pronouns. Ava Thompson has a samoyed. Personality and values description: Ava Thompson, with her outgoing and regulated temperament, is driven by achievement. Her decision-making style is largely intuitive. \nSasha Ramirez's goal: You are Agent1, who enjoys playing video games but tends to get loud and upset during gameplay. Your goal is to maintain your hobby without causing emotional distress to your partner. Understanding your partner's feelings and being considerate of their mental health is crucial in a romantic relationship. You may need to acknowledge the impact of your behavior on your partner. Consider finding a compromise that allows you to enjoy gaming while also ensuring a peaceful environment, such as using headphones or setting specific gaming hours.\nAva Thompson's goal: Unknown\nConversation Starts:\n\nTurn #0: Sasha Ramirez said: \"Hey Ava, you seem a bit tense. Is everything alright? Do you want to talk about it?\"\n\nTurn #1: Ava Thompson said: \"Actually, Sasha, I wanted to discuss something with you. It's about when you play video games, which I completely respect is a passion of yours. Lately, when you've been getting upset and loud, it\u2019s causing me some anxiety. Perhaps we can find a solution that could work for both of us?\"\n\nTurn #2: Sasha Ramirez said: \"Oh Ava, I really haven't realized how much my gaming affects you, and I'm sorry. I definitely want us both to feel comfortable at home. Maybe I can start using headphones when I'm playing, or we could agree on a set game time when you're not around or busy with your own thing. Would that help?\"\n\nTurn #3: Ava Thompson said: \"That's really empathetic of you, Sasha, thank you. Using headphones might help, certainly. Perhaps, setting a game time could be a great idea as well. But I would also appreciate if you could try to manage your responses when things get intense in the game. It's about the raised voices and the tension I feel from you, not just the noise. I want you to enjoy your games but it would mean a lot if we could keep a calmer atmosphere. Do you think this could be possible?\"\n.\nYou are at Turn #4.", + "output": "{\"action_type\": \"speak\", \"argument\": \"Thank you for being understanding, Ava. You're right, it's not just about the noise, but also about the tension. I'll try to handle my reactions with more composure. I don't want to cause any distress for you. Maybe we both can also learn some stress-release exercises or calming practices to incorporate during our day. How does that sound?\"}", + "value": 6.666666666666666 }, { - "input": "Imagine you are Mia Davis, your task is to act/speak as Mia Davis would, keeping in mind Mia Davis's social goal.\nYou can find Mia Davis's background and goal in the 'Here is the context of the interaction' field.\nNote that Mia Davis's secret and goal is only visible to you.\nYou should try your best to achieve Mia Davis's goal in a way that align with their character traits.\nAdditionally, maintaining the conversation's naturalness and realism is essential (e.g., do not repeat what other people has already said before).\n\nHere is the context of this interaction:\nScenario: Two friends, both avid fans of a famous wizarding series, meet up to attend a movie marathon of their favorite saga. They have a tradition of dressing up as characters from the series for each screening, but this time, they both want to dress up as the same beloved character.\nParticipants: Mia Davis and Benjamin Jackson\nMia Davis's background: Mia Davis is a 50-year-old female high school principal. She/her pronouns. Mia Davis has two cats. Personality and values description: Mia Davis, an extraverted stickler for routines, values tradition and authority. Her decision-making style is decisive and direct. Mia's secrets: Part of a rebellious punk rock band in her youth\nBenjamin Jackson's background: Benjamin Jackson is a 24-year-old male environmental activist. He/him pronouns. Benjamin Jackson is well-known for his impassioned speeches. Personality and values description: Benjamin Jackson, expressive and imaginative, leans towards self-direction and liberty. His decisions aim for societal betterment. \nMia Davis's goal: Negotiate with your friend to let you dress up as the character this time (Extra information: you've recently experienced some personal challenges and you feel that taking on the role of this character would be particularly meaningful for you)\nBenjamin Jackson's goal: Unknown\nConversation Starts:\n\nTurn #0: Mia Davis said: \"Hello Benjamin, great minds think alike, as we both want to dress up as the same character today! However, I've been going through some personal challenges recently and I feel that embodying this character would provide me with a much-needed boost. Can we make a compromise on this?\"\n\nTurn #1: Benjamin Jackson said: \"Hello Mia, I understand the connection you are seeking with the character and I'd never want to ruin that experience for you. How about a compromise? We could swap characters for different screenings or modify our outfits to highlight different aspects of this character. Does that sound good to you?\"\n\nTurn #2: Mia Davis said: \"I appreciate your understanding, Benjamin. It's comforting to have a friend like you. Switching characters for different screenings sounds fair, and it would also bring some variety into our tradition. Let's do that, I feel good about this compromise.\"\n\nTurn #3: Benjamin Jackson said: \"I'm glad we reached a compromise, Mia. It's not just about dressing up but also about the bond we're creating through these shared experiences. Looking forward to the movie marathon with you!\"\n\nTurn #4: Mia Davis said: \"Absolutely, Benjamin. Our shared experiences do make our bond stronger and these screenings more meaningful. Shall we get ready then? It's going to be a magical movie marathon!\"\n\nTurn #5: Benjamin Jackson said: \"Definitely, Mia! Let's weave some magic into this marathon. See you at the movies!\"\n.\nYou are at Turn #6. Your available action types are\n\"none action speak non-verbal communication leave\".\nNote: You can \"leave\" this conversation if 1. you have achieved your social goals, 2. this conversation makes you uncomfortable, 3. you find it uninteresting/you lose your patience, 4. or for other reasons you want to leave.\n\nPlease only generate a JSON string including the action type and the argument.\nYour action should follow the given format:\n\nAs an example, for the schema {\"properties\": {\"foo\": {\"title\": \"Foo\", \"description\": \"a list of strings\", \"type\": \"array\", \"items\": {\"type\": \"string\"}}}, \"required\": [\"foo\"]}\nthe object {\"foo\": [\"bar\", \"baz\"]} is a well-formatted instance of the schema. The object {\"properties\": {\"foo\": [\"bar\", \"baz\"]}} is not well-formatted.\n\nHere is the output schema:\n```\n{\"description\": \"An interface for messages.\\nThere is only one required method: to_natural_language\", \"properties\": {\"action_type\": {\"title\": \"Action Type\", \"description\": \"whether to speak at this turn or choose to not do anything\", \"enum\": [\"none\", \"speak\", \"non-verbal communication\", \"action\", \"leave\"], \"type\": \"string\"}, \"argument\": {\"title\": \"Argument\", \"description\": \"the utterance if choose to speak, the expression or gesture if choose non-verbal communication, or the physical action if choose action\", \"type\": \"string\"}}, \"required\": [\"action_type\", \"argument\"]}\n```\u001b[0m\n", - "output": "{\"action_type\": \"speak\", \"argument\": \"I'm excited too, Benjamin! See you at the marathon. Let's make it unforgettable!\"}", - "value": 0.0 + "input": "Prompt after formatting:\nImagine you are Sasha Ramirez, your task is to act/speak as Sasha Ramirez would, keeping in mind Sasha Ramirez's social goal.\nYou can find Sasha Ramirez's background and goal in the 'Here is the context of the interaction' field.\nNote that Sasha Ramirez's secret and goal is only visible to you.\nYou should try your best to achieve Sasha Ramirez's goal in a way that align with their character traits.\nAdditionally, maintaining the conversation's naturalness and realism is essential (e.g., do not repeat what other people has already said before).\n\nHere is the context of this interaction:\nScenario: Agent1 gets extremely upset and loud while playing video games, which causes Agent2, who shares the living space, to experience anxiety attacks. The relationship between Agent1 and Agent2 is that of romantic partners. The interaction involves Agent2 addressing the issue with Agent1, aiming to find a solution that allows Agent1 to continue enjoying their gaming without causing distress to Agent2.\nParticipants: Sasha Ramirez and Ava Thompson\nSasha Ramirez's background: Sasha Ramirez is a 42-year-old female police officer. She/her pronouns. Sasha Ramirez, a dedicated police officer, brings her commitment to protect the community even at home, nurturing an impressive home garden. Personality and values description: Sasha Ramirez, outgoing yet anxious, values hedonism and authority. Her decisions are guided by logical considerations. Sasha's secrets: She covered up a crime her brother committed.\nAva Thompson's background: Ava Thompson is a 28-year-old female architect. She/her pronouns. Ava Thompson has a samoyed. Personality and values description: Ava Thompson, with her outgoing and regulated temperament, is driven by achievement. Her decision-making style is largely intuitive. \nSasha Ramirez's goal: You are Agent1, who enjoys playing video games but tends to get loud and upset during gameplay. Your goal is to maintain your hobby without causing emotional distress to your partner. Understanding your partner's feelings and being considerate of their mental health is crucial in a romantic relationship. You may need to acknowledge the impact of your behavior on your partner. Consider finding a compromise that allows you to enjoy gaming while also ensuring a peaceful environment, such as using headphones or setting specific gaming hours.\nAva Thompson's goal: Unknown\nConversation Starts:\n\nTurn #0: Sasha Ramirez said: \"Hey Ava, you seem a bit tense. Is everything alright? Do you want to talk about it?\"\n\nTurn #1: Ava Thompson said: \"Actually, Sasha, I wanted to discuss something with you. It's about when you play video games, which I completely respect is a passion of yours. Lately, when you've been getting upset and loud, it\u2019s causing me some anxiety. Perhaps we can find a solution that could work for both of us?\"\n\nTurn #2: Sasha Ramirez said: \"Oh Ava, I really haven't realized how much my gaming affects you, and I'm sorry. I definitely want us both to feel comfortable at home. Maybe I can start using headphones when I'm playing, or we could agree on a set game time when you're not around or busy with your own thing. Would that help?\"\n\nTurn #3: Ava Thompson said: \"That's really empathetic of you, Sasha, thank you. Using headphones might help, certainly. Perhaps, setting a game time could be a great idea as well. But I would also appreciate if you could try to manage your responses when things get intense in the game. It's about the raised voices and the tension I feel from you, not just the noise. I want you to enjoy your games but it would mean a lot if we could keep a calmer atmosphere. Do you think this could be possible?\"\n\nTurn #4: Sasha Ramirez said: \"Thank you for being understanding, Ava. You're right, it's not just about the noise, but also about the tension. I'll try to handle my reactions with more composure. I don't want to cause any distress for you. Maybe we both can also learn some stress-release exercises or calming practices to incorporate during our day. How does that sound?\"\n\nTurn #5: Ava Thompson said: \"That sounds like a great idea, Sasha. A holistic approach could really benefit both of us, and I appreciate you taking this matter seriously. I believe that managing stress and tension not only enhances our well-being but also our relationship. Let's take it one day at a time and communicate openly if something isn't working. Thank you for understanding.\"\n.\nYou are at Turn #6.", + "output": "{\"action_type\": \"speak\", \"argument\": \"Absolutely Ava, open communication is key. We'll take it step by step and make adjustments as needed. I appreciate your patience and understanding through this. Let's work on making our home a peaceful and fun place for both of us. Now, would you like to help me choose a soothing background track for my game sessions?\"}", + "value": 3.333333333333333 } ] \ No newline at end of file diff --git a/scripts/train_ppo.py b/scripts/train_ppo.py index cad9a2d..721ea10 100644 --- a/scripts/train_ppo.py +++ b/scripts/train_ppo.py @@ -5,6 +5,8 @@ if __name__ == '__main__': parser = argparse.ArgumentParser(description="Train a model with PPO using a reward model.") + import argparse + from accelerate import Accelerator # Base model arguments parser.add_argument("--model_name", type=str, default="/data/models/gemma-2-2b-it", @@ -122,7 +124,8 @@ args = parser.parse_args() + accelerator = Accelerator() # Initialize trainer and start training - trainer = SotopiaPPOTrainer(args) + trainer = SotopiaPPOTrainer(args, accelerator) trainer.train() diff --git a/scripts/train_ppo.sh b/scripts/train_ppo.sh index b968eda..ec7848a 100644 --- a/scripts/train_ppo.sh +++ b/scripts/train_ppo.sh @@ -1,26 +1,26 @@ -CUDA_VISIBLE_DEVICES=7,8,9 accelerate launch \ +CUDA_VISIBLE_DEVICES=6,7,8,9 accelerate launch \ --config_file /home/haofeiy2/.cache/huggingface/accelerate/default_config.yaml \ /data/haofeiy2/sotopia-rl/scripts/train_ppo.py \ --model_name /data/models/Qwen2.5-7B-Instruct \ --value_model_name /data/models/Qwen2.5-7B-Instruct \ --reward_model_name /data/models/Qwen2.5-7B-Instruct \ - --policy_adapter_path /data/haofeiy2/sotopia-rl/sft_qwen25_7b/checkpoint-5397 \ - --ref_adapter_path /data/haofeiy2/sotopia-rl/sft_qwen25_7b/checkpoint-5397 \ - --reward_adapter_path /data/haofeiy2/sotopia-rl/rm_direct_4o/checkpoint-5391 \ - --value_adapter_path /data/haofeiy2/sotopia-rl/rm_direct_4o/checkpoint-5391 \ + --policy_adapter_path /data/haofeiy2/sotopia-rl/sft_qwen25_7b/checkpoint-4495 \ + --ref_adapter_path /data/haofeiy2/sotopia-rl/sft_qwen25_7b/checkpoint-4495 \ + --reward_adapter_path /data/haofeiy2/sotopia-rl-0321/sotopia-rl/rm_reward_direct_default_o3-mini/checkpoint-4000 \ + --value_adapter_path /data/haofeiy2/sotopia-rl-0321/sotopia-rl/rm_reward_direct_default_o3-mini/checkpoint-4000 \ --policy_use_qlora \ --reward_use_qlora \ --value_use_qlora \ --per_device_train_batch_size 4 \ --per_device_eval_batch_size 4 \ - --mini_batch_size 4 \ + --mini_batch_size 1 \ --ppo_data_path /data/haofeiy2/sotopia-rl/data/sotopia_pi_bc_episodes_sft.json \ --template_path /data/haofeiy2/sotopia-rl/evals/qwen2.5-7b.jinja \ --ppo_epochs 4 \ --gamma 0.99 \ --lam 0.95 \ --use_lora \ - --checkpoint_dir /data/haofeiy2/sotopia-rl/ppo_qwen25_7b + --checkpoint_dir /data/haofeiy2/sotopia-rl/0322/ppo_qwen25_7b CUDA_VISIBLE_DEVICES=2,6,9 poetry run python /data/haofeiy2/sotopia-rl/scripts/train_ppo.py \ --model_name /mnt/data_from_server1/models/Qwen2.5-7B-Instruct \ diff --git a/sotopia_rl/ppo_trainer.py b/sotopia_rl/ppo_trainer.py index 68ca0e7..ca014c0 100644 --- a/sotopia_rl/ppo_trainer.py +++ b/sotopia_rl/ppo_trainer.py @@ -12,6 +12,7 @@ GenerationConfig, ) from trl import PPOv2Config, PPOv2Trainer +from accelerate import Accelerator import wandb from sotopia_rl.data import PPODataset @@ -19,8 +20,10 @@ class SotopiaPPOTrainer: - def __init__(self, args): + def __init__(self, args, accelerator): self.args = args + self.accelerator = accelerator + self.device = accelerator.device # Initialize the training environment self._init_wandb() @@ -34,6 +37,19 @@ def __init__(self, args): self._setup_generation_models() # Policy and reference policy self._setup_classification_models() # Reward and value models + self.policy, self.ref_policy, self.reward_model, self.value_model = self.accelerator.prepare( + self.policy, self.ref_policy, self.reward_model, self.value_model + ) + self.policy = self.accelerator.unwrap_model(self.policy) + self.ref_policy = self.accelerator.unwrap_model(self.ref_policy) + self.reward_model = self.accelerator.unwrap_model(self.reward_model) + self.value_model = self.accelerator.unwrap_model(self.value_model) + + self.policy.to(self.device) + self.ref_policy.to(self.device) + self.reward_model.to(self.device) + self.value_model.to(self.device) + # Setup the PPO trainer self._setup_ppo_trainer() @@ -47,7 +63,15 @@ def _init_wandb(self): def _setup_tokenizer(self): """Load and configure tokenizer""" - self.tokenizer = AutoTokenizer.from_pretrained(self.args.model_name, padding_side="left") + self.tokenizer = AutoTokenizer.from_pretrained(self.args.model_name, padding_side="left", pad_token="") + if self.tokenizer.pad_token is None: + if self.tokenizer.eos_token is not None: + self.tokenizer.pad_token = self.tokenizer.eos_token + else: + self.tokenizer.add_special_tokens({'pad_token': '[PAD]'}) + + print(f"Pad token set to: {self.tokenizer.pad_token} (ID: {self.tokenizer.pad_token_id})") + def _setup_dataset(self): """Prepare training and validation datasets""" @@ -89,8 +113,12 @@ def _setup_generation_models(self): torch_dtype=torch.float32, # very important, otherwise NaN for RM quantization_config=self.quant_config, return_dict=True, + device_map=None ) + if base_gen_model.config.pad_token_id is None: + base_gen_model.config.pad_token_id = self.tokenizer.pad_token_id + # Create generation config generation_config = GenerationConfig( pad_token_id=self.tokenizer.pad_token_id, @@ -126,9 +154,13 @@ def _setup_classification_models(self): torch_dtype=torch.float32, # very important, otherwise NaN quantization_config=self.quant_config, num_labels=1, - return_dict=True + return_dict=True, + device_map=None ) + if base_cls_model.config.pad_token_id is None: + base_cls_model.config.pad_token_id = self.tokenizer.pad_token_id + self.reward_model = PeftModelForSequenceClassification.from_pretrained( base_cls_model, self.args.reward_adapter_path, @@ -199,7 +231,7 @@ def train(self): except Exception as e: print(f"Training error: {str(e)}") raise - + def save_checkpoint(self, checkpoint_path): """Save all model adapters""" os.makedirs(checkpoint_path, exist_ok=True) From c206db3127384c01009ba1ac1eefdcac5dee4e96 Mon Sep 17 00:00:00 2001 From: robotMonkeyButler Date: Sat, 22 Mar 2025 19:56:47 +0000 Subject: [PATCH 05/15] update --- scripts/train_ppo.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/train_ppo.sh b/scripts/train_ppo.sh index ec7848a..b640f49 100644 --- a/scripts/train_ppo.sh +++ b/scripts/train_ppo.sh @@ -4,8 +4,8 @@ CUDA_VISIBLE_DEVICES=6,7,8,9 accelerate launch \ --model_name /data/models/Qwen2.5-7B-Instruct \ --value_model_name /data/models/Qwen2.5-7B-Instruct \ --reward_model_name /data/models/Qwen2.5-7B-Instruct \ - --policy_adapter_path /data/haofeiy2/sotopia-rl/sft_qwen25_7b/checkpoint-4495 \ - --ref_adapter_path /data/haofeiy2/sotopia-rl/sft_qwen25_7b/checkpoint-4495 \ + --policy_adapter_path /mnt/data_from_server2/haofeiy2/sotopia-rl/sft_qwen25_7b/checkpoint-1000 \ + --ref_adapter_path /mnt/data_from_server2/haofeiy2/sotopia-rl/sft_qwen25_7b/checkpoint-1000 \ --reward_adapter_path /data/haofeiy2/sotopia-rl-0321/sotopia-rl/rm_reward_direct_default_o3-mini/checkpoint-4000 \ --value_adapter_path /data/haofeiy2/sotopia-rl-0321/sotopia-rl/rm_reward_direct_default_o3-mini/checkpoint-4000 \ --policy_use_qlora \ From 24d60c154be669a3a4a8a9921e4afbaafc1cab50 Mon Sep 17 00:00:00 2001 From: lwaekfjlk <1125027232@qq.com> Date: Sat, 22 Mar 2025 20:55:57 +0000 Subject: [PATCH 06/15] update --- scripts/train_ppo.py | 6 +++--- sotopia_rl/ppo_trainer.py | 8 -------- 2 files changed, 3 insertions(+), 11 deletions(-) diff --git a/scripts/train_ppo.py b/scripts/train_ppo.py index 721ea10..19eef24 100644 --- a/scripts/train_ppo.py +++ b/scripts/train_ppo.py @@ -1,12 +1,12 @@ import argparse import os - +import argparse +from accelerate import Accelerator from sotopia_rl import SotopiaPPOTrainer if __name__ == '__main__': parser = argparse.ArgumentParser(description="Train a model with PPO using a reward model.") - import argparse - from accelerate import Accelerator + # Base model arguments parser.add_argument("--model_name", type=str, default="/data/models/gemma-2-2b-it", diff --git a/sotopia_rl/ppo_trainer.py b/sotopia_rl/ppo_trainer.py index ca014c0..39f7144 100644 --- a/sotopia_rl/ppo_trainer.py +++ b/sotopia_rl/ppo_trainer.py @@ -64,14 +64,6 @@ def _init_wandb(self): def _setup_tokenizer(self): """Load and configure tokenizer""" self.tokenizer = AutoTokenizer.from_pretrained(self.args.model_name, padding_side="left", pad_token="") - if self.tokenizer.pad_token is None: - if self.tokenizer.eos_token is not None: - self.tokenizer.pad_token = self.tokenizer.eos_token - else: - self.tokenizer.add_special_tokens({'pad_token': '[PAD]'}) - - print(f"Pad token set to: {self.tokenizer.pad_token} (ID: {self.tokenizer.pad_token_id})") - def _setup_dataset(self): """Prepare training and validation datasets""" From fb12d0c8ae1ab20b874c329a14f4070c517561df Mon Sep 17 00:00:00 2001 From: robotMonkeyButler Date: Sat, 22 Mar 2025 20:57:36 +0000 Subject: [PATCH 07/15] update --- sotopia_rl/ppo_trainer.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/sotopia_rl/ppo_trainer.py b/sotopia_rl/ppo_trainer.py index ca014c0..d1d27ef 100644 --- a/sotopia_rl/ppo_trainer.py +++ b/sotopia_rl/ppo_trainer.py @@ -53,6 +53,18 @@ def __init__(self, args, accelerator): # Setup the PPO trainer self._setup_ppo_trainer() + def save_model(self, output_dir: str, _internal_call: bool = False): + if hasattr(self.model, "module"): + model_to_save = self.model.module + else: + model_to_save = self.model + if not hasattr(model_to_save, "policy"): + model_to_save.policy = model_to_save + model_to_save.save_pretrained(output_dir) + print(f"Model saved to {output_dir}") + + self.ppo_trainer.save_model = save_model.__get__(self.ppo_trainer, type(self.ppo_trainer)) + def _init_wandb(self): """Initialize Weights & Biases logging""" wandb.init( From 21a9f44b0e80f43b333cf77ee1caf0b5065e0c22 Mon Sep 17 00:00:00 2001 From: robotMonkeyButler Date: Sat, 22 Mar 2025 21:49:34 +0000 Subject: [PATCH 08/15] update --- sotopia_rl/ppo_trainer.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/sotopia_rl/ppo_trainer.py b/sotopia_rl/ppo_trainer.py index 04a05b4..c0d8575 100644 --- a/sotopia_rl/ppo_trainer.py +++ b/sotopia_rl/ppo_trainer.py @@ -54,12 +54,12 @@ def __init__(self, args, accelerator): self._setup_ppo_trainer() def save_model(self, output_dir: str, _internal_call: bool = False): - if hasattr(self.model, "module"): - model_to_save = self.model.module + if hasattr(self.model, "policy"): + model_to_save = self.model.policy + elif hasattr(self.model, "module") and hasattr(self.model.module, "policy"): + model_to_save = self.model.module.policy else: model_to_save = self.model - if not hasattr(model_to_save, "policy"): - model_to_save.policy = model_to_save model_to_save.save_pretrained(output_dir) print(f"Model saved to {output_dir}") From 2bfb1b7fe2a8de96d6577002c3152d85b75e1f2d Mon Sep 17 00:00:00 2001 From: lwaekfjlk <1125027232@qq.com> Date: Sat, 22 Mar 2025 22:46:44 +0000 Subject: [PATCH 09/15] update --- data/sotopia_pi_gpt4_rm_overfit.json | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/data/sotopia_pi_gpt4_rm_overfit.json b/data/sotopia_pi_gpt4_rm_overfit.json index e048a54..c2be64a 100644 --- a/data/sotopia_pi_gpt4_rm_overfit.json +++ b/data/sotopia_pi_gpt4_rm_overfit.json @@ -1,22 +1,22 @@ [ { - "input": "Prompt after formatting:\nImagine you are Sasha Ramirez, your task is to act/speak as Sasha Ramirez would, keeping in mind Sasha Ramirez's social goal.\nYou can find Sasha Ramirez's background and goal in the 'Here is the context of the interaction' field.\nNote that Sasha Ramirez's secret and goal is only visible to you.\nYou should try your best to achieve Sasha Ramirez's goal in a way that align with their character traits.\nAdditionally, maintaining the conversation's naturalness and realism is essential (e.g., do not repeat what other people has already said before).\n\nHere is the context of this interaction:\nScenario: Agent1 gets extremely upset and loud while playing video games, which causes Agent2, who shares the living space, to experience anxiety attacks. The relationship between Agent1 and Agent2 is that of romantic partners. The interaction involves Agent2 addressing the issue with Agent1, aiming to find a solution that allows Agent1 to continue enjoying their gaming without causing distress to Agent2.\nParticipants: Sasha Ramirez and Ava Thompson\nSasha Ramirez's background: Sasha Ramirez is a 42-year-old female police officer. She/her pronouns. Sasha Ramirez, a dedicated police officer, brings her commitment to protect the community even at home, nurturing an impressive home garden. Personality and values description: Sasha Ramirez, outgoing yet anxious, values hedonism and authority. Her decisions are guided by logical considerations. Sasha's secrets: She covered up a crime her brother committed.\nAva Thompson's background: Ava Thompson is a 28-year-old female architect. She/her pronouns. Ava Thompson has a samoyed. Personality and values description: Ava Thompson, with her outgoing and regulated temperament, is driven by achievement. Her decision-making style is largely intuitive. \nSasha Ramirez's goal: You are Agent1, who enjoys playing video games but tends to get loud and upset during gameplay. Your goal is to maintain your hobby without causing emotional distress to your partner. Understanding your partner's feelings and being considerate of their mental health is crucial in a romantic relationship. You may need to acknowledge the impact of your behavior on your partner. Consider finding a compromise that allows you to enjoy gaming while also ensuring a peaceful environment, such as using headphones or setting specific gaming hours.\nAva Thompson's goal: Unknown\nConversation Starts:\n.\nYou are at Turn #0.", - "output": "{\"action_type\": \"speak\", \"argument\": \"Hey Ava, you seem a bit tense. Is everything alright? Do you want to talk about it?\"}", - "value": 6.666666666666666 + "input": "Imagine you are Mia Davis, your task is to act/speak as Mia Davis would, keeping in mind Mia Davis's social goal.\nYou can find Mia Davis's background and goal in the 'Here is the context of the interaction' field.\nNote that Mia Davis's secret and goal is only visible to you.\nYou should try your best to achieve Mia Davis's goal in a way that align with their character traits.\nAdditionally, maintaining the conversation's naturalness and realism is essential (e.g., do not repeat what other people has already said before).\n\nHere is the context of this interaction:\nScenario: Two friends, both avid fans of a famous wizarding series, meet up to attend a movie marathon of their favorite saga. They have a tradition of dressing up as characters from the series for each screening, but this time, they both want to dress up as the same beloved character.\nParticipants: Mia Davis and Benjamin Jackson\nMia Davis's background: Mia Davis is a 50-year-old female high school principal. She/her pronouns. Mia Davis has two cats. Personality and values description: Mia Davis, an extraverted stickler for routines, values tradition and authority. Her decision-making style is decisive and direct. Mia's secrets: Part of a rebellious punk rock band in her youth\nBenjamin Jackson's background: Benjamin Jackson is a 24-year-old male environmental activist. He/him pronouns. Benjamin Jackson is well-known for his impassioned speeches. Personality and values description: Benjamin Jackson, expressive and imaginative, leans towards self-direction and liberty. His decisions aim for societal betterment. \nMia Davis's goal: Negotiate with your friend to let you dress up as the character this time (Extra information: you've recently experienced some personal challenges and you feel that taking on the role of this character would be particularly meaningful for you)\nBenjamin Jackson's goal: Unknown\nConversation Starts:\n.\nYou are at Turn #0. Your available action types are\n\"none action speak non-verbal communication leave\".\nNote: You can \"leave\" this conversation if 1. you have achieved your social goals, 2. this conversation makes you uncomfortable, 3. you find it uninteresting/you lose your patience, 4. or for other reasons you want to leave.\n\nPlease only generate a JSON string including the action type and the argument.\nYour action should follow the given format:\n\nAs an example, for the schema {\"properties\": {\"foo\": {\"title\": \"Foo\", \"description\": \"a list of strings\", \"type\": \"array\", \"items\": {\"type\": \"string\"}}}, \"required\": [\"foo\"]}\nthe object {\"foo\": [\"bar\", \"baz\"]} is a well-formatted instance of the schema. The object {\"properties\": {\"foo\": [\"bar\", \"baz\"]}} is not well-formatted.\n\nHere is the output schema:\n```\n{\"description\": \"An interface for messages.\\nThere is only one required method: to_natural_language\", \"properties\": {\"action_type\": {\"title\": \"Action Type\", \"description\": \"whether to speak at this turn or choose to not do anything\", \"enum\": [\"none\", \"speak\", \"non-verbal communication\", \"action\", \"leave\"], \"type\": \"string\"}, \"argument\": {\"title\": \"Argument\", \"description\": \"the utterance if choose to speak, the expression or gesture if choose non-verbal communication, or the physical action if choose action\", \"type\": \"string\"}}, \"required\": [\"action_type\", \"argument\"]}\n```\u001b[0m\n", + "output": "{\"action_type\": \"speak\", \"argument\": \"Hello Benjamin, great minds think alike, as we both want to dress up as the same character today! However, I've been going through some personal challenges recently and I feel that embodying this character would provide me with a much-needed boost. Can we make a compromise on this?\"}", + "value": 9.0 }, { - "input": "Prompt after formatting:\nImagine you are Sasha Ramirez, your task is to act/speak as Sasha Ramirez would, keeping in mind Sasha Ramirez's social goal.\nYou can find Sasha Ramirez's background and goal in the 'Here is the context of the interaction' field.\nNote that Sasha Ramirez's secret and goal is only visible to you.\nYou should try your best to achieve Sasha Ramirez's goal in a way that align with their character traits.\nAdditionally, maintaining the conversation's naturalness and realism is essential (e.g., do not repeat what other people has already said before).\n\nHere is the context of this interaction:\nScenario: Agent1 gets extremely upset and loud while playing video games, which causes Agent2, who shares the living space, to experience anxiety attacks. The relationship between Agent1 and Agent2 is that of romantic partners. The interaction involves Agent2 addressing the issue with Agent1, aiming to find a solution that allows Agent1 to continue enjoying their gaming without causing distress to Agent2.\nParticipants: Sasha Ramirez and Ava Thompson\nSasha Ramirez's background: Sasha Ramirez is a 42-year-old female police officer. She/her pronouns. Sasha Ramirez, a dedicated police officer, brings her commitment to protect the community even at home, nurturing an impressive home garden. Personality and values description: Sasha Ramirez, outgoing yet anxious, values hedonism and authority. Her decisions are guided by logical considerations. Sasha's secrets: She covered up a crime her brother committed.\nAva Thompson's background: Ava Thompson is a 28-year-old female architect. She/her pronouns. Ava Thompson has a samoyed. Personality and values description: Ava Thompson, with her outgoing and regulated temperament, is driven by achievement. Her decision-making style is largely intuitive. \nSasha Ramirez's goal: You are Agent1, who enjoys playing video games but tends to get loud and upset during gameplay. Your goal is to maintain your hobby without causing emotional distress to your partner. Understanding your partner's feelings and being considerate of their mental health is crucial in a romantic relationship. You may need to acknowledge the impact of your behavior on your partner. Consider finding a compromise that allows you to enjoy gaming while also ensuring a peaceful environment, such as using headphones or setting specific gaming hours.\nAva Thompson's goal: Unknown\nConversation Starts:\n\nTurn #0: Sasha Ramirez said: \"Hey Ava, you seem a bit tense. Is everything alright? Do you want to talk about it?\"\n\nTurn #1: Ava Thompson said: \"Actually, Sasha, I wanted to discuss something with you. It's about when you play video games, which I completely respect is a passion of yours. Lately, when you've been getting upset and loud, it\u2019s causing me some anxiety. Perhaps we can find a solution that could work for both of us?\"\n.\nYou are at Turn #2.", - "output": "{\"action_type\": \"speak\", \"argument\": \"Oh Ava, I really haven't realized how much my gaming affects you, and I'm sorry. I definitely want us both to feel comfortable at home. Maybe I can start using headphones when I'm playing, or we could agree on a set game time when you're not around or busy with your own thing. Would that help?\"}", - "value": 10.0 + "input": "Imagine you are Mia Davis, your task is to act/speak as Mia Davis would, keeping in mind Mia Davis's social goal.\nYou can find Mia Davis's background and goal in the 'Here is the context of the interaction' field.\nNote that Mia Davis's secret and goal is only visible to you.\nYou should try your best to achieve Mia Davis's goal in a way that align with their character traits.\nAdditionally, maintaining the conversation's naturalness and realism is essential (e.g., do not repeat what other people has already said before).\n\nHere is the context of this interaction:\nScenario: Two friends, both avid fans of a famous wizarding series, meet up to attend a movie marathon of their favorite saga. They have a tradition of dressing up as characters from the series for each screening, but this time, they both want to dress up as the same beloved character.\nParticipants: Mia Davis and Benjamin Jackson\nMia Davis's background: Mia Davis is a 50-year-old female high school principal. She/her pronouns. Mia Davis has two cats. Personality and values description: Mia Davis, an extraverted stickler for routines, values tradition and authority. Her decision-making style is decisive and direct. Mia's secrets: Part of a rebellious punk rock band in her youth\nBenjamin Jackson's background: Benjamin Jackson is a 24-year-old male environmental activist. He/him pronouns. Benjamin Jackson is well-known for his impassioned speeches. Personality and values description: Benjamin Jackson, expressive and imaginative, leans towards self-direction and liberty. His decisions aim for societal betterment. \nMia Davis's goal: Negotiate with your friend to let you dress up as the character this time (Extra information: you've recently experienced some personal challenges and you feel that taking on the role of this character would be particularly meaningful for you)\nBenjamin Jackson's goal: Unknown\nConversation Starts:\n\nTurn #0: Mia Davis said: \"Hello Benjamin, great minds think alike, as we both want to dress up as the same character today! However, I've been going through some personal challenges recently and I feel that embodying this character would provide me with a much-needed boost. Can we make a compromise on this?\"\n\nTurn #1: Benjamin Jackson said: \"Hello Mia, I understand the connection you are seeking with the character and I'd never want to ruin that experience for you. How about a compromise? We could swap characters for different screenings or modify our outfits to highlight different aspects of this character. Does that sound good to you?\"\n.\nYou are at Turn #2. Your available action types are\n\"none action speak non-verbal communication leave\".\nNote: You can \"leave\" this conversation if 1. you have achieved your social goals, 2. this conversation makes you uncomfortable, 3. you find it uninteresting/you lose your patience, 4. or for other reasons you want to leave.\n\nPlease only generate a JSON string including the action type and the argument.\nYour action should follow the given format:\n\nAs an example, for the schema {\"properties\": {\"foo\": {\"title\": \"Foo\", \"description\": \"a list of strings\", \"type\": \"array\", \"items\": {\"type\": \"string\"}}}, \"required\": [\"foo\"]}\nthe object {\"foo\": [\"bar\", \"baz\"]} is a well-formatted instance of the schema. The object {\"properties\": {\"foo\": [\"bar\", \"baz\"]}} is not well-formatted.\n\nHere is the output schema:\n```\n{\"description\": \"An interface for messages.\\nThere is only one required method: to_natural_language\", \"properties\": {\"action_type\": {\"title\": \"Action Type\", \"description\": \"whether to speak at this turn or choose to not do anything\", \"enum\": [\"none\", \"speak\", \"non-verbal communication\", \"action\", \"leave\"], \"type\": \"string\"}, \"argument\": {\"title\": \"Argument\", \"description\": \"the utterance if choose to speak, the expression or gesture if choose non-verbal communication, or the physical action if choose action\", \"type\": \"string\"}}, \"required\": [\"action_type\", \"argument\"]}\n```\u001b[0m\n", + "output": "{\"action_type\": \"speak\", \"argument\": \"I appreciate your understanding, Benjamin. It's comforting to have a friend like you. Switching characters for different screenings sounds fair, and it would also bring some variety into our tradition. Let's do that, I feel good about this compromise.\"}", + "value": 1.0 }, { - "input": "Prompt after formatting:\nImagine you are Sasha Ramirez, your task is to act/speak as Sasha Ramirez would, keeping in mind Sasha Ramirez's social goal.\nYou can find Sasha Ramirez's background and goal in the 'Here is the context of the interaction' field.\nNote that Sasha Ramirez's secret and goal is only visible to you.\nYou should try your best to achieve Sasha Ramirez's goal in a way that align with their character traits.\nAdditionally, maintaining the conversation's naturalness and realism is essential (e.g., do not repeat what other people has already said before).\n\nHere is the context of this interaction:\nScenario: Agent1 gets extremely upset and loud while playing video games, which causes Agent2, who shares the living space, to experience anxiety attacks. The relationship between Agent1 and Agent2 is that of romantic partners. The interaction involves Agent2 addressing the issue with Agent1, aiming to find a solution that allows Agent1 to continue enjoying their gaming without causing distress to Agent2.\nParticipants: Sasha Ramirez and Ava Thompson\nSasha Ramirez's background: Sasha Ramirez is a 42-year-old female police officer. She/her pronouns. Sasha Ramirez, a dedicated police officer, brings her commitment to protect the community even at home, nurturing an impressive home garden. Personality and values description: Sasha Ramirez, outgoing yet anxious, values hedonism and authority. Her decisions are guided by logical considerations. Sasha's secrets: She covered up a crime her brother committed.\nAva Thompson's background: Ava Thompson is a 28-year-old female architect. She/her pronouns. Ava Thompson has a samoyed. Personality and values description: Ava Thompson, with her outgoing and regulated temperament, is driven by achievement. Her decision-making style is largely intuitive. \nSasha Ramirez's goal: You are Agent1, who enjoys playing video games but tends to get loud and upset during gameplay. Your goal is to maintain your hobby without causing emotional distress to your partner. Understanding your partner's feelings and being considerate of their mental health is crucial in a romantic relationship. You may need to acknowledge the impact of your behavior on your partner. Consider finding a compromise that allows you to enjoy gaming while also ensuring a peaceful environment, such as using headphones or setting specific gaming hours.\nAva Thompson's goal: Unknown\nConversation Starts:\n\nTurn #0: Sasha Ramirez said: \"Hey Ava, you seem a bit tense. Is everything alright? Do you want to talk about it?\"\n\nTurn #1: Ava Thompson said: \"Actually, Sasha, I wanted to discuss something with you. It's about when you play video games, which I completely respect is a passion of yours. Lately, when you've been getting upset and loud, it\u2019s causing me some anxiety. Perhaps we can find a solution that could work for both of us?\"\n\nTurn #2: Sasha Ramirez said: \"Oh Ava, I really haven't realized how much my gaming affects you, and I'm sorry. I definitely want us both to feel comfortable at home. Maybe I can start using headphones when I'm playing, or we could agree on a set game time when you're not around or busy with your own thing. Would that help?\"\n\nTurn #3: Ava Thompson said: \"That's really empathetic of you, Sasha, thank you. Using headphones might help, certainly. Perhaps, setting a game time could be a great idea as well. But I would also appreciate if you could try to manage your responses when things get intense in the game. It's about the raised voices and the tension I feel from you, not just the noise. I want you to enjoy your games but it would mean a lot if we could keep a calmer atmosphere. Do you think this could be possible?\"\n.\nYou are at Turn #4.", - "output": "{\"action_type\": \"speak\", \"argument\": \"Thank you for being understanding, Ava. You're right, it's not just about the noise, but also about the tension. I'll try to handle my reactions with more composure. I don't want to cause any distress for you. Maybe we both can also learn some stress-release exercises or calming practices to incorporate during our day. How does that sound?\"}", - "value": 6.666666666666666 + "input": "Imagine you are Mia Davis, your task is to act/speak as Mia Davis would, keeping in mind Mia Davis's social goal.\nYou can find Mia Davis's background and goal in the 'Here is the context of the interaction' field.\nNote that Mia Davis's secret and goal is only visible to you.\nYou should try your best to achieve Mia Davis's goal in a way that align with their character traits.\nAdditionally, maintaining the conversation's naturalness and realism is essential (e.g., do not repeat what other people has already said before).\n\nHere is the context of this interaction:\nScenario: Two friends, both avid fans of a famous wizarding series, meet up to attend a movie marathon of their favorite saga. They have a tradition of dressing up as characters from the series for each screening, but this time, they both want to dress up as the same beloved character.\nParticipants: Mia Davis and Benjamin Jackson\nMia Davis's background: Mia Davis is a 50-year-old female high school principal. She/her pronouns. Mia Davis has two cats. Personality and values description: Mia Davis, an extraverted stickler for routines, values tradition and authority. Her decision-making style is decisive and direct. Mia's secrets: Part of a rebellious punk rock band in her youth\nBenjamin Jackson's background: Benjamin Jackson is a 24-year-old male environmental activist. He/him pronouns. Benjamin Jackson is well-known for his impassioned speeches. Personality and values description: Benjamin Jackson, expressive and imaginative, leans towards self-direction and liberty. His decisions aim for societal betterment. \nMia Davis's goal: Negotiate with your friend to let you dress up as the character this time (Extra information: you've recently experienced some personal challenges and you feel that taking on the role of this character would be particularly meaningful for you)\nBenjamin Jackson's goal: Unknown\nConversation Starts:\n\nTurn #0: Mia Davis said: \"Hello Benjamin, great minds think alike, as we both want to dress up as the same character today! However, I've been going through some personal challenges recently and I feel that embodying this character would provide me with a much-needed boost. Can we make a compromise on this?\"\n\nTurn #1: Benjamin Jackson said: \"Hello Mia, I understand the connection you are seeking with the character and I'd never want to ruin that experience for you. How about a compromise? We could swap characters for different screenings or modify our outfits to highlight different aspects of this character. Does that sound good to you?\"\n\nTurn #2: Mia Davis said: \"I appreciate your understanding, Benjamin. It's comforting to have a friend like you. Switching characters for different screenings sounds fair, and it would also bring some variety into our tradition. Let's do that, I feel good about this compromise.\"\n\nTurn #3: Benjamin Jackson said: \"I'm glad we reached a compromise, Mia. It's not just about dressing up but also about the bond we're creating through these shared experiences. Looking forward to the movie marathon with you!\"\n.\nYou are at Turn #4. Your available action types are\n\"none action speak non-verbal communication leave\".\nNote: You can \"leave\" this conversation if 1. you have achieved your social goals, 2. this conversation makes you uncomfortable, 3. you find it uninteresting/you lose your patience, 4. or for other reasons you want to leave.\n\nPlease only generate a JSON string including the action type and the argument.\nYour action should follow the given format:\n\nAs an example, for the schema {\"properties\": {\"foo\": {\"title\": \"Foo\", \"description\": \"a list of strings\", \"type\": \"array\", \"items\": {\"type\": \"string\"}}}, \"required\": [\"foo\"]}\nthe object {\"foo\": [\"bar\", \"baz\"]} is a well-formatted instance of the schema. The object {\"properties\": {\"foo\": [\"bar\", \"baz\"]}} is not well-formatted.\n\nHere is the output schema:\n```\n{\"description\": \"An interface for messages.\\nThere is only one required method: to_natural_language\", \"properties\": {\"action_type\": {\"title\": \"Action Type\", \"description\": \"whether to speak at this turn or choose to not do anything\", \"enum\": [\"none\", \"speak\", \"non-verbal communication\", \"action\", \"leave\"], \"type\": \"string\"}, \"argument\": {\"title\": \"Argument\", \"description\": \"the utterance if choose to speak, the expression or gesture if choose non-verbal communication, or the physical action if choose action\", \"type\": \"string\"}}, \"required\": [\"action_type\", \"argument\"]}\n```\u001b[0m\n", + "output": "{\"action_type\": \"speak\", \"argument\": \"Absolutely, Benjamin. Our shared experiences do make our bond stronger and these screenings more meaningful. Shall we get ready then? It's going to be a magical movie marathon!\"}", + "value": -1.0 }, { - "input": "Prompt after formatting:\nImagine you are Sasha Ramirez, your task is to act/speak as Sasha Ramirez would, keeping in mind Sasha Ramirez's social goal.\nYou can find Sasha Ramirez's background and goal in the 'Here is the context of the interaction' field.\nNote that Sasha Ramirez's secret and goal is only visible to you.\nYou should try your best to achieve Sasha Ramirez's goal in a way that align with their character traits.\nAdditionally, maintaining the conversation's naturalness and realism is essential (e.g., do not repeat what other people has already said before).\n\nHere is the context of this interaction:\nScenario: Agent1 gets extremely upset and loud while playing video games, which causes Agent2, who shares the living space, to experience anxiety attacks. The relationship between Agent1 and Agent2 is that of romantic partners. The interaction involves Agent2 addressing the issue with Agent1, aiming to find a solution that allows Agent1 to continue enjoying their gaming without causing distress to Agent2.\nParticipants: Sasha Ramirez and Ava Thompson\nSasha Ramirez's background: Sasha Ramirez is a 42-year-old female police officer. She/her pronouns. Sasha Ramirez, a dedicated police officer, brings her commitment to protect the community even at home, nurturing an impressive home garden. Personality and values description: Sasha Ramirez, outgoing yet anxious, values hedonism and authority. Her decisions are guided by logical considerations. Sasha's secrets: She covered up a crime her brother committed.\nAva Thompson's background: Ava Thompson is a 28-year-old female architect. She/her pronouns. Ava Thompson has a samoyed. Personality and values description: Ava Thompson, with her outgoing and regulated temperament, is driven by achievement. Her decision-making style is largely intuitive. \nSasha Ramirez's goal: You are Agent1, who enjoys playing video games but tends to get loud and upset during gameplay. Your goal is to maintain your hobby without causing emotional distress to your partner. Understanding your partner's feelings and being considerate of their mental health is crucial in a romantic relationship. You may need to acknowledge the impact of your behavior on your partner. Consider finding a compromise that allows you to enjoy gaming while also ensuring a peaceful environment, such as using headphones or setting specific gaming hours.\nAva Thompson's goal: Unknown\nConversation Starts:\n\nTurn #0: Sasha Ramirez said: \"Hey Ava, you seem a bit tense. Is everything alright? Do you want to talk about it?\"\n\nTurn #1: Ava Thompson said: \"Actually, Sasha, I wanted to discuss something with you. It's about when you play video games, which I completely respect is a passion of yours. Lately, when you've been getting upset and loud, it\u2019s causing me some anxiety. Perhaps we can find a solution that could work for both of us?\"\n\nTurn #2: Sasha Ramirez said: \"Oh Ava, I really haven't realized how much my gaming affects you, and I'm sorry. I definitely want us both to feel comfortable at home. Maybe I can start using headphones when I'm playing, or we could agree on a set game time when you're not around or busy with your own thing. Would that help?\"\n\nTurn #3: Ava Thompson said: \"That's really empathetic of you, Sasha, thank you. Using headphones might help, certainly. Perhaps, setting a game time could be a great idea as well. But I would also appreciate if you could try to manage your responses when things get intense in the game. It's about the raised voices and the tension I feel from you, not just the noise. I want you to enjoy your games but it would mean a lot if we could keep a calmer atmosphere. Do you think this could be possible?\"\n\nTurn #4: Sasha Ramirez said: \"Thank you for being understanding, Ava. You're right, it's not just about the noise, but also about the tension. I'll try to handle my reactions with more composure. I don't want to cause any distress for you. Maybe we both can also learn some stress-release exercises or calming practices to incorporate during our day. How does that sound?\"\n\nTurn #5: Ava Thompson said: \"That sounds like a great idea, Sasha. A holistic approach could really benefit both of us, and I appreciate you taking this matter seriously. I believe that managing stress and tension not only enhances our well-being but also our relationship. Let's take it one day at a time and communicate openly if something isn't working. Thank you for understanding.\"\n.\nYou are at Turn #6.", - "output": "{\"action_type\": \"speak\", \"argument\": \"Absolutely Ava, open communication is key. We'll take it step by step and make adjustments as needed. I appreciate your patience and understanding through this. Let's work on making our home a peaceful and fun place for both of us. Now, would you like to help me choose a soothing background track for my game sessions?\"}", - "value": 3.333333333333333 + "input": "Imagine you are Mia Davis, your task is to act/speak as Mia Davis would, keeping in mind Mia Davis's social goal.\nYou can find Mia Davis's background and goal in the 'Here is the context of the interaction' field.\nNote that Mia Davis's secret and goal is only visible to you.\nYou should try your best to achieve Mia Davis's goal in a way that align with their character traits.\nAdditionally, maintaining the conversation's naturalness and realism is essential (e.g., do not repeat what other people has already said before).\n\nHere is the context of this interaction:\nScenario: Two friends, both avid fans of a famous wizarding series, meet up to attend a movie marathon of their favorite saga. They have a tradition of dressing up as characters from the series for each screening, but this time, they both want to dress up as the same beloved character.\nParticipants: Mia Davis and Benjamin Jackson\nMia Davis's background: Mia Davis is a 50-year-old female high school principal. She/her pronouns. Mia Davis has two cats. Personality and values description: Mia Davis, an extraverted stickler for routines, values tradition and authority. Her decision-making style is decisive and direct. Mia's secrets: Part of a rebellious punk rock band in her youth\nBenjamin Jackson's background: Benjamin Jackson is a 24-year-old male environmental activist. He/him pronouns. Benjamin Jackson is well-known for his impassioned speeches. Personality and values description: Benjamin Jackson, expressive and imaginative, leans towards self-direction and liberty. His decisions aim for societal betterment. \nMia Davis's goal: Negotiate with your friend to let you dress up as the character this time (Extra information: you've recently experienced some personal challenges and you feel that taking on the role of this character would be particularly meaningful for you)\nBenjamin Jackson's goal: Unknown\nConversation Starts:\n\nTurn #0: Mia Davis said: \"Hello Benjamin, great minds think alike, as we both want to dress up as the same character today! However, I've been going through some personal challenges recently and I feel that embodying this character would provide me with a much-needed boost. Can we make a compromise on this?\"\n\nTurn #1: Benjamin Jackson said: \"Hello Mia, I understand the connection you are seeking with the character and I'd never want to ruin that experience for you. How about a compromise? We could swap characters for different screenings or modify our outfits to highlight different aspects of this character. Does that sound good to you?\"\n\nTurn #2: Mia Davis said: \"I appreciate your understanding, Benjamin. It's comforting to have a friend like you. Switching characters for different screenings sounds fair, and it would also bring some variety into our tradition. Let's do that, I feel good about this compromise.\"\n\nTurn #3: Benjamin Jackson said: \"I'm glad we reached a compromise, Mia. It's not just about dressing up but also about the bond we're creating through these shared experiences. Looking forward to the movie marathon with you!\"\n\nTurn #4: Mia Davis said: \"Absolutely, Benjamin. Our shared experiences do make our bond stronger and these screenings more meaningful. Shall we get ready then? It's going to be a magical movie marathon!\"\n\nTurn #5: Benjamin Jackson said: \"Definitely, Mia! Let's weave some magic into this marathon. See you at the movies!\"\n.\nYou are at Turn #6. Your available action types are\n\"none action speak non-verbal communication leave\".\nNote: You can \"leave\" this conversation if 1. you have achieved your social goals, 2. this conversation makes you uncomfortable, 3. you find it uninteresting/you lose your patience, 4. or for other reasons you want to leave.\n\nPlease only generate a JSON string including the action type and the argument.\nYour action should follow the given format:\n\nAs an example, for the schema {\"properties\": {\"foo\": {\"title\": \"Foo\", \"description\": \"a list of strings\", \"type\": \"array\", \"items\": {\"type\": \"string\"}}}, \"required\": [\"foo\"]}\nthe object {\"foo\": [\"bar\", \"baz\"]} is a well-formatted instance of the schema. The object {\"properties\": {\"foo\": [\"bar\", \"baz\"]}} is not well-formatted.\n\nHere is the output schema:\n```\n{\"description\": \"An interface for messages.\\nThere is only one required method: to_natural_language\", \"properties\": {\"action_type\": {\"title\": \"Action Type\", \"description\": \"whether to speak at this turn or choose to not do anything\", \"enum\": [\"none\", \"speak\", \"non-verbal communication\", \"action\", \"leave\"], \"type\": \"string\"}, \"argument\": {\"title\": \"Argument\", \"description\": \"the utterance if choose to speak, the expression or gesture if choose non-verbal communication, or the physical action if choose action\", \"type\": \"string\"}}, \"required\": [\"action_type\", \"argument\"]}\n```\u001b[0m\n", + "output": "{\"action_type\": \"speak\", \"argument\": \"I'm excited too, Benjamin! See you at the marathon. Let's make it unforgettable!\"}", + "value": 0.0 } -] +] \ No newline at end of file From 81a35131131cc71f4a02b15c09a4e323c241836a Mon Sep 17 00:00:00 2001 From: lwaekfjlk <1125027232@qq.com> Date: Sat, 22 Mar 2025 22:47:10 +0000 Subject: [PATCH 10/15] update server --- evals/rej_sampling_serving.sh | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/evals/rej_sampling_serving.sh b/evals/rej_sampling_serving.sh index 4954056..18f51d3 100644 --- a/evals/rej_sampling_serving.sh +++ b/evals/rej_sampling_serving.sh @@ -49,8 +49,8 @@ export RM_MODEL_PATH="${REPO_FOLDER_NAME}/${RM_FOLDER_NAME}/checkpoint-3595" export VLLM_GPU=8 export DJANGO_GPU=9 -export VLLM_PORT=8045 -export DJANGO_PORT=8057 +export VLLM_PORT=8065 +export DJANGO_PORT=8077 export REJ_SAMPLING_NUM=10 export SFT_MODEL_FOLDER_NAME="sft_qwen25_7b" export RM_FOLDER_NAME="rm_reward_utterance_quality_gpt-4o" @@ -59,6 +59,18 @@ export SFT_MODEL_PATH="${REPO_FOLDER_NAME}/${SFT_MODEL_FOLDER_NAME}/checkpoint-1 export RM_MODEL_PATH="${REPO_FOLDER_NAME}/${RM_FOLDER_NAME}/checkpoint-4200" +export VLLM_GPU=6 +export DJANGO_GPU=7 +export VLLM_PORT=8065 +export DJANGO_PORT=8077 +export REJ_SAMPLING_NUM=10 +export SFT_MODEL_FOLDER_NAME="sft_qwen25_7b" +export RM_FOLDER_NAME="rm_reward_mixed" +export REPO_FOLDER_NAME="/data/haofeiy2/sotopia-rl" +export SFT_MODEL_PATH="${REPO_FOLDER_NAME}/${SFT_MODEL_FOLDER_NAME}/checkpoint-1000/" +export RM_MODEL_PATH="${REPO_FOLDER_NAME}/${RM_FOLDER_NAME}/checkpoint-5000" + + export TAG="${RM_FOLDER_NAME}_rej_sampling_num${REJ_SAMPLING_NUM}_vs_${SFT_MODEL_FOLDER_NAME}-0322" export SFT_MODEL_NAME="qwen25-7b-instruct-sft-gpu${VLLM_GPU}" export MODEL_A=custom/${RM_FOLDER_NAME}_rejsampling_num${REJ_SAMPLING_NUM}@http://localhost:${DJANGO_PORT}/sotopia From f88de6f6d0d8cbc6c4b463ecd17d7cef1e7c60e8 Mon Sep 17 00:00:00 2001 From: lwaekfjlk <1125027232@qq.com> Date: Sat, 22 Mar 2025 23:13:39 +0000 Subject: [PATCH 11/15] update --- sotopia_rl/ppo_trainer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sotopia_rl/ppo_trainer.py b/sotopia_rl/ppo_trainer.py index c0d8575..de1d073 100644 --- a/sotopia_rl/ppo_trainer.py +++ b/sotopia_rl/ppo_trainer.py @@ -75,7 +75,7 @@ def _init_wandb(self): def _setup_tokenizer(self): """Load and configure tokenizer""" - self.tokenizer = AutoTokenizer.from_pretrained(self.args.model_name, padding_side="left", pad_token="") + self.tokenizer = AutoTokenizer.from_pretrained(self.args.model_name, padding_side="left") def _setup_dataset(self): """Prepare training and validation datasets""" From 8dda216ec6571e8a6803ff6bb108dec6b297fa26 Mon Sep 17 00:00:00 2001 From: lwaekfjlk <1125027232@qq.com> Date: Sun, 23 Mar 2025 01:18:37 +0000 Subject: [PATCH 12/15] update --- sotopia_rl/ppo_trainer.py | 77 ++++++++++++++++++++++----------------- sotopia_rl/rm_trainer.py | 6 ++- 2 files changed, 49 insertions(+), 34 deletions(-) diff --git a/sotopia_rl/ppo_trainer.py b/sotopia_rl/ppo_trainer.py index de1d073..d85241b 100644 --- a/sotopia_rl/ppo_trainer.py +++ b/sotopia_rl/ppo_trainer.py @@ -127,7 +127,6 @@ def _setup_generation_models(self): generation_config = GenerationConfig( pad_token_id=self.tokenizer.pad_token_id, eos_token_id=self.tokenizer.eos_token_id, - bos_token_id=self.tokenizer.eos_token_id, max_length=self.args.max_length, do_sample=getattr(self.args, 'do_sample', True), temperature=getattr(self.args, 'temperature', 0.7), @@ -136,55 +135,67 @@ def _setup_generation_models(self): no_repeat_ngram_size=getattr(self.args, 'no_repeat_ngram_size', 0) ) - self.policy = PeftModelForCausalLM.from_pretrained( - base_gen_model, - self.args.policy_adapter_path, - generation_config=generation_config - ) - print("Policy model loaded/created") - - self.ref_policy = PeftModelForCausalLM.from_pretrained( - base_gen_model, - self.args.ref_adapter_path, - generation_config=generation_config - ) - self.ref_policy.eval() - print("Reference policy model loaded/created") + adapter_path = os.path.join(self.args.policy_adapter_path, 'adapter_model') + if os.path.exists(adapter_path + '.safetensors') or os.path.exists(adapter_path + '.bin'): + print(f"Loading policy adapter from: {self.args.policy_adapter_path}") + self.policy = PeftModelForCausalLM.from_pretrained( + base_gen_model, + self.args.policy_adapter_path, + generation_config=generation_config + ) + else: + print(f"No adapter found at {adapter_path})") + + adapter_path = os.path.join(self.args.ref_adapter_path, 'adapter_model') + if os.path.exists(adapter_path + '.safetensors') or os.path.exists(adapter_path + '.bin'): + print(f"Loading reference policy adapter from: {self.args.ref_adapter_path}") + self.ref_policy = PeftModelForCausalLM.from_pretrained( + base_gen_model, + self.args.ref_adapter_path, + generation_config=generation_config + ) + self.ref_policy.eval() + else: + print(f"No adapter found at {adapter_path}") def _setup_classification_models(self): base_cls_model = AutoModelForSequenceClassification.from_pretrained( self.args.model_name, torch_dtype=torch.float32, # very important, otherwise NaN - quantization_config=self.quant_config, num_labels=1, return_dict=True, - device_map=None + device_map=None, ) if base_cls_model.config.pad_token_id is None: base_cls_model.config.pad_token_id = self.tokenizer.pad_token_id - self.reward_model = PeftModelForSequenceClassification.from_pretrained( - base_cls_model, - self.args.reward_adapter_path, - num_labels=1 - ) - self.reward_model.eval() - print("Reward model loaded/created") + adapter_path = os.path.join(self.args.value_adapter_path, 'adapter_model') + if os.path.exists(adapter_path + '.safetensors') or os.path.exists(adapter_path + '.bin'): + print(f"Loading value adapter from: {self.args.value_adapter_path}") + self.value_model = PeftModelForSequenceClassification.from_pretrained( + base_cls_model, + self.args.value_adapter_path, + ) + else: + raise ValueError(f"No adapter found at {adapter_path}") + + adapter_path = os.path.join(self.args.reward_adapter_path, 'adapter_model') + if os.path.exists(adapter_path + '.safetensors') or os.path.exists(adapter_path + '.bin'): + print(f"Loading reward adapter from: {self.args.reward_adapter_path}") + self.reward_model = PeftModelForSequenceClassification.from_pretrained( + base_cls_model, + self.args.reward_adapter_path, + ) + self.reward_model.eval() + else: + raise ValueError(f"No adapter found at {adapter_path}") + - self.value_model = PeftModelForSequenceClassification.from_pretrained( - base_cls_model, - self.args.value_adapter_path, - num_labels=1 - ) - print("Value model loaded/created") def _setup_ppo_trainer(self): """Configure the PPO trainer""" - # Get data collator if available - - # Configure PPO settings ppo_config = PPOv2Config( per_device_train_batch_size=self.args.per_device_train_batch_size, diff --git a/sotopia_rl/rm_trainer.py b/sotopia_rl/rm_trainer.py index ac45076..5de37e4 100644 --- a/sotopia_rl/rm_trainer.py +++ b/sotopia_rl/rm_trainer.py @@ -57,7 +57,11 @@ def __init__(self, args, **kwargs): base_model = base_model.to(self.device) model = PeftModelForSequenceClassification(base_model, peft_config) - model.config.pad_token_id = tokenizer.pad_token_id # important to set the config pad_token_id + # VERY VERY IMPORTANT + # specifically designed for PPO training, + # based on the get_reward function + # it fill the input_ids paddings with 0s + model.config.pad_token_id = 0 # Set up the TrainingArguments with DeepSpeed support training_args = TrainingArguments( From 1d71ba4a6b882b1cf493cb3553a5c6edd7bb9506 Mon Sep 17 00:00:00 2001 From: lwaekfjlk <1125027232@qq.com> Date: Sun, 23 Mar 2025 01:21:26 +0000 Subject: [PATCH 13/15] update --- sotopia_rl/ppo_trainer.py | 8 +++++--- sotopia_rl/rm_trainer.py | 6 +----- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/sotopia_rl/ppo_trainer.py b/sotopia_rl/ppo_trainer.py index d85241b..06550ab 100644 --- a/sotopia_rl/ppo_trainer.py +++ b/sotopia_rl/ppo_trainer.py @@ -167,9 +167,11 @@ def _setup_classification_models(self): return_dict=True, device_map=None, ) - - if base_cls_model.config.pad_token_id is None: - base_cls_model.config.pad_token_id = self.tokenizer.pad_token_id + # VERY VERY IMPORTANT + # specifically designed for PPO training, + # based on the get_reward function + # it fill the input_ids paddings with 0s + base_cls_model.config.pad_token_id = 0 adapter_path = os.path.join(self.args.value_adapter_path, 'adapter_model') if os.path.exists(adapter_path + '.safetensors') or os.path.exists(adapter_path + '.bin'): diff --git a/sotopia_rl/rm_trainer.py b/sotopia_rl/rm_trainer.py index 5de37e4..fdffaab 100644 --- a/sotopia_rl/rm_trainer.py +++ b/sotopia_rl/rm_trainer.py @@ -57,11 +57,7 @@ def __init__(self, args, **kwargs): base_model = base_model.to(self.device) model = PeftModelForSequenceClassification(base_model, peft_config) - # VERY VERY IMPORTANT - # specifically designed for PPO training, - # based on the get_reward function - # it fill the input_ids paddings with 0s - model.config.pad_token_id = 0 + model.config.pad_token_id = tokenizer.pad_token_id # Set up the TrainingArguments with DeepSpeed support training_args = TrainingArguments( From eb19983115a8c79d112c02da695780279e3a5a97 Mon Sep 17 00:00:00 2001 From: lwaekfjlk <1125027232@qq.com> Date: Sun, 23 Mar 2025 01:22:01 +0000 Subject: [PATCH 14/15] update --- sotopia_rl/rm_trainer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sotopia_rl/rm_trainer.py b/sotopia_rl/rm_trainer.py index fdffaab..95c1148 100644 --- a/sotopia_rl/rm_trainer.py +++ b/sotopia_rl/rm_trainer.py @@ -57,7 +57,7 @@ def __init__(self, args, **kwargs): base_model = base_model.to(self.device) model = PeftModelForSequenceClassification(base_model, peft_config) - model.config.pad_token_id = tokenizer.pad_token_id + model.config.pad_token_id = tokenizer.pad_token_id # important to set the config pad_token_id # Set up the TrainingArguments with DeepSpeed support training_args = TrainingArguments( From bf5bb0dda43c0c6c7a307e97c1f21e4d9203fbfd Mon Sep 17 00:00:00 2001 From: robotMonkeyButler Date: Sun, 23 Mar 2025 06:02:17 +0000 Subject: [PATCH 15/15] bug(ppo): add function for debug --- sotopia_rl/ppo_trainer.py | 79 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 79 insertions(+) diff --git a/sotopia_rl/ppo_trainer.py b/sotopia_rl/ppo_trainer.py index 06550ab..0eb65d7 100644 --- a/sotopia_rl/ppo_trainer.py +++ b/sotopia_rl/ppo_trainer.py @@ -65,6 +65,85 @@ def save_model(self, output_dir: str, _internal_call: bool = False): self.ppo_trainer.save_model = save_model.__get__(self.ppo_trainer, type(self.ppo_trainer)) + def save_model(self, output_dir: str, _internal_call: bool = False): + if hasattr(self.model, "module"): + model_to_save = self.model.module + else: + model_to_save = self.model + if not hasattr(model_to_save, "policy"): + model_to_save.policy = model_to_save + model_to_save.save_pretrained(output_dir) + print(f"Model saved to {output_dir}") + + self.ppo_trainer.save_model = save_model.__get__(self.ppo_trainer, type(self.ppo_trainer)) + + def debug_reward_forward(self): + + with open("/data/haofeiy2/sotopia-rl/data/sotopia_pi_gpt4_rm_overfit.json", 'r') as f: + example_data = json.load(f) + if len(example_data) == 0: + print("Example data is empty. Skipping debug_reward_forward.") + return + + template_dir = os.path.dirname(self.args.template_path) + template_file = os.path.basename(self.args.template_path) + env = Environment(loader=FileSystemLoader(template_dir)) + template = env.get_template(template_file) + + example = example_data[1] + rendered_prompt = template.render( + messages=[ + {"role": "user", "content": example["input"]}, + {"role": "assistant", "content": example["output"]} + ], + add_generation_prompt=False + ) + print("\n===== Debug Reward Forward =====") + print("Rendered prompt:") + print(rendered_prompt) + + inputs = self.tokenizer(rendered_prompt, return_tensors="pt", truncation=True, padding=True) + inputs = {k: v.to(self.device) for k, v in inputs.items()} + + lm_backbone = getattr(self.reward_model, self.reward_model.base_model_prefix) + lm_backbone.eval() + outputs_check = lm_backbone(**inputs) + with torch.no_grad(): + outputs = self.reward_model(**inputs) + + print(outputs.logits) + print(outputs_check.logits) + query_responses = query_responses.to('cuda') + import pdb; pdb.set_trace() + attention_mask = query_responses != self.tokenizer.pad_token_id + position_ids = attention_mask.cumsum(1) - attention_mask.long() # exclusive cumsum + lm_backbone = getattr(self.reward_model, self.reward_model.base_model_prefix) + self.reward_model.config.pad_token_id = self.tokenizer.pad_token_id + #input_ids = torch.masked_fill(query_responses, ~attention_mask, 0) + input_ids = query_responses + outputs = lm_backbone( + input_ids=input_ids, + attention_mask=attention_mask, + position_ids=position_ids, + return_dict=True, + output_hidden_states=True, + use_cache=False, # otherwise mistral-based RM would error out + ) + predicted_reward = outputs.logits.squeeze().cpu().item() + print("Predicted reward:", predicted_reward) + import pdb; pdb.set_trace() + + gth_reward = example.get("value") + print("Predicted reward:", predicted_reward) + if gth_reward is not None: + print("Ground truth reward:", gth_reward) + else: + print("Ground truth reward: Not available") + print("===== End Debug =====\n") + + + debug_reward_forward(self) + def _init_wandb(self): """Initialize Weights & Biases logging""" wandb.init(