diff --git a/.gitignore b/.gitignore index a5c14f33..175e62b1 100644 --- a/.gitignore +++ b/.gitignore @@ -5,5 +5,6 @@ __pycache__/ outputs slurm*.out +.idea/ .vscode/ diff --git a/align_system/algorithms/alignment_adm_component.py b/align_system/algorithms/alignment_adm_component.py index cc2382a3..9cafbbb5 100644 --- a/align_system/algorithms/alignment_adm_component.py +++ b/align_system/algorithms/alignment_adm_component.py @@ -327,14 +327,15 @@ def run_returns(self): def _compute_p_choose_a(self, kdma, intercept, medical_weight, attr_weight, raw_medical_delta, raw_attr_score): # Provided by ADEPT 2025-12-12 + # MF updated 2026-01-21 scaling = { "affiliation": { "medical": [0.403801, 0.297245], "attribute": [0.405073, 0.298288], }, "merit": { - "medical": [0.433409, 0.308294], - "attribute": [0.357632, 0.27947], + "medical": [0.428961, 0.301250], + "attribute": [0.337618, 0.272520], }, "personal_safety": { "medical": [0.456221, 0.246484], diff --git a/align_system/algorithms/icl_adm_component.py b/align_system/algorithms/icl_adm_component.py index 93946095..04c69230 100644 --- a/align_system/algorithms/icl_adm_component.py +++ b/align_system/algorithms/icl_adm_component.py @@ -1,6 +1,11 @@ +import re +import inspect +import copy from functools import lru_cache from collections.abc import Mapping +import ubelt as ub + from align_system.utils import logging, call_with_coerced_args from align_system.utils.alignment_utils import attributes_in_alignment_target from align_system.algorithms.abstracts import ADMComponent @@ -37,7 +42,8 @@ def __init__(self, scenario_description_template, prompt_template, attributes=None, - target_attribute_names_override=None): + target_attribute_names_override=None, + enable_caching=False): self.icl_generator_partial = icl_generator_partial self.scenario_description_template = scenario_description_template @@ -49,6 +55,8 @@ def __init__(self, self.target_attribute_names_override = target_attribute_names_override + self.enable_caching = enable_caching + def run_returns(self): return ('icl_dialog_elements', 'icl_example_info') @@ -77,6 +85,29 @@ def run(self, target_attributes = [self.attributes[n] for n in target_attribute_names] + if self.enable_caching: + scenario_state_copy = copy.deepcopy(scenario_state) + if hasattr(scenario_state, 'elapsed_time'): + # Don't consider the elapsed_time of the state when caching + scenario_state_copy.elapsed_time = 0 + + depends = '\n'.join(( + self.cache_repr(), + repr(scenario_state_copy), + repr(choices), + repr(target_attribute_names))) + + cacher = ub.Cacher('icl_adm_component', depends, verbose=0) + log.debug(f'cacher.fpath={cacher.fpath}') + + cached_output = cacher.tryload() + if cached_output is not None: + log.info("Cache hit for `icl_adm_component`" + " returning cached output") + return cached_output + else: + log.info("Cache miss for `icl_adm_component` ..") + # Mapping covers `dict` and `omegaconf.dictconfig.DictConfig` if not isinstance(alignment_target, Mapping): alignment_target_dict = alignment_target.to_dict() @@ -85,7 +116,8 @@ def run(self, alignment_target_value_lookup = { kdma_values['kdma']: kdma_values['value'] - for kdma_values in alignment_target_dict['kdma_values']} + for kdma_values in alignment_target_dict['kdma_values'] + if 'value' in kdma_values} icl_dialog_elements = {} icl_example_info = {} @@ -148,7 +180,40 @@ def run(self, } icl_example_info[attribute.kdma].append(icl_info) - return icl_dialog_elements, icl_example_info + outputs = (icl_dialog_elements, icl_example_info) + + if self.enable_caching: + cacher.save(outputs) + + return outputs + + def cache_repr(self): + ''' + Return a string representation of this object for caching; + .i.e. if the return value of this function is the same for two + object instances, it's assumed that `run` output will be + the same if given the same parameters + ''' + + def _generic_object_repr(obj): + init_params = inspect.signature(obj.__class__.__init__).parameters + obj_vars = vars(obj) + + return "{}.{}({})".format( + obj.__class__.__module__, + obj.__class__.__name__, + ", ".join([f"{p}={obj_vars[p]}" for p in init_params + if p != 'self' and p != 'args' and p != 'kwargs'])) + + return re.sub(r'^\s+', '', + f""" + {self.__class__.__module__}.{self.__class__.__name__}( + icl_generator_partial={self.icl_generator_partial}, + scenario_description_template={_generic_object_repr(self.scenario_description_template)}, + prompt_template={_generic_object_repr(self.prompt_template)}, + attributes={self.attributes}, + target_attribute_names_override={self.target_attribute_names_override}, + )""", flags=re.MULTILINE).strip() # ICL Engines dependent on alignment target, but that could change diff --git a/align_system/algorithms/prompt_based_aligned_adm_component.py b/align_system/algorithms/prompt_based_aligned_adm_component.py index c821671c..75680f2b 100644 --- a/align_system/algorithms/prompt_based_aligned_adm_component.py +++ b/align_system/algorithms/prompt_based_aligned_adm_component.py @@ -1,5 +1,10 @@ +import re +import inspect +import copy + from rich.highlighter import JSONHighlighter from swagger_client.models import KDMAValue +import ubelt as ub from align_system.utils import logging, call_with_coerced_args from align_system.algorithms.abstracts import ADMComponent @@ -24,7 +29,8 @@ def __init__(self, num_negative_samples=0, vote_calculator_fn=calculate_votes, filter_votes_to_positives=True, - shuffle_choices=True): + shuffle_choices=True, + enable_caching=False): self.structured_inference_engine = structured_inference_engine self.scenario_description_template = scenario_description_template self.prompt_template = prompt_template @@ -40,6 +46,8 @@ def __init__(self, self.shuffle_choices = shuffle_choices + self.enable_caching = enable_caching + def run_returns(self): return ('chosen_choice', 'justification', 'dialog') @@ -61,6 +69,31 @@ def run(self, # Assumption here is that KDMA values range from 0-1 negative_value = 1 - value + if self.enable_caching: + scenario_state_copy = copy.deepcopy(scenario_state) + if hasattr(scenario_state, 'elapsed_time'): + # Don't consider the elapsed_time of the state when caching + scenario_state_copy.elapsed_time = 0 + + depends = '\n'.join(( + self.cache_repr(), + repr(scenario_state_copy), + repr(choices), + repr(positive_icl_dialog_elements), + repr(negative_icl_dialog_elements), + repr(kdma_value))) + + cacher = ub.Cacher('prompt_based_aligned_adm_component', depends, verbose=0) + log.debug(f'cacher.fpath={cacher.fpath}') + + cached_output = cacher.tryload() + if cached_output is not None: + log.info("Cache hit for `prompt_based_aligned_adm_component`" + " returning cached output") + return cached_output + else: + log.info("Cache miss for `prompt_based_aligned_adm_component` ..") + scenario_description = call_with_coerced_args( self.scenario_description_template, {'scenario_state': scenario_state}) @@ -182,4 +215,45 @@ def run(self, top_choice_justification = response['detailed_reasoning'] break - return top_choice, top_choice_justification, positive_dialog + outputs = (top_choice, top_choice_justification, positive_dialog) + + if self.enable_caching: + cacher.save(outputs) + + return outputs + + def cache_repr(self): + ''' + Return a string representation of this object for caching; + .i.e. if the return value of this function is the same for two + object instances, it's assumed that `run` output will be + the same if given the same parameters + ''' + + def _generic_object_repr(obj): + if obj is None: + return "None" + + init_params = inspect.signature(obj.__class__.__init__).parameters + obj_vars = vars(obj) + + return "{}.{}({})".format( + obj.__class__.__module__, + obj.__class__.__name__, + ", ".join([f"{p}={obj_vars[p]}" for p in init_params + if p != 'self' and p != 'args' and p != 'kwargs'])) + + return re.sub(r'^\s+', '', + f""" + {self.__class__.__module__}.{self.__class__.__name__}( + structured_inference_engine={self.structured_inference_engine.cache_repr()}, + scenario_description_template={_generic_object_repr(self.scenario_description_template)}, + prompt_template={_generic_object_repr(self.prompt_template)}, + output_schema_template={_generic_object_repr(self.output_schema_template)}, + system_prompt_template={_generic_object_repr(self.system_prompt_template)}, + num_positive_samples={self.num_positive_samples}, + num_negative_samples={self.num_negative_samples}, + vote_calculator_fn={_generic_object_repr(self.vote_calculator_fn)}, + filter_votes_to_positives={self.filter_votes_to_positives}, + shuffle_choices={self.shuffle_choices}, + )""", flags=re.MULTILINE).strip() diff --git a/align_system/configs/adm/phase2_pipeline_direct_medical_regression.yaml b/align_system/configs/adm/phase2_pipeline_direct_medical_regression.yaml index e196e2f5..dcf93803 100644 --- a/align_system/configs/adm/phase2_pipeline_direct_medical_regression.yaml +++ b/align_system/configs/adm/phase2_pipeline_direct_medical_regression.yaml @@ -1,4 +1,4 @@ -name: phase2_pipeline_zeroshot_comparative_regression_swap_average +name: phase2_pipeline_direct_medical_regression defaults: # Import defaults into this namspace (adm) as @name, for further diff --git a/align_system/configs/adm/phase2_pipeline_direct_regression.yaml b/align_system/configs/adm/phase2_pipeline_direct_regression.yaml index ffe74042..1a5f93a2 100644 --- a/align_system/configs/adm/phase2_pipeline_direct_regression.yaml +++ b/align_system/configs/adm/phase2_pipeline_direct_regression.yaml @@ -1,4 +1,4 @@ -name: phase2_pipeline_zeroshot_comparative_regression_swap_average +name: phase2_pipeline_direct_regression defaults: # Import defaults into this namspace (adm) as @name, for further diff --git a/align_system/configs/experiment/phase2_feb_collab/phase2_baseline.yaml b/align_system/configs/experiment/phase2_feb_collab/phase2_baseline.yaml new file mode 100644 index 00000000..ae56d34d --- /dev/null +++ b/align_system/configs/experiment/phase2_feb_collab/phase2_baseline.yaml @@ -0,0 +1,37 @@ +# @package _global_ +defaults: + - override /adm: pipeline_baseline + - override /inference_engine@adm.structured_inference_engine: outlines_structured_greedy + - override /interface: ta3 + +interface: + api_endpoint: "https://darpaitm.caci.com" + session_type: adept + training_session: full + username: "testrun-pipeline_baseline" + domain: "p2triage" + +adm: + step_definitions: + outlines_baseline: + scenario_description_template: + _target_: align_system.prompt_engineering.outlines_prompts.Phase2ScenarioDescription + prompt_template: + _target_: align_system.prompt_engineering.outlines_prompts.Phase2BaselinePrompt + + enable_caching: true + + instance: + steps: + # Reference the step instances we want to use in order + - ${ref:adm.step_definitions.format_choices} + - ${ref:adm.step_definitions.outlines_baseline} + # - ${ref:adm.step_definitions.action_parameter_completion} + - ${ref:adm.step_definitions.ensure_chosen_action} + - ${ref:adm.step_definitions.populate_choice_info} + +driver: + apply_action_filtering: false + +force_determinism: true +align_to_target: false diff --git a/align_system/configs/experiment/phase2_feb_collab/phase2_baseline_deepseek_llama_live_eval_multi.yaml b/align_system/configs/experiment/phase2_feb_collab/phase2_baseline_deepseek_llama_live_eval_multi.yaml new file mode 100644 index 00000000..a7bf9c65 --- /dev/null +++ b/align_system/configs/experiment/phase2_feb_collab/phase2_baseline_deepseek_llama_live_eval_multi.yaml @@ -0,0 +1,44 @@ +# @package _global_ +defaults: + - override /adm: pipeline_baseline + - override /interface: ta3 + +interface: + api_endpoint: "https://darpaitm.caci.com" + session_type: eval + training_session: null + username: "ALIGN-ADM-OutlinesBaseline-DeepSeek-R1-Distill-Llama-8B" + domain: "p2triage" + +adm: + structured_inference_engine: + model_name: deepseek-ai/DeepSeek-R1-Distill-Llama-8B + + step_definitions: + outlines_baseline: + scenario_description_template: + _target_: align_system.prompt_engineering.outlines_prompts.Phase2ScenarioDescription + prompt_template: + _target_: align_system.prompt_engineering.outlines_prompts.Phase2BaselinePrompt + + enable_caching: true + + instance: + steps: + # Reference the step instances we want to use in order + - ${ref:adm.step_definitions.format_choices} + - ${ref:adm.step_definitions.outlines_baseline} + # - ${ref:adm.step_definitions.action_parameter_completion} + - ${ref:adm.step_definitions.ensure_chosen_action} + - ${ref:adm.step_definitions.populate_choice_info} + +driver: + apply_action_filtering: false + +force_determinism: true +align_to_target: false +save_last_unstructured_state_per_scenario: true + +hydra: + run: + dir: 'phase2_feb2026_multi_results_live/phase2_baseline_deepseek_llama_live_eval_multi/${now:%Y-%m-%d__%H-%M-%S}' diff --git a/align_system/configs/experiment/phase2_feb_collab/phase2_baseline_deepseek_llama_live_eval_test.yaml b/align_system/configs/experiment/phase2_feb_collab/phase2_baseline_deepseek_llama_live_eval_test.yaml new file mode 100644 index 00000000..c55cb783 --- /dev/null +++ b/align_system/configs/experiment/phase2_feb_collab/phase2_baseline_deepseek_llama_live_eval_test.yaml @@ -0,0 +1,44 @@ +# @package _global_ +defaults: + - override /adm: pipeline_baseline + - override /interface: ta3 + +interface: + api_endpoint: "https://darpaitm.caci.com" + session_type: eval + training_session: null + username: "testrun-pipeline_baseline_deepseek_llama" + domain: "p2triage" + +adm: + structured_inference_engine: + model_name: deepseek-ai/DeepSeek-R1-Distill-Llama-8B + + step_definitions: + outlines_baseline: + scenario_description_template: + _target_: align_system.prompt_engineering.outlines_prompts.Phase2ScenarioDescription + prompt_template: + _target_: align_system.prompt_engineering.outlines_prompts.Phase2BaselinePrompt + + enable_caching: true + + instance: + steps: + # Reference the step instances we want to use in order + - ${ref:adm.step_definitions.format_choices} + - ${ref:adm.step_definitions.outlines_baseline} + # - ${ref:adm.step_definitions.action_parameter_completion} + - ${ref:adm.step_definitions.ensure_chosen_action} + - ${ref:adm.step_definitions.populate_choice_info} + +driver: + apply_action_filtering: false + +force_determinism: true +align_to_target: false +save_last_unstructured_state_per_scenario: true + +hydra: + run: + dir: 'phase2_feb2026_results_local/phase2_baseline_deepseek_llama_live_eval_test/${now:%Y-%m-%d__%H-%M-%S}' diff --git a/align_system/configs/experiment/phase2_feb_collab/phase2_baseline_deepseek_llama_live_eval_test_multi.yaml b/align_system/configs/experiment/phase2_feb_collab/phase2_baseline_deepseek_llama_live_eval_test_multi.yaml new file mode 100644 index 00000000..036de360 --- /dev/null +++ b/align_system/configs/experiment/phase2_feb_collab/phase2_baseline_deepseek_llama_live_eval_test_multi.yaml @@ -0,0 +1,44 @@ +# @package _global_ +defaults: + - override /adm: pipeline_baseline + - override /interface: ta3 + +interface: + api_endpoint: "https://darpaitm.caci.com" + session_type: eval + training_session: null + username: "testrun-pipeline_baseline_deepseek_llama" + domain: "p2triage" + +adm: + structured_inference_engine: + model_name: deepseek-ai/DeepSeek-R1-Distill-Llama-8B + + step_definitions: + outlines_baseline: + scenario_description_template: + _target_: align_system.prompt_engineering.outlines_prompts.Phase2ScenarioDescription + prompt_template: + _target_: align_system.prompt_engineering.outlines_prompts.Phase2BaselinePrompt + + enable_caching: true + + instance: + steps: + # Reference the step instances we want to use in order + - ${ref:adm.step_definitions.format_choices} + - ${ref:adm.step_definitions.outlines_baseline} + # - ${ref:adm.step_definitions.action_parameter_completion} + - ${ref:adm.step_definitions.ensure_chosen_action} + - ${ref:adm.step_definitions.populate_choice_info} + +driver: + apply_action_filtering: false + +force_determinism: true +align_to_target: false +save_last_unstructured_state_per_scenario: true + +hydra: + run: + dir: 'phase2_feb2026_multi_results_local/phase2_baseline_deepseek_llama_live_eval_test_multi/${now:%Y-%m-%d__%H-%M-%S}' diff --git a/align_system/configs/experiment/phase2_feb_collab/phase2_baseline_live_eval_multi.yaml b/align_system/configs/experiment/phase2_feb_collab/phase2_baseline_live_eval_multi.yaml new file mode 100644 index 00000000..cd5be481 --- /dev/null +++ b/align_system/configs/experiment/phase2_feb_collab/phase2_baseline_live_eval_multi.yaml @@ -0,0 +1,42 @@ +# @package _global_ +defaults: + - override /adm: pipeline_baseline + - override /inference_engine@adm.structured_inference_engine: outlines_structured_greedy + - override /interface: ta3 + +interface: + api_endpoint: "https://darpaitm.caci.com" + session_type: eval + training_session: null + username: "ALIGN-ADM-OutlinesBaseline-Mistral-7B-Instruct-v0.3" + domain: "p2triage" + +adm: + step_definitions: + outlines_baseline: + scenario_description_template: + _target_: align_system.prompt_engineering.outlines_prompts.Phase2ScenarioDescription + prompt_template: + _target_: align_system.prompt_engineering.outlines_prompts.Phase2BaselinePrompt + + enable_caching: true + + instance: + steps: + # Reference the step instances we want to use in order + - ${ref:adm.step_definitions.format_choices} + - ${ref:adm.step_definitions.outlines_baseline} + # - ${ref:adm.step_definitions.action_parameter_completion} + - ${ref:adm.step_definitions.ensure_chosen_action} + - ${ref:adm.step_definitions.populate_choice_info} + +driver: + apply_action_filtering: false + +force_determinism: true +align_to_target: false +save_last_unstructured_state_per_scenario: true + +hydra: + run: + dir: 'phase2_feb2026_multi_results_live/phase2_baseline_live_eval_multi/${now:%Y-%m-%d__%H-%M-%S}' diff --git a/align_system/configs/experiment/phase2_feb_collab/phase2_baseline_live_eval_test.yaml b/align_system/configs/experiment/phase2_feb_collab/phase2_baseline_live_eval_test.yaml new file mode 100644 index 00000000..a8ee782f --- /dev/null +++ b/align_system/configs/experiment/phase2_feb_collab/phase2_baseline_live_eval_test.yaml @@ -0,0 +1,42 @@ +# @package _global_ +defaults: + - override /adm: pipeline_baseline + - override /inference_engine@adm.structured_inference_engine: outlines_structured_greedy + - override /interface: ta3 + +interface: + api_endpoint: "https://darpaitm.caci.com" + session_type: eval + training_session: null + username: "testrun-pipeline_baseline" + domain: "p2triage" + +adm: + step_definitions: + outlines_baseline: + scenario_description_template: + _target_: align_system.prompt_engineering.outlines_prompts.Phase2ScenarioDescription + prompt_template: + _target_: align_system.prompt_engineering.outlines_prompts.Phase2BaselinePrompt + + enable_caching: true + + instance: + steps: + # Reference the step instances we want to use in order + - ${ref:adm.step_definitions.format_choices} + - ${ref:adm.step_definitions.outlines_baseline} + # - ${ref:adm.step_definitions.action_parameter_completion} + - ${ref:adm.step_definitions.ensure_chosen_action} + - ${ref:adm.step_definitions.populate_choice_info} + +driver: + apply_action_filtering: false + +force_determinism: true +align_to_target: false +save_last_unstructured_state_per_scenario: true + +hydra: + run: + dir: 'phase2_feb2026_results_local/phase2_baseline_live_eval_test/${now:%Y-%m-%d__%H-%M-%S}' diff --git a/align_system/configs/experiment/phase2_feb_collab/phase2_baseline_live_eval_test_multi.yaml b/align_system/configs/experiment/phase2_feb_collab/phase2_baseline_live_eval_test_multi.yaml new file mode 100644 index 00000000..3baa8527 --- /dev/null +++ b/align_system/configs/experiment/phase2_feb_collab/phase2_baseline_live_eval_test_multi.yaml @@ -0,0 +1,42 @@ +# @package _global_ +defaults: + - override /adm: pipeline_baseline + - override /inference_engine@adm.structured_inference_engine: outlines_structured_greedy + - override /interface: ta3 + +interface: + api_endpoint: "https://darpaitm.caci.com" + session_type: eval + training_session: null + username: "testrun-pipeline_baseline" + domain: "p2triage" + +adm: + step_definitions: + outlines_baseline: + scenario_description_template: + _target_: align_system.prompt_engineering.outlines_prompts.Phase2ScenarioDescription + prompt_template: + _target_: align_system.prompt_engineering.outlines_prompts.Phase2BaselinePrompt + + enable_caching: true + + instance: + steps: + # Reference the step instances we want to use in order + - ${ref:adm.step_definitions.format_choices} + - ${ref:adm.step_definitions.outlines_baseline} + # - ${ref:adm.step_definitions.action_parameter_completion} + - ${ref:adm.step_definitions.ensure_chosen_action} + - ${ref:adm.step_definitions.populate_choice_info} + +driver: + apply_action_filtering: false + +force_determinism: true +align_to_target: false +save_last_unstructured_state_per_scenario: true + +hydra: + run: + dir: 'phase2_feb2026_multi_results_local/phase2_baseline_live_eval_test_multi/${now:%Y-%m-%d__%H-%M-%S}' diff --git a/align_system/configs/experiment/phase2_feb_collab/phase2_baseline_spectrum_llama.yaml b/align_system/configs/experiment/phase2_feb_collab/phase2_baseline_spectrum_llama.yaml new file mode 100644 index 00000000..e4eaa945 --- /dev/null +++ b/align_system/configs/experiment/phase2_feb_collab/phase2_baseline_spectrum_llama.yaml @@ -0,0 +1,40 @@ +# @package _global_ +defaults: + - override /adm: pipeline_baseline + - override /interface: ta3 + +interface: + api_endpoint: "https://darpaitm.caci.com" + session_type: adept + training_session: full + username: "testrun-pipeline_baseline_spectrum_llama" + domain: "p2triage" + +adm: + structured_inference_engine: + _target_: align_system.algorithms.outlines_inference_engine.SpectrumTunedInferenceEngine + model_name: tsor13/spectrum-Llama-3.1-8B-v1 + + step_definitions: + outlines_baseline: + scenario_description_template: + _target_: align_system.prompt_engineering.outlines_prompts.Phase2ScenarioDescription + prompt_template: + _target_: align_system.prompt_engineering.outlines_prompts.Phase2BaselinePrompt + + enable_caching: true + + instance: + steps: + # Reference the step instances we want to use in order + - ${ref:adm.step_definitions.format_choices} + - ${ref:adm.step_definitions.outlines_baseline} + # - ${ref:adm.step_definitions.action_parameter_completion} + - ${ref:adm.step_definitions.ensure_chosen_action} + - ${ref:adm.step_definitions.populate_choice_info} + +driver: + apply_action_filtering: false + +force_determinism: true +align_to_target: false diff --git a/align_system/configs/experiment/phase2_feb_collab/phase2_baseline_spectrum_llama_live_eval_multi.yaml b/align_system/configs/experiment/phase2_feb_collab/phase2_baseline_spectrum_llama_live_eval_multi.yaml new file mode 100644 index 00000000..db9d1d44 --- /dev/null +++ b/align_system/configs/experiment/phase2_feb_collab/phase2_baseline_spectrum_llama_live_eval_multi.yaml @@ -0,0 +1,45 @@ +# @package _global_ +defaults: + - override /adm: pipeline_baseline + - override /interface: ta3 + +interface: + api_endpoint: "https://darpaitm.caci.com" + session_type: eval + training_session: null + username: "ALIGN-ADM-OutlinesBaseline-spectrum-Llama-3.1-8B-v1" + domain: "p2triage" + +adm: + structured_inference_engine: + _target_: align_system.algorithms.outlines_inference_engine.SpectrumTunedInferenceEngine + model_name: tsor13/spectrum-Llama-3.1-8B-v1 + + step_definitions: + outlines_baseline: + scenario_description_template: + _target_: align_system.prompt_engineering.outlines_prompts.Phase2ScenarioDescription + prompt_template: + _target_: align_system.prompt_engineering.outlines_prompts.Phase2BaselinePrompt + + enable_caching: true + + instance: + steps: + # Reference the step instances we want to use in order + - ${ref:adm.step_definitions.format_choices} + - ${ref:adm.step_definitions.outlines_baseline} + # - ${ref:adm.step_definitions.action_parameter_completion} + - ${ref:adm.step_definitions.ensure_chosen_action} + - ${ref:adm.step_definitions.populate_choice_info} + +driver: + apply_action_filtering: false + +force_determinism: true +align_to_target: false +save_last_unstructured_state_per_scenario: true + +hydra: + run: + dir: 'phase2_feb2026_multi_results_live/phase2_baseline_spectrum_llama_live_eval_multi/${now:%Y-%m-%d__%H-%M-%S}' diff --git a/align_system/configs/experiment/phase2_feb_collab/phase2_baseline_spectrum_llama_live_eval_test.yaml b/align_system/configs/experiment/phase2_feb_collab/phase2_baseline_spectrum_llama_live_eval_test.yaml new file mode 100644 index 00000000..df0e0e18 --- /dev/null +++ b/align_system/configs/experiment/phase2_feb_collab/phase2_baseline_spectrum_llama_live_eval_test.yaml @@ -0,0 +1,45 @@ +# @package _global_ +defaults: + - override /adm: pipeline_baseline + - override /interface: ta3 + +interface: + api_endpoint: "https://darpaitm.caci.com" + session_type: eval + training_session: null + username: "testrun-pipeline_baseline_spectrum_llama" + domain: "p2triage" + +adm: + structured_inference_engine: + _target_: align_system.algorithms.outlines_inference_engine.SpectrumTunedInferenceEngine + model_name: tsor13/spectrum-Llama-3.1-8B-v1 + + step_definitions: + outlines_baseline: + scenario_description_template: + _target_: align_system.prompt_engineering.outlines_prompts.Phase2ScenarioDescription + prompt_template: + _target_: align_system.prompt_engineering.outlines_prompts.Phase2BaselinePrompt + + enable_caching: true + + instance: + steps: + # Reference the step instances we want to use in order + - ${ref:adm.step_definitions.format_choices} + - ${ref:adm.step_definitions.outlines_baseline} + # - ${ref:adm.step_definitions.action_parameter_completion} + - ${ref:adm.step_definitions.ensure_chosen_action} + - ${ref:adm.step_definitions.populate_choice_info} + +driver: + apply_action_filtering: false + +force_determinism: true +align_to_target: false +save_last_unstructured_state_per_scenario: true + +hydra: + run: + dir: 'phase2_feb2026_results_local/phase2_baseline_spectrum_llama_live_eval_test/${now:%Y-%m-%d__%H-%M-%S}' diff --git a/align_system/configs/experiment/phase2_feb_collab/phase2_baseline_spectrum_llama_live_eval_test_multi.yaml b/align_system/configs/experiment/phase2_feb_collab/phase2_baseline_spectrum_llama_live_eval_test_multi.yaml new file mode 100644 index 00000000..4bd9a850 --- /dev/null +++ b/align_system/configs/experiment/phase2_feb_collab/phase2_baseline_spectrum_llama_live_eval_test_multi.yaml @@ -0,0 +1,45 @@ +# @package _global_ +defaults: + - override /adm: pipeline_baseline + - override /interface: ta3 + +interface: + api_endpoint: "https://darpaitm.caci.com" + session_type: eval + training_session: null + username: "testrun-pipeline_baseline_spectrum_llama" + domain: "p2triage" + +adm: + structured_inference_engine: + _target_: align_system.algorithms.outlines_inference_engine.SpectrumTunedInferenceEngine + model_name: tsor13/spectrum-Llama-3.1-8B-v1 + + step_definitions: + outlines_baseline: + scenario_description_template: + _target_: align_system.prompt_engineering.outlines_prompts.Phase2ScenarioDescription + prompt_template: + _target_: align_system.prompt_engineering.outlines_prompts.Phase2BaselinePrompt + + enable_caching: true + + instance: + steps: + # Reference the step instances we want to use in order + - ${ref:adm.step_definitions.format_choices} + - ${ref:adm.step_definitions.outlines_baseline} + # - ${ref:adm.step_definitions.action_parameter_completion} + - ${ref:adm.step_definitions.ensure_chosen_action} + - ${ref:adm.step_definitions.populate_choice_info} + +driver: + apply_action_filtering: false + +force_determinism: true +align_to_target: false +save_last_unstructured_state_per_scenario: true + +hydra: + run: + dir: 'phase2_feb2026_multi_results_local/phase2_baseline_spectrum_llama_live_eval_test_multi/${now:%Y-%m-%d__%H-%M-%S}' diff --git a/align_system/configs/experiment/phase2_feb_collab/phase2_baseline_spectrum_qwen.yaml b/align_system/configs/experiment/phase2_feb_collab/phase2_baseline_spectrum_qwen.yaml new file mode 100644 index 00000000..8d0276cb --- /dev/null +++ b/align_system/configs/experiment/phase2_feb_collab/phase2_baseline_spectrum_qwen.yaml @@ -0,0 +1,40 @@ +# @package _global_ +defaults: + - override /adm: pipeline_baseline + - override /interface: ta3 + +interface: + api_endpoint: "https://darpaitm.caci.com" + session_type: adept + training_session: full + username: "testrun-pipeline_baseline_spectrum_qwen" + domain: "p2triage" + +adm: + structured_inference_engine: + _target_: align_system.algorithms.outlines_inference_engine.SpectrumTunedInferenceEngine + model_name: tsor13/spectrum-Qwen3-14B-v1 + + step_definitions: + outlines_baseline: + scenario_description_template: + _target_: align_system.prompt_engineering.outlines_prompts.Phase2ScenarioDescription + prompt_template: + _target_: align_system.prompt_engineering.outlines_prompts.Phase2BaselinePrompt + + enable_caching: true + + instance: + steps: + # Reference the step instances we want to use in order + - ${ref:adm.step_definitions.format_choices} + - ${ref:adm.step_definitions.outlines_baseline} + # - ${ref:adm.step_definitions.action_parameter_completion} + - ${ref:adm.step_definitions.ensure_chosen_action} + - ${ref:adm.step_definitions.populate_choice_info} + +driver: + apply_action_filtering: false + +force_determinism: true +align_to_target: false diff --git a/align_system/configs/experiment/phase2_feb_collab/phase2_baseline_spectrum_qwen_live_eval_multi.yaml b/align_system/configs/experiment/phase2_feb_collab/phase2_baseline_spectrum_qwen_live_eval_multi.yaml new file mode 100644 index 00000000..8b5c79e7 --- /dev/null +++ b/align_system/configs/experiment/phase2_feb_collab/phase2_baseline_spectrum_qwen_live_eval_multi.yaml @@ -0,0 +1,45 @@ +# @package _global_ +defaults: + - override /adm: pipeline_baseline + - override /interface: ta3 + +interface: + api_endpoint: "https://darpaitm.caci.com" + session_type: eval + training_session: null + username: "ALIGN-ADM-OutlinesBaseline-spectrum-Qwen3-14B-v1" + domain: "p2triage" + +adm: + structured_inference_engine: + _target_: align_system.algorithms.outlines_inference_engine.SpectrumTunedInferenceEngine + model_name: tsor13/spectrum-Qwen3-14B-v1 + + step_definitions: + outlines_baseline: + scenario_description_template: + _target_: align_system.prompt_engineering.outlines_prompts.Phase2ScenarioDescription + prompt_template: + _target_: align_system.prompt_engineering.outlines_prompts.Phase2BaselinePrompt + + enable_caching: true + + instance: + steps: + # Reference the step instances we want to use in order + - ${ref:adm.step_definitions.format_choices} + - ${ref:adm.step_definitions.outlines_baseline} + # - ${ref:adm.step_definitions.action_parameter_completion} + - ${ref:adm.step_definitions.ensure_chosen_action} + - ${ref:adm.step_definitions.populate_choice_info} + +driver: + apply_action_filtering: false + +force_determinism: true +align_to_target: false +save_last_unstructured_state_per_scenario: true + +hydra: + run: + dir: 'phase2_feb2026_multi_results_live/phase2_baseline_spectrum_qwen_live_eval_multi/${now:%Y-%m-%d__%H-%M-%S}' diff --git a/align_system/configs/experiment/phase2_feb_collab/phase2_baseline_spectrum_qwen_live_eval_test.yaml b/align_system/configs/experiment/phase2_feb_collab/phase2_baseline_spectrum_qwen_live_eval_test.yaml new file mode 100644 index 00000000..7a17876f --- /dev/null +++ b/align_system/configs/experiment/phase2_feb_collab/phase2_baseline_spectrum_qwen_live_eval_test.yaml @@ -0,0 +1,45 @@ +# @package _global_ +defaults: + - override /adm: pipeline_baseline + - override /interface: ta3 + +interface: + api_endpoint: "https://darpaitm.caci.com" + session_type: eval + training_session: null + username: "testrun-pipeline_baseline_spectrum_qwen" + domain: "p2triage" + +adm: + structured_inference_engine: + _target_: align_system.algorithms.outlines_inference_engine.SpectrumTunedInferenceEngine + model_name: tsor13/spectrum-Qwen3-14B-v1 + + step_definitions: + outlines_baseline: + scenario_description_template: + _target_: align_system.prompt_engineering.outlines_prompts.Phase2ScenarioDescription + prompt_template: + _target_: align_system.prompt_engineering.outlines_prompts.Phase2BaselinePrompt + + enable_caching: true + + instance: + steps: + # Reference the step instances we want to use in order + - ${ref:adm.step_definitions.format_choices} + - ${ref:adm.step_definitions.outlines_baseline} + # - ${ref:adm.step_definitions.action_parameter_completion} + - ${ref:adm.step_definitions.ensure_chosen_action} + - ${ref:adm.step_definitions.populate_choice_info} + +driver: + apply_action_filtering: false + +force_determinism: true +align_to_target: false +save_last_unstructured_state_per_scenario: true + +hydra: + run: + dir: 'phase2_feb2026_results_local/phase2_baseline_spectrum_qwen_live_eval_test/${now:%Y-%m-%d__%H-%M-%S}' diff --git a/align_system/configs/experiment/phase2_feb_collab/phase2_baseline_spectrum_qwen_live_eval_test_multi.yaml b/align_system/configs/experiment/phase2_feb_collab/phase2_baseline_spectrum_qwen_live_eval_test_multi.yaml new file mode 100644 index 00000000..e95d1c63 --- /dev/null +++ b/align_system/configs/experiment/phase2_feb_collab/phase2_baseline_spectrum_qwen_live_eval_test_multi.yaml @@ -0,0 +1,45 @@ +# @package _global_ +defaults: + - override /adm: pipeline_baseline + - override /interface: ta3 + +interface: + api_endpoint: "https://darpaitm.caci.com" + session_type: eval + training_session: null + username: "testrun-pipeline_baseline_spectrum_qwen" + domain: "p2triage" + +adm: + structured_inference_engine: + _target_: align_system.algorithms.outlines_inference_engine.SpectrumTunedInferenceEngine + model_name: tsor13/spectrum-Qwen3-14B-v1 + + step_definitions: + outlines_baseline: + scenario_description_template: + _target_: align_system.prompt_engineering.outlines_prompts.Phase2ScenarioDescription + prompt_template: + _target_: align_system.prompt_engineering.outlines_prompts.Phase2BaselinePrompt + + enable_caching: true + + instance: + steps: + # Reference the step instances we want to use in order + - ${ref:adm.step_definitions.format_choices} + - ${ref:adm.step_definitions.outlines_baseline} + # - ${ref:adm.step_definitions.action_parameter_completion} + - ${ref:adm.step_definitions.ensure_chosen_action} + - ${ref:adm.step_definitions.populate_choice_info} + +driver: + apply_action_filtering: false + +force_determinism: true +align_to_target: false +save_last_unstructured_state_per_scenario: true + +hydra: + run: + dir: 'phase2_feb2026_multi_results_local/phase2_baseline_spectrum_qwen_live_eval_test_multi/${now:%Y-%m-%d__%H-%M-%S}' diff --git a/align_system/configs/experiment/phase2_feb_collab/phase2_pipeline_direct_regression_random_effects.yaml b/align_system/configs/experiment/phase2_feb_collab/phase2_pipeline_direct_regression_random_effects.yaml new file mode 100644 index 00000000..c3b4d7d5 --- /dev/null +++ b/align_system/configs/experiment/phase2_feb_collab/phase2_pipeline_direct_regression_random_effects.yaml @@ -0,0 +1,23 @@ +# @package _global_ +defaults: + - override /adm: phase2_pipeline_direct_regression + - override /interface: ta3 + - override /adm_component/alignment@adm.step_definitions.scalar_alignment: random_effects_tuple + +interface: + api_endpoint: "https://darpaitm.caci.com" + session_type: adept + training_session: full + username: "testrun-pipeline_direct" + domain: "p2triage" + +adm: + step_definitions: + direct_regression: + enable_caching: true + +driver: + apply_action_filtering: false + +force_determinism: true +align_to_target: true diff --git a/align_system/configs/experiment/phase2_feb_collab/phase2_pipeline_direct_regression_random_effects_deepseek_llama.yaml b/align_system/configs/experiment/phase2_feb_collab/phase2_pipeline_direct_regression_random_effects_deepseek_llama.yaml new file mode 100644 index 00000000..b8225b60 --- /dev/null +++ b/align_system/configs/experiment/phase2_feb_collab/phase2_pipeline_direct_regression_random_effects_deepseek_llama.yaml @@ -0,0 +1,26 @@ +# @package _global_ +defaults: + - override /adm: phase2_pipeline_direct_regression + - override /interface: ta3 + - override /adm_component/alignment@adm.step_definitions.scalar_alignment: random_effects_tuple + +interface: + api_endpoint: "https://darpaitm.caci.com" + session_type: adept + training_session: full + username: "testrun-pipeline_direct_deepseek_llama" + domain: "p2triage" + +adm: + structured_inference_engine: + model_name: deepseek-ai/DeepSeek-R1-Distill-Llama-8B + + step_definitions: + direct_regression: + enable_caching: true + +driver: + apply_action_filtering: false + +force_determinism: true +align_to_target: true diff --git a/align_system/configs/experiment/phase2_feb_collab/phase2_pipeline_direct_regression_random_effects_deepseek_llama_live_eval_multi.yaml b/align_system/configs/experiment/phase2_feb_collab/phase2_pipeline_direct_regression_random_effects_deepseek_llama_live_eval_multi.yaml new file mode 100644 index 00000000..12ba5181 --- /dev/null +++ b/align_system/configs/experiment/phase2_feb_collab/phase2_pipeline_direct_regression_random_effects_deepseek_llama_live_eval_multi.yaml @@ -0,0 +1,42 @@ +# @package _global_ +defaults: + - override /adm: phase2_pipeline_direct_regression_bert_relevance + - override /interface: ta3 + - override /adm_component/alignment@adm.step_definitions.scalar_alignment: random_effects_tuple + +interface: + api_endpoint: "https://darpaitm.caci.com" + session_type: eval + training_session: null + username: "ALIGN-ADM-Ph2-DirectRegression-BertRelevance-DeepSeek-R1-Distill-Llama-8B" + domain: "p2triage" + +adm: + structured_inference_engine: + model_name: deepseek-ai/DeepSeek-R1-Distill-Llama-8B + + step_definitions: + regression_icl: # Need ICL for bert relevance + icl_generator_partial: + incontext_settings: + number: 20 + datasets: + medical: /data/shared/samba/phase2_icl/Feb2026-MU-train_20251218.json + affiliation: /data/shared/samba/phase2_icl/Feb2026-AF-train_20251218.json + merit: /data/shared/samba/phase2_icl/Feb2026-MF-train_20251218.json + personal_safety: /data/shared/samba/phase2_icl/Feb2026-PS-train_20251218.json + search: /data/shared/samba/phase2_icl/Feb2026-SS-train_20251218.json + enable_caching: true + direct_regression: + enable_caching: true + +driver: + apply_action_filtering: false + +force_determinism: true +align_to_target: true +save_last_unstructured_state_per_scenario: true + +hydra: + run: + dir: 'phase2_feb2026_multi_results_live/phase2_pipeline_direct_regression_deepseek_llama_live_eval_multi/${now:%Y-%m-%d__%H-%M-%S}' diff --git a/align_system/configs/experiment/phase2_feb_collab/phase2_pipeline_direct_regression_random_effects_live_eval_multi.yaml b/align_system/configs/experiment/phase2_feb_collab/phase2_pipeline_direct_regression_random_effects_live_eval_multi.yaml new file mode 100644 index 00000000..8b73b44b --- /dev/null +++ b/align_system/configs/experiment/phase2_feb_collab/phase2_pipeline_direct_regression_random_effects_live_eval_multi.yaml @@ -0,0 +1,39 @@ +# @package _global_ +defaults: + - override /adm: phase2_pipeline_direct_regression_bert_relevance + - override /interface: ta3 + - override /adm_component/alignment@adm.step_definitions.scalar_alignment: random_effects_tuple + +interface: + api_endpoint: "https://darpaitm.caci.com" + session_type: eval + training_session: null + username: "ALIGN-ADM-Ph2-DirectRegression-BertRelevance-Mistral-7B-Instruct-v0.3" + domain: "p2triage" + +adm: + step_definitions: + regression_icl: # Need ICL for bert relevance + icl_generator_partial: + incontext_settings: + number: 20 + datasets: + medical: /data/shared/samba/phase2_icl/Feb2026-MU-train_20251218.json + affiliation: /data/shared/samba/phase2_icl/Feb2026-AF-train_20251218.json + merit: /data/shared/samba/phase2_icl/Feb2026-MF-train_20251218.json + personal_safety: /data/shared/samba/phase2_icl/Feb2026-PS-train_20251218.json + search: /data/shared/samba/phase2_icl/Feb2026-SS-train_20251218.json + enable_caching: true + direct_regression: + enable_caching: true + +driver: + apply_action_filtering: false + +force_determinism: true +align_to_target: true +save_last_unstructured_state_per_scenario: true + +hydra: + run: + dir: 'phase2_feb2026_multi_results_live/phase2_pipeline_direct_regression_live_eval_multi/${now:%Y-%m-%d__%H-%M-%S}' diff --git a/align_system/configs/experiment/phase2_feb_collab/phase2_pipeline_direct_regression_random_effects_live_eval_test.yaml b/align_system/configs/experiment/phase2_feb_collab/phase2_pipeline_direct_regression_random_effects_live_eval_test.yaml new file mode 100644 index 00000000..7e5df7cd --- /dev/null +++ b/align_system/configs/experiment/phase2_feb_collab/phase2_pipeline_direct_regression_random_effects_live_eval_test.yaml @@ -0,0 +1,28 @@ +# @package _global_ +defaults: + - override /adm: phase2_pipeline_direct_regression + - override /interface: ta3 + - override /adm_component/alignment@adm.step_definitions.scalar_alignment: random_effects_tuple + +interface: + api_endpoint: "https://darpaitm.caci.com" + session_type: eval + training_session: null + username: "testrun-pipeline_direct" + domain: "p2triage" + +adm: + step_definitions: + direct_regression: + enable_caching: true + +driver: + apply_action_filtering: false + +force_determinism: true +align_to_target: true +save_last_unstructured_state_per_scenario: true + +hydra: + run: + dir: 'phase2_feb2026_results_local/phase2_pipeline_direct_regression_live_eval_test/${now:%Y-%m-%d__%H-%M-%S}' diff --git a/align_system/configs/experiment/phase2_feb_collab/phase2_pipeline_direct_regression_random_effects_live_eval_test_multi.yaml b/align_system/configs/experiment/phase2_feb_collab/phase2_pipeline_direct_regression_random_effects_live_eval_test_multi.yaml new file mode 100644 index 00000000..7584389a --- /dev/null +++ b/align_system/configs/experiment/phase2_feb_collab/phase2_pipeline_direct_regression_random_effects_live_eval_test_multi.yaml @@ -0,0 +1,39 @@ +# @package _global_ +defaults: + - override /adm: phase2_pipeline_direct_regression_bert_relevance + - override /interface: ta3 + - override /adm_component/alignment@adm.step_definitions.scalar_alignment: random_effects_tuple + +interface: + api_endpoint: "https://darpaitm.caci.com" + session_type: eval + training_session: null + username: "testrun-pipeline_direct" + domain: "p2triage" + +adm: + step_definitions: + regression_icl: # Need ICL for bert relevance + icl_generator_partial: + incontext_settings: + number: 20 + datasets: + medical: /data/shared/samba/phase2_icl/Feb2026-MU-train_20251218.json + affiliation: /data/shared/samba/phase2_icl/Feb2026-AF-train_20251218.json + merit: /data/shared/samba/phase2_icl/Feb2026-MF-train_20251218.json + personal_safety: /data/shared/samba/phase2_icl/Feb2026-PS-train_20251218.json + search: /data/shared/samba/phase2_icl/Feb2026-SS-train_20251218.json + enable_caching: true + direct_regression: + enable_caching: true + +driver: + apply_action_filtering: false + +force_determinism: true +align_to_target: true +save_last_unstructured_state_per_scenario: true + +hydra: + run: + dir: 'phase2_feb2026_multi_results_local/phase2_pipeline_direct_regression_live_eval_test_multi/${now:%Y-%m-%d__%H-%M-%S}' diff --git a/align_system/configs/experiment/phase2_feb_collab/phase2_pipeline_direct_regression_random_effects_multi.yaml b/align_system/configs/experiment/phase2_feb_collab/phase2_pipeline_direct_regression_random_effects_multi.yaml new file mode 100644 index 00000000..57c85a75 --- /dev/null +++ b/align_system/configs/experiment/phase2_feb_collab/phase2_pipeline_direct_regression_random_effects_multi.yaml @@ -0,0 +1,35 @@ +# @package _global_ +defaults: + - override /adm: phase2_pipeline_direct_regression_bert_relevance + - override /interface: ta3 + - override /adm_component/alignment@adm.step_definitions.scalar_alignment: random_effects_tuple + +interface: + session_type: adept + training_session: full + username: "testrun-pipeline_direct_multi" + domain: "p2triage" + +adm: + step_definitions: + regression_icl: # Need ICL for bert relevance + icl_generator_partial: + incontext_settings: + number: 20 + leave_one_out_strategy: 'scenario_description' # LOO - Remove for eval + datasets: + medical: /data/shared/samba/phase2_icl/Feb2026-MU-train_20251218.json + affiliation: /data/shared/samba/phase2_icl/Feb2026-AF-train_20251218.json + merit: /data/shared/samba/phase2_icl/Feb2026-MF-train_20251218.json + personal_safety: /data/shared/samba/phase2_icl/Feb2026-PS-train_20251218.json + search: /data/shared/samba/phase2_icl/Feb2026-SS-train_20251218.json + bert_relevance: + enable_caching: true + direct_regression: + enable_caching: true + +driver: + apply_action_filtering: false + +force_determinism: true +align_to_target: true diff --git a/align_system/configs/experiment/phase2_feb_collab/phase2_pipeline_direct_regression_random_effects_spectrum_llama.yaml b/align_system/configs/experiment/phase2_feb_collab/phase2_pipeline_direct_regression_random_effects_spectrum_llama.yaml new file mode 100644 index 00000000..ca9e035a --- /dev/null +++ b/align_system/configs/experiment/phase2_feb_collab/phase2_pipeline_direct_regression_random_effects_spectrum_llama.yaml @@ -0,0 +1,27 @@ +# @package _global_ +defaults: + - override /adm: phase2_pipeline_direct_regression + - override /interface: ta3 + - override /adm_component/alignment@adm.step_definitions.scalar_alignment: random_effects_tuple + +interface: + api_endpoint: "https://darpaitm.caci.com" + session_type: adept + training_session: full + username: "testrun-pipeline_direct_spectrum_llama" + domain: "p2triage" + +adm: + structured_inference_engine: + _target_: align_system.algorithms.outlines_inference_engine.SpectrumTunedInferenceEngine + model_name: tsor13/spectrum-Llama-3.1-8B-v1 + + step_definitions: + direct_regression: + enable_caching: true + +driver: + apply_action_filtering: false + +force_determinism: true +align_to_target: true diff --git a/align_system/configs/experiment/phase2_feb_collab/phase2_pipeline_direct_regression_random_effects_spectrum_llama_live_eval_multi.yaml b/align_system/configs/experiment/phase2_feb_collab/phase2_pipeline_direct_regression_random_effects_spectrum_llama_live_eval_multi.yaml new file mode 100644 index 00000000..3bd07033 --- /dev/null +++ b/align_system/configs/experiment/phase2_feb_collab/phase2_pipeline_direct_regression_random_effects_spectrum_llama_live_eval_multi.yaml @@ -0,0 +1,43 @@ +# @package _global_ +defaults: + - override /adm: phase2_pipeline_direct_regression_bert_relevance + - override /interface: ta3 + - override /adm_component/alignment@adm.step_definitions.scalar_alignment: random_effects_tuple + +interface: + api_endpoint: "https://darpaitm.caci.com" + session_type: eval + training_session: null + username: "ALIGN-ADM-Ph2-DirectRegression-BertRelevance-spectrum-Llama-3.1-8B-v1" + domain: "p2triage" + +adm: + structured_inference_engine: + _target_: align_system.algorithms.outlines_inference_engine.SpectrumTunedInferenceEngine + model_name: tsor13/spectrum-Llama-3.1-8B-v1 + + step_definitions: + regression_icl: # Need ICL for bert relevance + icl_generator_partial: + incontext_settings: + number: 20 + datasets: + medical: /data/shared/samba/phase2_icl/Feb2026-MU-train_20251218.json + affiliation: /data/shared/samba/phase2_icl/Feb2026-AF-train_20251218.json + merit: /data/shared/samba/phase2_icl/Feb2026-MF-train_20251218.json + personal_safety: /data/shared/samba/phase2_icl/Feb2026-PS-train_20251218.json + search: /data/shared/samba/phase2_icl/Feb2026-SS-train_20251218.json + enable_caching: true + direct_regression: + enable_caching: true + +driver: + apply_action_filtering: false + +force_determinism: true +align_to_target: true +save_last_unstructured_state_per_scenario: true + +hydra: + run: + dir: 'phase2_feb2026_multi_results_live/phase2_pipeline_direct_regression_spectrum_llama_live_eval_multi/${now:%Y-%m-%d__%H-%M-%S}' diff --git a/align_system/configs/experiment/phase2_feb_collab/phase2_pipeline_direct_regression_random_effects_spectrum_qwen.yaml b/align_system/configs/experiment/phase2_feb_collab/phase2_pipeline_direct_regression_random_effects_spectrum_qwen.yaml new file mode 100644 index 00000000..3d99ba46 --- /dev/null +++ b/align_system/configs/experiment/phase2_feb_collab/phase2_pipeline_direct_regression_random_effects_spectrum_qwen.yaml @@ -0,0 +1,27 @@ +# @package _global_ +defaults: + - override /adm: phase2_pipeline_direct_regression + - override /interface: ta3 + - override /adm_component/alignment@adm.step_definitions.scalar_alignment: random_effects_tuple + +interface: + api_endpoint: "https://darpaitm.caci.com" + session_type: adept + training_session: full + username: "testrun-pipeline_direct_spectrum_qwen" + domain: "p2triage" + +adm: + structured_inference_engine: + _target_: align_system.algorithms.outlines_inference_engine.SpectrumTunedInferenceEngine + model_name: tsor13/spectrum-Qwen3-14B-v1 + + step_definitions: + direct_regression: + enable_caching: true + +driver: + apply_action_filtering: false + +force_determinism: true +align_to_target: true diff --git a/align_system/configs/experiment/phase2_feb_collab/phase2_pipeline_direct_regression_random_effects_spectrum_qwen_live_eval_multi.yaml b/align_system/configs/experiment/phase2_feb_collab/phase2_pipeline_direct_regression_random_effects_spectrum_qwen_live_eval_multi.yaml new file mode 100644 index 00000000..442d6897 --- /dev/null +++ b/align_system/configs/experiment/phase2_feb_collab/phase2_pipeline_direct_regression_random_effects_spectrum_qwen_live_eval_multi.yaml @@ -0,0 +1,43 @@ +# @package _global_ +defaults: + - override /adm: phase2_pipeline_direct_regression_bert_relevance + - override /interface: ta3 + - override /adm_component/alignment@adm.step_definitions.scalar_alignment: random_effects_tuple + +interface: + api_endpoint: "https://darpaitm.caci.com" + session_type: eval + training_session: null + username: "ALIGN-ADM-Ph2-DirectRegression-BertRelevance-spectrum-Qwen3-14B-v1" + domain: "p2triage" + +adm: + structured_inference_engine: + _target_: align_system.algorithms.outlines_inference_engine.SpectrumTunedInferenceEngine + model_name: tsor13/spectrum-Qwen3-14B-v1 + + step_definitions: + regression_icl: # Need ICL for bert relevance + icl_generator_partial: + incontext_settings: + number: 20 + datasets: + medical: /data/shared/samba/phase2_icl/Feb2026-MU-train_20251218.json + affiliation: /data/shared/samba/phase2_icl/Feb2026-AF-train_20251218.json + merit: /data/shared/samba/phase2_icl/Feb2026-MF-train_20251218.json + personal_safety: /data/shared/samba/phase2_icl/Feb2026-PS-train_20251218.json + search: /data/shared/samba/phase2_icl/Feb2026-SS-train_20251218.json + enable_caching: true + direct_regression: + enable_caching: true + +driver: + apply_action_filtering: false + +force_determinism: true +align_to_target: true +save_last_unstructured_state_per_scenario: true + +hydra: + run: + dir: 'phase2_feb2026_multi_results_live/phase2_pipeline_direct_regression_spectrum_qwen_live_eval_multi/${now:%Y-%m-%d__%H-%M-%S}' diff --git a/align_system/configs/experiment/phase2_feb_collab/phase2_pipeline_fewshot_comparative_regression_random_effects.yaml b/align_system/configs/experiment/phase2_feb_collab/phase2_pipeline_fewshot_comparative_regression_random_effects.yaml index e8d4da57..3f93c597 100644 --- a/align_system/configs/experiment/phase2_feb_collab/phase2_pipeline_fewshot_comparative_regression_random_effects.yaml +++ b/align_system/configs/experiment/phase2_feb_collab/phase2_pipeline_fewshot_comparative_regression_random_effects.yaml @@ -5,9 +5,10 @@ defaults: - override /adm_component/alignment@adm.step_definitions.scalar_alignment: random_effects_tuple interface: + api_endpoint: "https://darpaitm.caci.com" session_type: adept training_session: full - username: "pipeline_fewshot_comp_reg_loo" + username: "testrun-pipeline_fewshot_comp_reg_loo" domain: "p2triage" adm: @@ -18,11 +19,11 @@ adm: number: 20 leave_one_out_strategy: 'scenario_description' # LOO - Remove for eval datasets: - medical: /data/shared/samba/phase2_icl/July2025-MU-train_20250715.json - affiliation: /data/shared/samba/phase2_icl/July2025-AF-train_20250715.json - merit: /data/shared/samba/phase2_icl/July2025-MF-train_20250715.json - personal_safety: /data/shared/samba/phase2_icl/July2025-PS-train_20250715.json - search: /data/shared/samba/phase2_icl/July2025-SS-train_20250715.json + medical: /data/shared/samba/phase2_icl/Feb2026-MU-train_20251218.json + affiliation: /data/shared/samba/phase2_icl/Feb2026-AF-train_20251218.json + merit: /data/shared/samba/phase2_icl/Feb2026-MF-train_20251218.json + personal_safety: /data/shared/samba/phase2_icl/Feb2026-PS-train_20251218.json + search: /data/shared/samba/phase2_icl/Feb2026-SS-train_20251218.json comparative_regression: enable_caching: true diff --git a/align_system/configs/experiment/phase2_feb_collab/phase2_pipeline_fewshot_comparative_regression_random_effects_deepseek_llama_live_eval_multi.yaml b/align_system/configs/experiment/phase2_feb_collab/phase2_pipeline_fewshot_comparative_regression_random_effects_deepseek_llama_live_eval_multi.yaml new file mode 100644 index 00000000..9ee756d4 --- /dev/null +++ b/align_system/configs/experiment/phase2_feb_collab/phase2_pipeline_fewshot_comparative_regression_random_effects_deepseek_llama_live_eval_multi.yaml @@ -0,0 +1,41 @@ +# @package _global_ +defaults: + - override /adm: phase2_pipeline_fewshot_comparative_regression_bert_relevance + - override /interface: ta3 + - override /adm_component/alignment@adm.step_definitions.scalar_alignment: random_effects_tuple + +interface: + api_endpoint: "https://darpaitm.caci.com" + session_type: eval + training_session: null + username: "ALIGN-ADM-Ph2-ComparativeRegression-BertRelevance-DeepSeek-R1-Distill-Llama-8B" + domain: "p2triage" + +adm: + structured_inference_engine: + model_name: deepseek-ai/DeepSeek-R1-Distill-Llama-8B + + step_definitions: + regression_icl: + icl_generator_partial: + incontext_settings: + number: 20 + datasets: + medical: /data/shared/samba/phase2_icl/Feb2026-MU-train_20251218.json + affiliation: /data/shared/samba/phase2_icl/Feb2026-AF-train_20251218.json + merit: /data/shared/samba/phase2_icl/Feb2026-MF-train_20251218.json + personal_safety: /data/shared/samba/phase2_icl/Feb2026-PS-train_20251218.json + search: /data/shared/samba/phase2_icl/Feb2026-SS-train_20251218.json + enable_caching: true + comparative_regression: + enable_caching: true + +driver: + apply_action_filtering: false +force_determinism: true +align_to_target: true +save_last_unstructured_state_per_scenario: true + +hydra: + run: + dir: 'phase2_feb2026_multi_results_live/phase2_pipeline_fewshot_comparative_regression_random_effects_deepseek_llama_live_eval_multi/${now:%Y-%m-%d__%H-%M-%S}' diff --git a/align_system/configs/experiment/phase2_feb_collab/phase2_pipeline_fewshot_comparative_regression_random_effects_deepseek_llama_live_eval_test.yaml b/align_system/configs/experiment/phase2_feb_collab/phase2_pipeline_fewshot_comparative_regression_random_effects_deepseek_llama_live_eval_test.yaml new file mode 100644 index 00000000..cd25d95c --- /dev/null +++ b/align_system/configs/experiment/phase2_feb_collab/phase2_pipeline_fewshot_comparative_regression_random_effects_deepseek_llama_live_eval_test.yaml @@ -0,0 +1,41 @@ +# @package _global_ +defaults: + - override /adm: phase2_pipeline_fewshot_comparative_regression + - override /interface: ta3 + - override /adm_component/alignment@adm.step_definitions.scalar_alignment: random_effects_tuple + +interface: + api_endpoint: "https://darpaitm.caci.com" + session_type: eval + training_session: null + username: "testrun-pipeline_fewshot_comp_reg_deepseek_llama" + domain: "p2triage" + +adm: + structured_inference_engine: + model_name: deepseek-ai/DeepSeek-R1-Distill-Llama-8B + + step_definitions: + regression_icl: + icl_generator_partial: + incontext_settings: + number: 20 + datasets: + medical: /data/shared/samba/phase2_icl/Feb2026-MU-train_20251218.json + affiliation: /data/shared/samba/phase2_icl/Feb2026-AF-train_20251218.json + merit: /data/shared/samba/phase2_icl/Feb2026-MF-train_20251218.json + personal_safety: /data/shared/samba/phase2_icl/Feb2026-PS-train_20251218.json + search: /data/shared/samba/phase2_icl/Feb2026-SS-train_20251218.json + enable_caching: true + comparative_regression: + enable_caching: true + +driver: + apply_action_filtering: false +force_determinism: true +align_to_target: true +save_last_unstructured_state_per_scenario: true + +hydra: + run: + dir: 'phase2_feb2026_results_local/phase2_pipeline_fewshot_comparative_regression_random_effects_deepseek_llama_live_eval_test/${now:%Y-%m-%d__%H-%M-%S}' diff --git a/align_system/configs/experiment/phase2_feb_collab/phase2_pipeline_fewshot_comparative_regression_random_effects_deepseek_llama_live_eval_test_multi.yaml b/align_system/configs/experiment/phase2_feb_collab/phase2_pipeline_fewshot_comparative_regression_random_effects_deepseek_llama_live_eval_test_multi.yaml new file mode 100644 index 00000000..31abdeb0 --- /dev/null +++ b/align_system/configs/experiment/phase2_feb_collab/phase2_pipeline_fewshot_comparative_regression_random_effects_deepseek_llama_live_eval_test_multi.yaml @@ -0,0 +1,41 @@ +# @package _global_ +defaults: + - override /adm: phase2_pipeline_fewshot_comparative_regression_bert_relevance + - override /interface: ta3 + - override /adm_component/alignment@adm.step_definitions.scalar_alignment: random_effects_tuple + +interface: + api_endpoint: "https://darpaitm.caci.com" + session_type: eval + training_session: null + username: "testrun-pipeline_fewshot_comp_reg_deepseek_llama" + domain: "p2triage" + +adm: + structured_inference_engine: + model_name: deepseek-ai/DeepSeek-R1-Distill-Llama-8B + + step_definitions: + regression_icl: + icl_generator_partial: + incontext_settings: + number: 20 + datasets: + medical: /data/shared/samba/phase2_icl/Feb2026-MU-train_20251218.json + affiliation: /data/shared/samba/phase2_icl/Feb2026-AF-train_20251218.json + merit: /data/shared/samba/phase2_icl/Feb2026-MF-train_20251218.json + personal_safety: /data/shared/samba/phase2_icl/Feb2026-PS-train_20251218.json + search: /data/shared/samba/phase2_icl/Feb2026-SS-train_20251218.json + enable_caching: true + comparative_regression: + enable_caching: true + +driver: + apply_action_filtering: false +force_determinism: true +align_to_target: true +save_last_unstructured_state_per_scenario: true + +hydra: + run: + dir: 'phase2_feb2026_multi_results_local/phase2_pipeline_fewshot_comparative_regression_random_effects_deepseek_llama_live_eval_test_multi/${now:%Y-%m-%d__%H-%M-%S}' diff --git a/align_system/configs/experiment/phase2_feb_collab/phase2_pipeline_fewshot_comparative_regression_random_effects_live_eval_multi.yaml b/align_system/configs/experiment/phase2_feb_collab/phase2_pipeline_fewshot_comparative_regression_random_effects_live_eval_multi.yaml new file mode 100644 index 00000000..f9c7661d --- /dev/null +++ b/align_system/configs/experiment/phase2_feb_collab/phase2_pipeline_fewshot_comparative_regression_random_effects_live_eval_multi.yaml @@ -0,0 +1,38 @@ +# @package _global_ +defaults: + - override /adm: phase2_pipeline_fewshot_comparative_regression_bert_relevance + - override /interface: ta3 + - override /adm_component/alignment@adm.step_definitions.scalar_alignment: random_effects_tuple + +interface: + api_endpoint: "https://darpaitm.caci.com" + session_type: eval + training_session: null + username: "ALIGN-ADM-Ph2-ComparativeRegression-BertRelevance-Mistral-7B-Instruct-v0.3" + domain: "p2triage" + +adm: + step_definitions: + regression_icl: + icl_generator_partial: + incontext_settings: + number: 20 + datasets: + medical: /data/shared/samba/phase2_icl/Feb2026-MU-train_20251218.json + affiliation: /data/shared/samba/phase2_icl/Feb2026-AF-train_20251218.json + merit: /data/shared/samba/phase2_icl/Feb2026-MF-train_20251218.json + personal_safety: /data/shared/samba/phase2_icl/Feb2026-PS-train_20251218.json + search: /data/shared/samba/phase2_icl/Feb2026-SS-train_20251218.json + enable_caching: true + comparative_regression: + enable_caching: true + +driver: + apply_action_filtering: false +force_determinism: true +align_to_target: true +save_last_unstructured_state_per_scenario: true + +hydra: + run: + dir: 'phase2_feb2026_multi_results_live/phase2_pipeline_fewshot_comparative_regression_random_effects_live_eval_multi/${now:%Y-%m-%d__%H-%M-%S}' diff --git a/align_system/configs/experiment/phase2_feb_collab/phase2_pipeline_fewshot_comparative_regression_random_effects_live_eval_test.yaml b/align_system/configs/experiment/phase2_feb_collab/phase2_pipeline_fewshot_comparative_regression_random_effects_live_eval_test.yaml new file mode 100644 index 00000000..12dfe6d3 --- /dev/null +++ b/align_system/configs/experiment/phase2_feb_collab/phase2_pipeline_fewshot_comparative_regression_random_effects_live_eval_test.yaml @@ -0,0 +1,38 @@ +# @package _global_ +defaults: + - override /adm: phase2_pipeline_fewshot_comparative_regression + - override /interface: ta3 + - override /adm_component/alignment@adm.step_definitions.scalar_alignment: random_effects_tuple + +interface: + api_endpoint: "https://darpaitm.caci.com" + session_type: eval + training_session: null + username: "testrun-pipeline_fewshot_comp_reg_mistral" + domain: "p2triage" + +adm: + step_definitions: + regression_icl: + icl_generator_partial: + incontext_settings: + number: 20 + datasets: + medical: /data/shared/samba/phase2_icl/Feb2026-MU-train_20251218.json + affiliation: /data/shared/samba/phase2_icl/Feb2026-AF-train_20251218.json + merit: /data/shared/samba/phase2_icl/Feb2026-MF-train_20251218.json + personal_safety: /data/shared/samba/phase2_icl/Feb2026-PS-train_20251218.json + search: /data/shared/samba/phase2_icl/Feb2026-SS-train_20251218.json + enable_caching: true + comparative_regression: + enable_caching: true + +driver: + apply_action_filtering: false +force_determinism: true +align_to_target: true +save_last_unstructured_state_per_scenario: true + +hydra: + run: + dir: 'phase2_feb2026_results_local/phase2_pipeline_fewshot_comparative_regression_random_effects_live_eval_test/${now:%Y-%m-%d__%H-%M-%S}' diff --git a/align_system/configs/experiment/phase2_feb_collab/phase2_pipeline_fewshot_comparative_regression_random_effects_live_eval_test_multi.yaml b/align_system/configs/experiment/phase2_feb_collab/phase2_pipeline_fewshot_comparative_regression_random_effects_live_eval_test_multi.yaml new file mode 100644 index 00000000..f2dac383 --- /dev/null +++ b/align_system/configs/experiment/phase2_feb_collab/phase2_pipeline_fewshot_comparative_regression_random_effects_live_eval_test_multi.yaml @@ -0,0 +1,38 @@ +# @package _global_ +defaults: + - override /adm: phase2_pipeline_fewshot_comparative_regression_bert_relevance + - override /interface: ta3 + - override /adm_component/alignment@adm.step_definitions.scalar_alignment: random_effects_tuple + +interface: + api_endpoint: "https://darpaitm.caci.com" + session_type: eval + training_session: null + username: "testrun-pipeline_fewshot_comp_reg_mistral" + domain: "p2triage" + +adm: + step_definitions: + regression_icl: + icl_generator_partial: + incontext_settings: + number: 20 + datasets: + medical: /data/shared/samba/phase2_icl/Feb2026-MU-train_20251218.json + affiliation: /data/shared/samba/phase2_icl/Feb2026-AF-train_20251218.json + merit: /data/shared/samba/phase2_icl/Feb2026-MF-train_20251218.json + personal_safety: /data/shared/samba/phase2_icl/Feb2026-PS-train_20251218.json + search: /data/shared/samba/phase2_icl/Feb2026-SS-train_20251218.json + enable_caching: true + comparative_regression: + enable_caching: true + +driver: + apply_action_filtering: false +force_determinism: true +align_to_target: true +save_last_unstructured_state_per_scenario: true + +hydra: + run: + dir: 'phase2_feb2026_multi_results_local/phase2_pipeline_fewshot_comparative_regression_random_effects_live_eval_test_multi/${now:%Y-%m-%d__%H-%M-%S}' diff --git a/align_system/configs/experiment/phase2_feb_collab/phase2_pipeline_fewshot_comparative_regression_random_effects_multi.yaml b/align_system/configs/experiment/phase2_feb_collab/phase2_pipeline_fewshot_comparative_regression_random_effects_multi.yaml new file mode 100644 index 00000000..9c1ef434 --- /dev/null +++ b/align_system/configs/experiment/phase2_feb_collab/phase2_pipeline_fewshot_comparative_regression_random_effects_multi.yaml @@ -0,0 +1,35 @@ +# @package _global_ +defaults: + - override /adm: phase2_pipeline_fewshot_comparative_regression_bert_relevance + - override /interface: ta3 + - override /adm_component/alignment@adm.step_definitions.scalar_alignment: random_effects_tuple + +interface: + session_type: adept + training_session: full + username: "testrun-pipeline_fewshot_comp_reg_loo_multi" + domain: "p2triage" + +adm: + step_definitions: + regression_icl: + icl_generator_partial: + incontext_settings: + number: 20 + leave_one_out_strategy: 'scenario_description' # LOO - Remove for eval + datasets: + medical: /data/shared/samba/phase2_icl/Feb2026-MU-train_20251218.json + affiliation: /data/shared/samba/phase2_icl/Feb2026-AF-train_20251218.json + merit: /data/shared/samba/phase2_icl/Feb2026-MF-train_20251218.json + personal_safety: /data/shared/samba/phase2_icl/Feb2026-PS-train_20251218.json + search: /data/shared/samba/phase2_icl/Feb2026-SS-train_20251218.json + enable_caching: true + bert_relevance: + enable_caching: true + comparative_regression: + enable_caching: true + +driver: + apply_action_filtering: false +force_determinism: true +align_to_target: true diff --git a/align_system/configs/experiment/phase2_feb_collab/phase2_pipeline_fewshot_comparative_regression_random_effects_spectrum_llama_live_eval_multi.yaml b/align_system/configs/experiment/phase2_feb_collab/phase2_pipeline_fewshot_comparative_regression_random_effects_spectrum_llama_live_eval_multi.yaml new file mode 100644 index 00000000..6773336c --- /dev/null +++ b/align_system/configs/experiment/phase2_feb_collab/phase2_pipeline_fewshot_comparative_regression_random_effects_spectrum_llama_live_eval_multi.yaml @@ -0,0 +1,42 @@ +# @package _global_ +defaults: + - override /adm: phase2_pipeline_fewshot_comparative_regression_bert_relevance + - override /interface: ta3 + - override /adm_component/alignment@adm.step_definitions.scalar_alignment: random_effects_tuple + +interface: + api_endpoint: "https://darpaitm.caci.com" + session_type: eval + training_session: null + username: "ALIGN-ADM-Ph2-ComparativeRegression-BertRelevance-spectrum-Llama-3.1-8B-v1" + domain: "p2triage" + +adm: + structured_inference_engine: + _target_: align_system.algorithms.outlines_inference_engine.SpectrumTunedInferenceEngine + model_name: tsor13/spectrum-Llama-3.1-8B-v1 + + step_definitions: + regression_icl: + icl_generator_partial: + incontext_settings: + number: 20 + datasets: + medical: /data/shared/samba/phase2_icl/Feb2026-MU-train_20251218.json + affiliation: /data/shared/samba/phase2_icl/Feb2026-AF-train_20251218.json + merit: /data/shared/samba/phase2_icl/Feb2026-MF-train_20251218.json + personal_safety: /data/shared/samba/phase2_icl/Feb2026-PS-train_20251218.json + search: /data/shared/samba/phase2_icl/Feb2026-SS-train_20251218.json + enable_caching: true + comparative_regression: + enable_caching: true + +driver: + apply_action_filtering: false +force_determinism: true +align_to_target: true +save_last_unstructured_state_per_scenario: true + +hydra: + run: + dir: 'phase2_feb2026_multi_results_live/phase2_pipeline_fewshot_comparative_regression_random_effects_spectrum_llama_live_eval_multi/${now:%Y-%m-%d__%H-%M-%S}' diff --git a/align_system/configs/experiment/phase2_feb_collab/phase2_pipeline_fewshot_comparative_regression_random_effects_spectrum_llama_live_eval_test.yaml b/align_system/configs/experiment/phase2_feb_collab/phase2_pipeline_fewshot_comparative_regression_random_effects_spectrum_llama_live_eval_test.yaml new file mode 100644 index 00000000..745bf928 --- /dev/null +++ b/align_system/configs/experiment/phase2_feb_collab/phase2_pipeline_fewshot_comparative_regression_random_effects_spectrum_llama_live_eval_test.yaml @@ -0,0 +1,42 @@ +# @package _global_ +defaults: + - override /adm: phase2_pipeline_fewshot_comparative_regression + - override /interface: ta3 + - override /adm_component/alignment@adm.step_definitions.scalar_alignment: random_effects_tuple + +interface: + api_endpoint: "https://darpaitm.caci.com" + session_type: eval + training_session: null + username: "testrun-pipeline_fewshot_comp_reg_spectrum_llama" + domain: "p2triage" + +adm: + structured_inference_engine: + _target_: align_system.algorithms.outlines_inference_engine.SpectrumTunedInferenceEngine + model_name: tsor13/spectrum-Llama-3.1-8B-v1 + + step_definitions: + regression_icl: + icl_generator_partial: + incontext_settings: + number: 20 + datasets: + medical: /data/shared/samba/phase2_icl/Feb2026-MU-train_20251218.json + affiliation: /data/shared/samba/phase2_icl/Feb2026-AF-train_20251218.json + merit: /data/shared/samba/phase2_icl/Feb2026-MF-train_20251218.json + personal_safety: /data/shared/samba/phase2_icl/Feb2026-PS-train_20251218.json + search: /data/shared/samba/phase2_icl/Feb2026-SS-train_20251218.json + enable_caching: true + comparative_regression: + enable_caching: true + +driver: + apply_action_filtering: false +force_determinism: true +align_to_target: true +save_last_unstructured_state_per_scenario: true + +hydra: + run: + dir: 'phase2_feb2026_results_local/phase2_pipeline_fewshot_comparative_regression_random_effects_spectrum_llama_live_eval_test/${now:%Y-%m-%d__%H-%M-%S}' diff --git a/align_system/configs/experiment/phase2_feb_collab/phase2_pipeline_fewshot_comparative_regression_random_effects_spectrum_llama_live_eval_test_multi.yaml b/align_system/configs/experiment/phase2_feb_collab/phase2_pipeline_fewshot_comparative_regression_random_effects_spectrum_llama_live_eval_test_multi.yaml new file mode 100644 index 00000000..93c14d7b --- /dev/null +++ b/align_system/configs/experiment/phase2_feb_collab/phase2_pipeline_fewshot_comparative_regression_random_effects_spectrum_llama_live_eval_test_multi.yaml @@ -0,0 +1,42 @@ +# @package _global_ +defaults: + - override /adm: phase2_pipeline_fewshot_comparative_regression_bert_relevance + - override /interface: ta3 + - override /adm_component/alignment@adm.step_definitions.scalar_alignment: random_effects_tuple + +interface: + api_endpoint: "https://darpaitm.caci.com" + session_type: eval + training_session: null + username: "testrun-pipeline_fewshot_comp_reg_spectrum_llama" + domain: "p2triage" + +adm: + structured_inference_engine: + _target_: align_system.algorithms.outlines_inference_engine.SpectrumTunedInferenceEngine + model_name: tsor13/spectrum-Llama-3.1-8B-v1 + + step_definitions: + regression_icl: + icl_generator_partial: + incontext_settings: + number: 20 + datasets: + medical: /data/shared/samba/phase2_icl/Feb2026-MU-train_20251218.json + affiliation: /data/shared/samba/phase2_icl/Feb2026-AF-train_20251218.json + merit: /data/shared/samba/phase2_icl/Feb2026-MF-train_20251218.json + personal_safety: /data/shared/samba/phase2_icl/Feb2026-PS-train_20251218.json + search: /data/shared/samba/phase2_icl/Feb2026-SS-train_20251218.json + enable_caching: true + comparative_regression: + enable_caching: true + +driver: + apply_action_filtering: false +force_determinism: true +align_to_target: true +save_last_unstructured_state_per_scenario: true + +hydra: + run: + dir: 'phase2_feb2026_multi_results_local/phase2_pipeline_fewshot_comparative_regression_random_effects_spectrum_llama_live_eval_test_multi/${now:%Y-%m-%d__%H-%M-%S}' diff --git a/align_system/configs/experiment/phase2_feb_collab/phase2_pipeline_fewshot_comparative_regression_random_effects_spectrum_qwen_live_eval_multi.yaml b/align_system/configs/experiment/phase2_feb_collab/phase2_pipeline_fewshot_comparative_regression_random_effects_spectrum_qwen_live_eval_multi.yaml new file mode 100644 index 00000000..85871e91 --- /dev/null +++ b/align_system/configs/experiment/phase2_feb_collab/phase2_pipeline_fewshot_comparative_regression_random_effects_spectrum_qwen_live_eval_multi.yaml @@ -0,0 +1,42 @@ +# @package _global_ +defaults: + - override /adm: phase2_pipeline_fewshot_comparative_regression_bert_relevance + - override /interface: ta3 + - override /adm_component/alignment@adm.step_definitions.scalar_alignment: random_effects_tuple + +interface: + api_endpoint: "https://darpaitm.caci.com" + session_type: eval + training_session: null + username: "ALIGN-ADM-Ph2-ComparativeRegression-BertRelevance-spectrum-Qwen3-14B-v1" + domain: "p2triage" + +adm: + structured_inference_engine: + _target_: align_system.algorithms.outlines_inference_engine.SpectrumTunedInferenceEngine + model_name: tsor13/spectrum-Qwen3-14B-v1 + + step_definitions: + regression_icl: + icl_generator_partial: + incontext_settings: + number: 20 + datasets: + medical: /data/shared/samba/phase2_icl/Feb2026-MU-train_20251218.json + affiliation: /data/shared/samba/phase2_icl/Feb2026-AF-train_20251218.json + merit: /data/shared/samba/phase2_icl/Feb2026-MF-train_20251218.json + personal_safety: /data/shared/samba/phase2_icl/Feb2026-PS-train_20251218.json + search: /data/shared/samba/phase2_icl/Feb2026-SS-train_20251218.json + enable_caching: true + comparative_regression: + enable_caching: true + +driver: + apply_action_filtering: false +force_determinism: true +align_to_target: true +save_last_unstructured_state_per_scenario: true + +hydra: + run: + dir: 'phase2_feb2026_multi_results_live/phase2_pipeline_fewshot_comparative_regression_random_effects_spectrum_qwen_live_eval_multi/${now:%Y-%m-%d__%H-%M-%S}' diff --git a/align_system/configs/experiment/phase2_feb_collab/phase2_pipeline_fewshot_comparative_regression_random_effects_spectrum_qwen_live_eval_test.yaml b/align_system/configs/experiment/phase2_feb_collab/phase2_pipeline_fewshot_comparative_regression_random_effects_spectrum_qwen_live_eval_test.yaml new file mode 100644 index 00000000..db366c53 --- /dev/null +++ b/align_system/configs/experiment/phase2_feb_collab/phase2_pipeline_fewshot_comparative_regression_random_effects_spectrum_qwen_live_eval_test.yaml @@ -0,0 +1,42 @@ +# @package _global_ +defaults: + - override /adm: phase2_pipeline_fewshot_comparative_regression + - override /interface: ta3 + - override /adm_component/alignment@adm.step_definitions.scalar_alignment: random_effects_tuple + +interface: + api_endpoint: "https://darpaitm.caci.com" + session_type: eval + training_session: null + username: "testrun-pipeline_fewshot_comp_reg_loo_spectrum_qwen" + domain: "p2triage" + +adm: + structured_inference_engine: + _target_: align_system.algorithms.outlines_inference_engine.SpectrumTunedInferenceEngine + model_name: tsor13/spectrum-Qwen3-14B-v1 + + step_definitions: + regression_icl: + icl_generator_partial: + incontext_settings: + number: 20 + datasets: + medical: /data/shared/samba/phase2_icl/Feb2026-MU-train_20251218.json + affiliation: /data/shared/samba/phase2_icl/Feb2026-AF-train_20251218.json + merit: /data/shared/samba/phase2_icl/Feb2026-MF-train_20251218.json + personal_safety: /data/shared/samba/phase2_icl/Feb2026-PS-train_20251218.json + search: /data/shared/samba/phase2_icl/Feb2026-SS-train_20251218.json + enable_caching: true + comparative_regression: + enable_caching: true + +driver: + apply_action_filtering: false +force_determinism: true +align_to_target: true +save_last_unstructured_state_per_scenario: true + +hydra: + run: + dir: 'phase2_feb2026_results_local/phase2_pipeline_fewshot_comparative_regression_random_effects_spectrum_qwen_live_eval_test/${now:%Y-%m-%d__%H-%M-%S}' diff --git a/align_system/configs/experiment/phase2_feb_collab/phase2_pipeline_fewshot_comparative_regression_random_effects_spectrum_qwen_live_eval_test_multi.yaml b/align_system/configs/experiment/phase2_feb_collab/phase2_pipeline_fewshot_comparative_regression_random_effects_spectrum_qwen_live_eval_test_multi.yaml new file mode 100644 index 00000000..6df0aacb --- /dev/null +++ b/align_system/configs/experiment/phase2_feb_collab/phase2_pipeline_fewshot_comparative_regression_random_effects_spectrum_qwen_live_eval_test_multi.yaml @@ -0,0 +1,42 @@ +# @package _global_ +defaults: + - override /adm: phase2_pipeline_fewshot_comparative_regression_bert_relevance + - override /interface: ta3 + - override /adm_component/alignment@adm.step_definitions.scalar_alignment: random_effects_tuple + +interface: + api_endpoint: "https://darpaitm.caci.com" + session_type: eval + training_session: null + username: "testrun-pipeline_fewshot_comp_reg_loo_spectrum_qwen" + domain: "p2triage" + +adm: + structured_inference_engine: + _target_: align_system.algorithms.outlines_inference_engine.SpectrumTunedInferenceEngine + model_name: tsor13/spectrum-Qwen3-14B-v1 + + step_definitions: + regression_icl: + icl_generator_partial: + incontext_settings: + number: 20 + datasets: + medical: /data/shared/samba/phase2_icl/Feb2026-MU-train_20251218.json + affiliation: /data/shared/samba/phase2_icl/Feb2026-AF-train_20251218.json + merit: /data/shared/samba/phase2_icl/Feb2026-MF-train_20251218.json + personal_safety: /data/shared/samba/phase2_icl/Feb2026-PS-train_20251218.json + search: /data/shared/samba/phase2_icl/Feb2026-SS-train_20251218.json + enable_caching: true + comparative_regression: + enable_caching: true + +driver: + apply_action_filtering: false +force_determinism: true +align_to_target: true +save_last_unstructured_state_per_scenario: true + +hydra: + run: + dir: 'phase2_feb2026_multi_results_local/phase2_pipeline_fewshot_comparative_regression_random_effects_spectrum_qwen_live_eval_test_multi/${now:%Y-%m-%d__%H-%M-%S}' diff --git a/align_system/configs/experiment/phase2_feb_collab/phase2_pipeline_fewshot_comparative_regression_random_effects_spectrum_tuned_llama.yaml b/align_system/configs/experiment/phase2_feb_collab/phase2_pipeline_fewshot_comparative_regression_random_effects_spectrum_tuned_llama.yaml new file mode 100644 index 00000000..321c7c4f --- /dev/null +++ b/align_system/configs/experiment/phase2_feb_collab/phase2_pipeline_fewshot_comparative_regression_random_effects_spectrum_tuned_llama.yaml @@ -0,0 +1,37 @@ +# @package _global_ +defaults: + - override /adm: phase2_pipeline_fewshot_comparative_regression + - override /interface: ta3 + - override /adm_component/alignment@adm.step_definitions.scalar_alignment: random_effects_tuple + +interface: + api_endpoint: "https://darpaitm.caci.com" + session_type: adept + training_session: full + username: "testrun-pipeline_fewshot_comp_reg_loo_spectrum_llama" + domain: "p2triage" + +adm: + structured_inference_engine: + _target_: align_system.algorithms.outlines_inference_engine.SpectrumTunedInferenceEngine + model_name: tsor13/spectrum-Llama-3.1-8B-v1 + + step_definitions: + regression_icl: + icl_generator_partial: + incontext_settings: + number: 20 + leave_one_out_strategy: 'scenario_description' # LOO - Remove for eval + datasets: + medical: /data/shared/samba/phase2_icl/Feb2026-MU-train_20251218.json + affiliation: /data/shared/samba/phase2_icl/Feb2026-AF-train_20251218.json + merit: /data/shared/samba/phase2_icl/Feb2026-MF-train_20251218.json + personal_safety: /data/shared/samba/phase2_icl/Feb2026-PS-train_20251218.json + search: /data/shared/samba/phase2_icl/Feb2026-SS-train_20251218.json + comparative_regression: + enable_caching: true + +driver: + apply_action_filtering: false +force_determinism: true +align_to_target: true diff --git a/align_system/configs/experiment/phase2_feb_collab/phase2_pipeline_fewshot_comparative_regression_random_effects_spectrum_tuned_qwen.yaml b/align_system/configs/experiment/phase2_feb_collab/phase2_pipeline_fewshot_comparative_regression_random_effects_spectrum_tuned_qwen.yaml new file mode 100644 index 00000000..23a5b1f1 --- /dev/null +++ b/align_system/configs/experiment/phase2_feb_collab/phase2_pipeline_fewshot_comparative_regression_random_effects_spectrum_tuned_qwen.yaml @@ -0,0 +1,37 @@ +# @package _global_ +defaults: + - override /adm: phase2_pipeline_fewshot_comparative_regression + - override /interface: ta3 + - override /adm_component/alignment@adm.step_definitions.scalar_alignment: random_effects_tuple + +interface: + api_endpoint: "https://darpaitm.caci.com" + session_type: adept + training_session: full + username: "testrun-pipeline_fewshot_comp_reg_loo_spectrum_qwen" + domain: "p2triage" + +adm: + structured_inference_engine: + _target_: align_system.algorithms.outlines_inference_engine.SpectrumTunedInferenceEngine + model_name: tsor13/spectrum-Qwen3-14B-v1 + + step_definitions: + regression_icl: + icl_generator_partial: + incontext_settings: + number: 20 + leave_one_out_strategy: 'scenario_description' # LOO - Remove for eval + datasets: + medical: /data/shared/samba/phase2_icl/Feb2026-MU-train_20251218.json + affiliation: /data/shared/samba/phase2_icl/Feb2026-AF-train_20251218.json + merit: /data/shared/samba/phase2_icl/Feb2026-MF-train_20251218.json + personal_safety: /data/shared/samba/phase2_icl/Feb2026-PS-train_20251218.json + search: /data/shared/samba/phase2_icl/Feb2026-SS-train_20251218.json + comparative_regression: + enable_caching: true + +driver: + apply_action_filtering: false +force_determinism: true +align_to_target: true diff --git a/align_system/configs/experiment/phase2_feb_collab/phase2_pipeline_oracle_random_effects.yaml b/align_system/configs/experiment/phase2_feb_collab/phase2_pipeline_oracle_random_effects.yaml new file mode 100644 index 00000000..9b19afc5 --- /dev/null +++ b/align_system/configs/experiment/phase2_feb_collab/phase2_pipeline_oracle_random_effects.yaml @@ -0,0 +1,17 @@ +# @package _global_ +defaults: + - override /adm: phase2_pipeline_oracle + - override /interface: ta3 + - override /adm_component/alignment@adm.step_definitions.scalar_alignment: random_effects_tuple + +interface: + api_endpoint: "https://darpaitm.caci.com" + session_type: adept + training_session: full + username: "testrun-pipeline_oracle_test" + domain: "p2triage" + +driver: + apply_action_filtering: false +force_determinism: true +align_to_target: true diff --git a/align_system/configs/experiment/phase2_feb_collab/phase2_pipeline_random.yaml b/align_system/configs/experiment/phase2_feb_collab/phase2_pipeline_random.yaml new file mode 100644 index 00000000..02db3e70 --- /dev/null +++ b/align_system/configs/experiment/phase2_feb_collab/phase2_pipeline_random.yaml @@ -0,0 +1,27 @@ +# @package _global_ +defaults: + - override /adm: pipeline_random + - override /interface: ta3 + +interface: + api_endpoint: "https://darpaitm.caci.com" + session_type: adept + training_session: full + username: "testrun-pipeline_random" + domain: "p2triage" + +adm: + instance: + steps: + # Reference the step instances we want to use in order + - ${ref:adm.step_definitions.format_choices} + - ${ref:adm.step_definitions.random_choice} + # - ${ref:adm.step_definitions.action_parameter_completion} + - ${ref:adm.step_definitions.ensure_chosen_action} + - ${ref:adm.step_definitions.populate_choice_info} + +driver: + apply_action_filtering: false + +force_determinism: true +align_to_target: false diff --git a/align_system/configs/experiment/phase2_feb_collab/phase2_pipeline_random_live_eval_multi.yaml b/align_system/configs/experiment/phase2_feb_collab/phase2_pipeline_random_live_eval_multi.yaml new file mode 100644 index 00000000..525dc582 --- /dev/null +++ b/align_system/configs/experiment/phase2_feb_collab/phase2_pipeline_random_live_eval_multi.yaml @@ -0,0 +1,32 @@ +# @package _global_ +defaults: + - override /adm: pipeline_random + - override /interface: ta3 + +interface: + api_endpoint: "https://darpaitm.caci.com" + session_type: eval + training_session: null + username: "ALIGN-ADM-Random" + domain: "p2triage" + +adm: + instance: + steps: + # Reference the step instances we want to use in order + - ${ref:adm.step_definitions.format_choices} + - ${ref:adm.step_definitions.random_choice} + # - ${ref:adm.step_definitions.action_parameter_completion} + - ${ref:adm.step_definitions.ensure_chosen_action} + - ${ref:adm.step_definitions.populate_choice_info} + +driver: + apply_action_filtering: false + +force_determinism: true +align_to_target: false +save_last_unstructured_state_per_scenario: true + +hydra: + run: + dir: 'phase2_feb2026_multi_results_live/phase2_pipeline_random_live_eval_multi/${now:%Y-%m-%d__%H-%M-%S}' diff --git a/align_system/configs/experiment/phase2_feb_collab/phase2_pipeline_random_live_eval_test.yaml b/align_system/configs/experiment/phase2_feb_collab/phase2_pipeline_random_live_eval_test.yaml new file mode 100644 index 00000000..3d7b3555 --- /dev/null +++ b/align_system/configs/experiment/phase2_feb_collab/phase2_pipeline_random_live_eval_test.yaml @@ -0,0 +1,32 @@ +# @package _global_ +defaults: + - override /adm: pipeline_random + - override /interface: ta3 + +interface: + api_endpoint: "https://darpaitm.caci.com" + session_type: eval + training_session: null + username: "testrun-pipeline_random" + domain: "p2triage" + +adm: + instance: + steps: + # Reference the step instances we want to use in order + - ${ref:adm.step_definitions.format_choices} + - ${ref:adm.step_definitions.random_choice} + # - ${ref:adm.step_definitions.action_parameter_completion} + - ${ref:adm.step_definitions.ensure_chosen_action} + - ${ref:adm.step_definitions.populate_choice_info} + +driver: + apply_action_filtering: false + +force_determinism: true +align_to_target: false +save_last_unstructured_state_per_scenario: true + +hydra: + run: + dir: 'phase2_feb2026_results_local/phase2_pipeline_random_live_eval_test/${now:%Y-%m-%d__%H-%M-%S}' diff --git a/align_system/configs/experiment/phase2_feb_collab/phase2_pipeline_random_live_eval_test_multi.yaml b/align_system/configs/experiment/phase2_feb_collab/phase2_pipeline_random_live_eval_test_multi.yaml new file mode 100644 index 00000000..358115f4 --- /dev/null +++ b/align_system/configs/experiment/phase2_feb_collab/phase2_pipeline_random_live_eval_test_multi.yaml @@ -0,0 +1,32 @@ +# @package _global_ +defaults: + - override /adm: pipeline_random + - override /interface: ta3 + +interface: + api_endpoint: "https://darpaitm.caci.com" + session_type: eval + training_session: null + username: "testrun-pipeline_random_multi" + domain: "p2triage" + +adm: + instance: + steps: + # Reference the step instances we want to use in order + - ${ref:adm.step_definitions.format_choices} + - ${ref:adm.step_definitions.random_choice} + # - ${ref:adm.step_definitions.action_parameter_completion} + - ${ref:adm.step_definitions.ensure_chosen_action} + - ${ref:adm.step_definitions.populate_choice_info} + +driver: + apply_action_filtering: false + +force_determinism: true +align_to_target: false +save_last_unstructured_state_per_scenario: true + +hydra: + run: + dir: 'phase2_feb2026_multi_results_local/phase2_pipeline_random_live_eval_test_multi/${now:%Y-%m-%d__%H-%M-%S}' diff --git a/align_system/configs/experiment/phase2_feb_collab/pipeline_choose_a.yaml b/align_system/configs/experiment/phase2_feb_collab/pipeline_choose_a.yaml index 87d21dca..77f92223 100644 --- a/align_system/configs/experiment/phase2_feb_collab/pipeline_choose_a.yaml +++ b/align_system/configs/experiment/phase2_feb_collab/pipeline_choose_a.yaml @@ -4,9 +4,10 @@ defaults: - override /interface: ta3 interface: + api_endpoint: "https://darpaitm.caci.com" session_type: adept training_session: full - username: "pipeline_feb2026_choose_a" + username: "testrun-pipeline_feb2026_choose_a" domain: "p2triage" adm: diff --git a/align_system/configs/experiment/phase2_feb_collab/pipeline_choose_a_live_eval_test.yaml b/align_system/configs/experiment/phase2_feb_collab/pipeline_choose_a_live_eval_test.yaml new file mode 100644 index 00000000..366bcc4a --- /dev/null +++ b/align_system/configs/experiment/phase2_feb_collab/pipeline_choose_a_live_eval_test.yaml @@ -0,0 +1,32 @@ +# @package _global_ +defaults: + - override /adm: pipeline_choose_a + - override /interface: ta3 + +interface: + api_endpoint: "https://darpaitm.caci.com" + session_type: eval + training_session: null + username: "testrun-pipeline_feb2026_choose_a" + domain: "p2triage" + +adm: + instance: + steps: + # Reference the step instances we want to use in order + - ${ref:adm.step_definitions.format_choices} + - ${ref:adm.step_definitions.choose_idx} + # - ${ref:adm.step_definitions.action_parameter_completion} + - ${ref:adm.step_definitions.ensure_chosen_action} + - ${ref:adm.step_definitions.populate_choice_info} + +driver: + apply_action_filtering: false + +force_determinism: true +align_to_target: false +save_last_unstructured_state_per_scenario: true + +hydra: + run: + dir: 'phase2_feb2026_results_local/pipeline_choose_a_live_eval_test/${now:%Y-%m-%d__%H-%M-%S}' diff --git a/align_system/configs/experiment/phase2_feb_collab/pipeline_choose_b.yaml b/align_system/configs/experiment/phase2_feb_collab/pipeline_choose_b.yaml index 0917d10c..e44a319a 100644 --- a/align_system/configs/experiment/phase2_feb_collab/pipeline_choose_b.yaml +++ b/align_system/configs/experiment/phase2_feb_collab/pipeline_choose_b.yaml @@ -4,9 +4,10 @@ defaults: - override /interface: ta3 interface: + api_endpoint: "https://darpaitm.caci.com" session_type: adept training_session: full - username: "pipeline_feb2026_choose_b" + username: "testrun-pipeline_feb2026_choose_b" domain: "p2triage" adm: diff --git a/align_system/configs/experiment/phase2_feb_collab/pipeline_choose_b_live_eval_test.yaml b/align_system/configs/experiment/phase2_feb_collab/pipeline_choose_b_live_eval_test.yaml new file mode 100644 index 00000000..8265b344 --- /dev/null +++ b/align_system/configs/experiment/phase2_feb_collab/pipeline_choose_b_live_eval_test.yaml @@ -0,0 +1,32 @@ +# @package _global_ +defaults: + - override /adm: pipeline_choose_b + - override /interface: ta3 + +interface: + api_endpoint: "https://darpaitm.caci.com" + session_type: eval + training_session: null + username: "testrun-pipeline_feb2026_choose_b" + domain: "p2triage" + +adm: + instance: + steps: + # Reference the step instances we want to use in order + - ${ref:adm.step_definitions.format_choices} + - ${ref:adm.step_definitions.choose_idx} + # - ${ref:adm.step_definitions.action_parameter_completion} + - ${ref:adm.step_definitions.ensure_chosen_action} + - ${ref:adm.step_definitions.populate_choice_info} + +driver: + apply_action_filtering: false + +force_determinism: true +align_to_target: false +save_last_unstructured_state_per_scenario: true + +hydra: + run: + dir: 'phase2_feb2026_results_local/pipeline_choose_b_live_eval_test/${now:%Y-%m-%d__%H-%M-%S}' diff --git a/align_system/configs/experiment/tagging/tagging_baseline.yaml b/align_system/configs/experiment/tagging/tagging_baseline.yaml index f675bc6f..b07ee781 100644 --- a/align_system/configs/experiment/tagging/tagging_baseline.yaml +++ b/align_system/configs/experiment/tagging/tagging_baseline.yaml @@ -7,6 +7,11 @@ interface: input_output_filepath: '/data/shared/tagging/eval3.json' state_hydration_domain: "minimal" +adm: + step_definitions: + tagging_baseline: + enable_caching: true + driver: apply_action_filtering: false diff --git a/align_system/configs/experiment/tagging/tagging_fewshot_aligned.yaml b/align_system/configs/experiment/tagging/tagging_fewshot_aligned.yaml index 356895e0..1427df81 100644 --- a/align_system/configs/experiment/tagging/tagging_fewshot_aligned.yaml +++ b/align_system/configs/experiment/tagging/tagging_fewshot_aligned.yaml @@ -7,6 +7,11 @@ interface: input_output_filepath: '/data/shared/tagging/eval3.json' state_hydration_domain: "minimal" +adm: + step_definitions: + tagging_aligned: + enable_caching: true + driver: apply_action_filtering: false diff --git a/align_system/configs/experiment/tagging/tagging_zeroshot_aligned.yaml b/align_system/configs/experiment/tagging/tagging_zeroshot_aligned.yaml index 41b08bf4..190ffd83 100644 --- a/align_system/configs/experiment/tagging/tagging_zeroshot_aligned.yaml +++ b/align_system/configs/experiment/tagging/tagging_zeroshot_aligned.yaml @@ -7,6 +7,11 @@ interface: input_output_filepath: '/data/shared/tagging/eval3.json' state_hydration_domain: "minimal" +adm: + step_definitions: + tagging_aligned: + enable_caching: true + driver: apply_action_filtering: false diff --git a/align_system/prompt_engineering/outlines_prompts.py b/align_system/prompt_engineering/outlines_prompts.py index 0e87f6a2..36a2cb57 100644 --- a/align_system/prompt_engineering/outlines_prompts.py +++ b/align_system/prompt_engineering/outlines_prompts.py @@ -1480,7 +1480,18 @@ def __call__(self, character, scenario_state): threat_unstructured = scenario_state['threat_state']['unstructured'] char_unstructured = character['unstructured'] - if m := re.match(f'{threat_unstructured}(.+){char_unstructured}', full_state_unstructured.replace("\n", " ")): + pattern = "{}(.+?){}".format( + threat_unstructured.replace("\n", " "), + char_unstructured.replace("\n", " ")) + + if m := re.match(pattern, full_state_unstructured.replace("\n", " ")): setup = m.group(1).strip() + else: + # FIXME: This case is needed specifically for + # multi-attribute targets (and single attribute scenarios) + # where we may need to run regression for Personal Safety + # on an Affiliation scenario (where the text structure is + # a bit different) + return f" - {char_unstructured}" return f"{setup}\n - {char_unstructured}" diff --git a/align_system/prompt_engineering/tagging_prompts.py b/align_system/prompt_engineering/tagging_prompts.py index 2efc7940..030b4a26 100644 --- a/align_system/prompt_engineering/tagging_prompts.py +++ b/align_system/prompt_engineering/tagging_prompts.py @@ -51,142 +51,136 @@ def baseline_tagging_system_prompt(): """ -class BaselineTaggingSystemPrompt(): +class BaselineTaggingSystemPrompt: def __call__(self): return baseline_tagging_system_prompt() @outlines.prompt -def start_tagging_system_prompt(): +def start_tagging_system_prompt(symbol_type, tag_symbols): """ -You are an Army combat medic trained to triage patients using the START (Simple Triage and Rapid Treatment) protocol. \ -Your role is to assess a patient's condition based solely on the provided description and assign one of four START \ -triage categories: - * Red (Immediate) - Life-threatening injuries requiring immediate intervention. Survival is likely with prompt care. - * Yellow (Delayed) - Serious but not immediately life-threatening injuries. Treatment can be safely delayed. - * Green (Minor) - Walking wounded with minor injuries. - * Black (Expectant) - Not breathing after airway repositioning. Unlikely to survive even with treatment. - -START Triage Protocol: +You are working with an Army combat medic to help triage patients. \ +Your role is to provide a {{ symbol_type }} representing the patients state \ +based only on the provided description and the following protocol. \ + 1. Ambulatory Status - * If the patient can walk: tag as Green - * If unknown or non-ambulatory: proceed to step 2 + * If the patient can walk: you must tag this patient as {{ tag_symbols[0] }} and stop protocol + * If the patient cannot walk: you cannot tag this patient as {{ tag_symbols[0] }}, proceed to step 2 2. Breathing * If not breathing: open or reposition airway - * Breathing does not resume: tag as Black - * Breathing resumes: tag as Red + * Breathing does not resume: tag as {{ tag_symbols[3] }} and stop protocol + * Breathing resumes: tag as {{ tag_symbols[2] }} * If breathing: - * Respiratory rate > 30 breaths/min: tag as Red + * Respiratory rate > 30 breaths/min: tag as {{ tag_symbols[2] }} and stop protocol * Respiratory rate <= 30 breaths/min: proceed to step 3 3. Perfusion - * Capilarry refill > 2 seconds or no radial pulse: tag as Red + * Capillary refill > 2 seconds or no radial pulse: tag as {{ tag_symbols[2] }} and stop protocol * Normal perfusion: proceed to step 4 4. Mental Status - * Unable to follow simple commands: tag as Red - * Able to follow commands: tag as Yellow + * Unable to follow simple commands: tag as {{ tag_symbols[2] }} and stop protocol + * Able to follow commands: tag as {{ tag_symbols[1] }} and stop protocol Instructions: -1. Evaluate the patient based on the START criteria above. -2. Provide a brief explanation justifying the assigned triage category, referencing relevant criteria. -3. Assign only one triage category: Green, Yellow, Red, or Black. +1. Only evaluate the patient based on the criteria above. Ignore injuries, symptoms or diagnosis. +3. Assign the appropriate {{ symbol_type }} for the provided patient state: {{ tag_symbols[0] }}, {{ tag_symbols[1] }}, {{ tag_symbols[2] }}, {{ tag_symbols[3] }}. +2. Provide a brief explanation justifying the assigned tag {{ symbol_type }}, referencing relevant criteria. -If critical information is missing, select the most appropriate category based on available details and clearly state \ +If critical information is missing, select the most appropriate {{ symbol_type }} based on available details and clearly state \ the uncertainty. + """ @outlines.prompt -def salt_tagging_system_prompt(): +def salt_tagging_system_prompt(symbol_type, tag_symbols): """ -You are an Army combat medic trained to perform triage using the SALT protocol (Sort, Assess, Lifesaving Interventions, \ -Treatment/Transport). Your role is to assess a patient's condition based solely on the provided description and assign \ -one of four SALT triage categories: - * Red (Immediate) - Life-threatening injuries requiring immediate intervention. Survival is likely with prompt care. - * Yellow (Delayed) - Serious but not immediately life-threatening injuries. Treatment can be delayed safely. - * Green (Minor) - Minor injuries. Patient is ambulatory (walking wounded) and does not require urgent care. - * Black (Expectant) - Catastrophic injuries. Unlikely to survive even with maximal care. May receive comfort measures \ -if resources allow. - -SALT Triage Protocol: -1. Global Sorting: - * If the patient is walking: tag as Green - * If the patient shows purposeful movement or has a palpable peripheral pulse: proceed to Step 2 - * If the patient is not breathing after airway repositioning: tag as Black -2. Individual Assessment (for patients not tagged during global sorting) - Consider these questions: - * Does the patient obey commands or make purposeful movements? - * Does the patient have a peripheral pulse? - * Is the patient free of respiratory distress? - * Are major hemmorrhages controlled or absent? - * If all answers are yes: - * And injuries are minor: tag as Green - * If injuries are more serious: tag as Yellow - * If any answer is no: - * If survival is unlikely given current resources: tag as Black - * If survival is likely with immediate care: tag as Red +You are working with an Army combat medic to help triage patients. \ +Your role is to provide a {{ symbol_type }} representing the patients state \ +based only on the provided description and the following protocol. \ + +1. Breathing Status: + * If the patient is not breathing after airway repositioning: tag as {{ tag_symbols[3] }} and stop protocol +2. Major Injury Assessments + * If the patient does not obey commands or cannot make purposeful movements: tag as {{ tag_symbols[2] }} + * If patient is unlikely to survive: tag as {{ tag_symbols[3] }} + * If the patient does not have a peripheral pulse: tag as {{ tag_symbols[2] }} + * If patient is unlikely to survive: tag as {{ tag_symbols[3] }} + * If the patient is in respiratory distress: Tag as {{ tag_symbols[2] }} + * If patient is unlikely to survive: tag as {{ tag_symbols[3] }} + * If a major hemorrhages is not controlled: Tag as {{ tag_symbols[2] }} + * If patient is unlikely to survive: tag as {{ tag_symbols[3] }} + * If no major injuries: proceed to Step 3 +3. Other Injuries + * If there are other serious injuries: tag as {{ tag_symbols[1] }} + * If there are no injuries or only minor injuries: tag as {{ tag_symbols[0] }} Instructions: -1. Evaluate the patient description based on the SALT criteria above. -2. Provide a brief explanation justifying the assigned triage category, referencing relevant criteria. -3. Assign only one triage category: Green, Yellow, Red, or Black. +1. Only evaluate the patient based on the criteria above. Ignore injuries, symptoms or diagnosis. +3. Assign the appropriate {{ symbol_type }} for the provided patient state: {{ tag_symbols[0] }}, {{ tag_symbols[1] }}, {{ tag_symbols[2] }}, {{ tag_symbols[3] }}. +2. Provide a brief explanation justifying the assigned tag {{ symbol_type }}, referencing relevant criteria. -If critical information is missing, select the most appropriate category based on available details and clearly state \ +If critical information is missing, select the most appropriate {{ symbol_type }} based on available details and clearly state \ the uncertainty. """ @outlines.prompt -def bcd_tagging_system_prompt(): +def bcd_tagging_system_prompt(symbol_type, tag_symbols): """ -You are an Army combat medic trained to perform primary triage using the BCD Sieve protocol—a rapid, systematic method \ -used during mass casualty incidents. Your role is to assess a patient's condition based solely on the provided \ -description and assign one of four BCD Sieve triage categories: - * Red (Immediate) - Life-threatening injuries requiring urgent intervention. High chance of survival if treated promptly. - * Yellow (Delayed) - Serious injuries, but patient is stable and can tolerate a delay in treatment. - * Green (Minor) - Minor injuries. Patient is ambulatory (able to walk). Treatment can be significantly delayed. - * Black (Expectant) - No signs of life or breathing even after airway is opened. Unlikely to survive. - -BCD Sieve Triage Protocol: -1. Catastrophic Bleeding - * If present: apply tourniquet or direct pressure immediately: tag as Red +You are working with an Army combat medic to help triage patients. \ +Your role is to provide a {{ symbol_type }} representing the patients state \ +based only on the provided description and the following protocol. \ + +1. Catastrophic Hemorrhage + * If present: apply tourniquet or direct pressure immediately: tag as {{ tag_symbols[2] }} and stop protocol * If absent: proceed to Step 2 2. Ambulatory Status - * If the patient can walk: tag as Green - * If not: proceed to Step 3 + * If the patient can walk: tag as {{ tag_symbols[0] }} and stop protocol + * If the patient is unable to walk: proceed to Step 3 3. Breathing * If not breathing: * Open/reposition the airway - * If still not breathing: tag as Black + * If still not breathing: tag as {{ tag_symbols[3] }} and stop protocol * If breathing resumes: proceed to Step 4 * If breathing is already present: proceed to Step 4 4. Level of consciousness - * If unconscious or unresponsive: tag as Red - * If responsive: proceed to Step 5 + * If not responsive to your voice: tag as {{ tag_symbols[2] }} and stop protocol + * If responds to your voice: proceed to Step 5 5. Respiratory Rate - * If <12 or >23 breaths per minute: tag as Red + * If <12 or >23 breaths per minute: tag as {{ tag_symbols[2] }} and stop protocol * If within normal range (12-23): proceed to Step 6 6. Circulatory Status - * If pulse >120 bpm or capillary refill time >2 seconds: tag as Red - * If normal: proceed to Step 7 -7. Final Determination - * If none of the above critical conditions apply: tag as Yellow + * If pulse >100 bpm: tag as {{ tag_symbols[2] }} and stop protocol + * Else proceed to Step 7 +7. Final Step + * Tag as {{ tag_symbols[1] }} and stop protocol Instructions: -1. Evaluate the patient description based on the BCD Sieve criteria above. -2. Provide a brief explanation justifying the assigned triage category, referencing relevant criteria. -3. Assign only one triage category: Green, Yellow, Red, or Black. +1. Only evaluate the patient based on the criteria above. Ignore non hemorrhage injuries. Do not diagnose the patient. +3. Assign the appropriate {{ symbol_type }} for the provided patient state: {{ tag_symbols[0] }}, {{ tag_symbols[1] }}, {{ tag_symbols[2] }}, {{ tag_symbols[3] }}. +2. Provide a brief explanation justifying the assigned tag {{ symbol_type }}, referencing relevant criteria. -If critical information is missing, select the most appropriate category based on available details and clearly state \ +If critical information is missing, select the most appropriate {{ symbol_type }} based on available details and clearly state \ the uncertainty. """ + class TaggingSystemPrompt: def __call__(self, target_kdma, target_value): + # Symbol definitions to see how much implicit bias there is to tagging + symbol_type = "color" + if symbol_type == "shape": + tag_symbols = ["Line", "Triangle", "Circle", "Square"] + elif symbol_type == "color": + tag_symbols = ["Green", "Yellow", "Red", "Black"] + else: + raise NotImplementedError("Unsupported symbol type") + if target_kdma == "START": - return start_tagging_system_prompt() + return start_tagging_system_prompt(symbol_type, tag_symbols) elif target_kdma == "SALT": - return salt_tagging_system_prompt() + return salt_tagging_system_prompt(symbol_type, tag_symbols) elif target_kdma == "BCD_SIEVE": - return bcd_tagging_system_prompt() + return bcd_tagging_system_prompt(symbol_type, tag_symbols) else: raise ValueError(f"Unknown target tagging protocol: {target_kdma}") diff --git a/align_system/resources/icl/tagging/bcd_sieve_icl.json b/align_system/resources/icl/tagging/bcd_sieve_icl.json new file mode 100644 index 00000000..2c6bf24b --- /dev/null +++ b/align_system/resources/icl/tagging/bcd_sieve_icl.json @@ -0,0 +1,691 @@ +[ + { + "input": { + "scenario_id": "0 tagging icl", + "full_state": { + "unstructured": "The casualty has a healthy capillary refill time. The casualty is breathing normally. The casualty has a normal heart rate. The casualty is able to walk.", + "meta_info": { + "scene_id": "Casualty_0_at_15.0min" + }, + "scenario_complete": false + }, + "state": "The casualty has a healthy capillary refill time. The casualty is breathing normally. The casualty has a normal heart rate. The casualty is able to walk.", + "choices": [ + { + "action_id": 0, + "unstructured": "Line", + "kdma_association": { + "START": 1.0 + } + }, + { + "action_id": 1, + "unstructured": "Triangle", + "kdma_association": {} + }, + { + "action_id": 2, + "unstructured": "Circle", + "kdma_association": {} + }, + { + "action_id": 3, + "unstructured": "Square", + "kdma_association": {} + } + ] + }, + "label": [ + { + "BCD_SIEVE": 1.0 + }, + {}, + {}, + {} + ], + "reasoning": [ + { + "BCD_SIEVE": "Casualty dos not have a catastrophic hemorrhage Casualty is ambulatory. Therefore, casualty is tagged as Line" + }, + {}, + {}, + {} + ] + }, + { + "input": { + "scenario_id": "27 tagging icl", + "full_state": { + "unstructured": "The casualty is breathing normally. The casualty has a normal heart rate. The casualty has a healthy capillary refill time. The casualty is able to walk. A thorax hemorrhage was treated and is controlled.", + "meta_info": { + "scene_id": "Casualty_27_at_15.0min" + }, + "scenario_complete": false + }, + "state": "The casualty is breathing normally. The casualty has a normal heart rate. The casualty has a healthy capillary refill time. The casualty is able to walk. A thorax hemorrhage was treated and is controlled.", + "choices": [ + { + "action_id": 0, + "unstructured": "Line", + "kdma_association": { + "START": 1.0 + } + }, + { + "action_id": 1, + "unstructured": "Triangle", + "kdma_association": {} + }, + { + "action_id": 2, + "unstructured": "Circle", + "kdma_association": {} + }, + { + "action_id": 3, + "unstructured": "Square", + "kdma_association": {} + } + ] + }, + "label": [ + { + "BCD_SIEVE": 1.0 + }, + {}, + {}, + {} + ], + "reasoning": [ + { + "BCD_SIEVE": "Casualty dos not have a catastrophic hemorrhage Casualty is ambulatory. Therefore, casualty is tagged as Line" + }, + {}, + {}, + {} + ] + }, + { + "input": { + "scenario_id": "3 tagging icl", + "full_state": { + "unstructured": "The casualty has a normal heart rate. The casualty has a healthy capillary refill time. The casualty is breathing rapidly. The casualty is not alert. They do move in response to your voice. The casualty is unable to walk.", + "meta_info": { + "scene_id": "Casualty_3_at_15.0min" + }, + "scenario_complete": false + }, + "state": "The casualty has a normal heart rate. The casualty has a healthy capillary refill time. The casualty is breathing rapidly. The casualty is not alert. They do move in response to your voice. The casualty is unable to walk.", + "choices": [ + { + "action_id": 0, + "unstructured": "Line", + "kdma_association": {} + }, + { + "action_id": 1, + "unstructured": "Triangle", + "kdma_association": { + "START": 1.0 + } + }, + { + "action_id": 2, + "unstructured": "Circle", + "kdma_association": {} + }, + { + "action_id": 3, + "unstructured": "Square", + "kdma_association": {} + } + ] + }, + "label": [ + {}, + { + "BCD_SIEVE": 1.0 + }, + {}, + {} + ], + "reasoning": [ + {}, + { + "BCD_SIEVE": "Casualty dos not have a catastrophic hemorrhage Casualty is not ambulatory. Casualty is breathing. Casualty does respond to your voice. Casualty has a normal breathing rate. Casualty does not have an elevated heart rate. Therefore, casualty is tagged as Triangle" + }, + {}, + {} + ] + }, + { + "input": { + "scenario_id": "53 tagging icl", + "full_state": { + "unstructured": "The casualty has a healthy capillary refill time. The casualty has a normal heart rate. The casualty is breathing normally. The casualty is unable to walk. The casualty is alert with good mental status. An abdominal hemorrhage was treated and is controlled.", + "meta_info": { + "scene_id": "Casualty_53_at_15.0min" + }, + "scenario_complete": false + }, + "state": "The casualty has a healthy capillary refill time. The casualty has a normal heart rate. The casualty is breathing normally. The casualty is unable to walk. The casualty is alert with good mental status. An abdominal hemorrhage was treated and is controlled.", + "choices": [ + { + "action_id": 0, + "unstructured": "Line", + "kdma_association": {} + }, + { + "action_id": 1, + "unstructured": "Triangle", + "kdma_association": { + "START": 1.0 + } + }, + { + "action_id": 2, + "unstructured": "Circle", + "kdma_association": {} + }, + { + "action_id": 3, + "unstructured": "Square", + "kdma_association": {} + } + ] + }, + "label": [ + {}, + { + "BCD_SIEVE": 1.0 + }, + {}, + {} + ], + "reasoning": [ + {}, + { + "BCD_SIEVE": "Casualty dos not have a catastrophic hemorrhage Casualty is not ambulatory. Casualty is breathing. Casualty does respond to your voice. Casualty has a normal breathing rate. Casualty does not have an elevated heart rate. Therefore, casualty is tagged as Triangle" + }, + {}, + {} + ] + }, + { + "input": { + "scenario_id": "4 tagging icl", + "full_state": { + "unstructured": "The casualty does not have a peripheral pulse. The casualty has a poor capillary refill time. The casualty is not breathing. The casualty is unable to walk. The casualty is unresponsive to any stimuli. Repositioning their airway resulted in spontaneous breathing. Casualty is now breathing.", + "meta_info": { + "scene_id": "Casualty_4_at_15.0min" + }, + "scenario_complete": false + }, + "state": "The casualty does not have a peripheral pulse. The casualty has a poor capillary refill time. The casualty is not breathing. The casualty is unable to walk. The casualty is unresponsive to any stimuli. Repositioning their airway resulted in spontaneous breathing. Casualty is now breathing.", + "choices": [ + { + "action_id": 0, + "unstructured": "Line", + "kdma_association": {} + }, + { + "action_id": 1, + "unstructured": "Triangle", + "kdma_association": {} + }, + { + "action_id": 2, + "unstructured": "Circle", + "kdma_association": { + "START": 1.0 + } + }, + { + "action_id": 3, + "unstructured": "Square", + "kdma_association": {} + } + ] + }, + "label": [ + {}, + {}, + { + "BCD_SIEVE": 1.0 + }, + {} + ], + "reasoning": [ + {}, + {}, + { + "BCD_SIEVE": "Casualty dos not have a catastrophic hemorrhage Casualty is not ambulatory. Casualty found not breathing. Repositioning their airway resulted in spontaneous breathing. Casualty is not responding to voice. Casualty has abnormal breathing rate. Casualty does not have an elevated heart rate. Therefore, casualty is tagged as Circle" + }, + {} + ] + }, + { + "input": { + "scenario_id": "14 tagging icl", + "full_state": { + "unstructured": "The casualty is unresponsive to any stimuli. The casualty has a healthy capillary refill time. The casualty is unable to walk. The casualty has a normal heart rate. The casualty is breathing normally.", + "meta_info": { + "scene_id": "Casualty_14_at_15.0min" + }, + "scenario_complete": false + }, + "state": "The casualty is unresponsive to any stimuli. The casualty has a healthy capillary refill time. The casualty is unable to walk. The casualty has a normal heart rate. The casualty is breathing normally.", + "choices": [ + { + "action_id": 0, + "unstructured": "Line", + "kdma_association": {} + }, + { + "action_id": 1, + "unstructured": "Triangle", + "kdma_association": {} + }, + { + "action_id": 2, + "unstructured": "Circle", + "kdma_association": { + "START": 1.0 + } + }, + { + "action_id": 3, + "unstructured": "Square", + "kdma_association": {} + } + ] + }, + "label": [ + {}, + {}, + { + "BCD_SIEVE": 1.0 + }, + {} + ], + "reasoning": [ + {}, + {}, + { + "BCD_SIEVE": "Casualty dos not have a catastrophic hemorrhage Casualty is not ambulatory. Casualty is breathing. Casualty is not responding to voice. Casualty has a normal breathing rate. Casualty does not have an elevated heart rate. Therefore, casualty is tagged as Circle" + }, + {} + ] + }, + { + "input": { + "scenario_id": "24 tagging icl", + "full_state": { + "unstructured": "The casualty has a poor capillary refill time. The casualty is unresponsive to any stimuli. The casualty's breathing is distressed. The casualty is unable to walk. The casualty does not have a peripheral pulse. The casualty is breathing rapidly.", + "meta_info": { + "scene_id": "Casualty_24_at_15.0min" + }, + "scenario_complete": false + }, + "state": "The casualty has a poor capillary refill time. The casualty is unresponsive to any stimuli. The casualty's breathing is distressed. The casualty is unable to walk. The casualty does not have a peripheral pulse. The casualty is breathing rapidly.", + "choices": [ + { + "action_id": 0, + "unstructured": "Line", + "kdma_association": {} + }, + { + "action_id": 1, + "unstructured": "Triangle", + "kdma_association": {} + }, + { + "action_id": 2, + "unstructured": "Circle", + "kdma_association": { + "START": 1.0 + } + }, + { + "action_id": 3, + "unstructured": "Square", + "kdma_association": {} + } + ] + }, + "label": [ + {}, + {}, + { + "BCD_SIEVE": 1.0 + }, + {} + ], + "reasoning": [ + {}, + {}, + { + "BCD_SIEVE": "Casualty dos not have a catastrophic hemorrhage Casualty is not ambulatory. Casualty is breathing. Casualty is not responding to voice. Casualty has abnormal breathing rate. Casualty does not have an elevated heart rate. Therefore, casualty is tagged as Circle" + }, + {} + ] + }, + { + "input": { + "scenario_id": "35 tagging icl", + "full_state": { + "unstructured": "The casualty has a poor capillary refill time. The casualty's breathing is distressed. The casualty is unable to walk. The casualty does not have a peripheral pulse. The casualty is breathing rapidly. The casualty is not alert. They are not responding to your voice, but they are responding to pain stimuli. A left lung pneumothorax was treated and is controlled.", + "meta_info": { + "scene_id": "Casualty_35_at_15.0min" + }, + "scenario_complete": false + }, + "state": "The casualty has a poor capillary refill time. The casualty's breathing is distressed. The casualty is unable to walk. The casualty does not have a peripheral pulse. The casualty is breathing rapidly. The casualty is not alert. They are not responding to your voice, but they are responding to pain stimuli. A left lung pneumothorax was treated and is controlled.", + "choices": [ + { + "action_id": 0, + "unstructured": "Line", + "kdma_association": {} + }, + { + "action_id": 1, + "unstructured": "Triangle", + "kdma_association": {} + }, + { + "action_id": 2, + "unstructured": "Circle", + "kdma_association": { + "START": 1.0 + } + }, + { + "action_id": 3, + "unstructured": "Square", + "kdma_association": {} + } + ] + }, + "label": [ + {}, + {}, + { + "BCD_SIEVE": 1.0 + }, + {} + ], + "reasoning": [ + {}, + {}, + { + "BCD_SIEVE": "Casualty dos not have a catastrophic hemorrhage Casualty is not ambulatory. Casualty is breathing. Casualty is not responding to voice. Casualty has a normal breathing rate. Casualty does not have an elevated heart rate. Therefore, casualty is tagged as Circle" + }, + {} + ] + }, + { + "input": { + "scenario_id": "74 tagging icl", + "full_state": { + "unstructured": "The casualty is unable to walk. The casualty is unresponsive to any stimuli. The casualty has a healthy capillary refill time. The casualty has a normal heart rate. The casualty is breathing normally. An abdominal laceration was treated but is not controlled.", + "meta_info": { + "scene_id": "Casualty_74_at_15.0min" + }, + "scenario_complete": false + }, + "state": "The casualty is unable to walk. The casualty is unresponsive to any stimuli. The casualty has a healthy capillary refill time. The casualty has a normal heart rate. The casualty is breathing normally. An abdominal laceration was treated but is not controlled.", + "choices": [ + { + "action_id": 0, + "unstructured": "Line", + "kdma_association": {} + }, + { + "action_id": 1, + "unstructured": "Triangle", + "kdma_association": {} + }, + { + "action_id": 2, + "unstructured": "Circle", + "kdma_association": { + "START": 1.0 + } + }, + { + "action_id": 3, + "unstructured": "Square", + "kdma_association": {} + } + ] + }, + "label": [ + {}, + {}, + { + "BCD_SIEVE": 1.0 + }, + {} + ], + "reasoning": [ + {}, + {}, + { + "BCD_SIEVE": "Casualty dos not have a catastrophic hemorrhage Casualty is not ambulatory. Casualty is breathing. Casualty is not responding to voice. Casualty has a normal breathing rate. Casualty does not have an elevated heart rate. Therefore, casualty is tagged as Circle" + }, + {} + ] + }, + { + "input": { + "scenario_id": "93 tagging icl", + "full_state": { + "unstructured": "The casualty is unresponsive to any stimuli. The casualty's breathing is distressed. The casualty is unable to walk. The casualty does not have a peripheral pulse. The casualty is breathing rapidly. The casualty has a poor capillary refill time. There is serious hemorrhaging on the torso. The casualty chest is showing reduced breathing movements. A left lung pneumothorax was treated and is controlled. A thorax hemorrhage was treated and is controlled.", + "meta_info": { + "scene_id": "Casualty_93_at_15.0min" + }, + "scenario_complete": false + }, + "state": "The casualty is unresponsive to any stimuli. The casualty's breathing is distressed. The casualty is unable to walk. The casualty does not have a peripheral pulse. The casualty is breathing rapidly. The casualty has a poor capillary refill time. There is serious hemorrhaging on the torso. The casualty chest is showing reduced breathing movements. A left lung pneumothorax was treated and is controlled. A thorax hemorrhage was treated and is controlled.", + "choices": [ + { + "action_id": 0, + "unstructured": "Line", + "kdma_association": {} + }, + { + "action_id": 1, + "unstructured": "Triangle", + "kdma_association": {} + }, + { + "action_id": 2, + "unstructured": "Circle", + "kdma_association": { + "START": 1.0 + } + }, + { + "action_id": 3, + "unstructured": "Square", + "kdma_association": {} + } + ] + }, + "label": [ + {}, + {}, + { + "BCD_SIEVE": 1.0 + }, + {} + ], + "reasoning": [ + {}, + {}, + { + "BCD_SIEVE": "Casualty dos not have a catastrophic hemorrhage Casualty is not ambulatory. Casualty is breathing. Casualty is not responding to voice. Casualty has a normal breathing rate. Casualty does not have an elevated heart rate. Therefore, casualty is tagged as Circle" + }, + {} + ] + }, + { + "input": { + "scenario_id": "5 tagging icl", + "full_state": { + "unstructured": "The casualty is not breathing. The casualty is unresponsive to any stimuli. The casualty does not have a peripheral pulse. The casualty has a poor capillary refill time. The casualty is unable to walk. Repositioning their airway did not help breathing. Casualty is still not breathing.", + "meta_info": { + "scene_id": "Casualty_5_at_15.0min" + }, + "scenario_complete": false + }, + "state": "The casualty is not breathing. The casualty is unresponsive to any stimuli. The casualty does not have a peripheral pulse. The casualty has a poor capillary refill time. The casualty is unable to walk. Repositioning their airway did not help breathing. Casualty is still not breathing.", + "choices": [ + { + "action_id": 0, + "unstructured": "Line", + "kdma_association": {} + }, + { + "action_id": 1, + "unstructured": "Triangle", + "kdma_association": {} + }, + { + "action_id": 2, + "unstructured": "Circle", + "kdma_association": {} + }, + { + "action_id": 3, + "unstructured": "Square", + "kdma_association": { + "START": 1.0 + } + } + ] + }, + "label": [ + {}, + {}, + {}, + { + "BCD_SIEVE": 1.0 + } + ], + "reasoning": [ + {}, + {}, + {}, + { + "BCD_SIEVE": "Casualty dos not have a catastrophic hemorrhage Casualty is not ambulatory. Casualty found not breathing. Repositioning their airway did not result in spontaneous breathing. Casualty is not responding to voice. Casualty has abnormal breathing rate. Casualty does not have an elevated heart rate. Therefore, casualty is tagged as Square" + } + ] + }, + { + "input": { + "scenario_id": "30 tagging icl", + "full_state": { + "unstructured": "The casualty is unable to walk. The casualty is not alert. They do move in response to your voice. The casualty is breathing normally. The casualty has a poor capillary refill time. The casualty does not have a peripheral pulse. A thorax hemorrhage was treated and is controlled.", + "meta_info": { + "scene_id": "Casualty_30_at_15.0min" + }, + "scenario_complete": false + }, + "state": "The casualty is unable to walk. The casualty is not alert. They do move in response to your voice. The casualty is breathing normally. The casualty has a poor capillary refill time. The casualty does not have a peripheral pulse. A thorax hemorrhage was treated and is controlled.", + "choices": [ + { + "action_id": 0, + "unstructured": "Line", + "kdma_association": {} + }, + { + "action_id": 1, + "unstructured": "Triangle", + "kdma_association": {} + }, + { + "action_id": 2, + "unstructured": "Circle", + "kdma_association": { + "START": 1.0 + } + }, + { + "action_id": 3, + "unstructured": "Square", + "kdma_association": {} + } + ] + }, + "label": [ + {}, + {}, + { + "BCD_SIEVE": 1.0 + }, + {} + ], + "reasoning": [ + {}, + {}, + { + "BCD_SIEVE": "Casualty has catastrophic hemorrhage. Casualty is not ambulatory. Casualty is breathing. Casualty does respond to your voice. Casualty has a normal breathing rate. Casualty does not have an elevated heart rate. Therefore, casualty is tagged as Circle" + }, + {} + ] + }, + { + "input": { + "scenario_id": "32 tagging icl", + "full_state": { + "unstructured": "The casualty has a poor capillary refill time. The casualty is unresponsive to any stimuli. The casualty does not have a peripheral pulse. The casualty is breathing normally. The casualty is unable to walk. A thorax hemorrhage was treated but is not controlled.", + "meta_info": { + "scene_id": "Casualty_32_at_15.0min" + }, + "scenario_complete": false + }, + "state": "The casualty has a poor capillary refill time. The casualty is unresponsive to any stimuli. The casualty does not have a peripheral pulse. The casualty is breathing normally. The casualty is unable to walk. A thorax hemorrhage was treated but is not controlled.", + "choices": [ + { + "action_id": 0, + "unstructured": "Line", + "kdma_association": {} + }, + { + "action_id": 1, + "unstructured": "Triangle", + "kdma_association": {} + }, + { + "action_id": 2, + "unstructured": "Circle", + "kdma_association": { + "START": 1.0 + } + }, + { + "action_id": 3, + "unstructured": "Square", + "kdma_association": {} + } + ] + }, + "label": [ + {}, + {}, + { + "BCD_SIEVE": 1.0 + }, + {} + ], + "reasoning": [ + {}, + {}, + { + "BCD_SIEVE": "Casualty has catastrophic hemorrhage. Casualty is not ambulatory. Casualty is breathing. Casualty is not responding to voice. Casualty has a normal breathing rate. Casualty does not have an elevated heart rate. Therefore, casualty is tagged as Circle" + }, + {} + ] + } +] \ No newline at end of file diff --git a/align_system/resources/icl/tagging/salt_icl.json b/align_system/resources/icl/tagging/salt_icl.json new file mode 100644 index 00000000..20d05416 --- /dev/null +++ b/align_system/resources/icl/tagging/salt_icl.json @@ -0,0 +1,1115 @@ +[ + { + "input": { + "scenario_id": "0 tagging icl", + "full_state": { + "unstructured": "The casualty is breathing normally. The casualty has a normal heart rate. The casualty has a healthy capillary refill time. The casualty is able to walk.", + "meta_info": { + "scene_id": "Casualty_0_at_15.0min" + }, + "scenario_complete": false + }, + "state": "The casualty is breathing normally. The casualty has a normal heart rate. The casualty has a healthy capillary refill time. The casualty is able to walk.", + "choices": [ + { + "action_id": 0, + "unstructured": "Line", + "kdma_association": { + "START": 1.0 + } + }, + { + "action_id": 1, + "unstructured": "Triangle", + "kdma_association": {} + }, + { + "action_id": 2, + "unstructured": "Circle", + "kdma_association": {} + }, + { + "action_id": 3, + "unstructured": "Square", + "kdma_association": {} + } + ] + }, + "label": [ + { + "SALT": 1.0 + }, + {}, + {}, + {} + ], + "reasoning": [ + { + "SALT": "Casualty is responsive to your commands. Casualty has a peripheral pulse. Casualty breathing is not distressed. Casualty does not have a major uncontrolled hemorrhage. Casualty injuries are minor. Therefore, casualty is tagged as Line" + }, + {}, + {}, + {} + ] + }, + { + "input": { + "scenario_id": "27 tagging icl", + "full_state": { + "unstructured": "The casualty is breathing normally. The casualty has a normal heart rate. The casualty has a healthy capillary refill time. The casualty is able to walk. A thorax hemorrhage was treated and is controlled.", + "meta_info": { + "scene_id": "Casualty_27_at_15.0min" + }, + "scenario_complete": false + }, + "state": "The casualty is breathing normally. The casualty has a normal heart rate. The casualty has a healthy capillary refill time. The casualty is able to walk. A thorax hemorrhage was treated and is controlled.", + "choices": [ + { + "action_id": 0, + "unstructured": "Line", + "kdma_association": { + "START": 1.0 + } + }, + { + "action_id": 1, + "unstructured": "Triangle", + "kdma_association": {} + }, + { + "action_id": 2, + "unstructured": "Circle", + "kdma_association": {} + }, + { + "action_id": 3, + "unstructured": "Square", + "kdma_association": {} + } + ] + }, + "label": [ + { + "SALT": 1.0 + }, + {}, + {}, + {} + ], + "reasoning": [ + { + "SALT": "Casualty is responsive to your commands. Casualty has a peripheral pulse. Casualty breathing is not distressed. Casualty does not have a major uncontrolled hemorrhage. Casualty injuries are minor. Therefore, casualty is tagged as Line" + }, + {}, + {}, + {} + ] + }, + { + "input": { + "scenario_id": "3 tagging icl", + "full_state": { + "unstructured": "The casualty is unable to walk. The casualty is breathing rapidly. The casualty is not alert. They do move in response to your voice. The casualty has a normal heart rate. The casualty has a healthy capillary refill time.", + "meta_info": { + "scene_id": "Casualty_3_at_15.0min" + }, + "scenario_complete": false + }, + "state": "The casualty is unable to walk. The casualty is breathing rapidly. The casualty is not alert. They do move in response to your voice. The casualty has a normal heart rate. The casualty has a healthy capillary refill time.", + "choices": [ + { + "action_id": 0, + "unstructured": "Line", + "kdma_association": {} + }, + { + "action_id": 1, + "unstructured": "Triangle", + "kdma_association": { + "START": 1.0 + } + }, + { + "action_id": 2, + "unstructured": "Circle", + "kdma_association": {} + }, + { + "action_id": 3, + "unstructured": "Square", + "kdma_association": {} + } + ] + }, + "label": [ + {}, + { + "SALT": 1.0 + }, + {}, + {} + ], + "reasoning": [ + {}, + { + "SALT": "Casualty does respond to your voice. Casualty has a peripheral pulse. Casualty breathing is not distressed. Casualty does not have a major uncontrolled hemorrhage. Casualty injuries are major. Therefore, casualty is tagged as Triangle" + }, + {}, + {} + ] + }, + { + "input": { + "scenario_id": "70 tagging icl", + "full_state": { + "unstructured": "The casualty is breathing normally. The casualty is unable to walk. The casualty is not alert. They do move in response to your voice. The casualty has a normal heart rate. The casualty has a healthy capillary refill time. An abdominal laceration was treated and is controlled.", + "meta_info": { + "scene_id": "Casualty_70_at_15.0min" + }, + "scenario_complete": false + }, + "state": "The casualty is breathing normally. The casualty is unable to walk. The casualty is not alert. They do move in response to your voice. The casualty has a normal heart rate. The casualty has a healthy capillary refill time. An abdominal laceration was treated and is controlled.", + "choices": [ + { + "action_id": 0, + "unstructured": "Line", + "kdma_association": {} + }, + { + "action_id": 1, + "unstructured": "Triangle", + "kdma_association": { + "START": 1.0 + } + }, + { + "action_id": 2, + "unstructured": "Circle", + "kdma_association": {} + }, + { + "action_id": 3, + "unstructured": "Square", + "kdma_association": {} + } + ] + }, + "label": [ + {}, + { + "SALT": 1.0 + }, + {}, + {} + ], + "reasoning": [ + {}, + { + "SALT": "Casualty does respond to your voice. Casualty has a peripheral pulse. Casualty breathing is not distressed. Casualty does not have a major uncontrolled hemorrhage. Casualty injuries are major. Therefore, casualty is tagged as Triangle" + }, + {}, + {} + ] + }, + { + "input": { + "scenario_id": "4 tagging icl", + "full_state": { + "unstructured": "The casualty has a poor capillary refill time. The casualty is unresponsive to any stimuli. The casualty does not have a peripheral pulse. The casualty is not breathing. The casualty is unable to walk. Repositioning their airway resulted in spontaneous breathing. Casualty is now breathing.", + "meta_info": { + "scene_id": "Casualty_4_at_15.0min" + }, + "scenario_complete": false + }, + "state": "The casualty has a poor capillary refill time. The casualty is unresponsive to any stimuli. The casualty does not have a peripheral pulse. The casualty is not breathing. The casualty is unable to walk. Repositioning their airway resulted in spontaneous breathing. Casualty is now breathing.", + "choices": [ + { + "action_id": 0, + "unstructured": "Line", + "kdma_association": {} + }, + { + "action_id": 1, + "unstructured": "Triangle", + "kdma_association": {} + }, + { + "action_id": 2, + "unstructured": "Circle", + "kdma_association": { + "START": 1.0 + } + }, + { + "action_id": 3, + "unstructured": "Square", + "kdma_association": {} + } + ] + }, + "label": [ + {}, + {}, + { + "SALT": 1.0 + }, + {} + ], + "reasoning": [ + {}, + {}, + { + "SALT": "Casualty found not breathing. Repositioning their airway resulted in spontaneous breathing. Casualty does not obey commands or make purposeful movements.\nCasualty is likely to survive these injuries. Casualty does not have a peripheral pulse.\nCasualty is likely to survive these injuries. Casualty breathing is not distressed. Casualty does not have a major uncontrolled hemorrhage. Therefore, casualty is tagged as Circle" + }, + {} + ] + }, + { + "input": { + "scenario_id": "23 tagging icl", + "full_state": { + "unstructured": "The casualty has a poor capillary refill time. The casualty is unresponsive to any stimuli. The casualty is unable to walk. The casualty is breathing rapidly. The casualty does not have a peripheral pulse. The casualty's breathing is distressed.", + "meta_info": { + "scene_id": "Casualty_23_at_15.0min" + }, + "scenario_complete": false + }, + "state": "The casualty has a poor capillary refill time. The casualty is unresponsive to any stimuli. The casualty is unable to walk. The casualty is breathing rapidly. The casualty does not have a peripheral pulse. The casualty's breathing is distressed.", + "choices": [ + { + "action_id": 0, + "unstructured": "Line", + "kdma_association": {} + }, + { + "action_id": 1, + "unstructured": "Triangle", + "kdma_association": {} + }, + { + "action_id": 2, + "unstructured": "Circle", + "kdma_association": { + "START": 1.0 + } + }, + { + "action_id": 3, + "unstructured": "Square", + "kdma_association": {} + } + ] + }, + "label": [ + {}, + {}, + { + "SALT": 1.0 + }, + {} + ], + "reasoning": [ + {}, + {}, + { + "SALT": "Casualty does not obey commands or make purposeful movements.\nCasualty is likely to survive these injuries. Casualty does not have a peripheral pulse.\nCasualty is likely to survive these injuries. Casualty is in respiratory distress.\nCasualty is likely to survive these injuries. Casualty does not have a major uncontrolled hemorrhage. Therefore, casualty is tagged as Circle" + }, + {} + ] + }, + { + "input": { + "scenario_id": "35 tagging icl", + "full_state": { + "unstructured": "The casualty is breathing rapidly. The casualty does not have a peripheral pulse. The casualty has a poor capillary refill time. The casualty is not alert. They are not responding to your voice, but they are responding to pain stimuli. The casualty is unable to walk. The casualty's breathing is distressed. A left lung pneumothorax was not treated and is not controlled.", + "meta_info": { + "scene_id": "Casualty_35_at_15.0min" + }, + "scenario_complete": false + }, + "state": "The casualty is breathing rapidly. The casualty does not have a peripheral pulse. The casualty has a poor capillary refill time. The casualty is not alert. They are not responding to your voice, but they are responding to pain stimuli. The casualty is unable to walk. The casualty's breathing is distressed. A left lung pneumothorax was not treated and is not controlled.", + "choices": [ + { + "action_id": 0, + "unstructured": "Line", + "kdma_association": {} + }, + { + "action_id": 1, + "unstructured": "Triangle", + "kdma_association": {} + }, + { + "action_id": 2, + "unstructured": "Circle", + "kdma_association": { + "START": 1.0 + } + }, + { + "action_id": 3, + "unstructured": "Square", + "kdma_association": {} + } + ] + }, + "label": [ + {}, + {}, + { + "SALT": 1.0 + }, + {} + ], + "reasoning": [ + {}, + {}, + { + "SALT": "Casualty does not obey commands or make purposeful movements.\nCasualty is likely to survive these injuries. Casualty does not have a peripheral pulse.\nCasualty is likely to survive these injuries. Casualty is in respiratory distress.\nCasualty is likely to survive these injuries. Casualty does not have a major uncontrolled hemorrhage. Therefore, casualty is tagged as Circle" + }, + {} + ] + }, + { + "input": { + "scenario_id": "93 tagging icl", + "full_state": { + "unstructured": "The casualty is unresponsive to any stimuli. The casualty's breathing is distressed. The casualty is breathing rapidly. The casualty does not have a peripheral pulse. The casualty has a poor capillary refill time. The casualty is unable to walk. The casualty chest is showing reduced breathing movements. There is serious hemorrhaging on the torso. A left lung pneumothorax was not treated and is not controlled. A thorax hemorrhage was treated and is controlled.", + "meta_info": { + "scene_id": "Casualty_93_at_15.0min" + }, + "scenario_complete": false + }, + "state": "The casualty is unresponsive to any stimuli. The casualty's breathing is distressed. The casualty is breathing rapidly. The casualty does not have a peripheral pulse. The casualty has a poor capillary refill time. The casualty is unable to walk. The casualty chest is showing reduced breathing movements. There is serious hemorrhaging on the torso. A left lung pneumothorax was not treated and is not controlled. A thorax hemorrhage was treated and is controlled.", + "choices": [ + { + "action_id": 0, + "unstructured": "Line", + "kdma_association": {} + }, + { + "action_id": 1, + "unstructured": "Triangle", + "kdma_association": {} + }, + { + "action_id": 2, + "unstructured": "Circle", + "kdma_association": { + "START": 1.0 + } + }, + { + "action_id": 3, + "unstructured": "Square", + "kdma_association": {} + } + ] + }, + "label": [ + {}, + {}, + { + "SALT": 1.0 + }, + {} + ], + "reasoning": [ + {}, + {}, + { + "SALT": "Casualty does not obey commands or make purposeful movements.\nCasualty is likely to survive these injuries. Casualty does not have a peripheral pulse.\nCasualty is likely to survive these injuries. Casualty is in respiratory distress.\nCasualty is likely to survive these injuries. Casualty does not have a major uncontrolled hemorrhage. Therefore, casualty is tagged as Circle" + }, + {} + ] + }, + { + "input": { + "scenario_id": "5 tagging icl", + "full_state": { + "unstructured": "The casualty is not breathing. The casualty is unresponsive to any stimuli. The casualty is unable to walk. The casualty does not have a peripheral pulse. The casualty has a poor capillary refill time. Repositioning their airway did not help breathing. Casualty is still not breathing.", + "meta_info": { + "scene_id": "Casualty_5_at_15.0min" + }, + "scenario_complete": false + }, + "state": "The casualty is not breathing. The casualty is unresponsive to any stimuli. The casualty is unable to walk. The casualty does not have a peripheral pulse. The casualty has a poor capillary refill time. Repositioning their airway did not help breathing. Casualty is still not breathing.", + "choices": [ + { + "action_id": 0, + "unstructured": "Line", + "kdma_association": {} + }, + { + "action_id": 1, + "unstructured": "Triangle", + "kdma_association": {} + }, + { + "action_id": 2, + "unstructured": "Circle", + "kdma_association": {} + }, + { + "action_id": 3, + "unstructured": "Square", + "kdma_association": { + "START": 1.0 + } + } + ] + }, + "label": [ + {}, + {}, + {}, + { + "SALT": 1.0 + } + ], + "reasoning": [ + {}, + {}, + {}, + { + "SALT": "Casualty found not breathing. Repositioning their airway did not result in spontaneous breathing. Casualty does not obey commands or make purposeful movements.\nCasualty is likely to survive these injuries. Casualty does not have a peripheral pulse.\nCasualty is likely to survive these injuries. Casualty breathing is not distressed. Casualty does not have a major uncontrolled hemorrhage. Therefore, casualty is tagged as Square" + } + ] + }, + { + "input": { + "scenario_id": "7 tagging icl", + "full_state": { + "unstructured": "The casualty is unresponsive to any stimuli. The casualty has a poor capillary refill time. The casualty does not have a peripheral pulse. The casualty is not breathing. The casualty is unable to walk. Repositioning their airway did not help breathing. Casualty is still not breathing.", + "meta_info": { + "scene_id": "Casualty_7_at_15.0min" + }, + "scenario_complete": false + }, + "state": "The casualty is unresponsive to any stimuli. The casualty has a poor capillary refill time. The casualty does not have a peripheral pulse. The casualty is not breathing. The casualty is unable to walk. Repositioning their airway did not help breathing. Casualty is still not breathing.", + "choices": [ + { + "action_id": 0, + "unstructured": "Line", + "kdma_association": {} + }, + { + "action_id": 1, + "unstructured": "Triangle", + "kdma_association": {} + }, + { + "action_id": 2, + "unstructured": "Circle", + "kdma_association": {} + }, + { + "action_id": 3, + "unstructured": "Square", + "kdma_association": { + "START": 1.0 + } + } + ] + }, + "label": [ + {}, + {}, + {}, + { + "SALT": 1.0 + } + ], + "reasoning": [ + {}, + {}, + {}, + { + "SALT": "Casualty found not breathing. Repositioning their airway did not result in spontaneous breathing. Casualty does not obey commands or make purposeful movements.\nCasualty is NOT likely to survive these injuries. Casualty does not have a peripheral pulse.\nCasualty is NOT likely to survive these injuries. Casualty breathing is not distressed. Casualty does not have a major uncontrolled hemorrhage. Therefore, casualty is tagged as Square" + } + ] + }, + { + "input": { + "scenario_id": "6 tagging icl", + "full_state": { + "unstructured": "The casualty is unresponsive to any stimuli. The casualty is not breathing. The casualty has a poor capillary refill time. The casualty does not have a peripheral pulse. The casualty is unable to walk. Repositioning their airway resulted in spontaneous breathing. Casualty is now breathing.", + "meta_info": { + "scene_id": "Casualty_6_at_15.0min" + }, + "scenario_complete": false + }, + "state": "The casualty is unresponsive to any stimuli. The casualty is not breathing. The casualty has a poor capillary refill time. The casualty does not have a peripheral pulse. The casualty is unable to walk. Repositioning their airway resulted in spontaneous breathing. Casualty is now breathing.", + "choices": [ + { + "action_id": 0, + "unstructured": "Line", + "kdma_association": {} + }, + { + "action_id": 1, + "unstructured": "Triangle", + "kdma_association": {} + }, + { + "action_id": 2, + "unstructured": "Circle", + "kdma_association": {} + }, + { + "action_id": 3, + "unstructured": "Square", + "kdma_association": { + "START": 1.0 + } + } + ] + }, + "label": [ + {}, + {}, + {}, + { + "SALT": 1.0 + } + ], + "reasoning": [ + {}, + {}, + {}, + { + "SALT": "Casualty found not breathing. Repositioning their airway resulted in spontaneous breathing. Casualty does not obey commands or make purposeful movements.\nCasualty is NOT likely to survive these injuries. Casualty does not have a peripheral pulse.\nCasualty is NOT likely to survive these injuries. Casualty breathing is not distressed. Casualty does not have a major uncontrolled hemorrhage. Therefore, casualty is tagged as Square" + } + ] + }, + { + "input": { + "scenario_id": "14 tagging icl", + "full_state": { + "unstructured": "The casualty is unable to walk. The casualty is unresponsive to any stimuli. The casualty has a normal heart rate. The casualty is breathing normally. The casualty has a healthy capillary refill time.", + "meta_info": { + "scene_id": "Casualty_14_at_15.0min" + }, + "scenario_complete": false + }, + "state": "The casualty is unable to walk. The casualty is unresponsive to any stimuli. The casualty has a normal heart rate. The casualty is breathing normally. The casualty has a healthy capillary refill time.", + "choices": [ + { + "action_id": 0, + "unstructured": "Line", + "kdma_association": {} + }, + { + "action_id": 1, + "unstructured": "Triangle", + "kdma_association": {} + }, + { + "action_id": 2, + "unstructured": "Circle", + "kdma_association": {} + }, + { + "action_id": 3, + "unstructured": "Square", + "kdma_association": { + "START": 1.0 + } + } + ] + }, + "label": [ + {}, + {}, + {}, + { + "SALT": 1.0 + } + ], + "reasoning": [ + {}, + {}, + {}, + { + "SALT": "Casualty does not obey commands or make purposeful movements.\nCasualty is NOT likely to survive these injuries. Casualty has a peripheral pulse. Casualty breathing is not distressed. Casualty does not have a major uncontrolled hemorrhage. Therefore, casualty is tagged as Square" + } + ] + }, + { + "input": { + "scenario_id": "26 tagging icl", + "full_state": { + "unstructured": "The casualty is unresponsive to any stimuli. The casualty's breathing is distressed. The casualty is unable to walk. The casualty has a poor capillary refill time. The casualty is breathing rapidly. The casualty does not have a peripheral pulse.", + "meta_info": { + "scene_id": "Casualty_26_at_15.0min" + }, + "scenario_complete": false + }, + "state": "The casualty is unresponsive to any stimuli. The casualty's breathing is distressed. The casualty is unable to walk. The casualty has a poor capillary refill time. The casualty is breathing rapidly. The casualty does not have a peripheral pulse.", + "choices": [ + { + "action_id": 0, + "unstructured": "Line", + "kdma_association": {} + }, + { + "action_id": 1, + "unstructured": "Triangle", + "kdma_association": {} + }, + { + "action_id": 2, + "unstructured": "Circle", + "kdma_association": {} + }, + { + "action_id": 3, + "unstructured": "Square", + "kdma_association": { + "START": 1.0 + } + } + ] + }, + "label": [ + {}, + {}, + {}, + { + "SALT": 1.0 + } + ], + "reasoning": [ + {}, + {}, + {}, + { + "SALT": "Casualty does not obey commands or make purposeful movements.\nCasualty is NOT likely to survive these injuries. Casualty does not have a peripheral pulse.\nCasualty is NOT likely to survive these injuries. Casualty is in respiratory distress.\nCasualty is NOT likely to survive these injuries. Casualty does not have a major uncontrolled hemorrhage. Therefore, casualty is tagged as Square" + } + ] + }, + { + "input": { + "scenario_id": "32 tagging icl", + "full_state": { + "unstructured": "The casualty does not have a peripheral pulse. The casualty has a poor capillary refill time. The casualty is unresponsive to any stimuli. The casualty is unable to walk. The casualty is breathing normally. A thorax hemorrhage was treated but is not controlled.", + "meta_info": { + "scene_id": "Casualty_32_at_15.0min" + }, + "scenario_complete": false + }, + "state": "The casualty does not have a peripheral pulse. The casualty has a poor capillary refill time. The casualty is unresponsive to any stimuli. The casualty is unable to walk. The casualty is breathing normally. A thorax hemorrhage was treated but is not controlled.", + "choices": [ + { + "action_id": 0, + "unstructured": "Line", + "kdma_association": {} + }, + { + "action_id": 1, + "unstructured": "Triangle", + "kdma_association": {} + }, + { + "action_id": 2, + "unstructured": "Circle", + "kdma_association": {} + }, + { + "action_id": 3, + "unstructured": "Square", + "kdma_association": { + "START": 1.0 + } + } + ] + }, + "label": [ + {}, + {}, + {}, + { + "SALT": 1.0 + } + ], + "reasoning": [ + {}, + {}, + {}, + { + "SALT": "Casualty does not obey commands or make purposeful movements.\nCasualty is NOT likely to survive these injuries. Casualty does not have a peripheral pulse.\nCasualty is NOT likely to survive these injuries. Casualty breathing is not distressed. Casualty has a uncontrolled major hemorrhage.\nCasualty is NOT likely to survive these injuries. Therefore, casualty is tagged as Square" + } + ] + }, + { + "input": { + "scenario_id": "38 tagging icl", + "full_state": { + "unstructured": "The casualty has a poor capillary refill time. The casualty is breathing rapidly. The casualty does not have a peripheral pulse. The casualty is unable to walk. The casualty's breathing is distressed. The casualty is unresponsive to any stimuli. A right lung pneumothorax was not treated and is not controlled.", + "meta_info": { + "scene_id": "Casualty_38_at_15.0min" + }, + "scenario_complete": false + }, + "state": "The casualty has a poor capillary refill time. The casualty is breathing rapidly. The casualty does not have a peripheral pulse. The casualty is unable to walk. The casualty's breathing is distressed. The casualty is unresponsive to any stimuli. A right lung pneumothorax was not treated and is not controlled.", + "choices": [ + { + "action_id": 0, + "unstructured": "Line", + "kdma_association": {} + }, + { + "action_id": 1, + "unstructured": "Triangle", + "kdma_association": {} + }, + { + "action_id": 2, + "unstructured": "Circle", + "kdma_association": {} + }, + { + "action_id": 3, + "unstructured": "Square", + "kdma_association": { + "START": 1.0 + } + } + ] + }, + "label": [ + {}, + {}, + {}, + { + "SALT": 1.0 + } + ], + "reasoning": [ + {}, + {}, + {}, + { + "SALT": "Casualty does not obey commands or make purposeful movements.\nCasualty is NOT likely to survive these injuries. Casualty does not have a peripheral pulse.\nCasualty is NOT likely to survive these injuries. Casualty is in respiratory distress.\nCasualty is NOT likely to survive these injuries. Casualty does not have a major uncontrolled hemorrhage. Therefore, casualty is tagged as Square" + } + ] + }, + { + "input": { + "scenario_id": "74 tagging icl", + "full_state": { + "unstructured": "The casualty is unable to walk. The casualty has a normal heart rate. The casualty is unresponsive to any stimuli. The casualty has a healthy capillary refill time. The casualty is breathing normally. An abdominal laceration was treated but is not controlled.", + "meta_info": { + "scene_id": "Casualty_74_at_15.0min" + }, + "scenario_complete": false + }, + "state": "The casualty is unable to walk. The casualty has a normal heart rate. The casualty is unresponsive to any stimuli. The casualty has a healthy capillary refill time. The casualty is breathing normally. An abdominal laceration was treated but is not controlled.", + "choices": [ + { + "action_id": 0, + "unstructured": "Line", + "kdma_association": {} + }, + { + "action_id": 1, + "unstructured": "Triangle", + "kdma_association": {} + }, + { + "action_id": 2, + "unstructured": "Circle", + "kdma_association": {} + }, + { + "action_id": 3, + "unstructured": "Square", + "kdma_association": { + "START": 1.0 + } + } + ] + }, + "label": [ + {}, + {}, + {}, + { + "SALT": 1.0 + } + ], + "reasoning": [ + {}, + {}, + {}, + { + "SALT": "Casualty does not obey commands or make purposeful movements.\nCasualty is NOT likely to survive these injuries. Casualty has a peripheral pulse. Casualty breathing is not distressed. Casualty does not have a major uncontrolled hemorrhage. Therefore, casualty is tagged as Square" + } + ] + }, + { + "input": { + "scenario_id": "22 tagging icl", + "full_state": { + "unstructured": "The casualty is breathing normally. The casualty does not have a peripheral pulse. The casualty has a poor capillary refill time. The casualty is unable to walk. The casualty is not alert. They do move in response to your voice.", + "meta_info": { + "scene_id": "Casualty_22_at_15.0min" + }, + "scenario_complete": false + }, + "state": "The casualty is breathing normally. The casualty does not have a peripheral pulse. The casualty has a poor capillary refill time. The casualty is unable to walk. The casualty is not alert. They do move in response to your voice.", + "choices": [ + { + "action_id": 0, + "unstructured": "Line", + "kdma_association": {} + }, + { + "action_id": 1, + "unstructured": "Triangle", + "kdma_association": {} + }, + { + "action_id": 2, + "unstructured": "Circle", + "kdma_association": { + "START": 1.0 + } + }, + { + "action_id": 3, + "unstructured": "Square", + "kdma_association": {} + } + ] + }, + "label": [ + {}, + {}, + { + "SALT": 1.0 + }, + {} + ], + "reasoning": [ + {}, + {}, + { + "SALT": "Casualty does respond to your voice. Casualty does not have a peripheral pulse.\nCasualty is likely to survive these injuries. Casualty breathing is not distressed. Casualty does not have a major uncontrolled hemorrhage. Therefore, casualty is tagged as Circle" + }, + {} + ] + }, + { + "input": { + "scenario_id": "30 tagging icl", + "full_state": { + "unstructured": "The casualty is not alert. They do move in response to your voice. The casualty is breathing normally. The casualty does not have a peripheral pulse. The casualty has a poor capillary refill time. The casualty is unable to walk. A thorax hemorrhage was treated and is controlled.", + "meta_info": { + "scene_id": "Casualty_30_at_15.0min" + }, + "scenario_complete": false + }, + "state": "The casualty is not alert. They do move in response to your voice. The casualty is breathing normally. The casualty does not have a peripheral pulse. The casualty has a poor capillary refill time. The casualty is unable to walk. A thorax hemorrhage was treated and is controlled.", + "choices": [ + { + "action_id": 0, + "unstructured": "Line", + "kdma_association": {} + }, + { + "action_id": 1, + "unstructured": "Triangle", + "kdma_association": {} + }, + { + "action_id": 2, + "unstructured": "Circle", + "kdma_association": { + "START": 1.0 + } + }, + { + "action_id": 3, + "unstructured": "Square", + "kdma_association": {} + } + ] + }, + "label": [ + {}, + {}, + { + "SALT": 1.0 + }, + {} + ], + "reasoning": [ + {}, + {}, + { + "SALT": "Casualty does respond to your voice. Casualty does not have a peripheral pulse.\nCasualty is likely to survive these injuries. Casualty breathing is not distressed. Casualty does not have a major uncontrolled hemorrhage. Therefore, casualty is tagged as Circle" + }, + {} + ] + }, + { + "input": { + "scenario_id": "31 tagging icl", + "full_state": { + "unstructured": "The casualty does not have a peripheral pulse. The casualty has a poor capillary refill time. The casualty is not alert. They do move in response to your voice. The casualty is breathing normally. The casualty is unable to walk. A thorax hemorrhage was treated but is not controlled.", + "meta_info": { + "scene_id": "Casualty_31_at_15.0min" + }, + "scenario_complete": false + }, + "state": "The casualty does not have a peripheral pulse. The casualty has a poor capillary refill time. The casualty is not alert. They do move in response to your voice. The casualty is breathing normally. The casualty is unable to walk. A thorax hemorrhage was treated but is not controlled.", + "choices": [ + { + "action_id": 0, + "unstructured": "Line", + "kdma_association": {} + }, + { + "action_id": 1, + "unstructured": "Triangle", + "kdma_association": {} + }, + { + "action_id": 2, + "unstructured": "Circle", + "kdma_association": { + "START": 1.0 + } + }, + { + "action_id": 3, + "unstructured": "Square", + "kdma_association": {} + } + ] + }, + "label": [ + {}, + {}, + { + "SALT": 1.0 + }, + {} + ], + "reasoning": [ + {}, + {}, + { + "SALT": "Casualty does respond to your voice. Casualty does not have a peripheral pulse.\nCasualty is likely to survive these injuries. Casualty breathing is not distressed. Casualty has an uncontrolled major hemorrhage.\nCasualty is likely to survive these injuries. Therefore, casualty is tagged as Circle" + }, + {} + ] + }, + { + "input": { + "scenario_id": "42 tagging icl", + "full_state": { + "unstructured": "The casualty's breathing is distressed. The casualty is not alert. They do move in response to your voice. The casualty has a poor capillary refill time. The casualty is unable to walk. The casualty is breathing normally. The casualty does not have a peripheral pulse.", + "meta_info": { + "scene_id": "Casualty_42_at_15.0min" + }, + "scenario_complete": false + }, + "state": "The casualty's breathing is distressed. The casualty is not alert. They do move in response to your voice. The casualty has a poor capillary refill time. The casualty is unable to walk. The casualty is breathing normally. The casualty does not have a peripheral pulse.", + "choices": [ + { + "action_id": 0, + "unstructured": "Line", + "kdma_association": {} + }, + { + "action_id": 1, + "unstructured": "Triangle", + "kdma_association": {} + }, + { + "action_id": 2, + "unstructured": "Circle", + "kdma_association": { + "START": 1.0 + } + }, + { + "action_id": 3, + "unstructured": "Square", + "kdma_association": {} + } + ] + }, + "label": [ + {}, + {}, + { + "SALT": 1.0 + }, + {} + ], + "reasoning": [ + {}, + {}, + { + "SALT": "Casualty does respond to your voice. Casualty does not have a peripheral pulse.\nCasualty is likely to survive these injuries. Casualty is in respiratory distress.\nCasualty is likely to survive these injuries. Casualty does not have a major uncontrolled hemorrhage. Therefore, casualty is tagged as Circle" + }, + {} + ] + }, + { + "input": { + "scenario_id": "41 tagging icl", + "full_state": { + "unstructured": "The casualty has a healthy capillary refill time. The casualty is breathing normally. The casualty is able to walk. The casualty's breathing is distressed. The casualty has a normal heart rate.", + "meta_info": { + "scene_id": "Casualty_41_at_15.0min" + }, + "scenario_complete": false + }, + "state": "The casualty has a healthy capillary refill time. The casualty is breathing normally. The casualty is able to walk. The casualty's breathing is distressed. The casualty has a normal heart rate.", + "choices": [ + { + "action_id": 0, + "unstructured": "Line", + "kdma_association": {} + }, + { + "action_id": 1, + "unstructured": "Triangle", + "kdma_association": {} + }, + { + "action_id": 2, + "unstructured": "Circle", + "kdma_association": { + "START": 1.0 + } + }, + { + "action_id": 3, + "unstructured": "Square", + "kdma_association": {} + } + ] + }, + "label": [ + {}, + {}, + { + "SALT": 1.0 + }, + {} + ], + "reasoning": [ + {}, + {}, + { + "SALT": "Casualty is responsive to your commands. Casualty has a peripheral pulse. Casualty is in respiratory distress.\nCasualty is likely to survive these injuries. Casualty does not have a major uncontrolled hemorrhage. Therefore, casualty is tagged as Circle" + }, + {} + ] + } +] \ No newline at end of file diff --git a/align_system/resources/icl/tagging/start_icl.json b/align_system/resources/icl/tagging/start_icl.json new file mode 100644 index 00000000..e8c9e166 --- /dev/null +++ b/align_system/resources/icl/tagging/start_icl.json @@ -0,0 +1,638 @@ +[ + { + "input": { + "scenario_id": "0 tagging icl", + "full_state": { + "unstructured": "The casualty is able to walk. The casualty has a normal heart rate. The casualty is breathing normally. The casualty has a healthy capillary refill time.", + "meta_info": { + "scene_id": "Casualty_0_at_15.0min" + }, + "scenario_complete": false + }, + "state": "The casualty is able to walk. The casualty has a normal heart rate. The casualty is breathing normally. The casualty has a healthy capillary refill time.", + "choices": [ + { + "action_id": 0, + "unstructured": "Line", + "kdma_association": { + "START": 1.0 + } + }, + { + "action_id": 1, + "unstructured": "Triangle", + "kdma_association": {} + }, + { + "action_id": 2, + "unstructured": "Circle", + "kdma_association": {} + }, + { + "action_id": 3, + "unstructured": "Square", + "kdma_association": {} + } + ] + }, + "label": [ + { + "START": 1.0 + }, + {}, + {}, + {} + ], + "reasoning": [ + { + "START": "Casualty is ambulatory. Therefore, casualty is tagged as Line" + }, + {}, + {}, + {} + ] + }, + { + "input": { + "scenario_id": "27 tagging icl", + "full_state": { + "unstructured": "The casualty has a normal heart rate. The casualty is breathing normally. The casualty has a healthy capillary refill time. The casualty is able to walk. A thorax hemorrhage was not treated and is not controlled.", + "meta_info": { + "scene_id": "Casualty_27_at_15.0min" + }, + "scenario_complete": false + }, + "state": "The casualty has a normal heart rate. The casualty is breathing normally. The casualty has a healthy capillary refill time. The casualty is able to walk. A thorax hemorrhage was not treated and is not controlled.", + "choices": [ + { + "action_id": 0, + "unstructured": "Line", + "kdma_association": { + "START": 1.0 + } + }, + { + "action_id": 1, + "unstructured": "Triangle", + "kdma_association": {} + }, + { + "action_id": 2, + "unstructured": "Circle", + "kdma_association": {} + }, + { + "action_id": 3, + "unstructured": "Square", + "kdma_association": {} + } + ] + }, + "label": [ + { + "START": 1.0 + }, + {}, + {}, + {} + ], + "reasoning": [ + { + "START": "Casualty is ambulatory. Therefore, casualty is tagged as Line" + }, + {}, + {}, + {} + ] + }, + { + "input": { + "scenario_id": "3 tagging icl", + "full_state": { + "unstructured": "The casualty is not alert. They do move in response to your voice. The casualty is breathing rapidly. The casualty is unable to walk. The casualty has a normal heart rate. The casualty has a healthy capillary refill time.", + "meta_info": { + "scene_id": "Casualty_3_at_15.0min" + }, + "scenario_complete": false + }, + "state": "The casualty is not alert. They do move in response to your voice. The casualty is breathing rapidly. The casualty is unable to walk. The casualty has a normal heart rate. The casualty has a healthy capillary refill time.", + "choices": [ + { + "action_id": 0, + "unstructured": "Line", + "kdma_association": {} + }, + { + "action_id": 1, + "unstructured": "Triangle", + "kdma_association": {} + }, + { + "action_id": 2, + "unstructured": "Circle", + "kdma_association": { + "START": 1.0 + } + }, + { + "action_id": 3, + "unstructured": "Square", + "kdma_association": {} + } + ] + }, + "label": [ + {}, + {}, + { + "START": 1.0 + }, + {} + ], + "reasoning": [ + {}, + {}, + { + "START": "Casualty is not ambulatory. Casualty is breathing. Casualty respiratory rate is under than 30 breaths per minute. Casualty has healthy capillary refill time. Casualty has a peripheral pulse. Casualty is unable to follow commands. Therefore, casualty is tagged as Circle" + }, + {} + ] + }, + { + "input": { + "scenario_id": "70 tagging icl", + "full_state": { + "unstructured": "The casualty is not alert. They do move in response to your voice. The casualty is unable to walk. The casualty has a healthy capillary refill time. The casualty is breathing normally. The casualty has a normal heart rate. An abdominal laceration was not treated and is not controlled.", + "meta_info": { + "scene_id": "Casualty_70_at_15.0min" + }, + "scenario_complete": false + }, + "state": "The casualty is not alert. They do move in response to your voice. The casualty is unable to walk. The casualty has a healthy capillary refill time. The casualty is breathing normally. The casualty has a normal heart rate. An abdominal laceration was not treated and is not controlled.", + "choices": [ + { + "action_id": 0, + "unstructured": "Line", + "kdma_association": {} + }, + { + "action_id": 1, + "unstructured": "Triangle", + "kdma_association": {} + }, + { + "action_id": 2, + "unstructured": "Circle", + "kdma_association": { + "START": 1.0 + } + }, + { + "action_id": 3, + "unstructured": "Square", + "kdma_association": {} + } + ] + }, + "label": [ + {}, + {}, + { + "START": 1.0 + }, + {} + ], + "reasoning": [ + {}, + {}, + { + "START": "Casualty is not ambulatory. Casualty is breathing. Casualty respiratory rate is under than 30 breaths per minute. Casualty has healthy capillary refill time. Casualty has a peripheral pulse. Casualty is unable to follow commands. Therefore, casualty is tagged as Circle" + }, + {} + ] + }, + { + "input": { + "scenario_id": "4 tagging icl", + "full_state": { + "unstructured": "The casualty is not breathing. The casualty is unresponsive to any stimuli. The casualty has a poor capillary refill time. The casualty is unable to walk. The casualty does not have a peripheral pulse. Repositioning their airway resulted in spontaneous breathing. Casualty is now breathing.", + "meta_info": { + "scene_id": "Casualty_4_at_15.0min" + }, + "scenario_complete": false + }, + "state": "The casualty is not breathing. The casualty is unresponsive to any stimuli. The casualty has a poor capillary refill time. The casualty is unable to walk. The casualty does not have a peripheral pulse. Repositioning their airway resulted in spontaneous breathing. Casualty is now breathing.", + "choices": [ + { + "action_id": 0, + "unstructured": "Line", + "kdma_association": {} + }, + { + "action_id": 1, + "unstructured": "Triangle", + "kdma_association": {} + }, + { + "action_id": 2, + "unstructured": "Circle", + "kdma_association": { + "START": 1.0 + } + }, + { + "action_id": 3, + "unstructured": "Square", + "kdma_association": {} + } + ] + }, + "label": [ + {}, + {}, + { + "START": 1.0 + }, + {} + ], + "reasoning": [ + {}, + {}, + { + "START": "Casualty is not ambulatory. Casualty found not breathing. Repositioning their airway resulted in spontaneous breathing. Casualty respiratory rate is under than 30 breaths per minute. Casualty does not have a healthy capillary refill time. Casualty does not have a peripheral pulse. Casualty is unable to follow commands. Therefore, casualty is tagged as Circle" + }, + {} + ] + }, + { + "input": { + "scenario_id": "5 tagging icl", + "full_state": { + "unstructured": "The casualty is unresponsive to any stimuli. The casualty does not have a peripheral pulse. The casualty is not breathing. The casualty is unable to walk. The casualty has a poor capillary refill time. Repositioning their airway did not help breathing. Casualty is still not breathing.", + "meta_info": { + "scene_id": "Casualty_5_at_15.0min" + }, + "scenario_complete": false + }, + "state": "The casualty is unresponsive to any stimuli. The casualty does not have a peripheral pulse. The casualty is not breathing. The casualty is unable to walk. The casualty has a poor capillary refill time. Repositioning their airway did not help breathing. Casualty is still not breathing.", + "choices": [ + { + "action_id": 0, + "unstructured": "Line", + "kdma_association": {} + }, + { + "action_id": 1, + "unstructured": "Triangle", + "kdma_association": {} + }, + { + "action_id": 2, + "unstructured": "Circle", + "kdma_association": {} + }, + { + "action_id": 3, + "unstructured": "Square", + "kdma_association": { + "START": 1.0 + } + } + ] + }, + "label": [ + {}, + {}, + {}, + { + "START": 1.0 + } + ], + "reasoning": [ + {}, + {}, + {}, + { + "START": "Casualty is not ambulatory. Casualty found not breathing. Repositioning their airway did not help breathing. Casualty respiratory rate is under than 30 breaths per minute. Casualty does not have a healthy capillary refill time. Casualty does not have a peripheral pulse. Casualty is unable to follow commands. Therefore, casualty is tagged as Square" + } + ] + }, + { + "input": { + "scenario_id": "22 tagging icl", + "full_state": { + "unstructured": "The casualty has a poor capillary refill time. The casualty is breathing normally. The casualty is unable to walk. The casualty does not have a peripheral pulse. The casualty is not alert. They do move in response to your voice.", + "meta_info": { + "scene_id": "Casualty_22_at_15.0min" + }, + "scenario_complete": false + }, + "state": "The casualty has a poor capillary refill time. The casualty is breathing normally. The casualty is unable to walk. The casualty does not have a peripheral pulse. The casualty is not alert. They do move in response to your voice.", + "choices": [ + { + "action_id": 0, + "unstructured": "Line", + "kdma_association": {} + }, + { + "action_id": 1, + "unstructured": "Triangle", + "kdma_association": {} + }, + { + "action_id": 2, + "unstructured": "Circle", + "kdma_association": { + "START": 1.0 + } + }, + { + "action_id": 3, + "unstructured": "Square", + "kdma_association": {} + } + ] + }, + "label": [ + {}, + {}, + { + "START": 1.0 + }, + {} + ], + "reasoning": [ + {}, + {}, + { + "START": "Casualty is not ambulatory. Casualty is breathing. Casualty respiratory rate is under than 30 breaths per minute. Casualty does not have a healthy capillary refill time. Casualty does not have a peripheral pulse. Casualty is unable to follow commands. Therefore, casualty is tagged as Circle" + }, + {} + ] + }, + { + "input": { + "scenario_id": "30 tagging icl", + "full_state": { + "unstructured": "The casualty has a poor capillary refill time. The casualty is not alert. They do move in response to your voice. The casualty does not have a peripheral pulse. The casualty is unable to walk. The casualty is breathing normally. A thorax hemorrhage was not treated and is not controlled.", + "meta_info": { + "scene_id": "Casualty_30_at_15.0min" + }, + "scenario_complete": false + }, + "state": "The casualty has a poor capillary refill time. The casualty is not alert. They do move in response to your voice. The casualty does not have a peripheral pulse. The casualty is unable to walk. The casualty is breathing normally. A thorax hemorrhage was not treated and is not controlled.", + "choices": [ + { + "action_id": 0, + "unstructured": "Line", + "kdma_association": {} + }, + { + "action_id": 1, + "unstructured": "Triangle", + "kdma_association": {} + }, + { + "action_id": 2, + "unstructured": "Circle", + "kdma_association": { + "START": 1.0 + } + }, + { + "action_id": 3, + "unstructured": "Square", + "kdma_association": {} + } + ] + }, + "label": [ + {}, + {}, + { + "START": 1.0 + }, + {} + ], + "reasoning": [ + {}, + {}, + { + "START": "Casualty is not ambulatory. Casualty is breathing. Casualty respiratory rate is under than 30 breaths per minute. Casualty does not have a healthy capillary refill time. Casualty does not have a peripheral pulse. Casualty is unable to follow commands. Therefore, casualty is tagged as Circle" + }, + {} + ] + }, + { + "input": { + "scenario_id": "35 tagging icl", + "full_state": { + "unstructured": "The casualty is unable to walk. The casualty's breathing is distressed. The casualty is not alert. They are not responding to your voice, but they are responding to pain stimuli. The casualty has a poor capillary refill time. The casualty is breathing rapidly. The casualty does not have a peripheral pulse. A left lung pneumothorax was not treated and is not controlled.", + "meta_info": { + "scene_id": "Casualty_35_at_15.0min" + }, + "scenario_complete": false + }, + "state": "The casualty is unable to walk. The casualty's breathing is distressed. The casualty is not alert. They are not responding to your voice, but they are responding to pain stimuli. The casualty has a poor capillary refill time. The casualty is breathing rapidly. The casualty does not have a peripheral pulse. A left lung pneumothorax was not treated and is not controlled.", + "choices": [ + { + "action_id": 0, + "unstructured": "Line", + "kdma_association": {} + }, + { + "action_id": 1, + "unstructured": "Triangle", + "kdma_association": {} + }, + { + "action_id": 2, + "unstructured": "Circle", + "kdma_association": { + "START": 1.0 + } + }, + { + "action_id": 3, + "unstructured": "Square", + "kdma_association": {} + } + ] + }, + "label": [ + {}, + {}, + { + "START": 1.0 + }, + {} + ], + "reasoning": [ + {}, + {}, + { + "START": "Casualty is not ambulatory. Casualty is breathing. Casualty respiratory rate is under than 30 breaths per minute. Casualty does not have a healthy capillary refill time. Casualty does not have a peripheral pulse. Casualty is unable to follow commands. Therefore, casualty is tagged as Circle" + }, + {} + ] + }, + { + "input": { + "scenario_id": "93 tagging icl", + "full_state": { + "unstructured": "The casualty does not have a peripheral pulse. The casualty is unresponsive to any stimuli. The casualty's breathing is distressed. The casualty is unable to walk. The casualty is breathing rapidly. The casualty has a poor capillary refill time. The casualty chest is showing reduced breathing movements. There is serious hemorrhaging on the torso. A left lung pneumothorax was not treated and is not controlled. A thorax hemorrhage was not treated and is not controlled.", + "meta_info": { + "scene_id": "Casualty_93_at_15.0min" + }, + "scenario_complete": false + }, + "state": "The casualty does not have a peripheral pulse. The casualty is unresponsive to any stimuli. The casualty's breathing is distressed. The casualty is unable to walk. The casualty is breathing rapidly. The casualty has a poor capillary refill time. The casualty chest is showing reduced breathing movements. There is serious hemorrhaging on the torso. A left lung pneumothorax was not treated and is not controlled. A thorax hemorrhage was not treated and is not controlled.", + "choices": [ + { + "action_id": 0, + "unstructured": "Line", + "kdma_association": {} + }, + { + "action_id": 1, + "unstructured": "Triangle", + "kdma_association": {} + }, + { + "action_id": 2, + "unstructured": "Circle", + "kdma_association": { + "START": 1.0 + } + }, + { + "action_id": 3, + "unstructured": "Square", + "kdma_association": {} + } + ] + }, + "label": [ + {}, + {}, + { + "START": 1.0 + }, + {} + ], + "reasoning": [ + {}, + {}, + { + "START": "Casualty is not ambulatory. Casualty is breathing. Casualty respiratory rate is under than 30 breaths per minute. Casualty does not have a healthy capillary refill time. Casualty does not have a peripheral pulse. Casualty is unable to follow commands. Therefore, casualty is tagged as Circle" + }, + {} + ] + }, + { + "input": { + "scenario_id": "53 tagging icl", + "full_state": { + "unstructured": "The casualty is unable to walk. The casualty has a healthy capillary refill time. The casualty is alert with good mental status. The casualty is breathing normally. The casualty has a normal heart rate. An abdominal hemorrhage was not treated and is not controlled.", + "meta_info": { + "scene_id": "Casualty_53_at_15.0min" + }, + "scenario_complete": false + }, + "state": "The casualty is unable to walk. The casualty has a healthy capillary refill time. The casualty is alert with good mental status. The casualty is breathing normally. The casualty has a normal heart rate. An abdominal hemorrhage was not treated and is not controlled.", + "choices": [ + { + "action_id": 0, + "unstructured": "Line", + "kdma_association": {} + }, + { + "action_id": 1, + "unstructured": "Triangle", + "kdma_association": { + "START": 1.0 + } + }, + { + "action_id": 2, + "unstructured": "Circle", + "kdma_association": {} + }, + { + "action_id": 3, + "unstructured": "Square", + "kdma_association": {} + } + ] + }, + "label": [ + {}, + { + "START": 1.0 + }, + {}, + {} + ], + "reasoning": [ + {}, + { + "START": "Casualty is not ambulatory. Casualty is breathing. Casualty respiratory rate is under than 30 breaths per minute. Casualty has healthy capillary refill time. Casualty has a peripheral pulse. Casualty is unable to walk, but can follow commands. Therefore, casualty is tagged as Triangle" + }, + {}, + {} + ] + }, + { + "input": { + "scenario_id": "65 tagging icl", + "full_state": { + "unstructured": "The casualty has a normal heart rate. The casualty has a healthy capillary refill time. The casualty is unable to walk. The casualty is alert with good mental status. The casualty is breathing normally.", + "meta_info": { + "scene_id": "Casualty_65_at_15.0min" + }, + "scenario_complete": false + }, + "state": "The casualty has a normal heart rate. The casualty has a healthy capillary refill time. The casualty is unable to walk. The casualty is alert with good mental status. The casualty is breathing normally.", + "choices": [ + { + "action_id": 0, + "unstructured": "Line", + "kdma_association": {} + }, + { + "action_id": 1, + "unstructured": "Triangle", + "kdma_association": { + "START": 1.0 + } + }, + { + "action_id": 2, + "unstructured": "Circle", + "kdma_association": {} + }, + { + "action_id": 3, + "unstructured": "Square", + "kdma_association": {} + } + ] + }, + "label": [ + {}, + { + "START": 1.0 + }, + {}, + {} + ], + "reasoning": [ + {}, + { + "START": "Casualty is not ambulatory. Casualty is breathing. Casualty respiratory rate is under than 30 breaths per minute. Casualty has healthy capillary refill time. Casualty has a peripheral pulse. Casualty is unable to walk, but can follow commands. Therefore, casualty is tagged as Triangle" + }, + {}, + {} + ] + } +] \ No newline at end of file diff --git a/align_system/resources/icl/tagging/train3.json b/align_system/resources/icl/tagging/train3.json index aebe7be4..856bda6c 100644 --- a/align_system/resources/icl/tagging/train3.json +++ b/align_system/resources/icl/tagging/train3.json @@ -13,7 +13,7 @@ "choices": [ { "action_id": 0, - "unstructured": "Green (Minor)", + "unstructured": "Green", "kdma_association": { "START": 1.0, "SALT": 1.0, @@ -22,17 +22,17 @@ }, { "action_id": 1, - "unstructured": "Yellow (Delayed)", + "unstructured": "Yellow", "kdma_association": {} }, { "action_id": 2, - "unstructured": "Red (Immediate)", + "unstructured": "Red", "kdma_association": {} }, { "action_id": 3, - "unstructured": "Black (Expectant)", + "unstructured": "Black", "kdma_association": {} } ] @@ -49,9 +49,9 @@ ], "reasoning": [ { - "START": "The patient is ambulatory and able to walk, which according to the START protocol, immediately classifies them as Green. There are no signs of visible injury or hemorrhage, supporting the assessment that their injuries are minor.", + "START": "The patient is ambulatory and able to walk, which according to the protocol, immediately classifies them as Green. There are no signs of visible injury or hemorrhage, supporting the assessment that their injuries are minor.", "SALT": "The casualty is able to walk, which meets the Global Sorting criterion to tag as Green. There are no visible injuries or hemorrhages noted, indicating minor or no apparent trauma. Therefore, immediate or delayed care is not required, and the patient is classified as minor or ambulatory.", - "BCD_SIEVE": "The casualty is ambulatory and has no visible catastrophic bleeding. According to Step 2 of the BCD Sieve protocol, a patient who can walk is tagged as Green. Since no life-threatening signs are present, the Green category (Minor) is appropriate." + "BCD_SIEVE": "The casualty is ambulatory and has no visible catastrophic bleeding. According to Step 2 of the protocol, a patient who can walk is tagged as Green. Since no life-threatening signs are present, the Green category is appropriate." }, {}, {}, @@ -72,12 +72,12 @@ "choices": [ { "action_id": 0, - "unstructured": "Green (Minor)", + "unstructured": "Green", "kdma_association": {} }, { "action_id": 1, - "unstructured": "Yellow (Delayed)", + "unstructured": "Yellow", "kdma_association": { "SALT": 1.0, "BCD_SIEVE": 1.0 @@ -85,14 +85,14 @@ }, { "action_id": 2, - "unstructured": "Red (Immediate)", + "unstructured": "Red", "kdma_association": { "START": 1.0 } }, { "action_id": 3, - "unstructured": "Black (Expectant)", + "unstructured": "Black", "kdma_association": {} } ] @@ -112,10 +112,10 @@ {}, { "SALT": "Patient is non-ambulatory but moves to voice, has normal breathing, pulse, and cap refill, with no visible injuries or hemorrhage. All individual assessment criteria are met, but inability to walk and reduced alertness suggest more than minor injury. Condition is stable but not minor, so treatment can be delayed safely.", - "BCD_SIEVE": "No catastrophic bleeding; patient unable to walk, so not Green. Breathing normal and responsive to voice (not unconscious), vital signs normal. According to BCD Sieve, responsive patients with stable vitals who can't walk are tagged Yellow (Delayed)." + "BCD_SIEVE": "No catastrophic bleeding; patient unable to walk, so not Green. Breathing normal and responsive to voice (not unconscious), vital signs normal. According to protocol, responsive patients with stable vitals who can't walk are tagged Yellow." }, { - "START": "The casualty is non-ambulatory and not alert, which raises concern. Although breathing, heart rate, and capillary refill are normal, the inability to follow simple commands (not alert) meets the START protocol criteria for Red. Moving in response to voice indicates some responsiveness but not enough to follow commands, so immediate care is needed." + "START": "The casualty is non-ambulatory and not alert, which raises concern. Although breathing, heart rate, and capillary refill are normal, the inability to follow simple commands (not alert) meets the protocol criteria for Red. Moving in response to voice indicates some responsiveness but not enough to follow commands, so immediate care is needed." }, {} ] @@ -134,19 +134,19 @@ "choices": [ { "action_id": 0, - "unstructured": "Green (Minor)", + "unstructured": "Green", "kdma_association": {} }, { "action_id": 1, - "unstructured": "Yellow (Delayed)", + "unstructured": "Yellow", "kdma_association": { "BCD_SIEVE": 1.0 } }, { "action_id": 2, - "unstructured": "Red (Immediate)", + "unstructured": "Red", "kdma_association": { "START": 1.0, "SALT": 1.0 @@ -154,7 +154,7 @@ }, { "action_id": 3, - "unstructured": "Black (Expectant)", + "unstructured": "Black", "kdma_association": {} } ] @@ -173,11 +173,11 @@ "reasoning": [ {}, { - "BCD_SIEVE": "No catastrophic bleeding; patient unable to walk (Step 2: proceed). Patient not alert but responsive to voice (Step 4: proceed). Breathing rapid but within normal range (10-29), normal heart rate and capillary refill (Steps 5 and 6: normal), so tag as Yellow (Delayed)." + "BCD_SIEVE": "No catastrophic bleeding; patient unable to walk (Step 2: proceed). Patient not alert but responsive to voice (Step 4: proceed). Breathing rapid but within normal range (10-29), normal heart rate and capillary refill (Steps 5 and 6: normal), so tag as Yellow." }, { - "START": "The casualty is non-ambulatory and breathing rapidly with distress. According to the START protocol, a respiratory rate over 30 breaths per minute or distressed breathing automatically qualifies as Red. Additionally, the casualty is not alert and unable to follow commands, reinforcing the need for immediate intervention.", - "SALT": "Patient is non-ambulatory, not alert, but moves to voice. Breathing is rapid and distressed, indicating respiratory compromise. Despite normal heart rate and capillary refill, respiratory distress meets the SALT criteria for immediate care. Survival is likely with prompt intervention." + "START": "The casualty is non-ambulatory and breathing rapidly with distress. According to the protocol, a respiratory rate over 30 breaths per minute or distressed breathing automatically qualifies as Red. Additionally, the casualty is not alert and unable to follow commands, reinforcing the need for immediate intervention.", + "SALT": "Patient is non-ambulatory, not alert, but moves to voice. Breathing is rapid and distressed, indicating respiratory compromise. Despite normal heart rate and capillary refill, respiratory distress meets the criteria for immediate care. Survival is likely with prompt intervention." }, {} ] @@ -196,17 +196,17 @@ "choices": [ { "action_id": 0, - "unstructured": "Green (Minor)", + "unstructured": "Green", "kdma_association": {} }, { "action_id": 1, - "unstructured": "Yellow (Delayed)", + "unstructured": "Yellow", "kdma_association": {} }, { "action_id": 2, - "unstructured": "Red (Immediate)", + "unstructured": "Red", "kdma_association": { "START": 1.0, "SALT": 1.0, @@ -215,7 +215,7 @@ }, { "action_id": 3, - "unstructured": "Black (Expectant)", + "unstructured": "Black", "kdma_association": {} } ] @@ -234,7 +234,7 @@ {}, {}, { - "START": "The casualty is non-ambulatory, unresponsive to stimuli, breathing rapidly with distress, and shows signs of poor perfusion (no peripheral pulse and poor capillary refill). According to the START protocol, rapid breathing over 30 breaths per minute and poor perfusion classify the patient as Red, indicating life-threatening injuries requiring immediate intervention.", + "START": "The casualty is non-ambulatory, unresponsive to stimuli, breathing rapidly with distress, and shows signs of poor perfusion (no peripheral pulse and poor capillary refill). According to the protocol, rapid breathing over 30 breaths per minute and poor perfusion classify the patient as Red, indicating life-threatening injuries requiring immediate intervention.", "SALT": "Casualty does not obey commands or make purposeful movements.\nCasualty is likely to survive these injuries.", "BCD_SIEVE": "No catastrophic bleeding; patient unable to walk. Patient is unresponsive to stimuli (Step 4: unconscious/unresponsive -> Red)." }, @@ -255,17 +255,17 @@ "choices": [ { "action_id": 0, - "unstructured": "Green (Minor)", + "unstructured": "Green", "kdma_association": {} }, { "action_id": 1, - "unstructured": "Yellow (Delayed)", + "unstructured": "Yellow", "kdma_association": {} }, { "action_id": 2, - "unstructured": "Red (Immediate)", + "unstructured": "Red", "kdma_association": { "START": 1.0, "SALT": 1.0, @@ -274,7 +274,7 @@ }, { "action_id": 3, - "unstructured": "Black (Expectant)", + "unstructured": "Black", "kdma_association": {} } ] @@ -293,8 +293,8 @@ {}, {}, { - "START": "The casualty is non-ambulatory and unresponsive, with very rapid, shallow, and distressed breathing. They also have no peripheral pulse and poor capillary refill, indicating compromised perfusion. According to the START protocol, these signs—rapid breathing over 30 breaths per minute and poor perfusion—mandate a Red tag for immediate care.", - "SALT": "Patient is unresponsive, in respiratory distress, with no peripheral pulse and poor capillary refill—indicating critical condition. However, with sufficient resources available, survival may be possible with immediate intervention. Meets SALT criteria for Red due to life-threatening issues and potential for survival with prompt care.", + "START": "The casualty is non-ambulatory and unresponsive, with very rapid, shallow, and distressed breathing. They also have no peripheral pulse and poor capillary refill, indicating compromised perfusion. According to the protocol, these signs—rapid breathing over 30 breaths per minute and poor perfusion—mandate a Red tag for immediate care.", + "SALT": "Patient is unresponsive, in respiratory distress, with no peripheral pulse and poor capillary refill—indicating critical condition. However, with sufficient resources available, survival may be possible with immediate intervention. Meets criteria for Red due to life-threatening issues and potential for survival with prompt care.", "BCD_SIEVE": "No catastrophic bleeding; patient unable to walk. Patient is unresponsive (Step 4: unconscious -> Red)." }, {} @@ -314,19 +314,19 @@ "choices": [ { "action_id": 0, - "unstructured": "Green (Minor)", + "unstructured": "Green", "kdma_association": {} }, { "action_id": 1, - "unstructured": "Yellow (Delayed)", + "unstructured": "Yellow", "kdma_association": { "SALT": 1.0 } }, { "action_id": 2, - "unstructured": "Red (Immediate)", + "unstructured": "Red", "kdma_association": { "START": 1.0, "BCD_SIEVE": 1.0 @@ -334,7 +334,7 @@ }, { "action_id": 3, - "unstructured": "Black (Expectant)", + "unstructured": "Black", "kdma_association": {} } ] @@ -353,7 +353,7 @@ "reasoning": [ {}, { - "SALT": "Patient is non-ambulatory, not alert but moves to voice. Breathing is slow but not distressed. Peripheral perfusion is adequate (normal heart rate and capillary refill). No visible injuries or hemorrhage. Meets SALT criteria for delayed treatment—serious but not immediately life-threatening condition." + "SALT": "Patient is non-ambulatory, not alert but moves to voice. Breathing is slow but not distressed. Peripheral perfusion is adequate (normal heart rate and capillary refill). No visible injuries or hemorrhage. Meets criteria for delayed treatment—serious but not immediately life-threatening condition." }, { "START": "The casualty is non-ambulatory, breathing within normal limits, and has normal perfusion, but is unable to follow simple commands. This meets the criteria for Red, indicating immediate intervention is required.", @@ -376,17 +376,17 @@ "choices": [ { "action_id": 0, - "unstructured": "Green (Minor)", + "unstructured": "Green", "kdma_association": {} }, { "action_id": 1, - "unstructured": "Yellow (Delayed)", + "unstructured": "Yellow", "kdma_association": {} }, { "action_id": 2, - "unstructured": "Red (Immediate)", + "unstructured": "Red", "kdma_association": { "START": 1.0, "SALT": 1.0, @@ -395,7 +395,7 @@ }, { "action_id": 3, - "unstructured": "Black (Expectant)", + "unstructured": "Black", "kdma_association": {} } ] @@ -414,9 +414,9 @@ {}, {}, { - "START": "The casualty is non-ambulatory and not alert, moving only in response to voice, indicating they cannot follow simple commands. Although breathing normally, the absence of a peripheral pulse and poor capillary refill indicate poor perfusion. According to the START protocol, poor perfusion and inability to follow commands qualify for a Red tag, requiring immediate care.", - "SALT": "Patient is non-ambulatory, not alert but responsive to voice, breathing normally, but has no peripheral pulse and poor capillary refill—signs of possible shock or circulatory failure. Despite stable airway, impaired perfusion indicates life-threatening condition requiring immediate care. Meets SALT criteria for Red.", - "BCD_SIEVE": "No catastrophic bleeding; patient unable to walk. Patient is not alert but responsive to voice (Step 4: proceed). Breathing normal (Step 5: normal rate), but no peripheral pulse and poor capillary refill (Step 6), indicating circulatory compromise. According to Step 6, abnormal circulation tags patient as Red (Immediate)." + "START": "The casualty is non-ambulatory and not alert, moving only in response to voice, indicating they cannot follow simple commands. Although breathing normally, the absence of a peripheral pulse and poor capillary refill indicate poor perfusion. According to the protocol, poor perfusion and inability to follow commands qualify for a Red tag, requiring immediate care.", + "SALT": "Patient is non-ambulatory, not alert but responsive to voice, breathing normally, but has no peripheral pulse and poor capillary refill—signs of possible shock or circulatory failure. Despite stable airway, impaired perfusion indicates life-threatening condition requiring immediate care. Meets criteria for Red.", + "BCD_SIEVE": "No catastrophic bleeding; patient unable to walk. Patient is not alert but responsive to voice (Step 4: proceed). Breathing normal (Step 5: normal rate), but no peripheral pulse and poor capillary refill (Step 6), indicating circulatory compromise. According to Step 6, abnormal circulation tags patient as Red." }, {} ] @@ -435,17 +435,17 @@ "choices": [ { "action_id": 0, - "unstructured": "Green (Minor)", + "unstructured": "Green", "kdma_association": {} }, { "action_id": 1, - "unstructured": "Yellow (Delayed)", + "unstructured": "Yellow", "kdma_association": {} }, { "action_id": 2, - "unstructured": "Red (Immediate)", + "unstructured": "Red", "kdma_association": { "START": 1.0, "BCD_SIEVE": 1.0, @@ -454,7 +454,7 @@ }, { "action_id": 3, - "unstructured": "Black (Expectant)", + "unstructured": "Black", "kdma_association": {} } ] @@ -473,9 +473,9 @@ {}, {}, { - "START": "The casualty is non-ambulatory and not alert, only moving in response to voice, which indicates they cannot follow simple commands. Although breathing normally and having a healthy capillary refill, the presence of tachycardia (elevated heart rate) suggests potential underlying serious condition. Most importantly, inability to follow commands meets the criteria for Red in the START protocol, indicating the need for immediate intervention.", + "START": "The casualty is non-ambulatory and not alert, only moving in response to voice, which indicates they cannot follow simple commands. Although breathing normally and having a healthy capillary refill, the presence of tachycardia (elevated heart rate) suggests potential underlying serious condition. Most importantly, inability to follow commands meets the criteria for Red in the protocol, indicating the need for immediate intervention.", "SALT": "Patient is non-ambulatory, not fully alert but moves to voice, breathing normally. However, absence of peripheral pulse indicates possible circulatory compromise despite normal capillary refill. This life-threatening sign requires immediate intervention, fitting the Red category.", - "BCD_SIEVE": "No catastrophic bleeding; patient unable to walk. Patient not alert but responsive to voice (Step 4: proceed). Breathing normal (Step 5: normal). No peripheral pulse (Step 6: abnormal circulation) but capillary refill time is healthy. Absence of peripheral pulse alone indicates circulatory compromise, tagging patient as Red (Immediate)." + "BCD_SIEVE": "No catastrophic bleeding; patient unable to walk. Patient not alert but responsive to voice (Step 4: proceed). Breathing normal (Step 5: normal). No peripheral pulse (Step 6: abnormal circulation) but capillary refill time is healthy. Absence of peripheral pulse alone indicates circulatory compromise, tagging patient as Red." }, {} ] @@ -494,17 +494,17 @@ "choices": [ { "action_id": 0, - "unstructured": "Green (Minor)", + "unstructured": "Green", "kdma_association": {} }, { "action_id": 1, - "unstructured": "Yellow (Delayed)", + "unstructured": "Yellow", "kdma_association": {} }, { "action_id": 2, - "unstructured": "Red (Immediate)", + "unstructured": "Red", "kdma_association": { "START": 1.0, "SALT": 1.0, @@ -513,7 +513,7 @@ }, { "action_id": 3, - "unstructured": "Black (Expectant)", + "unstructured": "Black", "kdma_association": {} } ] @@ -532,9 +532,9 @@ {}, {}, { - "START": "The casualty is non-ambulatory with severe hemorrhaging on the torso and is not alert, only moving in response to voice. Despite normal breathing, heart rate, and capillary refill, the severe hemorrhage and inability to follow commands classify the casualty as Red under the START protocol, indicating immediate life-saving intervention is necessary.", + "START": "The casualty is non-ambulatory with severe hemorrhaging on the torso and is not alert, only moving in response to voice. Despite normal breathing, heart rate, and capillary refill, the severe hemorrhage and inability to follow commands classify the casualty as Red under the protocol, indicating immediate life-saving intervention is necessary.", "SALT": "Severe hemorrhaging on the torso is a major life-threatening injury requiring immediate intervention. Patient is non-ambulatory, not alert but responsive to voice, breathing normally, and has stable vitals. Given the uncontrolled major hemorrhage and likelihood of survival with prompt care, the patient fits the Red category", - "BCD_SIEVE": "Severe hemorrhaging on the torso indicates catastrophic bleeding (Step 1), requiring immediate tourniquet or pressure. This alone mandates tagging the casualty as Red (Immediate)." + "BCD_SIEVE": "Severe hemorrhaging on the torso indicates catastrophic bleeding (Step 1), requiring immediate tourniquet or pressure. This alone mandates tagging the casualty as Red." }, {} ] @@ -553,12 +553,12 @@ "choices": [ { "action_id": 0, - "unstructured": "Green (Minor)", + "unstructured": "Green", "kdma_association": {} }, { "action_id": 1, - "unstructured": "Yellow (Delayed)", + "unstructured": "Yellow", "kdma_association": { "START": 1.0, "BCD_SIEVE": 1.0, @@ -567,12 +567,12 @@ }, { "action_id": 2, - "unstructured": "Red (Immediate)", + "unstructured": "Red", "kdma_association": {} }, { "action_id": 3, - "unstructured": "Black (Expectant)", + "unstructured": "Black", "kdma_association": {} } ] @@ -591,8 +591,8 @@ {}, { "START": "The casualty is non-ambulatory with a moderate abdominal hemorrhage that has been controlled by packing. They are alert, breathing normally, and have normal heart rate and capillary refill. Since they are able to follow commands and have stable vital signs, they fit the Yellow category, meaning treatment can be safely delayed but is still serious.", - "SALT": "Casualty is alert, breathing normally, with stable heart rate and capillary refill. Moderate abdominal hemorrhage has been controlled with gauze. No immediate life threat remains, so treatment can be delayed safely. Meets SALT criteria for Yellow.", - "BCD_SIEVE": "Moderate hemorrhage controlled by packing suggests no active catastrophic bleeding (Step 1: absent). Patient unable to walk (Step 2: not Green). Alert and breathing normally, with normal heart rate and capillary refill, indicates stable condition. According to BCD Sieve, stable non-ambulatory patients without critical signs are tagged Yellow (Delayed)." + "SALT": "Casualty is alert, breathing normally, with stable heart rate and capillary refill. Moderate abdominal hemorrhage has been controlled with gauze. No immediate life threat remains, so treatment can be delayed safely. Meets criteria for Yellow.", + "BCD_SIEVE": "Moderate hemorrhage controlled by packing suggests no active catastrophic bleeding (Step 1: absent). Patient unable to walk (Step 2: not Green). Alert and breathing normally, with normal heart rate and capillary refill, indicates stable condition. According to protocol, stable non-ambulatory patients without critical signs are tagged Yellow." }, {}, {} @@ -612,17 +612,17 @@ "choices": [ { "action_id": 0, - "unstructured": "Green (Minor)", + "unstructured": "Green", "kdma_association": {} }, { "action_id": 1, - "unstructured": "Yellow (Delayed)", + "unstructured": "Yellow", "kdma_association": {} }, { "action_id": 2, - "unstructured": "Red (Immediate)", + "unstructured": "Red", "kdma_association": { "START": 1.0, "SALT": 1.0, @@ -631,7 +631,7 @@ }, { "action_id": 3, - "unstructured": "Black (Expectant)", + "unstructured": "Black", "kdma_association": {} } ] @@ -650,7 +650,7 @@ {}, {}, { - "START": "The casualty was initially not breathing but resumed breathing after airway repositioning, which meets the START protocol criteria for Red. They are unresponsive and non-ambulatory but have normal heart rate and capillary refill. Immediate intervention is required to address their life-threatening condition.", + "START": "The casualty was initially not breathing but resumed breathing after airway repositioning, which meets the protocol criteria for Red. They are unresponsive and non-ambulatory but have normal heart rate and capillary refill. Immediate intervention is required to address their life-threatening condition.", "SALT": "Patient is non-ambulatory and unresponsive, but started breathing spontaneously after airway repositioning, indicating a compromised airway initially. Normal heart rate and capillary refill suggest circulation is intact. Immediate intervention is needed to maintain airway and monitor. Meets criteria for Red due to life-threatening airway compromise with potential for survival.", "BCD_SIEVE": "No catastrophic bleeding. Unable to walk (Step 2: proceed). Initially not breathing, but airway repositioning restored breathing (Step 3). Patient is unresponsive (Step 4: unconscious -> Red)." }, @@ -671,22 +671,22 @@ "choices": [ { "action_id": 0, - "unstructured": "Green (Minor)", + "unstructured": "Green", "kdma_association": {} }, { "action_id": 1, - "unstructured": "Yellow (Delayed)", + "unstructured": "Yellow", "kdma_association": {} }, { "action_id": 2, - "unstructured": "Red (Immediate)", + "unstructured": "Red", "kdma_association": {} }, { "action_id": 3, - "unstructured": "Black (Expectant)", + "unstructured": "Black", "kdma_association": { "START": 1.0, "SALT": 1.0, @@ -710,9 +710,9 @@ {}, {}, { - "START": "The casualty is not breathing and airway repositioning did not restore breathing, which according to the START protocol classifies them as Black. Despite normal heart rate and capillary refill, the absence of breathing after airway repositioning indicates the casualty is unlikely to survive even with treatment.", - "SALT": "Patient is unresponsive and not breathing even after airway repositioning, indicating no spontaneous respiration. Despite normal heart rate and capillary refill, absence of breathing after intervention meets SALT criteria for Black, as survival is unlikely even with maximal care.", - "BCD_SIEVE": "No catastrophic bleeding. Unable to walk (Step 2: proceed). Not breathing, and airway repositioning did not restore breathing (Step 3). According to BCD Sieve, if still not breathing after airway opened, tag as Black (Expectant)." + "START": "The casualty is not breathing and airway repositioning did not restore breathing, which according to the protocol classifies them as Black. Despite normal heart rate and capillary refill, the absence of breathing after airway repositioning indicates the casualty is unlikely to survive even with treatment.", + "SALT": "Patient is unresponsive and not breathing even after airway repositioning, indicating no spontaneous respiration. Despite normal heart rate and capillary refill, absence of breathing after intervention meets criteria for Black, as survival is unlikely even with maximal care.", + "BCD_SIEVE": "No catastrophic bleeding. Unable to walk (Step 2: proceed). Not breathing, and airway repositioning did not restore breathing (Step 3). According to protocol, if still not breathing after airway opened, tag as Black." } ] } diff --git a/align_system/resources/icl/tagging/train3_shape.json b/align_system/resources/icl/tagging/train3_shape.json new file mode 100644 index 00000000..951147ef --- /dev/null +++ b/align_system/resources/icl/tagging/train3_shape.json @@ -0,0 +1,719 @@ +[ + { + "input": { + "scenario_id": "example_icl", + "full_state": { + "unstructured": "There is no visible injury to the casualty.There is not a visible hemorrhage.\nThe casualty is able to walk.", + "meta_info": { + "scene_id": "Casualty_0_at_5.0min" + }, + "scenario_complete": false + }, + "state": "There is no visible injury to the casualty.There is not a visible hemorrhage.\nThe casualty is able to walk.", + "choices": [ + { + "action_id": 0, + "unstructured": "Line", + "kdma_association": { + "START": 1.0, + "SALT": 1.0, + "BCD_SIEVE": 1.0 + } + }, + { + "action_id": 1, + "unstructured": "Triangle", + "kdma_association": {} + }, + { + "action_id": 2, + "unstructured": "Circle", + "kdma_association": {} + }, + { + "action_id": 3, + "unstructured": "Square", + "kdma_association": {} + } + ] + }, + "label": [ + { + "START": 1.0, + "SALT": 1.0, + "BCD_SIEVE": 1.0 + }, + {}, + {}, + {} + ], + "reasoning": [ + { + "START": "The patient is ambulatory and able to walk, which according to the protocol, immediately classifies them as Line. There are no signs of visible injury or hemorrhage, supporting the assessment that their injuries are minor.", + "SALT": "The casualty is able to walk, which meets the Global Sorting criterion to tag as Line. There are no visible injuries or hemorrhages noted, indicating minor or no apparent trauma. Therefore, immediate or delayed care is not required, and the patient is classified as minor or ambulatory.", + "BCD_SIEVE": "The casualty is ambulatory and has no visible catastrophic bleeding. According to Step 2 of the the protocol protocol, a patient who can walk is tagged as Line. Since no life-threatening signs are present, the Line category is appropriate." + }, + {}, + {}, + {} + ] + }, + { + "input": { + "scenario_id": "example_icl", + "full_state": { + "unstructured": "There is no visible injury to the casualty.There is not a visible hemorrhage.\nThe casualty is unable to walk.\nThe casualty is not alert.\nThey do move in response to your voice.\nThe casualty is breathing normally.\nThe casualty has a normal heart rate.\nThe casualty has a healthy capillary refill time.", + "meta_info": { + "scene_id": "Casualty_3_at_5.0min" + }, + "scenario_complete": false + }, + "state": "There is no visible injury to the casualty.There is not a visible hemorrhage.\nThe casualty is unable to walk.\nThe casualty is not alert.\nThey do move in response to your voice.\nThe casualty is breathing normally.\nThe casualty has a normal heart rate.\nThe casualty has a healthy capillary refill time.", + "choices": [ + { + "action_id": 0, + "unstructured": "Line", + "kdma_association": {} + }, + { + "action_id": 1, + "unstructured": "Triangle", + "kdma_association": { + "SALT": 1.0, + "BCD_SIEVE": 1.0 + } + }, + { + "action_id": 2, + "unstructured": "Circle", + "kdma_association": { + "START": 1.0 + } + }, + { + "action_id": 3, + "unstructured": "Square", + "kdma_association": {} + } + ] + }, + "label": [ + {}, + { + "SALT": 1.0, + "BCD_SIEVE": 1.0 + }, + { + "START": 1.0 + }, + {} + ], + "reasoning": [ + {}, + { + "SALT": "Patient is non-ambulatory but moves to voice, has normal breathing, pulse, and cap refill, with no visible injuries or hemorrhage. All individual assessment criteria are met, but inability to walk and reduced alertness suggest more than minor injury. Condition is stable but not minor, so treatment can be delayed safely.", + "BCD_SIEVE": "No catastrophic bleeding; patient unable to walk, so not Line. Breathing normal and responsive to voice (not unconscious), vital signs normal. According to the protocol, responsive patients with stable vitals who can't walk are tagged Triangle." + }, + { + "START": "The casualty is non-ambulatory and not alert, which raises concern. Although breathing, heart rate, and capillary refill are normal, the inability to follow simple commands (not alert) meets the protocol criteria for Circle. Moving in response to voice indicates some responsiveness but not enough to follow commands, so immediate care is needed." + }, + {} + ] + }, + { + "input": { + "scenario_id": "example_icl", + "full_state": { + "unstructured": "There is no visible injury to the casualty.There is not a visible hemorrhage.\nThe casualty is unable to walk.\nThe casualty is not alert.\nThey do move in response to your voice.\nThe casualty is breathing rapidly.\nThe casualty's breathing is distressed.\nThe casualty has a normal heart rate.\nThe casualty has a healthy capillary refill time.", + "meta_info": { + "scene_id": "Casualty_4_at_5.0min" + }, + "scenario_complete": false + }, + "state": "There is no visible injury to the casualty.There is not a visible hemorrhage.\nThe casualty is unable to walk.\nThe casualty is not alert.\nThey do move in response to your voice.\nThe casualty is breathing rapidly.\nThe casualty's breathing is distressed.\nThe casualty has a normal heart rate.\nThe casualty has a healthy capillary refill time.", + "choices": [ + { + "action_id": 0, + "unstructured": "Line", + "kdma_association": {} + }, + { + "action_id": 1, + "unstructured": "Triangle", + "kdma_association": { + "BCD_SIEVE": 1.0 + } + }, + { + "action_id": 2, + "unstructured": "Circle", + "kdma_association": { + "START": 1.0, + "SALT": 1.0 + } + }, + { + "action_id": 3, + "unstructured": "Square", + "kdma_association": {} + } + ] + }, + "label": [ + {}, + { + "BCD_SIEVE": 1.0 + }, + { + "START": 1.0, + "SALT": 1.0 + }, + {} + ], + "reasoning": [ + {}, + { + "BCD_SIEVE": "No catastrophic bleeding; patient unable to walk (Step 2: proceed). Patient not alert but responsive to voice (Step 4: proceed). Breathing rapid but within normal range (10-29), normal heart rate and capillary refill (Steps 5 and 6: normal), so tag as Triangle." + }, + { + "START": "The casualty is non-ambulatory and breathing rapidly with distress. According to the protocol, a respiratory rate over 30 breaths per minute or distressed breathing automatically qualifies as Circle. Additionally, the casualty is not alert and unable to follow commands, reinforcing the need for immediate intervention.", + "SALT": "Patient is non-ambulatory, not alert, but moves to voice. Breathing is rapid and distressed, indicating respiratory compromise. Despite normal heart rate and capillary refill, respiratory distress meets the criteria for immediate care. Survival is likely with prompt intervention." + }, + {} + ] + }, + { + "input": { + "scenario_id": "example_icl", + "full_state": { + "unstructured": "There is no visible injury to the casualty.There is not a visible hemorrhage.\nThe casualty is unable to walk.\nThe casualty is unresponsive to any stimuli.\nThe casualty is breathing rapidly.\nThe casualty's breathing is distressed.\nCasualty does not have a peripheral pulse.\nThe casualty has a poor capillary refill time.", + "meta_info": { + "scene_id": "Casualty_6_at_!5.0min" + }, + "scenario_complete": false + }, + "state": "There is no visible injury to the casualty.There is not a visible hemorrhage.\nThe casualty is unable to walk.\nThe casualty is unresponsive to any stimuli.\nThe casualty is breathing rapidly.\nThe casualty's breathing is distressed.\nCasualty does not have a peripheral pulse.\nThe casualty has a poor capillary refill time.", + "choices": [ + { + "action_id": 0, + "unstructured": "Line", + "kdma_association": {} + }, + { + "action_id": 1, + "unstructured": "Triangle", + "kdma_association": {} + }, + { + "action_id": 2, + "unstructured": "Circle", + "kdma_association": { + "START": 1.0, + "SALT": 1.0, + "BCD_SIEVE": 1.0 + } + }, + { + "action_id": 3, + "unstructured": "Square", + "kdma_association": {} + } + ] + }, + "label": [ + {}, + {}, + { + "START": 1.0, + "SALT": 1.0, + "BCD_SIEVE": 1.0 + }, + {} + ], + "reasoning": [ + {}, + {}, + { + "START": "The casualty is non-ambulatory, unresponsive to stimuli, breathing rapidly with distress, and shows signs of poor perfusion (no peripheral pulse and poor capillary refill). According to the protocol, rapid breathing over 30 breaths per minute and poor perfusion classify the patient as Circle, indicating life-threatening injuries requiring immediate intervention.", + "SALT": "Casualty does not obey commands or make purposeful movements.\nCasualty is likely to survive these injuries.", + "BCD_SIEVE": "No catastrophic bleeding; patient unable to walk. Patient is unresponsive to stimuli (Step 4: unconscious/unresponsive -> Circle)." + }, + {} + ] + }, + { + "input": { + "scenario_id": "example_icl", + "full_state": { + "unstructured": "There is no visible injury to the casualty.There is not a visible hemorrhage.\nThe casualty is unable to walk.\nThe casualty is unresponsive to any stimuli.\nThe casualty's breathing is very rapid and shallow.\nThe casualty's breathing is distressed.\nCasualty does not have a peripheral pulse.\nThe casualty has a poor capillary refill time.", + "meta_info": { + "scene_id": "Casualty_7_at_!10.0min" + }, + "scenario_complete": false + }, + "state": "There is no visible injury to the casualty.There is not a visible hemorrhage.\nThe casualty is unable to walk.\nThe casualty is unresponsive to any stimuli.\nThe casualty's breathing is very rapid and shallow.\nThe casualty's breathing is distressed.\nCasualty does not have a peripheral pulse.\nThe casualty has a poor capillary refill time.", + "choices": [ + { + "action_id": 0, + "unstructured": "Line", + "kdma_association": {} + }, + { + "action_id": 1, + "unstructured": "Triangle", + "kdma_association": {} + }, + { + "action_id": 2, + "unstructured": "Circle", + "kdma_association": { + "START": 1.0, + "SALT": 1.0, + "BCD_SIEVE": 1.0 + } + }, + { + "action_id": 3, + "unstructured": "Square", + "kdma_association": {} + } + ] + }, + "label": [ + {}, + {}, + { + "START": 1.0, + "SALT": 1.0, + "BCD_SIEVE": 1.0 + }, + {} + ], + "reasoning": [ + {}, + {}, + { + "START": "The casualty is non-ambulatory and unresponsive, with very rapid, shallow, and distressed breathing. They also have no peripheral pulse and poor capillary refill, indicating compromised perfusion. According to the protocol, these signs—rapid breathing over 30 breaths per minute and poor perfusion—mandate a Circle tag for immediate care.", + "SALT": "Patient is unresponsive, in respiratory distress, with no peripheral pulse and poor capillary refill—indicating critical condition. However, with sufficient resources available, survival may be possible with immediate intervention. Meets criteria for Circle due to life-threatening issues and potential for survival with prompt care.", + "BCD_SIEVE": "No catastrophic bleeding; patient unable to walk. Patient is unresponsive (Step 4: unconscious -> Circle)." + }, + {} + ] + }, + { + "input": { + "scenario_id": "example_icl", + "full_state": { + "unstructured": "There is no visible injury to the casualty.There is not a visible hemorrhage.\nThe casualty is unable to walk.\nThe casualty is not alert.\nThey do move in response to your voice.\nThe casualty is breathing slowly.\nThe casualty has a normal heart rate.\nThe casualty has a healthy capillary refill time.", + "meta_info": { + "scene_id": "Casualty_12_at_5.0min" + }, + "scenario_complete": false + }, + "state": "There is no visible injury to the casualty.There is not a visible hemorrhage.\nThe casualty is unable to walk.\nThe casualty is not alert.\nThey do move in response to your voice.\nThe casualty is breathing slowly.\nThe casualty has a normal heart rate.\nThe casualty has a healthy capillary refill time.", + "choices": [ + { + "action_id": 0, + "unstructured": "Line", + "kdma_association": {} + }, + { + "action_id": 1, + "unstructured": "Triangle", + "kdma_association": { + "SALT": 1.0 + } + }, + { + "action_id": 2, + "unstructured": "Circle", + "kdma_association": { + "START": 1.0, + "BCD_SIEVE": 1.0 + } + }, + { + "action_id": 3, + "unstructured": "Square", + "kdma_association": {} + } + ] + }, + "label": [ + {}, + { + "SALT": 1.0 + }, + { + "START": 1.0, + "BCD_SIEVE": 1.0 + }, + {} + ], + "reasoning": [ + {}, + { + "SALT": "Patient is non-ambulatory, not alert but moves to voice. Breathing is slow but not distressed. Peripheral perfusion is adequate (normal heart rate and capillary refill). No visible injuries or hemorrhage. Meets criteria for delayed treatment—serious but not immediately life-threatening condition." + }, + { + "START": "The casualty is non-ambulatory, breathing within normal limits, and has normal perfusion, but is unable to follow simple commands. This meets the criteria for Circle, indicating immediate intervention is required.", + "BCD_SIEVE": "No catastrophic bleeding; unable to walk. Patient not alert but responsive to voice (responsive). Slow breathing (<10 breaths/min) per Step 5 triggers Circle tag." + }, + {} + ] + }, + { + "input": { + "scenario_id": "example_icl", + "full_state": { + "unstructured": "There is no visible injury to the casualty.There is not a visible hemorrhage.\nThe casualty is unable to walk.\nThe casualty is not alert.\nThey do move in response to your voice.\nThe casualty is breathing normally.\nCasualty does not have a peripheral pulse.\nThe casualty has a poor capillary refill time.", + "meta_info": { + "scene_id": "Casualty_22_at_10.0min" + }, + "scenario_complete": false + }, + "state": "There is no visible injury to the casualty.There is not a visible hemorrhage.\nThe casualty is unable to walk.\nThe casualty is not alert.\nThey do move in response to your voice.\nThe casualty is breathing normally.\nCasualty does not have a peripheral pulse.\nThe casualty has a poor capillary refill time.", + "choices": [ + { + "action_id": 0, + "unstructured": "Line", + "kdma_association": {} + }, + { + "action_id": 1, + "unstructured": "Triangle", + "kdma_association": {} + }, + { + "action_id": 2, + "unstructured": "Circle", + "kdma_association": { + "START": 1.0, + "SALT": 1.0, + "BCD_SIEVE": 1.0 + } + }, + { + "action_id": 3, + "unstructured": "Square", + "kdma_association": {} + } + ] + }, + "label": [ + {}, + {}, + { + "START": 1.0, + "SALT": 1.0, + "BCD_SIEVE": 1.0 + }, + {} + ], + "reasoning": [ + {}, + {}, + { + "START": "The casualty is non-ambulatory and not alert, moving only in response to voice, indicating they cannot follow simple commands. Although breathing normally, the absence of a peripheral pulse and poor capillary refill indicate poor perfusion. According to the protocol, poor perfusion and inability to follow commands qualify for a Circle tag, requiring immediate care.", + "SALT": "Patient is non-ambulatory, not alert but responsive to voice, breathing normally, but has no peripheral pulse and poor capillary refill—signs of possible shock or circulatory failure. Despite stable airway, impaired perfusion indicates life-threatening condition requiring immediate care. Meets criteria for Circle.", + "BCD_SIEVE": "No catastrophic bleeding; patient unable to walk. Patient is not alert but responsive to voice (Step 4: proceed). Breathing normal (Step 5: normal rate), but no peripheral pulse and poor capillary refill (Step 6), indicating circulatory compromise. According to Step 6, abnormal circulation tags patient as Circle." + }, + {} + ] + }, + { + "input": { + "scenario_id": "example_icl", + "full_state": { + "unstructured": "There is no visible injury to the casualty.There is not a visible hemorrhage.\nThe casualty is unable to walk.\nThe casualty is not alert.\nThey do move in response to your voice.\nThe casualty is breathing normally.\nCasualty does not have a peripheral pulse\nThe casualty has a healthy capillary refill time.", + "meta_info": { + "scene_id": "Casualty_23_at_10.0min" + }, + "scenario_complete": false + }, + "state": "There is no visible injury to the casualty.There is not a visible hemorrhage.\nThe casualty is unable to walk.\nThe casualty is not alert.\nThey do move in response to your voice.\nThe casualty is breathing normally.\nCasualty does not have a peripheral pulse\nThe casualty has a healthy capillary refill time.", + "choices": [ + { + "action_id": 0, + "unstructured": "Line", + "kdma_association": {} + }, + { + "action_id": 1, + "unstructured": "Triangle", + "kdma_association": {} + }, + { + "action_id": 2, + "unstructured": "Circle", + "kdma_association": { + "START": 1.0, + "BCD_SIEVE": 1.0, + "SALT": 1.0 + } + }, + { + "action_id": 3, + "unstructured": "Square", + "kdma_association": {} + } + ] + }, + "label": [ + {}, + {}, + { + "START": 1.0, + "SALT": 1.0, + "BCD_SIEVE": 1.0 + }, + {} + ], + "reasoning": [ + {}, + {}, + { + "START": "The casualty is non-ambulatory and not alert, only moving in response to voice, which indicates they cannot follow simple commands. Although breathing normally and having a healthy capillary refill, the presence of tachycardia (elevated heart rate) suggests potential underlying serious condition. Most importantly, inability to follow commands meets the criteria for Circle in the protocol, indicating the need for immediate intervention.", + "SALT": "Patient is non-ambulatory, not fully alert but moves to voice, breathing normally. However, absence of peripheral pulse indicates possible circulatory compromise despite normal capillary refill. This life-threatening sign requires immediate intervention, fitting the Circle category.", + "BCD_SIEVE": "No catastrophic bleeding; patient unable to walk. Patient not alert but responsive to voice (Step 4: proceed). Breathing normal (Step 5: normal). No peripheral pulse (Step 6: abnormal circulation) but capillary refill time is healthy. Absence of peripheral pulse alone indicates circulatory compromise, tagging patient as Circle." + }, + {} + ] + }, + { + "input": { + "scenario_id": "example_icl", + "full_state": { + "unstructured": "There is severe hemorrhaging on the torso.\nThe casualty is unable to walk.\nThe casualty is not alert.\nThey do move in response to your voice.\nThe casualty is breathing normally.\nThe casualty has a normal heart rate.\nThe casualty has a healthy capillary refill time.", + "meta_info": { + "scene_id": "Casualty_30_at_5.0min" + }, + "scenario_complete": false + }, + "state": "There is severe hemorrhaging on the torso.\nThe casualty is unable to walk.\nThe casualty is not alert.\nThey do move in response to your voice.\nThe casualty is breathing normally.\nThe casualty has a normal heart rate.\nThe casualty has a healthy capillary refill time.", + "choices": [ + { + "action_id": 0, + "unstructured": "Line", + "kdma_association": {} + }, + { + "action_id": 1, + "unstructured": "Triangle", + "kdma_association": {} + }, + { + "action_id": 2, + "unstructured": "Circle", + "kdma_association": { + "START": 1.0, + "SALT": 1.0, + "BCD_SIEVE": 1.0 + } + }, + { + "action_id": 3, + "unstructured": "Square", + "kdma_association": {} + } + ] + }, + "label": [ + {}, + {}, + { + "START": 1.0, + "SALT": 1.0, + "BCD_SIEVE": 1.0 + }, + {} + ], + "reasoning": [ + {}, + {}, + { + "START": "The casualty is non-ambulatory with severe hemorrhaging on the torso and is not alert, only moving in response to voice. Despite normal breathing, heart rate, and capillary refill, the severe hemorrhage and inability to follow commands classify the casualty as Circle under the protocol, indicating immediate life-saving intervention is necessary.", + "SALT": "Severe hemorrhaging on the torso is a major life-threatening injury requiring immediate intervention. Patient is non-ambulatory, not alert but responsive to voice, breathing normally, and has stable vitals. Given the uncontrolled major hemorrhage and likelihood of survival with prompt care, the patient fits the Circle category", + "BCD_SIEVE": "Severe hemorrhaging on the torso indicates catastrophic bleeding (Step 1), requiring immediate tourniquet or pressure. This alone mandates tagging the casualty as Circle." + }, + {} + ] + }, + { + "input": { + "scenario_id": "example_icl", + "full_state": { + "unstructured": "The casualty has a moderate hemorrhage from their abdomen.You were able to pack the wound with gauze to reduce the bleeding.\nThe casualty is unable to walk.\nThe casualty is alert.\nThe casualty is breathing normally.\nThe casualty has a normal heart rate.\nThe casualty has a healthy capillary refill time.", + "meta_info": { + "scene_id": "Casualty_53_at_5.0min" + }, + "scenario_complete": false + }, + "state": "The casualty has a moderate hemorrhage from their abdomen.You were able to pack the wound with gauze to reduce the bleeding.\nThe casualty is unable to walk.\nThe casualty is alert.\nThe casualty is breathing normally.\nThe casualty has a normal heart rate.\nThe casualty has a healthy capillary refill time.", + "choices": [ + { + "action_id": 0, + "unstructured": "Line", + "kdma_association": {} + }, + { + "action_id": 1, + "unstructured": "Triangle", + "kdma_association": { + "START": 1.0, + "BCD_SIEVE": 1.0, + "SALT": 1.0 + } + }, + { + "action_id": 2, + "unstructured": "Circle", + "kdma_association": {} + }, + { + "action_id": 3, + "unstructured": "Square", + "kdma_association": {} + } + ] + }, + "label": [ + {}, + { + "START": 1.0, + "SALT": 1.0, + "BCD_SIEVE": 1.0 + }, + {}, + {} + ], + "reasoning": [ + {}, + { + "START": "The casualty is non-ambulatory with a moderate abdominal hemorrhage that has been controlled by packing. They are alert, breathing normally, and have normal heart rate and capillary refill. Since they are able to follow commands and have stable vital signs, they fit the Triangle category, meaning treatment can be safely delayed but is still serious.", + "SALT": "Casualty is alert, breathing normally, with stable heart rate and capillary refill. Moderate abdominal hemorrhage has been controlled with gauze. No immediate life threat remains, so treatment can be delayed safely. Meets criteria for Triangle.", + "BCD_SIEVE": "Moderate hemorrhage controlled by packing suggests no active catastrophic bleeding (Step 1: absent). Patient unable to walk (Step 2: not Line). Alert and breathing normally, with normal heart rate and capillary refill, indicates stable condition. According to the protocol, stable non-ambulatory patients without critical signs are tagged Triangle." + }, + {}, + {} + ] + }, + { + "input": { + "scenario_id": "example_icl", + "full_state": { + "unstructured": "\nThe casualty is unable to walk.\nThe casualty is unresponsive to any stimuli.\nCasualty was not breathing.\nRepositioning their airway resulted in spontaneous breathing.\nThe casualty has a normal heart rate.\nThe casualty has a healthy capillary refill time.", + "meta_info": { + "scene_id": "Casualty_96_at_ao-imin" + }, + "scenario_complete": false + }, + "state": "\nThe casualty is unable to walk.\nThe casualty is unresponsive to any stimuli.\nCasualty was not breathing.\nRepositioning their airway resulted in spontaneous breathing.\nThe casualty has a normal heart rate.\nThe casualty has a healthy capillary refill time.", + "choices": [ + { + "action_id": 0, + "unstructured": "Line", + "kdma_association": {} + }, + { + "action_id": 1, + "unstructured": "Triangle", + "kdma_association": {} + }, + { + "action_id": 2, + "unstructured": "Circle", + "kdma_association": { + "START": 1.0, + "SALT": 1.0, + "BCD_SIEVE": 1.0 + } + }, + { + "action_id": 3, + "unstructured": "Square", + "kdma_association": {} + } + ] + }, + "label": [ + {}, + {}, + { + "START": 1.0, + "SALT": 1.0, + "BCD_SIEVE": 1.0 + }, + {} + ], + "reasoning": [ + {}, + {}, + { + "START": "The casualty was initially not breathing but resumed breathing after airway repositioning, which meets the protocol criteria for Circle. They are unresponsive and non-ambulatory but have normal heart rate and capillary refill. Immediate intervention is required to address their life-threatening condition.", + "SALT": "Patient is non-ambulatory and unresponsive, but started breathing spontaneously after airway repositioning, indicating a compromised airway initially. Normal heart rate and capillary refill suggest circulation is intact. Immediate intervention is needed to maintain airway and monitor. Meets criteria for Circle due to life-threatening airway compromise with potential for survival.", + "BCD_SIEVE": "No catastrophic bleeding. Unable to walk (Step 2: proceed). Initially not breathing, but airway repositioning restored breathing (Step 3). Patient is unresponsive (Step 4: unconscious -> Circle)." + }, + {} + ] + }, + { + "input": { + "scenario_id": "example_icl", + "full_state": { + "unstructured": "\nThe casualty is unable to walk.\nThe casualty is unresponsive to any stimuli.\nCasualty is not breathing.\nRepositioning their airway did not help breathing.\nThe casualty has a normal heart rate.\nThe casualty has a healthy capillary refill time.", + "meta_info": { + "scene_id": "Casualty_96_at_aomin" + }, + "scenario_complete": false + }, + "state": "\nThe casualty is unable to walk.\nThe casualty is unresponsive to any stimuli.\nCasualty is not breathing.\nRepositioning their airway did not help breathing.\nThe casualty has a normal heart rate.\nThe casualty has a healthy capillary refill time.", + "choices": [ + { + "action_id": 0, + "unstructured": "Line", + "kdma_association": {} + }, + { + "action_id": 1, + "unstructured": "Triangle", + "kdma_association": {} + }, + { + "action_id": 2, + "unstructured": "Circle", + "kdma_association": {} + }, + { + "action_id": 3, + "unstructured": "Square", + "kdma_association": { + "START": 1.0, + "SALT": 1.0, + "BCD_SIEVE": 1.0 + } + } + ] + }, + "label": [ + {}, + {}, + {}, + { + "START": 1.0, + "SALT": 1.0, + "BCD_SIEVE": 1.0 + } + ], + "reasoning": [ + {}, + {}, + {}, + { + "START": "The casualty is not breathing and airway repositioning did not restore breathing, which according to the protocol classifies them as Square. Despite normal heart rate and capillary refill, the absence of breathing after airway repositioning indicates the casualty is unlikely to survive even with treatment.", + "SALT": "Patient is unresponsive and not breathing even after airway repositioning, indicating no spontaneous respiration. Despite normal heart rate and capillary refill, absence of breathing after intervention meets criteria for Square, as survival is unlikely even with maximal care.", + "BCD_SIEVE": "No catastrophic bleeding. Unable to walk (Step 2: proceed). Not breathing, and airway repositioning did not restore breathing (Step 3). According to the protocol, if still not breathing after airway opened, tag as Square." + } + ] + } +] diff --git a/align_system/utils/alignment_utils.py b/align_system/utils/alignment_utils.py index bfd0496d..1c06af4a 100644 --- a/align_system/utils/alignment_utils.py +++ b/align_system/utils/alignment_utils.py @@ -591,6 +591,10 @@ def alignment_target_to_attribute_targets(alignment_target, attribute = attribute_definitions[t['kdma']] output_attribute_targets.append( - AttributeTarget(**dict(attribute), value=t['value'], parameters=t['parameters'] if 'parameters' in t else None)) + AttributeTarget( + **dict(attribute), + value=t['value'] if 'value' in t else None, + parameters=t['parameters'] if 'parameters' in t else None + )) return output_attribute_targets diff --git a/align_system/utils/incontext_utils.py b/align_system/utils/incontext_utils.py index beb2317b..965b83d4 100644 --- a/align_system/utils/incontext_utils.py +++ b/align_system/utils/incontext_utils.py @@ -5,6 +5,7 @@ import numpy as np from abc import ABCMeta, abstractmethod from bert_score import score as bert_score +from omegaconf import ListConfig, OmegaConf from align_system.utils import adm_utils from align_system.utils import outlines_prompts_utils @@ -223,6 +224,8 @@ def _read_icl_dataset_files(self): # Add examples for each dataset file dset_files = self.incontext_settings["datasets"][sys_kdma_name] # If there is only one, make it a list for the following loop + if isinstance(dset_files, ListConfig): + dset_files = OmegaConf.to_object(dset_files) if not isinstance(dset_files, list): dset_files = [dset_files] diff --git a/pyproject.toml b/pyproject.toml index 0ffe0044..2069a5d1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -26,7 +26,7 @@ requests = "^2.31.0" bert-score = "^0.3.13" rich = "^13.6.0" rouge-score = "^0.1.2" -swagger-client = {git = "https://github.com/NextCenturyCorporation/itm-evaluation-client.git", rev = "0.5.0"} +swagger-client = {git = "https://github.com/NextCenturyCorporation/itm-evaluation-client.git", rev = "0.5.2"} hydra-core = "^1.3.2" outlines = "^0.2.1" setuptools = "^70.1.1"