Skip to content

Run batch

sweagent.run.run_batch.RunBatchConfig

Bases: BaseSettings

agent class-attribute instance-attribute 

agent: AgentConfig = Field(description='Agent options.')

env_var_path class-attribute instance-attribute 

env_var_path: Path | None = None

Path to a .env file to load environment variables from.

instances class-attribute instance-attribute 

instances: BatchInstanceSourceConfig = Field(description='Instances to run.')

num_workers class-attribute instance-attribute 

num_workers: int = Field(default=1)

Number of parallel workers to use.

output_dir class-attribute instance-attribute 

output_dir: Path = Field(default=Path('DEFAULT'), description='Output directory.')

progress_bar class-attribute instance-attribute 

progress_bar: bool = True

Whether to show a progress bar. Progress bar is never shown for human models. Progress bar is always shown for multi-worker runs.

raise_exceptions class-attribute instance-attribute 

raise_exceptions: bool = False

Raise exceptions instead of skipping instances.

random_delay_multiplier class-attribute instance-attribute 

random_delay_multiplier: float = 0.3

We will wait for a random amount of time between 0 and random_delay_multiplier times the number of workers at the start of each instance. This is to avoid any potential race conditions.

redo_existing class-attribute instance-attribute 

redo_existing: bool = False

Do not skip instances that already have a trajectory.

suffix class-attribute instance-attribute 

suffix: str = ''

Suffix to add to the output directory. Only used if output_dir is DEFAULT.

evaluate_and_redo_existing 

evaluate_and_redo_existing() -> Self
Source code in sweagent/run/run_batch.py 
113
114
115
116
117
118
119
120
121
122
123
124
@model_validator(mode="after")
def evaluate_and_redo_existing(self) -> Self:
    if not isinstance(self.instances, SWEBenchInstances):
        return self
    if self.instances.evaluate and self.redo_existing:
        msg = (
            "Cannot evaluate and redo existing at the same time. This would cause invalid results, because "
            "after the first merge_preds gives you a preds.json, this file would be submitted to SB-CLI, causing"
            "evaluation of old instances, which could then not be overwritten by the new ones."
        )
        raise ValueError(msg)
    return self

set_default_output_dir 

set_default_output_dir() -> None
Source code in sweagent/run/run_batch.py 
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
def set_default_output_dir(self) -> None:
    # Needs to be called explicitly, because self._config_files will be setup
    # post-init.
    if self.output_dir == Path("DEFAULT"):
        user_id = getpass.getuser()
        source_id = self.instances.id
        try:
            model_id = self.agent.model.id  # type: ignore[attr-defined]
        except AttributeError:
            model_id = "unknown"
        config_file = getattr(self, "_config_files", ["no_config"])[0]
        if config_file != "no_config":
            config_file = Path(config_file).stem
        suffix = f"__{self.suffix}" if self.suffix else ""
        self.output_dir = Path.cwd() / "trajectories" / user_id / f"{config_file}__{model_id}___{source_id}{suffix}"