Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -237,6 +237,10 @@ All notable changes to SkillOpt are documented here. This project adheres to
answer in SearchQA, LiveMathematicianBench and OfficeQA, as DocVQA already
does, instead of scoring the text between them and marking a correct answer
wrong (thanks @Shizoqua).
- Report hard selection accuracy in the run summary's `best_selection_hard`
and reused `final_selection_hard` instead of the gate-metric score, which
differs under the `soft` and `mixed` gate metrics or the semantic-density
bonus (thanks @Shizoqua).

### Security
- Prevent managed-identity credentials from being sent to non-Azure or
Expand Down
13 changes: 10 additions & 3 deletions skillopt/engine/trainer.py
Original file line number Diff line number Diff line change
Expand Up @@ -2289,10 +2289,14 @@ def _persist_runtime_state(last_completed_step: int) -> None:
# When final == best, reuse the existing val score (no rollout).
try:
if skill_hash(current_skill) == skill_hash(best_skill):
final_selection_hard, final_selection_soft = best_score, None
# Reuse the cached (hard, soft) selection scores; best_score is
# the gate-metric score. Fall back to it only when uncached.
final_selection_hard, final_selection_soft = sel_cache.get(
skill_hash(best_skill), (best_score, None),
)
print(
"\n [final skill == best skill] "
f"final_selection_hard={best_score:.4f} (reused)"
f"final_selection_hard={final_selection_hard:.4f} (reused)"
)
else:
fval_env, fval_n = _build_eval_env(
Expand All @@ -2303,6 +2307,7 @@ def _persist_runtime_state(last_completed_step: int) -> None:
fval_dir = os.path.join(out_root, "final_selection_eval")
fval_results = adapter.rollout(fval_env, current_skill, fval_dir)
final_selection_hard, final_selection_soft = compute_score(fval_results)
sel_cache[skill_hash(current_skill)] = (final_selection_hard, final_selection_soft)
final_gate_score = select_gate_score(
final_selection_hard, final_selection_soft,
gate_metric, gate_mixed_weight,
Expand Down Expand Up @@ -2521,7 +2526,9 @@ def _persist_runtime_state(last_completed_step: int) -> None:
"baseline_selection_hard": sel_cache.get(
skill_hash(skill_init), (None, None),
)[0],
"best_selection_hard": best_score,
"best_selection_hard": sel_cache.get(
skill_hash(best_skill), (best_score, None),
)[0],
"final_selection_hard": final_selection_hard,
"final_selection_soft": final_selection_soft,
"best_step": best_step,
Expand Down
62 changes: 62 additions & 0 deletions tests/test_trainer_summary_scores.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,62 @@
"""The run summary's *_selection_hard fields hold hard accuracy, not the gate score."""

from __future__ import annotations

from unittest import mock

import pytest

import skillopt.engine.trainer as trainer_mod
from skillopt.envs.base import EnvAdapter

# Two selection items: hard accuracy 0.5, soft score 0.75.
RESULTS = [
{"id": "1", "hard": 0, "soft": 0.5, "task_type": "qa"},
{"id": "2", "hard": 1, "soft": 1.0, "task_type": "qa"},
]


def _summary(tmp_path, gate_metric: str) -> dict:
skill = tmp_path / "skill.md"
skill.write_text("# Skill\n", encoding="utf-8")
cfg = {
"out_root": str(tmp_path / "out"),
"backend": "openai_chat",
"optimizer_backend": "openai_chat",
"target_backend": "openai_chat",
"skill_init": str(skill),
"num_epochs": 0, # baseline only: the initial skill is both best and final
"train_size": 1,
"batch_size": 1,
"accumulation": 1,
"merge_batch_size": 2,
"edit_budget": 2,
"seed": 42,
"sel_env_num": 2,
"test_env_num": 2,
"eval_test": True,
"gate_metric": gate_metric,
}
adapter = mock.create_autospec(EnvAdapter, instance=True)
adapter.requires_ray.return_value = False
adapter.get_dataloader.return_value = None
adapter.get_task_types.return_value = []
adapter.build_eval_env.return_value = ["1", "2"]
adapter.rollout.return_value = [dict(row) for row in RESULTS]
return trainer_mod.ReflACTTrainer(cfg, adapter).train()


@pytest.mark.parametrize("gate_metric", ["soft", "mixed"])
def test_selection_hard_fields_report_hard_accuracy_for_non_hard_gates(tmp_path, gate_metric):
summary = _summary(tmp_path, gate_metric)

assert summary["baseline_selection_hard"] == 0.5
assert summary["best_selection_hard"] == 0.5
assert summary["final_selection_hard"] == 0.5
assert summary["final_selection_soft"] == 0.75


def test_hard_gate_summary_is_unchanged(tmp_path):
summary = _summary(tmp_path, "hard")

assert summary["best_selection_hard"] == summary["final_selection_hard"] == 0.5