{
 "diagnosis": {
  "n_cases": 38,
  "n_healthy": 10,
  "n_faulty": 28,
  "categories": [
   "healthy",
   "diverged",
   "leakage",
   "sr_mismatch",
   "unnormalised",
   "label_shuffle",
   "crash_bad_path",
   "regression"
  ],
  "truth_counts": {
   "diverged": 4,
   "unnormalised": 4,
   "crash_bad_path": 4,
   "healthy": 10,
   "sr_mismatch": 4,
   "regression": 4,
   "label_shuffle": 4,
   "leakage": 4
  },
  "methods": {
   "rule": {
    "label": "Rule-based checker",
    "pending": false,
    "accuracy": {
     "mean": 0.6842,
     "lo": 0.5263,
     "hi": 0.8158
    },
    "n_correct": 26,
    "false_alarm_rate_on_healthy": {
     "mean": 0.5,
     "lo": 0.2,
     "hi": 0.8
    },
    "n_false_alarms": 5,
    "faulty_called_healthy_rate": {
     "mean": 0.0357,
     "lo": 0.0,
     "hi": 0.1071
    },
    "n_faulty_called_healthy": 1,
    "per_fault": {
     "diverged": [
      4,
      4
     ],
     "unnormalised": [
      2,
      4
     ],
     "crash_bad_path": [
      4,
      4
     ],
     "healthy": [
      5,
      10
     ],
     "sr_mismatch": [
      0,
      4
     ],
     "regression": [
      4,
      4
     ],
     "label_shuffle": [
      3,
      4
     ],
     "leakage": [
      4,
      4
     ]
    },
    "confusion": {
     "diverged": {
      "diverged": 4
     },
     "unnormalised": {
      "regression": 1,
      "diverged": 1,
      "unnormalised": 2
     },
     "crash_bad_path": {
      "crash_bad_path": 4
     },
     "healthy": {
      "healthy": 5,
      "regression": 5
     },
     "sr_mismatch": {
      "regression": 3,
      "healthy": 1
     },
     "regression": {
      "regression": 4
     },
     "label_shuffle": {
      "label_shuffle": 3,
      "unnormalised": 1
     },
     "leakage": {
      "leakage": 4
     }
    },
    "invalid_outputs": 0
   },
   "metric_only": {
    "label": "Llama 3.1 8B, final metrics only",
    "pending": false,
    "accuracy": {
     "mean": 0.2368,
     "lo": 0.1053,
     "hi": 0.3684
    },
    "n_correct": 9,
    "false_alarm_rate_on_healthy": {
     "mean": 1.0,
     "lo": 1.0,
     "hi": 1.0
    },
    "n_false_alarms": 10,
    "faulty_called_healthy_rate": {
     "mean": 0.0357,
     "lo": 0.0,
     "hi": 0.1071
    },
    "n_faulty_called_healthy": 1,
    "per_fault": {
     "diverged": [
      4,
      4
     ],
     "unnormalised": [
      0,
      4
     ],
     "crash_bad_path": [
      4,
      4
     ],
     "healthy": [
      0,
      10
     ],
     "sr_mismatch": [
      0,
      4
     ],
     "regression": [
      1,
      4
     ],
     "label_shuffle": [
      0,
      4
     ],
     "leakage": [
      0,
      4
     ]
    },
    "confusion": {
     "diverged": {
      "diverged": 4
     },
     "unnormalised": {
      "regression": 2,
      "diverged": 2
     },
     "crash_bad_path": {
      "crash_bad_path": 4
     },
     "healthy": {
      "regression": 10
     },
     "sr_mismatch": {
      "regression": 3,
      "leakage": 1
     },
     "regression": {
      "leakage": 3,
      "regression": 1
     },
     "label_shuffle": {
      "diverged": 4
     },
     "leakage": {
      "regression": 3,
      "healthy": 1
     }
    },
    "invalid_outputs": 0
   },
   "agent": {
    "label": "Llama 3.1 8B agent with tools",
    "pending": false,
    "accuracy": {
     "mean": 0.3947,
     "lo": 0.2368,
     "hi": 0.5526
    },
    "n_correct": 15,
    "false_alarm_rate_on_healthy": {
     "mean": 0.8,
     "lo": 0.5,
     "hi": 1.0
    },
    "n_false_alarms": 8,
    "faulty_called_healthy_rate": {
     "mean": 0.0,
     "lo": 0.0,
     "hi": 0.0
    },
    "n_faulty_called_healthy": 0,
    "per_fault": {
     "diverged": [
      2,
      4
     ],
     "unnormalised": [
      2,
      4
     ],
     "crash_bad_path": [
      4,
      4
     ],
     "healthy": [
      2,
      10
     ],
     "sr_mismatch": [
      0,
      4
     ],
     "regression": [
      1,
      4
     ],
     "label_shuffle": [
      0,
      4
     ],
     "leakage": [
      4,
      4
     ]
    },
    "confusion": {
     "diverged": {
      "leakage": 1,
      "regression": 1,
      "diverged": 2
     },
     "unnormalised": {
      "leakage": 2,
      "unnormalised": 2
     },
     "crash_bad_path": {
      "crash_bad_path": 4
     },
     "healthy": {
      "healthy": 2,
      "regression": 6,
      "leakage": 2
     },
     "sr_mismatch": {
      "leakage": 4
     },
     "regression": {
      "leakage": 3,
      "regression": 1
     },
     "label_shuffle": {
      "regression": 3,
      "leakage": 1
     },
     "leakage": {
      "leakage": 4
     }
    },
    "invalid_outputs": 0,
    "mean_tool_calls": 3.32,
    "tool_usage": {
     "compare_runs": 38,
     "get_metrics": 38,
     "get_audio_info": 11,
     "get_split_info": 30,
     "read_log": 4,
     "get_feature_stats": 4,
     "run_status": 1
    }
   },
   "agent70": {
    "label": "Llama 3.3 70B agent with tools",
    "pending": false,
    "accuracy": {
     "mean": 0.6316,
     "lo": 0.4737,
     "hi": 0.7895
    },
    "n_correct": 24,
    "false_alarm_rate_on_healthy": {
     "mean": 0.8,
     "lo": 0.5,
     "hi": 1.0
    },
    "n_false_alarms": 8,
    "faulty_called_healthy_rate": {
     "mean": 0.0,
     "lo": 0.0,
     "hi": 0.0
    },
    "n_faulty_called_healthy": 0,
    "per_fault": {
     "diverged": [
      3,
      4
     ],
     "unnormalised": [
      4,
      4
     ],
     "crash_bad_path": [
      4,
      4
     ],
     "healthy": [
      2,
      10
     ],
     "sr_mismatch": [
      2,
      4
     ],
     "regression": [
      4,
      4
     ],
     "label_shuffle": [
      1,
      4
     ],
     "leakage": [
      4,
      4
     ]
    },
    "confusion": {
     "diverged": {
      "diverged": 3,
      "unnormalised": 1
     },
     "unnormalised": {
      "unnormalised": 4
     },
     "crash_bad_path": {
      "crash_bad_path": 4
     },
     "healthy": {
      "regression": 8,
      "healthy": 2
     },
     "sr_mismatch": {
      "regression": 2,
      "sr_mismatch": 2
     },
     "regression": {
      "regression": 4
     },
     "label_shuffle": {
      "regression": 3,
      "label_shuffle": 1
     },
     "leakage": {
      "leakage": 4
     }
    },
    "invalid_outputs": 0,
    "mean_tool_calls": 2.89,
    "tool_usage": {
     "get_metrics": 41,
     "run_status": 1,
     "compare_runs": 33,
     "get_feature_stats": 9,
     "read_log": 4,
     "get_split_info": 20,
     "get_audio_info": 2
    }
   }
  },
  "paired_agent70_minus_rule": {
   "mean": -0.0526,
   "lo": -0.2105,
   "hi": 0.1053
  },
  "paired_agent8_minus_rule": {
   "mean": -0.2895,
   "lo": -0.4474,
   "hi": -0.1316
  },
  "paired_agent70_minus_agent8": {
   "mean": 0.2368,
   "lo": 0.0789,
   "hi": 0.4211
  },
  "paired_agent70_minus_metric_only": {
   "mean": 0.3947,
   "lo": 0.2105,
   "hi": 0.5526
  },
  "agent8_leakage_calls": {
   "n": 17,
   "wrong": 13,
   "after_split_info": 17
  },
  "paired_agent8_minus_metric_only": {
   "mean": 0.1579,
   "lo": 0.0,
   "hi": 0.3158
  },
  "healthy_val_acc_range": [
   0.486,
   0.72
  ],
  "baseline_val_acc": 0.722
 },
 "search": {
  "budget_runs": 10,
  "n_random": 30,
  "n_agent": 10,
  "n_default": 30,
  "random_test": {
   "mean": 0.807,
   "lo": 0.7984,
   "hi": 0.8151
  },
  "agent_test": {
   "mean": 0.7602,
   "lo": 0.7322,
   "hi": 0.7846
  },
  "default_test": {
   "mean": 0.8217,
   "lo": 0.8154,
   "hi": 0.8283
  },
  "random_first10_test": {
   "mean": 0.813,
   "lo": 0.8008,
   "hi": 0.824
  },
  "agent_minus_random30": {
   "mean": -0.0468,
   "lo": -0.0753,
   "hi": -0.0203
  },
  "agent_minus_random_paired10": {
   "mean": -0.0528,
   "lo": -0.0756,
   "hi": -0.0296
  },
  "random_val": {
   "mean": 0.7028,
   "lo": 0.6917,
   "hi": 0.714
  },
  "agent_val": {
   "mean": 0.554,
   "lo": 0.525,
   "hi": 0.579
  },
  "agent_wins_over_paired_random": 1,
  "agent_ties": 0,
  "agent_invalid_json": 0,
  "agent_rejected_configs": 0,
  "agent_denied_over_budget": 0,
  "agent_test_each": [
   0.804,
   0.772,
   0.716,
   0.73,
   0.818,
   0.756,
   0.67,
   0.76,
   0.778,
   0.798
  ],
  "random_first10_test_each": [
   0.822,
   0.844,
   0.81,
   0.82,
   0.814,
   0.826,
   0.772,
   0.824,
   0.794,
   0.804
  ],
  "median_config_agent": {
   "lr": 0.05,
   "hidden": 128.0,
   "epochs": 20.0,
   "n_mfcc": 10.0,
   "momentum": 0.5,
   "l2": 0.0
  },
  "median_config_random": {
   "lr": 0.0247,
   "hidden": 126.0,
   "epochs": 21.5,
   "n_mfcc": 18.0,
   "momentum": 0.4955,
   "l2": 0.0007
  },
  "best_config_example_agent": {
   "lr": 0.1,
   "hidden": 128,
   "l2": 0.0,
   "epochs": 30,
   "batch_size": 32,
   "n_mfcc": 10,
   "momentum": 0.5,
   "normalize": true,
   "seed": 0
  },
  "distinct_configs_per_agent_session": 9.4,
  "agent_runs": 100,
  "agent_runs_n_mfcc_10": 98,
  "agent_runs_n_mfcc_ge20": 2,
  "random_runs_n_mfcc_ge20_frac": 0.4233
 },
 "report_control": {
  "corrupted_numbers": 203,
  "flagged": 168,
  "detection_rate": 0.8276
 },
 "report": {
  "agent_sessions": {
   "n_reports": 10,
   "n_numbers": 145,
   "n_unsupported": 0,
   "rate": 0.0,
   "reports_with_any_unsupported": 0,
   "unsupported_examples": []
  },
  "random_sessions": {
   "n_reports": 10,
   "n_numbers": 144,
   "n_unsupported": 0,
   "rate": 0.0,
   "reports_with_any_unsupported": 0,
   "unsupported_examples": []
  },
  "all": {
   "n_reports": 20,
   "n_numbers": 289,
   "n_unsupported": 0,
   "rate": 0.0,
   "reports_with_any_unsupported": 0,
   "unsupported_examples": []
  },
  "per_report_rate_ci": {
   "mean": 0.0,
   "lo": 0.0,
   "hi": 0.0
  },
  "example_text": "A hyper-parameter search was conducted over 10 runs of a small audio digit classifier. The run with the best validation accuracy was s0_03 with a validation accuracy of 0.58. Its configuration included a learning rate of 0.1, 128 hidden units, no L2 regularization, 20 epochs, a batch size of 32, 10 MFCC features, a momentum of 0.5, and normalization enabled. The table suggests that increasing the number of epochs (from 10 to 20) and batch size (from 32 to 64) improved validation accuracy, while decreasing the learning rate (from 0.1 to 0.01) had a negative impact.",
  "example_check": {
   "n_numbers": 14,
   "unsupported": []
  }
 },
 "neurons": {
  "total": 5096.4,
  "by_tag": [
   {
    "tag": "diag_agent",
    "model": "llama-3.1-8b-instruct-fast",
    "neurons": 606.4,
    "calls": 159
   },
   {
    "tag": "diag_agent70",
    "model": "llama-3.3-70b-instruct-fp8-fast",
    "neurons": 3475.9,
    "calls": 140
   },
   {
    "tag": "diag_metric_only",
    "model": "llama-3.1-8b-instruct-fast",
    "neurons": 108.3,
    "calls": 38
   },
   {
    "tag": "probe",
    "model": "llama-3.1-8b-instruct-fast",
    "neurons": 30.9,
    "calls": 8
   },
   {
    "tag": "probe70",
    "model": "llama-3.3-70b-instruct-fp8-fast",
    "neurons": 205.8,
    "calls": 8
   },
   {
    "tag": "search_agent",
    "model": "llama-3.1-8b-instruct-fast",
    "neurons": 511.0,
    "calls": 100
   },
   {
    "tag": "search_report",
    "model": "llama-3.1-8b-instruct-fast",
    "neurons": 157.8,
    "calls": 20
   },
   {
    "tag": "smoke",
    "model": "llama-3.1-8b-instruct-fast",
    "neurons": 0.4,
    "calls": 1
   }
  ]
 },
 "registry": {
  "search_runs": 430,
  "diagnosis_runs": 39,
  "test_reveals": 70,
  "test_reveals_distinct_runs": 70,
  "code_hashes": [
   "003d2669dd32cbe7",
   "aaeaaffc6a6125f6"
  ],
  "data_hash": "0fbe6612aaa20a80ef4371894a46e060038337a518fe3d4728cb7eee7caa013c"
 },
 "data": {
  "clips": 3000,
  "speakers": 6,
  "train": 2000,
  "val": 500,
  "test": 500,
  "splits": {
   "train": [
    "george",
    "jackson",
    "nicolas",
    "theo"
   ],
   "val": [
    "lucas"
   ],
   "test": [
    "yweweler"
   ]
  }
 },
 "traces": {
  "good": {
   "case": "c14",
   "truth": "leakage",
   "diagnosis": "leakage",
   "model": "Llama 3.3 70B agent with tools",
   "steps": [
    {
     "step": 1,
     "tool": "get_metrics",
     "args": {
      "run_id": "c14"
     },
     "result": "{\"run_id\":\"c14\",\"status\":\"completed\",\"final\":{\"epoch\":31,\"train_acc\":0.9996,\"train_loss\":0.021,\"val_acc\":0.94,\"val_loss\":0.2047},\"best_val_acc\":0.944,\"nan_loss\":false,\"history_sample\":[{\"epoch\":1,\"train_acc\":0.928,\"train_loss\":0.4014,\"val_acc\":0.832,\"val_loss\":0.7496},{\"epoch\":16,\"train_acc\":0.9991,\"train_loss\":0.0337,\"val_acc\":0.928,\"val_loss\":0.2353},{\"epoch\":31,\"train_acc\":0.9996,\"train_loss\":0.021,\"val_acc\":0.94,\"val_loss\":0.2047}]}"
    },
    {
     "step": 2,
     "tool": "compare_runs",
     "args": {
      "run_a": "c14",
      "run_b": "base"
     },
     "result": "{\"config_diff_[a,b]\":{\"epochs\":[31,30],\"hidden\":[192,256],\"l2\":[0.002,0.003],\"lr\":[0.0044,0.005],\"n_mfcc\":[21,20],\"seed\":[2,0],\"split_mode\":[\"mixed\",\"speaker\"]},\"same_data_hash\":true,\"same_code_hash\":true,\"a_final\":{\"train_acc\":0.9996,\"val_acc\":0.94,\"train_loss\":0.021},\"a_status\":\"completed\",\"b_final\":{\"train_acc\":0.9995,\"val_acc\":0.722,\"train_loss\":0.0217},\"b_status\":\"completed\"}"
    },
    {
     "step": 3,
     "tool": "get_split_info",
     "args": {
      "run_id": "c14"
     },
     "result": "{\"run_id\":\"c14\",\"split\":{\"n_train\":2250,\"n_val\":250,\"overlap_train_val\":[\"lucas\"],\"train_speakers\":[\"george\",\"jackson\",\"lucas\",\"nicolas\",\"theo\"],\"val_speakers\":[\"lucas\"]}}"
    },
    {
     "step": 4,
     "final": {
      "diagnosis": "leakage",
      "evidence": "The validation set contains a speaker ('lucas') that is also present in the training set, as shown by the get_split_info tool, which indicates data leakage and inflated validation accuracy."
     }
    }
   ],
   "config": {
    "batch_size": 32,
    "data_dir": null,
    "declared_sr": 8000,
    "epochs": 31,
    "fmax": 4000,
    "hidden": 192,
    "l2": 0.002,
    "label_map": "identity",
    "lr": 0.0044,
    "momentum": 0.9,
    "n_mfcc": 21,
    "normalize": true,
    "resume_from": null,
    "seed": 2,
    "split_mode": "mixed"
   }
  },
  "failure": {
   "case": "c10",
   "truth": "sr_mismatch",
   "diagnosis": "regression",
   "model": "Llama 3.3 70B agent with tools",
   "steps": [
    {
     "step": 1,
     "tool": "get_metrics",
     "args": {
      "run_id": "c10"
     },
     "result": "{\"run_id\":\"c10\",\"status\":\"completed\",\"final\":{\"epoch\":20,\"train_acc\":0.999,\"train_loss\":0.0396,\"val_acc\":0.402,\"val_loss\":2.3592},\"best_val_acc\":0.402,\"nan_loss\":false,\"history_sample\":[{\"epoch\":1,\"train_acc\":0.8745,\"train_loss\":0.5135,\"val_acc\":0.286,\"val_loss\":2.2275},{\"epoch\":11,\"train_acc\":0.991,\"train_loss\":0.0713,\"val_acc\":0.386,\"val_loss\":2.3841},{\"epoch\":20,\"train_acc\":0.999,\"train_loss\":0.0396,\"val_acc\":0.402,\"val_loss\":2.3592}]}"
    },
    {
     "step": 2,
     "tool": "compare_runs",
     "args": {
      "run_a": "c10",
      "run_b": "base"
     },
     "result": "{\"config_diff_[a,b]\":{\"declared_sr\":[22050,8000],\"epochs\":[20,30],\"hidden\":[192,256],\"l2\":[0.0012,0.003],\"lr\":[0.006,0.005],\"n_mfcc\":[17,20],\"seed\":[2,0]},\"same_data_hash\":true,\"same_code_hash\":true,\"a_final\":{\"train_acc\":0.999,\"val_acc\":0.402,\"train_loss\":0.0396},\"a_status\":\"completed\",\"b_final\":{\"train_acc\":0.9995,\"val_acc\":0.722,\"train_loss\":0.0217},\"b_status\":\"completed\"}"
    },
    {
     "step": 3,
     "tool": "get_split_info",
     "args": {
      "run_id": "c10"
     },
     "result": "{\"run_id\":\"c10\",\"split\":{\"n_train\":2000,\"n_val\":500,\"overlap_train_val\":[],\"train_speakers\":[\"george\",\"jackson\",\"nicolas\",\"theo\"],\"val_speakers\":[\"lucas\"]}}"
    },
    {
     "step": 4,
     "final": {
      "diagnosis": "regression",
      "evidence": "The validation accuracy of run 'c10' (0.402) is lower than the baseline run 'base' (0.722), and the difference in configuration between the two runs, such as the declared sample rate, number of MFCC features, and hyperparameters, could be the cause of the regression, as indicated by the compare_runs tool result."
     }
    }
   ],
   "config": {
    "batch_size": 32,
    "data_dir": null,
    "declared_sr": 22050,
    "epochs": 20,
    "fmax": 4000,
    "hidden": 192,
    "l2": 0.0012,
    "label_map": "identity",
    "lr": 0.006,
    "momentum": 0.9,
    "n_mfcc": 17,
    "normalize": true,
    "resume_from": null,
    "seed": 2,
    "split_mode": "speaker"
   }
  }
 }
}