{
 "schema_version": "2.1.0",
 "report_id": "01a1a221-4aee-7e9b-bdb2-6d5588f14612",
 "dataset_card": {
  "citation": "Gebru et al., Datasheets for Datasets, arXiv:1803.09010.",
  "class_counts": {
   "GRR": 0,
   "LEAD": 171,
   "NON_LEAD": 375581
  },
  "collection": "Field verifications and utility records are separate label sources. Records can disagree with the field. This card does not merge them.",
  "composition": {
   "class_counts": {
    "GRR": 0,
    "LEAD": 171,
    "NON_LEAD": 375581
   },
   "equity_dimensions": [
    "decade_built"
   ],
   "rows": 375752
  },
  "distribution": {
   "licence": "not recorded",
   "source_url": "not recorded"
  },
  "maintenance": {
   "note": "Each EvalReport writes a new card. Hand-edited numbers are not accepted.",
   "versions": {
    "craftpine-eval": "0.0.0",
    "numpy": "2.3.5",
    "pandas": "3.1.0"
   }
  },
  "motivation": "The dataset supports evaluation of a service-line material model.",
  "n_verified": null,
  "preprocessing": {
   "split": "spatial_block",
   "target_encoding_outside_folds": []
  },
  "rows": 375752,
  "schema": "gebru-datasheets-2021",
  "uses": "Evaluation, release gating, and regression checks. The dataset is not itself an inventory update.",
  "rows_in_snapshot": 384888
 },
 "model_card": {
  "caveats": "If the homes we score are like the homes we checked, at least 95 of every 100 will have their true material inside the range shown.",
  "citation": "Mitchell et al., Model Cards for Model Reporting, arXiv:1810.03993.",
  "ethical_considerations": "A false non-lead call can leave a lead line in service. Share of residents of colour is reported for audit and is not a training feature. The model does not change the inventory.",
  "evaluation_data": {
   "split": "spatial_block"
  },
  "factors": {
   "audit_only": [
    "residents_of_colour"
   ],
   "feature_violation": [],
   "groups": [
    "decade_built"
   ]
  },
  "intended_use": {
   "out_of_scope": [
    "automatic changes to an official inventory category",
    "geocoding addresses",
    "use of race or ethnicity as a model feature"
   ],
   "primary": "Score unknown service lines and show where the score is too weak to call. A named person approves any inventory change."
  },
  "metrics": {
   "calibration_ece": 0.00012843958941498833,
   "conformal_coverage": 0.9999752496327365,
   "false_non_lead_rate": 0.00029734678206631246,
   "overall": [
    {
     "estimate": 0.0003388093114748336,
     "interval": {
      "level": 0.95,
      "lower": 0.00027376043595919556,
      "upper": 0.00040686849337091705
     },
     "name": "brier_lead"
    },
    {
     "estimate": 0.0006776186275699534,
     "interval": {
      "level": 0.95,
      "lower": 0.0005475208781304177,
      "upper": 0.0008137369891330517
     },
     "name": "brier_multiclass"
    },
    {
     "estimate": 2.5829643494591136e-08,
     "interval": {
      "level": 0.95,
      "lower": 8.126822123583593e-09,
      "upper": 1.4739123403693204e-07
     },
     "name": "brier_reliability"
    },
    {
     "estimate": 1.6667691164377228e-06,
     "interval": {
      "level": 0.95,
      "lower": 9.62514554837711e-07,
      "upper": 2.598000443924272e-06
     },
     "name": "brier_resolution"
    },
    {
     "estimate": 0.0004548802935311851,
     "interval": {
      "level": 0.95,
      "lower": 0.000347541736007994,
      "upper": 0.0005564586083579578
     },
     "name": "brier_uncertainty"
    },
    {
     "estimate": 162.6,
     "interval": {
      "level": 0.95,
      "lower": 124.955,
      "upper": 202.925
     },
     "name": "lead_found_top_decile"
    },
    {
     "estimate": 0.001407359502713376,
     "interval": {
      "level": 0.95,
      "lower": 0.001126620214151554,
      "upper": 0.0017404281703563879
     },
     "name": "log_loss"
    },
    {
     "estimate": 0.33231627302999106,
     "interval": {
      "level": 0.95,
      "lower": 0.271500527147005,
      "upper": 0.405540994286563
     },
     "name": "pr_auc_lead"
    },
    {
     "estimate": 0.6661461079496049,
     "interval": {
      "level": 0.95,
      "lower": 0.6357293451373062,
      "upper": 0.7027621149926631
     },
     "name": "pr_auc_macro"
    },
    {
     "estimate": 0.9999759428692192,
     "interval": {
      "level": 0.95,
      "lower": 0.9999575730641439,
      "upper": 0.9999901758055172
     },
     "name": "pr_auc_non_lead"
    },
    {
     "estimate": 0.0029239766081871343,
     "interval": {
      "level": 0.95,
      "lower": 0.001129167442726765,
      "upper": 0.02256451878707976
     },
     "name": "recall_at_precision_0_8"
    },
    {
     "estimate": 0.9717511789258875,
     "interval": {
      "level": 0.95,
      "lower": 0.9530041949488859,
      "upper": 0.9870737151474448
     },
     "name": "roc_auc_lead"
    },
    {
     "estimate": 0.971751240429039,
     "interval": {
      "level": 0.95,
      "lower": 0.9530042022040286,
      "upper": 0.9870736048235791
     },
     "name": "roc_auc_macro"
    },
    {
     "estimate": 0.9717513019321908,
     "interval": {
      "level": 0.95,
      "lower": 0.953004209459171,
      "upper": 0.9870734944997133
     },
     "name": "roc_auc_non_lead"
    }
   ],
   "shuffled_label_auc": 0.5313465215086408
  },
  "model_details": {
   "name": "lightgbm",
   "params": {
    "class_weight": null,
    "classes": [
     "LEAD",
     "GRR",
     "NON_LEAD"
    ],
    "learning_rate": 0.08,
    "min_child_samples": 5,
    "n_estimators": 40,
    "n_jobs": 1,
    "name": "lightgbm",
    "num_leaves": 15,
    "reg_lambda": 1.0,
    "seed": 0
   },
   "seeds": [
    0,
    1,
    2,
    3,
    4,
    5,
    6,
    7,
    8,
    9
   ],
   "type": "tabular classifier over LEAD, GRR, and NON_LEAD",
   "versions": {
    "craftpine-eval": "0.0.0",
    "numpy": "2.3.5",
    "pandas": "3.1.0"
   }
  },
  "quantitative_analyses": {
   "canary_detected": true,
   "flagged_features": [],
   "gate": "FAIL",
   "label_copies": [],
   "reasons": [
    "cleared share 0.9466 is not above the year-built rule's 1.0000; abstain rate 0.0534 against 0.0000",
    "false non-lead gap exceeds 0.01 for decade_built:1940",
    "release bound fails on records-only clears of lead or GRR: decade 1920 0.31% (upper 2.11%); decade 1940 2.07% (upper 3.88%)"
   ],
   "underpredicted": []
  },
  "schema": "mitchell-2019",
  "requested_algorithm": "lightgbm",
  "algorithm": "lightgbm",
  "gated_model": "lightgbm",
  "stored_model": null,
  "engine": "craftpine.eval.evaluate",
  "leaderboard": [
   {
    "name": "year_built_rule",
    "log_loss": 0.003657007796588621,
    "lower": 0.00291328014057915,
    "upper": 0.004358661507645339,
    "gate": "FAIL",
    "label_copies": [],
    "log_loss_improvement": {
     "estimate": 6.1100461854723e-06,
     "lower": 9.2215722203151e-07,
     "upper": 1.2965170600895273e-05,
     "excludes_zero": true
    }
   },
   {
    "name": "records_face_value",
    "log_loss": 0.003985536249806941,
    "lower": 0.0031583378080380625,
    "upper": 0.004770946904511324,
    "gate": "FAIL",
    "label_copies": [],
    "log_loss_improvement": {
     "estimate": -0.0003224184070328475,
     "lower": -0.0004123509578006287,
     "upper": -0.00022512361254117756,
     "excludes_zero": true
    }
   },
   {
    "name": "lightgbm",
    "log_loss": 0.001407359502713376,
    "lower": 0.001126620214151554,
    "upper": 0.0017404281703563879,
    "gate": "FAIL",
    "label_copies": [],
    "log_loss_improvement": {
     "estimate": 0.002255758340060717,
     "lower": 0.0017636582539805005,
     "upper": 0.0028218093843334125,
     "excludes_zero": true
    }
   },
   {
    "name": "xgboost",
    "log_loss": 0.0013642193286165223,
    "lower": 0.001074563571542154,
    "upper": 0.001716151168758506,
    "gate": "FAIL",
    "label_copies": [],
    "log_loss_improvement": {
     "estimate": 0.002298898514157571,
     "lower": 0.0017919998504436327,
     "upper": 0.0028753153789788117,
     "excludes_zero": true
    }
   },
   {
    "name": "catboost",
    "log_loss": 0.0023390627718766228,
    "lower": 0.0018787019278397865,
    "upper": 0.002830313309846144,
    "gate": "FAIL",
    "label_copies": [],
    "log_loss_improvement": {
     "estimate": 0.0013240550708974706,
     "lower": 0.001055841135150669,
     "upper": 0.0016352070802681665,
     "excludes_zero": true
    }
   }
  ]
 },
 "split_strategy": "spatial_block",
 "metrics": [
  {
   "scope": "GROUP",
   "group_key": "year_built_rule",
   "metrics": [
    {
     "name": "brier_lead",
     "estimate": 0.00045467234121589927,
     "interval": {
      "lower": 0.0003474837833019922,
      "upper": 0.0005561113803418318,
      "level": 0.95
     }
    },
    {
     "name": "brier_multiclass",
     "estimate": 0.0009093455594293233,
     "interval": {
      "lower": 0.0006949693508998964,
      "upper": 0.0011122225459367367,
      "level": 0.95
     }
    },
    {
     "name": "brier_reliability",
     "estimate": 3.7276801119812966e-08,
     "interval": {
      "lower": 5.023317937464364e-08,
      "upper": 1.273048394328107e-07,
      "level": 0.95
     }
    },
    {
     "name": "brier_resolution",
     "estimate": 3.0501817885761395e-07,
     "interval": {
      "lower": 1.5808591061295991e-07,
      "upper": 4.1903681937564093e-07,
      "level": 0.95
     }
    },
    {
     "name": "brier_uncertainty",
     "estimate": 0.0004548802935311851,
     "interval": {
      "lower": 0.000347541736007994,
      "upper": 0.0005564586083579578,
      "level": 0.95
     }
    },
    {
     "name": "lead_found_top_decile",
     "estimate": 28.4,
     "interval": {
      "lower": 25.1925,
      "upper": 42.0025,
      "level": 0.95
     }
    },
    {
     "name": "log_loss",
     "estimate": 0.003657007796588621,
     "interval": {
      "lower": 0.00291328014057915,
      "upper": 0.004358661507645339,
      "level": 0.95
     }
    },
    {
     "name": "pr_auc_lead",
     "estimate": 0.0007950385945487302,
     "interval": {
      "lower": 0.0007615320873460762,
      "upper": 0.001277504455149513,
      "level": 0.95
     }
    },
    {
     "name": "pr_auc_macro",
     "estimate": 0.500334712934656,
     "interval": {
      "lower": 0.5003287115400674,
      "upper": 0.5005592240472153,
      "level": 0.95
     }
    },
    {
     "name": "pr_auc_non_lead",
     "estimate": 0.9998743872747633,
     "interval": {
      "lower": 0.999816435303886,
      "upper": 0.9998949597381332,
      "level": 0.95
     }
    },
    {
     "name": "recall_at_precision_0_8",
     "estimate": 0.0,
     "interval": {
      "lower": 0.0,
      "upper": 0.0,
      "level": 0.95
     }
    },
    {
     "name": "roc_auc_lead",
     "estimate": 0.7577342587704778,
     "interval": {
      "lower": 0.740337984587592,
      "upper": 0.7744993010265514,
      "level": 0.95
     }
    },
    {
     "name": "roc_auc_macro",
     "estimate": 0.7576342608740414,
     "interval": {
      "lower": 0.7401660765920967,
      "upper": 0.7743824365047076,
      "level": 0.95
     }
    },
    {
     "name": "roc_auc_non_lead",
     "estimate": 0.757534262977605,
     "interval": {
      "lower": 0.7401064236423417,
      "upper": 0.774265571982864,
      "level": 0.95
     }
    }
   ]
  },
  {
   "scope": "GROUP",
   "group_key": "records_face_value",
   "metrics": [
    {
     "name": "brier_lead",
     "estimate": 0.0004549017489577208,
     "interval": {
      "lower": 0.0003475760748690686,
      "upper": 0.0005564909172617754,
      "level": 0.95
     }
    },
    {
     "name": "brier_multiclass",
     "estimate": 0.0009098035876743996,
     "interval": {
      "lower": 0.0006951527892755261,
      "upper": 0.0011129813617655588,
      "level": 0.95
     }
    },
    {
     "name": "brier_reliability",
     "estimate": 3.9394148854501153e-08,
     "interval": {
      "lower": 3.804933653266667e-08,
      "upper": 1.0024520752087647e-07,
      "level": 0.95
     }
    },
    {
     "name": "brier_resolution",
     "estimate": 1.7939789311017842e-08,
     "interval": {
      "lower": 1.9957079811693406e-08,
      "upper": 7.214444691139446e-08,
      "level": 0.95
     }
    },
    {
     "name": "brier_uncertainty",
     "estimate": 0.0004548802935311851,
     "interval": {
      "lower": 0.000347541736007994,
      "upper": 0.0005564586083579578,
      "level": 0.95
     }
    },
    {
     "name": "lead_found_top_decile",
     "estimate": 13.1,
     "interval": {
      "lower": 9.285,
      "upper": 18.4,
      "level": 0.95
     }
    },
    {
     "name": "log_loss",
     "estimate": 0.003985536249806941,
     "interval": {
      "lower": 0.0031583378080380625,
      "upper": 0.004770946904511324,
      "level": 0.95
     }
    },
    {
     "name": "pr_auc_lead",
     "estimate": 0.0003849467371134275,
     "interval": {
      "lower": 0.00033107510670731726,
      "upper": 0.0005648980605367901,
      "level": 0.95
     }
    },
    {
     "name": "pr_auc_macro",
     "estimate": 0.49995648846070806,
     "interval": {
      "lower": 0.49991719148598446,
      "upper": 0.5000069899246322,
      "level": 0.95
     }
    },
    {
     "name": "pr_auc_non_lead",
     "estimate": 0.9995280301843025,
     "interval": {
      "lower": 0.9993428307593488,
      "upper": 0.9996104216081556,
      "level": 0.95
     }
    },
    {
     "name": "recall_at_precision_0_8",
     "estimate": 0.0,
     "interval": {
      "lower": 0.0,
      "upper": 0.0,
      "level": 0.95
     }
    },
    {
     "name": "roc_auc_lead",
     "estimate": 0.4536623772188838,
     "interval": {
      "lower": 0.4347605630990849,
      "upper": 0.4742463290231076,
      "level": 0.95
     }
    },
    {
     "name": "roc_auc_macro",
     "estimate": 0.4536623772188838,
     "interval": {
      "lower": 0.4347605630990849,
      "upper": 0.4742463290231076,
      "level": 0.95
     }
    },
    {
     "name": "roc_auc_non_lead",
     "estimate": 0.4536623772188838,
     "interval": {
      "lower": 0.4347605630990849,
      "upper": 0.4742463290231076,
      "level": 0.95
     }
    }
   ]
  },
  {
   "scope": "GROUP",
   "group_key": "lightgbm",
   "metrics": [
    {
     "name": "brier_lead",
     "estimate": 0.0003388093114748336,
     "interval": {
      "lower": 0.00027376043595919556,
      "upper": 0.00040686849337091705,
      "level": 0.95
     }
    },
    {
     "name": "brier_multiclass",
     "estimate": 0.0006776186275699534,
     "interval": {
      "lower": 0.0005475208781304177,
      "upper": 0.0008137369891330517,
      "level": 0.95
     }
    },
    {
     "name": "brier_reliability",
     "estimate": 2.5829643494591136e-08,
     "interval": {
      "lower": 8.126822123583593e-09,
      "upper": 1.4739123403693204e-07,
      "level": 0.95
     }
    },
    {
     "name": "brier_resolution",
     "estimate": 1.6667691164377228e-06,
     "interval": {
      "lower": 9.62514554837711e-07,
      "upper": 2.598000443924272e-06,
      "level": 0.95
     }
    },
    {
     "name": "brier_uncertainty",
     "estimate": 0.0004548802935311851,
     "interval": {
      "lower": 0.000347541736007994,
      "upper": 0.0005564586083579578,
      "level": 0.95
     }
    },
    {
     "name": "lead_found_top_decile",
     "estimate": 162.6,
     "interval": {
      "lower": 124.955,
      "upper": 202.925,
      "level": 0.95
     }
    },
    {
     "name": "log_loss",
     "estimate": 0.001407359502713376,
     "interval": {
      "lower": 0.001126620214151554,
      "upper": 0.0017404281703563879,
      "level": 0.95
     }
    },
    {
     "name": "pr_auc_lead",
     "estimate": 0.33231627302999106,
     "interval": {
      "lower": 0.271500527147005,
      "upper": 0.405540994286563,
      "level": 0.95
     }
    },
    {
     "name": "pr_auc_macro",
     "estimate": 0.6661461079496049,
     "interval": {
      "lower": 0.6357293451373062,
      "upper": 0.7027621149926631,
      "level": 0.95
     }
    },
    {
     "name": "pr_auc_non_lead",
     "estimate": 0.9999759428692192,
     "interval": {
      "lower": 0.9999575730641439,
      "upper": 0.9999901758055172,
      "level": 0.95
     }
    },
    {
     "name": "recall_at_precision_0_8",
     "estimate": 0.0029239766081871343,
     "interval": {
      "lower": 0.001129167442726765,
      "upper": 0.02256451878707976,
      "level": 0.95
     }
    },
    {
     "name": "roc_auc_lead",
     "estimate": 0.9717511789258875,
     "interval": {
      "lower": 0.9530041949488859,
      "upper": 0.9870737151474448,
      "level": 0.95
     }
    },
    {
     "name": "roc_auc_macro",
     "estimate": 0.971751240429039,
     "interval": {
      "lower": 0.9530042022040286,
      "upper": 0.9870736048235791,
      "level": 0.95
     }
    },
    {
     "name": "roc_auc_non_lead",
     "estimate": 0.9717513019321908,
     "interval": {
      "lower": 0.953004209459171,
      "upper": 0.9870734944997133,
      "level": 0.95
     }
    }
   ]
  },
  {
   "scope": "GROUP",
   "group_key": "xgboost",
   "metrics": [
    {
     "name": "brier_lead",
     "estimate": 0.00032656654510556944,
     "interval": {
      "lower": 0.00026685776516735467,
      "upper": 0.0003920225684543943,
      "level": 0.95
     }
    },
    {
     "name": "brier_multiclass",
     "estimate": 0.0006531330930751898,
     "interval": {
      "lower": 0.0005337155386314189,
      "upper": 0.0007840451380229622,
      "level": 0.95
     }
    },
    {
     "name": "brier_reliability",
     "estimate": 7.396539070070737e-09,
     "interval": {
      "lower": 6.536141837432633e-09,
      "upper": 7.850149476363215e-08,
      "level": 0.95
     }
    },
    {
     "name": "brier_resolution",
     "estimate": 1.6303922997813477e-06,
     "interval": {
      "lower": 9.513564328845168e-07,
      "upper": 2.5687133613218405e-06,
      "level": 0.95
     }
    },
    {
     "name": "brier_uncertainty",
     "estimate": 0.0004548802935311851,
     "interval": {
      "lower": 0.000347541736007994,
      "upper": 0.0005564586083579578,
      "level": 0.95
     }
    },
    {
     "name": "lead_found_top_decile",
     "estimate": 161.4,
     "interval": {
      "lower": 124.45,
      "upper": 201.1425,
      "level": 0.95
     }
    },
    {
     "name": "log_loss",
     "estimate": 0.0013642193286165223,
     "interval": {
      "lower": 0.001074563571542154,
      "upper": 0.001716151168758506,
      "level": 0.95
     }
    },
    {
     "name": "pr_auc_lead",
     "estimate": 0.3467270351239734,
     "interval": {
      "lower": 0.28386501165799666,
      "upper": 0.42732982563313276,
      "level": 0.95
     }
    },
    {
     "name": "pr_auc_macro",
     "estimate": 0.6733521795222193,
     "interval": {
      "lower": 0.6419160345163007,
      "upper": 0.7136445902594979,
      "level": 0.95
     }
    },
    {
     "name": "pr_auc_non_lead",
     "estimate": 0.999977323920465,
     "interval": {
      "lower": 0.9999527539658956,
      "upper": 0.9999895894547787,
      "level": 0.95
     }
    },
    {
     "name": "recall_at_precision_0_8",
     "estimate": 0.009941520467836256,
     "interval": {
      "lower": 0.004342071611253197,
      "upper": 0.027601950354609932,
      "level": 0.95
     }
    },
    {
     "name": "roc_auc_lead",
     "estimate": 0.9686040719975514,
     "interval": {
      "lower": 0.9486998157850233,
      "upper": 0.9848241500832133,
      "level": 0.95
     }
    },
    {
     "name": "roc_auc_macro",
     "estimate": 0.9686040136084832,
     "interval": {
      "lower": 0.948699718408658,
      "upper": 0.9848241491718077,
      "level": 0.95
     }
    },
    {
     "name": "roc_auc_non_lead",
     "estimate": 0.9686039552194152,
     "interval": {
      "lower": 0.9486996210322928,
      "upper": 0.9848241482604022,
      "level": 0.95
     }
    }
   ]
  },
  {
   "scope": "GROUP",
   "group_key": "catboost",
   "metrics": [
    {
     "name": "brier_lead",
     "estimate": 0.00044570371806345514,
     "interval": {
      "lower": 0.00034219058458073514,
      "upper": 0.0005434408493556516,
      "level": 0.95
     }
    },
    {
     "name": "brier_multiclass",
     "estimate": 0.0008914074345330097,
     "interval": {
      "lower": 0.000684381170658552,
      "upper": 0.0010868816976358513,
      "level": 0.95
     }
    },
    {
     "name": "brier_reliability",
     "estimate": 4.68688321241797e-08,
     "interval": {
      "lower": 1.4903982577735087e-08,
      "upper": 2.5575119164091026e-07,
      "level": 0.95
     }
    },
    {
     "name": "brier_resolution",
     "estimate": 1.7059506944645388e-06,
     "interval": {
      "lower": 9.909418846012608e-07,
      "upper": 2.650168023495112e-06,
      "level": 0.95
     }
    },
    {
     "name": "brier_uncertainty",
     "estimate": 0.0004548802935311851,
     "interval": {
      "lower": 0.000347541736007994,
      "upper": 0.0005564586083579578,
      "level": 0.95
     }
    },
    {
     "name": "lead_found_top_decile",
     "estimate": 164.3,
     "interval": {
      "lower": 126.5625,
      "upper": 205.1025,
      "level": 0.95
     }
    },
    {
     "name": "log_loss",
     "estimate": 0.0023390627718766228,
     "interval": {
      "lower": 0.0018787019278397865,
      "upper": 0.002830313309846144,
      "level": 0.95
     }
    },
    {
     "name": "pr_auc_lead",
     "estimate": 0.035293198080371374,
     "interval": {
      "lower": 0.02953330093382105,
      "upper": 0.048283808938520655,
      "level": 0.95
     }
    },
    {
     "name": "pr_auc_macro",
     "estimate": 0.51763717450687,
     "interval": {
      "lower": 0.514757263276033,
      "upper": 0.5241264177807298,
      "level": 0.95
     }
    },
    {
     "name": "pr_auc_non_lead",
     "estimate": 0.9999811509333684,
     "interval": {
      "lower": 0.9999671715762892,
      "upper": 0.9999915893371184,
      "level": 0.95
     }
    },
    {
     "name": "recall_at_precision_0_8",
     "estimate": 0.0,
     "interval": {
      "lower": 0.0,
      "upper": 0.0016856313083222245,
      "level": 0.95
     }
    },
    {
     "name": "roc_auc_lead",
     "estimate": 0.9734080450264108,
     "interval": {
      "lower": 0.9577553488576661,
      "upper": 0.9856187593064704,
      "level": 0.95
     }
    },
    {
     "name": "roc_auc_macro",
     "estimate": 0.9734058682819544,
     "interval": {
      "lower": 0.9577514911296486,
      "upper": 0.9856185751748703,
      "level": 0.95
     }
    },
    {
     "name": "roc_auc_non_lead",
     "estimate": 0.973403691537498,
     "interval": {
      "lower": 0.9577476334016313,
      "upper": 0.98561839104327,
      "level": 0.95
     }
    }
   ]
  }
 ],
 "calibration": {
  "classwise_ece_equal_mass": 2.118991965284966e-05,
  "classwise_ece_equal_width": 8.563709065601014e-05,
  "counts": {
   "by_block": [
    {
     "actual_lead": 0.0,
     "block": "8912ea6cc57ffff",
     "n": 29.0,
     "predicted_lead": 0.0034954892960307837
    },
    {
     "actual_lead": 0.0,
     "block": "8912ea6cc1bffff",
     "n": 128.0,
     "predicted_lead": 0.003886162687631158
    },
    {
     "actual_lead": 0.0,
     "block": "8912ccd2523ffff",
     "n": 72.0,
     "predicted_lead": 0.0051810550435931555
    },
    {
     "actual_lead": 0.0,
     "block": "8912ea6cd57ffff",
     "n": 165.0,
     "predicted_lead": 0.004284619480386516
    },
    {
     "actual_lead": 0.0,
     "block": "8912ccd2973ffff",
     "n": 136.0,
     "predicted_lead": 0.07984811533602276
    },
    {
     "actual_lead": 0.0,
     "block": "8912ea6da6bffff",
     "n": 128.0,
     "predicted_lead": 0.3432528555442199
    },
    {
     "actual_lead": 1.0,
     "block": "8912ccd29b7ffff",
     "n": 122.0,
     "predicted_lead": 1.233472848142298
    },
    {
     "actual_lead": 0.0,
     "block": "8912ccc2987ffff",
     "n": 130.0,
     "predicted_lead": 0.007531477378963437
    },
    {
     "actual_lead": 0.0,
     "block": "8912ccd2d87ffff",
     "n": 98.0,
     "predicted_lead": 0.009289138943740916
    },
    {
     "actual_lead": 0.0,
     "block": "8912ea6d94fffff",
     "n": 105.0,
     "predicted_lead": 0.002675582303454338
    },
    {
     "actual_lead": 0.0,
     "block": "8912ccd01cfffff",
     "n": 83.0,
     "predicted_lead": 0.00785549710018405
    },
    {
     "actual_lead": 0.0,
     "block": "8912ccd60cbffff",
     "n": 90.0,
     "predicted_lead": 0.01706424919669776
    },
    {
     "actual_lead": 1.0,
     "block": "8912ccd2937ffff",
     "n": 69.0,
     "predicted_lead": 0.46851615290727244
    },
    {
     "actual_lead": 1.0,
     "block": "8912ccd2923ffff",
     "n": 129.0,
     "predicted_lead": 0.6920656647363065
    },
    {
     "actual_lead": 0.0,
     "block": "8912ccc740bffff",
     "n": 88.0,
     "predicted_lead": 0.002329536352212722
    },
    {
     "actual_lead": 1.0,
     "block": "8912ccd7217ffff",
     "n": 130.0,
     "predicted_lead": 1.0973652950830937
    },
    {
     "actual_lead": 0.0,
     "block": "8912ccd7543ffff",
     "n": 73.0,
     "predicted_lead": 0.4059262617973987
    },
    {
     "actual_lead": 0.0,
     "block": "8912ccd754fffff",
     "n": 149.0,
     "predicted_lead": 0.1714775006908035
    },
    {
     "actual_lead": 0.0,
     "block": "8912ccd08d3ffff",
     "n": 133.0,
     "predicted_lead": 0.011573602285413188
    },
    {
     "actual_lead": 0.0,
     "block": "8912ccd08d7ffff",
     "n": 199.0,
     "predicted_lead": 0.005923294795988503
    },
    {
     "actual_lead": 0.0,
     "block": "8912ccd089bffff",
     "n": 195.0,
     "predicted_lead": 0.012477741503651094
    },
    {
     "actual_lead": 0.0,
     "block": "8912ccd088bffff",
     "n": 126.0,
     "predicted_lead": 0.012497528195506194
    },
    {
     "actual_lead": 0.0,
     "block": "8912ccd0c63ffff",
     "n": 83.0,
     "predicted_lead": 0.0020041972033753848
    },
    {
     "actual_lead": 0.0,
     "block": "8912ccd0c6fffff",
     "n": 147.0,
     "predicted_lead": 0.011591712843190124
    },
    {
     "actual_lead": 0.0,
     "block": "8912ea6de67ffff",
     "n": 81.0,
     "predicted_lead": 0.25502085754183657
    },
    {
     "actual_lead": 0.0,
     "block": "8912ea6dac3ffff",
     "n": 63.0,
     "predicted_lead": 0.006959924606483003
    },
    {
     "actual_lead": 1.0,
     "block": "8912ccd2987ffff",
     "n": 160.0,
     "predicted_lead": 0.9170052509938058
    },
    {
     "actual_lead": 0.0,
     "block": "8912ccd2d37ffff",
     "n": 120.0,
     "predicted_lead": 0.07964362447737525
    },
    {
     "actual_lead": 0.0,
     "block": "8912ea6da1bffff",
     "n": 137.0,
     "predicted_lead": 0.4163014373091918
    },
    {
     "actual_lead": 0.0,
     "block": "8912ea6dab3ffff",
     "n": 101.0,
     "predicted_lead": 0.2970508404977556
    }
   ],
   "clipped_at_zero": false,
   "disagree": null,
   "model_based": {
    "expected_unknown": 181.58094652318448,
    "known_lead": 0.0,
    "level": 0.8,
    "lower": 166.5557010409139,
    "n_unknown": 375752.0,
    "p10": 166.5557010409139,
    "p50": 181.58094652318448,
    "p90": 196.60619200545506,
    "upper": 196.60619200545506,
    "variance": 137.4583040992509,
    "variance_calibration": 0.0,
    "variance_model": 0.0,
    "variance_poisson_binomial": 137.4583040992509
   }
  },
  "ece_bins": 10,
  "ece_confidence_equal_width": 0.00011779892969868269,
  "ece_lead_equal_mass": 3.0791842789648485e-05,
  "ece_lead_equal_width": 0.00012843958941498833,
  "ece_n": 375752,
  "intercept": 0.13939722843985436,
  "lead_curve": [
   1,
   157,
   157,
   158,
   161,
   161,
   161,
   161,
   161,
   161,
   161,
   161,
   161,
   161,
   161,
   163,
   163,
   163,
   163,
   163,
   163,
   163,
   163,
   163,
   163,
   163,
   163,
   163,
   163,
   163,
   163,
   163,
   163,
   163,
   163,
   163,
   163,
   163,
   163,
   163,
   163,
   164,
   164,
   164,
   164,
   164,
   164,
   165,
   165,
   165,
   165,
   165,
   165,
   165,
   165,
   165,
   165,
   165,
   165,
   165,
   165,
   165,
   165,
   165,
   165,
   165,
   165,
   165,
   165,
   165,
   165,
   165,
   165,
   165,
   165,
   165,
   165,
   165,
   165,
   165,
   165,
   165,
   165,
   165,
   165,
   165,
   165,
   165,
   165,
   165,
   165,
   165,
   165,
   166,
   166,
   166,
   166,
   166,
   167,
   167,
   167,
   167,
   167,
   167,
   168,
   169,
   169,
   169,
   169,
   169,
   169,
   169,
   169,
   169,
   169,
   169,
   169,
   169,
   169,
   169,
   169,
   169,
   169,
   169,
   169,
   170,
   170,
   170,
   170,
   170,
   170,
   170,
   170,
   170,
   170,
   170,
   170,
   170,
   170,
   170,
   170,
   171,
   171,
   171,
   171,
   171,
   171,
   171,
   171,
   171,
   171,
   171,
   171,
   171,
   171,
   171,
   171,
   171,
   171,
   171,
   171,
   171,
   171,
   171,
   171,
   171,
   171,
   171,
   171,
   171,
   171,
   171,
   171,
   171,
   171,
   171,
   171,
   171,
   171,
   171,
   171,
   171,
   171,
   171,
   171,
   171,
   171,
   171,
   171,
   171,
   171,
   171,
   171,
   171,
   171,
   171,
   171,
   171,
   171,
   171
  ],
  "log_loss_improvement": {
   "estimate": 0.002255758340060717,
   "excludes_zero": true,
   "level": 0.95,
   "lower": 0.0017636582539805005,
   "upper": 0.0028218093843334125
  },
  "reliability_equal_mass": [
   {
    "accuracy": 0.0,
    "confidence": 2.3983907478997205e-05,
    "lower": 0.0,
    "n": 28629.0,
    "upper": 2.4164083574807103e-05,
    "weight": 28629.0
   },
   {
    "accuracy": 0.0,
    "confidence": 2.6112995865099647e-05,
    "lower": 2.4164083574807103e-05,
    "n": 41452.0,
    "upper": 3.0277007335743112e-05,
    "weight": 41452.0
   },
   {
    "accuracy": 2.3535503306738213e-05,
    "confidence": 3.539676049922015e-05,
    "lower": 3.0277007335743112e-05,
    "n": 42489.0,
    "upper": 4.93514706683192e-05,
    "weight": 42489.0
   },
   {
    "accuracy": 0.0,
    "confidence": 5.274940337607684e-05,
    "lower": 4.93514706683192e-05,
    "n": 28430.0,
    "upper": 5.6852438315791794e-05,
    "weight": 28430.0
   },
   {
    "accuracy": 6.879471656576775e-05,
    "confidence": 5.745333438002795e-05,
    "lower": 5.6852438315791794e-05,
    "n": 43608.0,
    "upper": 5.88885479270972e-05,
    "weight": 43608.0
   },
   {
    "accuracy": 5.269120320362516e-05,
    "confidence": 5.990739182156484e-05,
    "lower": 5.88885479270972e-05,
    "n": 37957.0,
    "upper": 6.29491285047988e-05,
    "weight": 37957.0
   },
   {
    "accuracy": 0.0,
    "confidence": 6.39361331624722e-05,
    "lower": 6.29491285047988e-05,
    "n": 40099.0,
    "upper": 6.819321748678565e-05,
    "weight": 40099.0
   },
   {
    "accuracy": 5.321979776476849e-05,
    "confidence": 7.243078797968379e-05,
    "lower": 6.819321748678565e-05,
    "n": 37580.0,
    "upper": 8.004196057142703e-05,
    "weight": 37580.0
   },
   {
    "accuracy": 0.0,
    "confidence": 9.669360443466957e-05,
    "lower": 8.004196057142703e-05,
    "n": 37044.0,
    "upper": 0.00011842552323915867,
    "weight": 37044.0
   },
   {
    "accuracy": 0.004237728785357737,
    "confidence": 0.004241921408612811,
    "lower": 0.00011842552323915867,
    "n": 38464.0,
    "upper": 1.0,
    "weight": 38464.0
   }
  ],
  "reliability_equal_width": [
   {
    "accuracy": 0.00010390994471458069,
    "confidence": 0.00015781854749148655,
    "lower": 0.0,
    "n": 375325.0,
    "upper": 0.1,
    "weight": 375325.0
   },
   {
    "accuracy": 0.1830065359477124,
    "confidence": 0.14705414255018634,
    "lower": 0.1,
    "n": 153.0,
    "upper": 0.2,
    "weight": 153.0
   },
   {
    "accuracy": 0.32098765432098764,
    "confidence": 0.24714018580412295,
    "lower": 0.2,
    "n": 81.0,
    "upper": 0.30000000000000004,
    "weight": 81.0
   },
   {
    "accuracy": 0.42105263157894735,
    "confidence": 0.349971677153938,
    "lower": 0.30000000000000004,
    "n": 95.0,
    "upper": 0.4,
    "weight": 95.0
   },
   {
    "accuracy": 0.42857142857142855,
    "confidence": 0.4485277829475642,
    "lower": 0.4,
    "n": 70.0,
    "upper": 0.5,
    "weight": 70.0
   },
   {
    "accuracy": 0.23076923076923078,
    "confidence": 0.5304230161134964,
    "lower": 0.5,
    "n": 26.0,
    "upper": 0.6000000000000001,
    "weight": 26.0
   },
   {
    "accuracy": 1.0,
    "confidence": 0.6974043853626565,
    "lower": 0.6000000000000001,
    "n": 2.0,
    "upper": 0.7000000000000001,
    "weight": 2.0
   }
  ],
  "slope": 1.082277882863107,
  "validation": {
   "ban_year": 1988,
   "buffer_m": 200.0,
   "buffer_source": "config",
   "crossfit_folds": [
    {
     "crossfit_buffer_m": 200.0,
     "crossfit_covered_share": 1.0,
     "crossfit_models_fit": 4,
     "fold": 0,
     "seed": 0
    },
    {
     "crossfit_buffer_m": 200.0,
     "crossfit_covered_share": 1.0,
     "crossfit_models_fit": 4,
     "fold": 1,
     "seed": 0
    },
    {
     "crossfit_buffer_m": 200.0,
     "crossfit_covered_share": 1.0,
     "crossfit_models_fit": 4,
     "fold": 2,
     "seed": 0
    },
    {
     "crossfit_buffer_m": 200.0,
     "crossfit_covered_share": 1.0,
     "crossfit_models_fit": 4,
     "fold": 3,
     "seed": 0
    },
    {
     "crossfit_buffer_m": 200.0,
     "crossfit_covered_share": 1.0,
     "crossfit_models_fit": 4,
     "fold": 4,
     "seed": 0
    },
    {
     "crossfit_buffer_m": 200.0,
     "crossfit_covered_share": 1.0,
     "crossfit_models_fit": 4,
     "fold": 0,
     "seed": 1
    },
    {
     "crossfit_buffer_m": 200.0,
     "crossfit_covered_share": 1.0,
     "crossfit_models_fit": 4,
     "fold": 1,
     "seed": 1
    },
    {
     "crossfit_buffer_m": 200.0,
     "crossfit_covered_share": 1.0,
     "crossfit_models_fit": 4,
     "fold": 2,
     "seed": 1
    },
    {
     "crossfit_buffer_m": 200.0,
     "crossfit_covered_share": 1.0,
     "crossfit_models_fit": 4,
     "fold": 3,
     "seed": 1
    },
    {
     "crossfit_buffer_m": 200.0,
     "crossfit_covered_share": 1.0,
     "crossfit_models_fit": 4,
     "fold": 4,
     "seed": 1
    },
    {
     "crossfit_buffer_m": 200.0,
     "crossfit_covered_share": 1.0,
     "crossfit_models_fit": 4,
     "fold": 0,
     "seed": 2
    },
    {
     "crossfit_buffer_m": 200.0,
     "crossfit_covered_share": 1.0,
     "crossfit_models_fit": 4,
     "fold": 1,
     "seed": 2
    },
    {
     "crossfit_buffer_m": 200.0,
     "crossfit_covered_share": 1.0,
     "crossfit_models_fit": 4,
     "fold": 2,
     "seed": 2
    },
    {
     "crossfit_buffer_m": 200.0,
     "crossfit_covered_share": 1.0,
     "crossfit_models_fit": 4,
     "fold": 3,
     "seed": 2
    },
    {
     "crossfit_buffer_m": 200.0,
     "crossfit_covered_share": 1.0,
     "crossfit_models_fit": 4,
     "fold": 4,
     "seed": 2
    },
    {
     "crossfit_buffer_m": 200.0,
     "crossfit_covered_share": 1.0,
     "crossfit_models_fit": 4,
     "fold": 0,
     "seed": 3
    },
    {
     "crossfit_buffer_m": 200.0,
     "crossfit_covered_share": 1.0,
     "crossfit_models_fit": 4,
     "fold": 1,
     "seed": 3
    },
    {
     "crossfit_buffer_m": 200.0,
     "crossfit_covered_share": 1.0,
     "crossfit_models_fit": 4,
     "fold": 2,
     "seed": 3
    },
    {
     "crossfit_buffer_m": 200.0,
     "crossfit_covered_share": 1.0,
     "crossfit_models_fit": 4,
     "fold": 3,
     "seed": 3
    },
    {
     "crossfit_buffer_m": 200.0,
     "crossfit_covered_share": 1.0,
     "crossfit_models_fit": 4,
     "fold": 4,
     "seed": 3
    },
    {
     "crossfit_buffer_m": 200.0,
     "crossfit_covered_share": 1.0,
     "crossfit_models_fit": 4,
     "fold": 0,
     "seed": 4
    },
    {
     "crossfit_buffer_m": 200.0,
     "crossfit_covered_share": 1.0,
     "crossfit_models_fit": 4,
     "fold": 1,
     "seed": 4
    },
    {
     "crossfit_buffer_m": 200.0,
     "crossfit_covered_share": 1.0,
     "crossfit_models_fit": 4,
     "fold": 2,
     "seed": 4
    },
    {
     "crossfit_buffer_m": 200.0,
     "crossfit_covered_share": 1.0,
     "crossfit_models_fit": 4,
     "fold": 3,
     "seed": 4
    },
    {
     "crossfit_buffer_m": 200.0,
     "crossfit_covered_share": 1.0,
     "crossfit_models_fit": 4,
     "fold": 4,
     "seed": 4
    },
    {
     "crossfit_buffer_m": 200.0,
     "crossfit_covered_share": 1.0,
     "crossfit_models_fit": 4,
     "fold": 0,
     "seed": 5
    },
    {
     "crossfit_buffer_m": 200.0,
     "crossfit_covered_share": 1.0,
     "crossfit_models_fit": 4,
     "fold": 1,
     "seed": 5
    },
    {
     "crossfit_buffer_m": 200.0,
     "crossfit_covered_share": 1.0,
     "crossfit_models_fit": 4,
     "fold": 2,
     "seed": 5
    },
    {
     "crossfit_buffer_m": 200.0,
     "crossfit_covered_share": 1.0,
     "crossfit_models_fit": 4,
     "fold": 3,
     "seed": 5
    },
    {
     "crossfit_buffer_m": 200.0,
     "crossfit_covered_share": 1.0,
     "crossfit_models_fit": 4,
     "fold": 4,
     "seed": 5
    },
    {
     "crossfit_buffer_m": 200.0,
     "crossfit_covered_share": 1.0,
     "crossfit_models_fit": 4,
     "fold": 0,
     "seed": 6
    },
    {
     "crossfit_buffer_m": 200.0,
     "crossfit_covered_share": 1.0,
     "crossfit_models_fit": 4,
     "fold": 1,
     "seed": 6
    },
    {
     "crossfit_buffer_m": 200.0,
     "crossfit_covered_share": 1.0,
     "crossfit_models_fit": 4,
     "fold": 2,
     "seed": 6
    },
    {
     "crossfit_buffer_m": 200.0,
     "crossfit_covered_share": 1.0,
     "crossfit_models_fit": 4,
     "fold": 3,
     "seed": 6
    },
    {
     "crossfit_buffer_m": 200.0,
     "crossfit_covered_share": 1.0,
     "crossfit_models_fit": 4,
     "fold": 4,
     "seed": 6
    },
    {
     "crossfit_buffer_m": 200.0,
     "crossfit_covered_share": 1.0,
     "crossfit_models_fit": 4,
     "fold": 0,
     "seed": 7
    },
    {
     "crossfit_buffer_m": 200.0,
     "crossfit_covered_share": 1.0,
     "crossfit_models_fit": 4,
     "fold": 1,
     "seed": 7
    },
    {
     "crossfit_buffer_m": 200.0,
     "crossfit_covered_share": 1.0,
     "crossfit_models_fit": 4,
     "fold": 2,
     "seed": 7
    },
    {
     "crossfit_buffer_m": 200.0,
     "crossfit_covered_share": 1.0,
     "crossfit_models_fit": 4,
     "fold": 3,
     "seed": 7
    },
    {
     "crossfit_buffer_m": 200.0,
     "crossfit_covered_share": 1.0,
     "crossfit_models_fit": 4,
     "fold": 4,
     "seed": 7
    },
    {
     "crossfit_buffer_m": 200.0,
     "crossfit_covered_share": 1.0,
     "crossfit_models_fit": 4,
     "fold": 0,
     "seed": 8
    },
    {
     "crossfit_buffer_m": 200.0,
     "crossfit_covered_share": 1.0,
     "crossfit_models_fit": 4,
     "fold": 1,
     "seed": 8
    },
    {
     "crossfit_buffer_m": 200.0,
     "crossfit_covered_share": 1.0,
     "crossfit_models_fit": 4,
     "fold": 2,
     "seed": 8
    },
    {
     "crossfit_buffer_m": 200.0,
     "crossfit_covered_share": 1.0,
     "crossfit_models_fit": 4,
     "fold": 3,
     "seed": 8
    },
    {
     "crossfit_buffer_m": 200.0,
     "crossfit_covered_share": 1.0,
     "crossfit_models_fit": 4,
     "fold": 4,
     "seed": 8
    },
    {
     "crossfit_buffer_m": 200.0,
     "crossfit_covered_share": 1.0,
     "crossfit_models_fit": 4,
     "fold": 0,
     "seed": 9
    },
    {
     "crossfit_buffer_m": 200.0,
     "crossfit_covered_share": 1.0,
     "crossfit_models_fit": 4,
     "fold": 1,
     "seed": 9
    },
    {
     "crossfit_buffer_m": 200.0,
     "crossfit_covered_share": 1.0,
     "crossfit_models_fit": 4,
     "fold": 2,
     "seed": 9
    },
    {
     "crossfit_buffer_m": 200.0,
     "crossfit_covered_share": 1.0,
     "crossfit_models_fit": 4,
     "fold": 3,
     "seed": 9
    },
    {
     "crossfit_buffer_m": 200.0,
     "crossfit_covered_share": 1.0,
     "crossfit_models_fit": 4,
     "fold": 4,
     "seed": 9
    }
   ],
   "n_bootstrap": 200,
   "random_log_loss": 0.0013738618754338477,
   "sensitivity": [
    {
     "buffer_m": 0.0,
     "train_rows": 300601.0,
     "train_rows_kept": 300601.0
    },
    {
     "buffer_m": 100.0,
     "train_rows": 300601.0,
     "train_rows_kept": 238948.0
    },
    {
     "buffer_m": 250.0,
     "train_rows": 300601.0,
     "train_rows_kept": 126494.0
    },
    {
     "buffer_m": 500.0,
     "train_rows": 300601.0,
     "train_rows_kept": 26678.0
    },
    {
     "buffer_m": 200.0,
     "train_rows": 300601.0,
     "train_rows_kept": 159679.0
    },
    {
     "buffer_m": 250.0,
     "fold": 0.0,
     "suggested_buffer_m": 250.0,
     "train_rows": 300601.0,
     "train_rows_kept": 300601.0,
     "variogram": 0.0
    },
    {
     "buffer_m": 4425.002858060651,
     "fold": 1.0,
     "suggested_buffer_m": 4425.002858060651,
     "train_rows": 300602.0,
     "train_rows_kept": 300602.0,
     "variogram": 1.0
    },
    {
     "buffer_m": 7000.750462570675,
     "fold": 2.0,
     "suggested_buffer_m": 7000.750462570675,
     "train_rows": 300601.0,
     "train_rows_kept": 300601.0,
     "variogram": 1.0
    },
    {
     "buffer_m": 250.0,
     "fold": 3.0,
     "suggested_buffer_m": 250.0,
     "train_rows": 300602.0,
     "train_rows_kept": 300602.0,
     "variogram": 0.0
    },
    {
     "buffer_m": 4644.902134329875,
     "fold": 4.0,
     "suggested_buffer_m": 4644.902134329875,
     "train_rows": 300602.0,
     "train_rows_kept": 300602.0,
     "variogram": 1.0
    }
   ],
   "split_log_loss": 0.0013975001239139371
  }
 },
 "conformal": {
  "alpha": 0.05,
  "assumption": "The coverage claim needs the homes being scored to be exchangeable with the homes that were checked, or, for the weighted sets, to match them after the covariate weights. It fails when crews picked homes for a reason that is not in the data, and it fails when a group has too few checked homes for its own quantile.",
  "baseline_abstain": 3.1935957759373206e-05,
  "baseline_non_lead_calls": 375740,
  "beta": 0.01,
  "clearance": {
   "baseline_share": 0.9999680640422406,
   "min_share": 0.05,
   "non_lead_calls": 355700,
   "scored_rows": 375752,
   "sentence": null,
   "share": 0.9466350145840874,
   "useful": true
  },
  "coverage": 0.9999752496327365,
  "coverage_by_seed": [
   0.9999787093614938,
   0.9999760480316805,
   0.9999733867018672,
   0.9999760480316805,
   0.999981370691307,
   0.9999680640422406,
   0.9999733867018672,
   0.9999787093614938,
   0.9999733867018672,
   0.9999733867018672
  ],
  "false_non_lead_by_seed": [
   {
    "calls": 355700,
    "false": 105,
    "rate": 0.00029519257801518135,
    "upper": 0.0003425702734123058
   },
   {
    "calls": 363243,
    "false": 101,
    "rate": 0.00027805078143281495,
    "upper": 0.0003235527623009328
   },
   {
    "calls": 365729,
    "false": 103,
    "rate": 0.0002816292938213814,
    "upper": 0.0003272671575232929
   },
   {
    "calls": 360365,
    "false": 104,
    "rate": 0.0002885962843228393,
    "upper": 0.00033513759884934186
   },
   {
    "calls": 364719,
    "false": 104,
    "rate": 0.0002851510340837741,
    "upper": 0.0003311368196043731
   },
   {
    "calls": 371842,
    "false": 157,
    "rate": 0.0004222223417473013,
    "upper": 0.0004776372906265733
   },
   {
    "calls": 366894,
    "false": 102,
    "rate": 0.00027800945232137893,
    "upper": 0.0003232811056938329
   },
   {
    "calls": 365963,
    "false": 109,
    "rate": 0.00029784431759494813,
    "upper": 0.00034476222577236595
   },
   {
    "calls": 363748,
    "false": 122,
    "rate": 0.00033539703311083496,
    "upper": 0.0003853353286549824
   },
   {
    "calls": 364282,
    "false": 77,
    "rate": 0.0002113747042126704,
    "upper": 0.00025099238083840255
   }
  ],
  "false_non_lead_calls": 355700,
  "false_non_lead_rate": 0.00029734678206631246,
  "false_non_lead_seed_summary": {
   "failing_seeds": 0,
   "rate_max": 0.0004222223417473013,
   "rate_mean": 0.00029734678206631246,
   "rate_mean_ci95": [
    0.0002642540278145566,
    0.0003304395363180683
   ],
   "rate_min": 0.0002113747042126704,
   "rate_sd": 5.339208034886212e-05,
   "seeds": 10,
   "seeds_with_calls": 10,
   "upper_mean": 0.0003441672943276403
  },
  "false_non_lead_upper": 0.0003441672943276403,
  "fnl_aggregate": "seed_mean",
  "gate_checks": [
   {
    "detail": "log-loss improvement over the year-built rule excludes zero",
    "name": "beats_year_built",
    "status": "pass"
   },
   {
    "detail": "ECE 0.0001 is at most the adjusted bound 0.0351 (bar 0.03, B=10, N=375752)",
    "name": "calibration_ece",
    "status": "pass"
   },
   {
    "detail": "coverage 1.000 is above 0.95 by more than 0.02; over-coverage is not a gate failure",
    "name": "conformal_coverage",
    "status": "note"
   },
   {
    "detail": "every large group has coverage at least 0.90",
    "name": "mondrian_coverage",
    "status": "pass"
   },
   {
    "detail": "false non-lead rate 0.0003 stays within the bound",
    "name": "false_non_lead",
    "status": "pass"
   },
   {
    "detail": "cleared share 0.9466 is not above the year-built rule's 1.0000; abstain rate 0.0534 against 0.0000",
    "name": "usefulness",
    "status": "fail"
   },
   {
    "detail": "false non-lead gap exceeds 0.01 for decade_built:1940",
    "name": "equity_gap",
    "status": "fail"
   },
   {
    "detail": "the leakage canary was detected",
    "name": "canary",
    "status": "pass"
   },
   {
    "detail": "shuffled-label AUC 0.531 is within the 3-sigma null band 0.0662 (n_pos=171, n_neg=375581)",
    "name": "shuffled_labels",
    "shuffled_min_pos": 50,
    "status": "pass"
   },
   {
    "detail": "the three count estimators were not all available",
    "name": "count_agreement",
    "status": "skipped"
   },
   {
    "detail": "no feature is a copy of the label",
    "name": "label_copies",
    "status": "pass"
   },
   {
    "detail": "the named red-team suite was not supplied",
    "name": "red_team",
    "status": "skipped"
   },
   {
    "detail": "no regression comparison was supplied",
    "name": "regression",
    "status": "skipped"
   },
   {
    "detail": "release bound fails on records-only clears of lead or GRR: decade 1920 0.31% (upper 2.11%); decade 1940 2.07% (upper 3.88%)",
    "name": "release_bound",
    "status": "fail"
   }
  ],
  "group_coverage": [
   {
    "coverage": 1.0,
    "group": "1900",
    "n": 12.0
   },
   {
    "coverage": 0.990521327014218,
    "group": "1910",
    "n": 211.0
   },
   {
    "coverage": 0.9966101694915255,
    "group": "1920",
    "n": 295.0
   },
   {
    "coverage": 0.9996413199426112,
    "group": "1930",
    "n": 2788.0
   },
   {
    "coverage": 0.9806290356175796,
    "group": "1940",
    "n": 4801.0
   },
   {
    "coverage": 0.9999627199522816,
    "group": "1950",
    "n": 26824.0
   },
   {
    "coverage": 1.0,
    "group": "1960",
    "n": 32829.0
   },
   {
    "coverage": 0.9999842745042538,
    "group": "1970",
    "n": 63591.0
   },
   {
    "coverage": 0.9999412835417768,
    "group": "1980",
    "n": 34062.0
   },
   {
    "coverage": 0.9999644943102132,
    "group": "1990",
    "n": 56329.0
   },
   {
    "coverage": 0.999970696829397,
    "group": "2000",
    "n": 68252.0
   },
   {
    "coverage": 1.0,
    "group": "2010",
    "n": 49921.0
   },
   {
    "coverage": 1.0,
    "group": "2020",
    "n": 35837.0
   }
  ],
  "model_abstain": 0.053359662756286064,
  "plain": {
   "decision": "FAIL",
   "headline": "The model failed the check.",
   "notes": [
    "Records only: the city marks no lines as checked in the field. Of the model's not-lead calls on test lines, the city's records show 0.03% as lead or galvanized pipe needing replacement; with 98.33% confidence the share is at most 0.05%. This is agreement with the records, not what was found in the ground, so it cannot unlock answers about single homes. The model makes no call for lines built in the 1910s: too few labelled lines there to judge it."
   ],
   "reasons": [
    "A simple rule by year built clears 100.0% of the test lines and the model clears 94.7%, so the model adds nothing over the rule.",
    "For lines built in the 1940s, 2.00% of the lines the model called not lead were lead, against 0.03% overall. No group may be more than 1 percentage point above the overall rate.",
    "Lines built in the 1920s: across 10 test runs, the city's records show 9 of the model's 2,902 not-lead calls (0.31%) as lead or galvanized pipe needing replacement, at most 2.11% with 98.33% confidence.",
    "Lines built in the 1940s: across 10 test runs, the city's records show 963 of the model's 46,485 not-lead calls (2.07%) as lead or galvanized pipe needing replacement, at most 3.88% with 98.33% confidence.",
    "The limits are 1.00% for the share of misses and 2.00% for its upper bound, in each decade and over all decades judged."
   ]
  },
  "release_bound": {
   "alpha": 0.01666666666666668,
   "basis": "records_only",
   "beta": 0.01,
   "candidates": 3,
   "groups": [
    {
     "bootstrap_upper": 1.0,
     "cleared_calls_by_run": [
      0,
      0,
      0,
      0,
      0,
      0,
      0,
      0,
      0,
      0
     ],
     "cp_upper": 1.0,
     "group": "1900",
     "judged_calls": 0,
     "judged_calls_min": 0,
     "lead_misses": 0,
     "lead_point": 0.0,
     "lead_upper": 1.0,
     "misses": 0,
     "point": 0.0,
     "status": "no calls",
     "upper": 1.0
    },
    {
     "bootstrap_upper": 0.05128205128205128,
     "cleared_calls_by_run": [
      204,
      211,
      202,
      206,
      211,
      209,
      211,
      206,
      129,
      211
     ],
     "cp_upper": 0.05877542474847704,
     "group": "1910",
     "judged_calls": 2000,
     "judged_calls_min": 129,
     "lead_misses": 20,
     "lead_point": 0.01,
     "lead_upper": 0.05877542474847704,
     "misses": 20,
     "point": 0.01,
     "status": "withdrawn",
     "upper": 0.05877542474847704
    },
    {
     "bootstrap_upper": 0.013793103448275862,
     "cleared_calls_by_run": [
      292,
      294,
      288,
      293,
      284,
      293,
      295,
      284,
      288,
      291
     ],
     "cp_upper": 0.02110381734292397,
     "group": "1920",
     "judged_calls": 2902,
     "judged_calls_min": 284,
     "lead_misses": 9,
     "lead_point": 0.0031013094417643005,
     "lead_upper": 0.02110381734292397,
     "misses": 9,
     "point": 0.0031013094417643005,
     "status": "fail",
     "upper": 0.02110381734292397
    },
    {
     "bootstrap_upper": 0.001418942887548776,
     "cleared_calls_by_run": [
      2712,
      2733,
      2734,
      2735,
      2719,
      2715,
      2740,
      2745,
      2711,
      2659
     ],
     "cp_upper": 0.0022719827599924506,
     "group": "1930",
     "judged_calls": 27203,
     "judged_calls_min": 2659,
     "lead_misses": 10,
     "lead_point": 0.00036760651398742785,
     "lead_upper": 0.0022719827599924506,
     "misses": 10,
     "point": 0.00036760651398742785,
     "status": "pass",
     "upper": 0.0022719827599924506
    },
    {
     "bootstrap_upper": 0.038826384292962715,
     "cleared_calls_by_run": [
      4642,
      4649,
      4639,
      4646,
      4658,
      4765,
      4619,
      4662,
      4665,
      4540
     ],
     "cp_upper": 0.03594644413427296,
     "group": "1940",
     "judged_calls": 46485,
     "judged_calls_min": 4540,
     "lead_misses": 963,
     "lead_point": 0.020716360116166507,
     "lead_upper": 0.03594644413427296,
     "misses": 963,
     "point": 0.020716360116166507,
     "status": "fail",
     "upper": 0.038826384292962715
    },
    {
     "bootstrap_upper": 0.00023111590462617003,
     "cleared_calls_by_run": [
      25821,
      25812,
      25841,
      25907,
      25367,
      26639,
      26185,
      26093,
      26311,
      26093
     ],
     "cp_upper": 0.0003055965208390621,
     "group": "1950",
     "judged_calls": 260069,
     "judged_calls_min": 25367,
     "lead_misses": 19,
     "lead_point": 7.30575347311675e-05,
     "lead_upper": 0.0003055965208390621,
     "misses": 19,
     "point": 7.30575347311675e-05,
     "status": "pass",
     "upper": 0.0003055965208390621
    },
    {
     "bootstrap_upper": 0.0,
     "cleared_calls_by_run": [
      31527,
      32292,
      32442,
      31567,
      32014,
      32626,
      32048,
      32224,
      32351,
      31851
     ],
     "cp_upper": 0.00012985944466048505,
     "group": "1960",
     "judged_calls": 320942,
     "judged_calls_min": 31527,
     "lead_misses": 0,
     "lead_point": 0.0,
     "lead_upper": 0.00012985944466048505,
     "misses": 0,
     "point": 0.0,
     "status": "pass",
     "upper": 0.00012985944466048505
    },
    {
     "bootstrap_upper": 6.80399394444539e-05,
     "cleared_calls_by_run": [
      58992,
      60580,
      61666,
      61939,
      61808,
      62150,
      61262,
      60273,
      62240,
      61715
     ],
     "cp_upper": 0.00010249998403784946,
     "group": "1970",
     "judged_calls": 612625,
     "judged_calls_min": 58992,
     "lead_misses": 10,
     "lead_point": 1.6323199347072025e-05,
     "lead_upper": 0.00010249998403784946,
     "misses": 10,
     "point": 1.6323199347072025e-05,
     "status": "pass",
     "upper": 0.00010249998403784946
    },
    {
     "bootstrap_upper": 0.00017869907076483203,
     "cleared_calls_by_run": [
      31915,
      31667,
      33313,
      33538,
      33281,
      33717,
      33591,
      33096,
      32578,
      33036
     ],
     "cp_upper": 0.00024480505973156834,
     "group": "1980",
     "judged_calls": 329732,
     "judged_calls_min": 31667,
     "lead_misses": 17,
     "lead_point": 5.1557022066405446e-05,
     "lead_upper": 0.00024480505973156834,
     "misses": 17,
     "point": 5.1557022066405446e-05,
     "status": "pass",
     "upper": 0.00024480505973156834
    },
    {
     "bootstrap_upper": 0.00011013215859030837,
     "cleared_calls_by_run": [
      54242,
      54150,
      54015,
      53522,
      54423,
      56097,
      54566,
      55205,
      54584,
      54286
     ],
     "cp_upper": 0.00014484752759456055,
     "group": "1990",
     "judged_calls": 545090,
     "judged_calls_min": 53522,
     "lead_misses": 18,
     "lead_point": 3.3022069749949546e-05,
     "lead_upper": 0.00014484752759456055,
     "misses": 18,
     "point": 3.3022069749949546e-05,
     "status": "pass",
     "upper": 0.00014484752759456055
    },
    {
     "bootstrap_upper": 9.192866335723479e-05,
     "cleared_calls_by_run": [
      63412,
      66740,
      65756,
      63976,
      66032,
      67168,
      66131,
      66698,
      64821,
      65327
     ],
     "cp_upper": 0.00012225752782765798,
     "group": "2000",
     "judged_calls": 656061,
     "judged_calls_min": 63412,
     "lead_misses": 18,
     "lead_point": 2.7436473132833686e-05,
     "lead_upper": 0.00012225752782765798,
     "misses": 18,
     "point": 2.7436473132833686e-05,
     "status": "pass",
     "upper": 0.00012225752782765798
    },
    {
     "bootstrap_upper": 0.0,
     "cleared_calls_by_run": [
      46325,
      48820,
      49270,
      47765,
      49159,
      49753,
      49564,
      49067,
      48298,
      48913
     ],
     "cp_upper": 8.837913938974928e-05,
     "group": "2010",
     "judged_calls": 486934,
     "judged_calls_min": 46325,
     "lead_misses": 0,
     "lead_point": 0.0,
     "lead_upper": 8.837913938974928e-05,
     "misses": 0,
     "point": 0.0,
     "status": "pass",
     "upper": 8.837913938974928e-05
    },
    {
     "bootstrap_upper": 0.0,
     "cleared_calls_by_run": [
      35616,
      35295,
      35563,
      34271,
      34763,
      35710,
      35682,
      35410,
      34772,
      35360
     ],
     "cp_upper": 0.00011946251924355167,
     "group": "2020",
     "judged_calls": 352442,
     "judged_calls_min": 34271,
     "lead_misses": 0,
     "lead_point": 0.0,
     "lead_upper": 0.00011946251924355167,
     "misses": 0,
     "point": 0.0,
     "status": "pass",
     "upper": 0.00011946251924355167
    }
   ],
   "judged_groups": [
    "1920",
    "1930",
    "1940",
    "1950",
    "1960",
    "1970",
    "1980",
    "1990",
    "2000",
    "2010",
    "2020"
   ],
   "level": 0.95,
   "method": "Each release run's held-out calls of exactly NON_LEAD on the judged lines; a miss is a label in miss_classes; each run's bound is the larger of the exact Clopper-Pearson end and a block bootstrap over the split's blocks at one-sided level 1 - alpha; runs combine by the largest bound; decades under min_support judged calls in any run are withdrawn (ADR 0019).",
   "min_support": 203,
   "miss_classes": [
    "LEAD",
    "GRR"
   ],
   "overall": {
    "bootstrap_upper": 0.0005365540886762942,
    "calls": 3640485,
    "calls_min": 355496,
    "cp_upper": 0.0004944019474837295,
    "lead_misses": 1064,
    "lead_point": 0.0002922687499055758,
    "lead_upper": 0.0004944019474837295,
    "misses": 1064,
    "point": 0.0002922687499055758,
    "upper": 0.0005365540886762942
   },
   "overall_all": {
    "bootstrap_upper": 0.000547945205479452,
    "calls": 3642485,
    "calls_min": 355700,
    "cp_upper": 0.0004999593254389324,
    "lead_misses": 1084,
    "lead_point": 0.00029759902923416295,
    "lead_upper": 0.0004999593254389324,
    "misses": 1084,
    "point": 0.00029759902923416295,
    "upper": 0.000547945205479452
   },
   "plain_reasons": [
    "Lines built in the 1920s: across 10 test runs, the city's records show 9 of the model's 2,902 not-lead calls (0.31%) as lead or galvanized pipe needing replacement, at most 2.11% with 98.33% confidence.",
    "Lines built in the 1940s: across 10 test runs, the city's records show 963 of the model's 46,485 not-lead calls (2.07%) as lead or galvanized pipe needing replacement, at most 3.88% with 98.33% confidence.",
    "The limits are 1.00% for the share of misses and 2.00% for its upper bound, in each decade and over all decades judged."
   ],
   "plain_statement": "Records only: the city marks no lines as checked in the field. Of the model's not-lead calls on test lines, the city's records show 0.03% as lead or galvanized pipe needing replacement; with 98.33% confidence the share is at most 0.05%. This is agreement with the records, not what was found in the ground, so it cannot unlock answers about single homes. The model makes no call for lines built in the 1910s: too few labelled lines there to judge it.",
   "reasons": [
    "decade 1920: 9 of 2,902 judged clears were lead or GRR (0.31%), upper 2.11% at 98.33%; bars 1% and 2%",
    "decade 1940: 963 of 46,485 judged clears were lead or GRR (2.07%), upper 3.88% at 98.33%; bars 1% and 2%"
   ],
   "runs": [
    {
     "bootstrap_upper": 0.0003763299843766037,
     "calls": 355496,
     "cp_upper": 0.0003568496535563441,
     "misses": 103,
     "rate": 0.0002897360307851565,
     "seed": 0,
     "upper": 0.0003763299843766037
    },
    {
     "bootstrap_upper": 0.00035144348653565937,
     "calls": 363032,
     "cp_upper": 0.00033726276347093194,
     "misses": 99,
     "rate": 0.00027270323277286853,
     "seed": 1,
     "upper": 0.00035144348653565937
    },
    {
     "bootstrap_upper": 0.00036297032577982776,
     "calls": 365527,
     "cp_upper": 0.0003410117659262328,
     "misses": 101,
     "rate": 0.0002763133776711433,
     "seed": 2,
     "upper": 0.00036297032577982776
    },
    {
     "bootstrap_upper": 0.0003618242520928244,
     "calls": 360159,
     "cp_upper": 0.00034916264422832995,
     "misses": 102,
     "rate": 0.00028320824968972036,
     "seed": 3,
     "upper": 0.0003618242520928244
    },
    {
     "bootstrap_upper": 0.0003593836159478537,
     "calls": 364508,
     "cp_upper": 0.0003449968677678841,
     "misses": 102,
     "rate": 0.00027982924928945317,
     "seed": 4,
     "upper": 0.0003593836159478537
    },
    {
     "bootstrap_upper": 0.0005365540886762942,
     "calls": 371633,
     "cp_upper": 0.0004944019474837295,
     "misses": 155,
     "rate": 0.00041707813891661934,
     "seed": 5,
     "upper": 0.0005365540886762942
    },
    {
     "bootstrap_upper": 0.00036935984815805365,
     "calls": 366683,
     "cp_upper": 0.00033692148393698457,
     "misses": 100,
     "rate": 0.00027271512450809007,
     "seed": 6,
     "upper": 0.00036935984815805365
    },
    {
     "bootstrap_upper": 0.00040017541936191207,
     "calls": 365757,
     "cp_upper": 0.0003589055888611094,
     "misses": 107,
     "rate": 0.00029254395678004796,
     "seed": 7,
     "upper": 0.00040017541936191207
    },
    {
     "bootstrap_upper": 0.00042811175097706147,
     "calls": 363619,
     "cp_upper": 0.0004003243432745857,
     "misses": 120,
     "rate": 0.00033001575825245655,
     "seed": 8,
     "upper": 0.00042811175097706147
    },
    {
     "bootstrap_upper": 0.000281147299348175,
     "calls": 364071,
     "cp_upper": 0.0002628811297520957,
     "misses": 75,
     "rate": 0.00020600377398913948,
     "seed": 9,
     "upper": 0.000281147299348175
    }
   ],
   "statement": "records only: no field-checked lines. Among held-out lines the model cleared, the share the city's records call lead or galvanized pipe needing replacement is 0.03%, at most 0.05% with 98.33% confidence for each release run. This is agreement with records, not field truth. Only decades with at least 203 judged clears in every run are judged; the model must abstain in the others.",
   "status": "fail",
   "summary": "release bound fails on records-only clears of lead or GRR: decade 1920 0.31% (upper 2.11%); decade 1940 2.07% (upper 3.88%)",
   "upper_max": 0.02,
   "weighted": null,
   "withdrawn_calls": 2000,
   "withdrawn_calls_by_run": [
    204,
    211,
    202,
    206,
    211,
    209,
    211,
    206,
    129,
    211
   ],
   "withdrawn_groups": [
    "1910"
   ]
  },
  "sentence": "If the homes we score are like the homes we checked, at least 95 of every 100 will have their true material inside the range shown.",
  "sufficiency": {
   "lead_blocks_h3_7": 15,
   "min_lead_blocks": 5,
   "min_verified_leads": 100,
   "verified_leads": 171,
   "verified_mask": "absent"
  }
 },
 "bias_diagnostics": {
  "doubtful": false,
  "limit": "Weighting fixes selection on observed covariates only. If crews chose houses for reasons that are not in the data, only a probability sample fixes it.",
  "reason": "No verified mask was provided.",
  "status": "not_run"
 },
 "equity": {
  "absent_dimensions": [
   "income",
   "renter_share",
   "residents_of_colour",
   "neighbourhood"
  ],
  "allocation": {
   "reason": "No inspection assignment was provided.",
   "status": "not_run"
  },
  "backend": "local",
  "bookkeeping": {
   "backend": "local",
   "by_group": [
    {
     "group": "1900",
     "observed_prevalence": 0.0,
     "predicted_prevalence": 0.007312318950552396
    },
    {
     "group": "1910",
     "observed_prevalence": 0.009478672985781991,
     "predicted_prevalence": 0.006371327994629708
    },
    {
     "group": "1920",
     "observed_prevalence": 0.003389830508474576,
     "predicted_prevalence": 0.0018511955838309204
    },
    {
     "group": "1930",
     "observed_prevalence": 0.0003586800573888092,
     "predicted_prevalence": 0.0009719672254975687
    },
    {
     "group": "1940",
     "observed_prevalence": 0.032909810456154966,
     "predicted_prevalence": 0.031504219414177144
    },
    {
     "group": "1950",
     "observed_prevalence": 7.456009543692216e-05,
     "predicted_prevalence": 0.00014243284785627134
    },
    {
     "group": "1960",
     "observed_prevalence": 0.0,
     "predicted_prevalence": 8.734671024562898e-05
    },
    {
     "group": "1970",
     "observed_prevalence": 1.57254957462534e-05,
     "predicted_prevalence": 7.120237089563761e-05
    },
    {
     "group": "1980",
     "observed_prevalence": 5.8716458223239976e-05,
     "predicted_prevalence": 7.115188950386585e-05
    },
    {
     "group": "1990",
     "observed_prevalence": 3.550568978678833e-05,
     "predicted_prevalence": 5.740435123721733e-05
    },
    {
     "group": "2000",
     "observed_prevalence": 2.930317060305925e-05,
     "predicted_prevalence": 6.332854189857608e-05
    },
    {
     "group": "2010",
     "observed_prevalence": 0.0,
     "predicted_prevalence": 5.138362044420684e-05
    },
    {
     "group": "2020",
     "observed_prevalence": 0.0,
     "predicted_prevalence": 5.247825463558744e-05
    }
   ],
   "difference": {
    "observed_prevalence": 0.032909810456154966,
    "predicted_prevalence": 0.03145283579373294
   },
   "overall": {
    "observed_prevalence": 0.00045508739807106815,
    "predicted_prevalence": 0.000483246786505952
   }
  },
  "dimensions": [
   "decade_built"
  ],
  "feature_violation": [],
  "gaps": [
   {
    "calls": 0.0,
    "equity_min_calls": 50,
    "gap": 0.0,
    "group": "decade_built:1900",
    "group_rate": 0.0,
    "judged": false,
    "lower": -0.00029519257801518135,
    "n": 12.0,
    "n_verified_positives": 0,
    "overall_rate": 0.00029519257801518135,
    "status": "not judged",
    "upper": -0.00029519257801518135
   },
   {
    "calls": 204.0,
    "equity_min_calls": 50,
    "gap": 0.00950872899061227,
    "group": "decade_built:1910",
    "group_rate": 0.00980392156862745,
    "judged": false,
    "lower": -0.004004156410258723,
    "n": 211.0,
    "n_verified_positives": 2,
    "overall_rate": 0.00029519257801518135,
    "status": "not judged",
    "upper": 0.023021614391483265
   },
   {
    "calls": 292.0,
    "equity_min_calls": 50,
    "gap": 0.0031294649562313938,
    "group": "decade_built:1920",
    "group_rate": 0.003424657534246575,
    "judged": false,
    "lower": -0.0035659721337814743,
    "n": 295.0,
    "n_verified_positives": 1,
    "overall_rate": 0.00029519257801518135,
    "status": "not judged",
    "upper": 0.009824902046244262
   },
   {
    "calls": 2712.0,
    "equity_min_calls": 50,
    "gap": 7.353898540664756e-05,
    "group": "decade_built:1930",
    "group_rate": 0.0003687315634218289,
    "judged": false,
    "lower": -0.0006457169819041448,
    "n": 2788.0,
    "n_verified_positives": 1,
    "overall_rate": 0.00029519257801518135,
    "status": "not judged",
    "upper": 0.00079279495271744
   },
   {
    "calls": 4642.0,
    "equity_min_calls": 50,
    "gap": 0.0197392753237513,
    "group": "decade_built:1940",
    "group_rate": 0.02003446790176648,
    "judged": true,
    "lower": 0.015761045786852196,
    "n": 4801.0,
    "n_verified_positives": 158,
    "overall_rate": 0.00029519257801518135,
    "status": "judged",
    "upper": 0.023717504860650405
   },
   {
    "calls": 25821.0,
    "equity_min_calls": 50,
    "gap": 0.0,
    "group": "decade_built:1950",
    "group_rate": 3.872816699585609e-05,
    "judged": false,
    "lower": -0.0003465310222235819,
    "n": 26824.0,
    "n_verified_positives": 2,
    "overall_rate": 0.00029519257801518135,
    "status": "not judged",
    "upper": -0.00016639779981506865
   },
   {
    "calls": 31527.0,
    "equity_min_calls": 50,
    "gap": 0.0,
    "group": "decade_built:1960",
    "group_rate": 0.0,
    "judged": false,
    "lower": -0.00035164577348906933,
    "n": 32829.0,
    "n_verified_positives": 0,
    "overall_rate": 0.00029519257801518135,
    "status": "not judged",
    "upper": -0.00023873938254129334
   },
   {
    "calls": 58992.0,
    "equity_min_calls": 50,
    "gap": 0.0,
    "group": "decade_built:1970",
    "group_rate": 1.6951451044209385e-05,
    "judged": false,
    "lower": -0.0003408876277359767,
    "n": 63591.0,
    "n_verified_positives": 1,
    "overall_rate": 0.00029519257801518135,
    "status": "not judged",
    "upper": -0.0002155946262059672
   },
   {
    "calls": 31915.0,
    "equity_min_calls": 50,
    "gap": 0.0,
    "group": "decade_built:1980",
    "group_rate": 6.266645777847408e-05,
    "judged": false,
    "lower": -0.0003293554395028845,
    "n": 34062.0,
    "n_verified_positives": 2,
    "overall_rate": 0.00029519257801518135,
    "status": "not judged",
    "upper": -0.00013569680097053006
   },
   {
    "calls": 54242.0,
    "equity_min_calls": 50,
    "gap": 0.0,
    "group": "decade_built:1990",
    "group_rate": 3.6871796762656245e-05,
    "judged": false,
    "lower": -0.00032904373704834824,
    "n": 56329.0,
    "n_verified_positives": 2,
    "overall_rate": 0.00029519257801518135,
    "status": "not judged",
    "upper": -0.00018759782545670197
   },
   {
    "calls": 63412.0,
    "equity_min_calls": 50,
    "gap": 0.0,
    "group": "decade_built:2000",
    "group_rate": 3.153977165205324e-05,
    "judged": false,
    "lower": -0.0003301078395458397,
    "n": 68252.0,
    "n_verified_positives": 2,
    "overall_rate": 0.00029519257801518135,
    "status": "not judged",
    "upper": -0.00019719777318041648
   },
   {
    "calls": 46325.0,
    "equity_min_calls": 50,
    "gap": 0.0,
    "group": "decade_built:2010",
    "group_rate": 0.0,
    "judged": false,
    "lower": -0.00035164533603629654,
    "n": 49921.0,
    "n_verified_positives": 0,
    "overall_rate": 0.00029519257801518135,
    "status": "not judged",
    "upper": -0.00023873981999406613
   },
   {
    "calls": 35616.0,
    "equity_min_calls": 50,
    "gap": 0.0,
    "group": "decade_built:2020",
    "group_rate": 0.0,
    "judged": false,
    "lower": -0.00035164565665613657,
    "n": 35837.0,
    "n_verified_positives": 0,
    "overall_rate": 0.00029519257801518135,
    "status": "not judged",
    "upper": -0.00023873949937422613
   }
  ],
  "groups": [
   {
    "abstain_rate": 1.0,
    "calibration_error": 0.007312318950552396,
    "conformal_coverage": 1.0,
    "dimension": "decade_built",
    "excludes_zero": true,
    "false_non_lead_calls": 0.0,
    "false_non_lead_rate": 0.0,
    "group": "1900",
    "n": 12,
    "n_verified": 12,
    "observed_prevalence": 0.0,
    "passed": false,
    "predicted_prevalence": 0.007312318950552396,
    "prevalence_gap": 0.007312318950552396,
    "prevalence_gap_lower": 0.0005996893113672677,
    "prevalence_gap_upper": 0.01911696031318811,
    "status": "insufficient_data",
    "underpredicted": false,
    "verified_share": 1.0
   },
   {
    "abstain_rate": 0.03317535545023697,
    "calibration_error": 0.0031073449911522832,
    "conformal_coverage": 0.990521327014218,
    "dimension": "decade_built",
    "excludes_zero": false,
    "false_non_lead_calls": 204.0,
    "false_non_lead_rate": 0.00980392156862745,
    "group": "1910",
    "n": 211,
    "n_verified": 211,
    "observed_prevalence": 0.009478672985781991,
    "passed": true,
    "predicted_prevalence": 0.006371327994629708,
    "prevalence_gap": -0.0031073449911522824,
    "prevalence_gap_lower": -0.017570707609030314,
    "prevalence_gap_upper": 0.007107051996784977,
    "status": "scored",
    "underpredicted": false,
    "verified_share": 1.0
   },
   {
    "abstain_rate": 0.010169491525423728,
    "calibration_error": 0.0015386349246436558,
    "conformal_coverage": 0.9966101694915255,
    "dimension": "decade_built",
    "excludes_zero": false,
    "false_non_lead_calls": 292.0,
    "false_non_lead_rate": 0.003424657534246575,
    "group": "1920",
    "n": 295,
    "n_verified": 295,
    "observed_prevalence": 0.003389830508474576,
    "passed": true,
    "predicted_prevalence": 0.0018511955838309204,
    "prevalence_gap": -0.001538634924643656,
    "prevalence_gap_lower": -0.008468366412689884,
    "prevalence_gap_upper": 0.002067631313805189,
    "status": "scored",
    "underpredicted": false,
    "verified_share": 1.0
   },
   {
    "abstain_rate": 0.027259684361549498,
    "calibration_error": 0.0006132871681087595,
    "conformal_coverage": 0.9996413199426112,
    "dimension": "decade_built",
    "excludes_zero": false,
    "false_non_lead_calls": 2712.0,
    "false_non_lead_rate": 0.0003687315634218289,
    "group": "1930",
    "n": 2788,
    "n_verified": 2788,
    "observed_prevalence": 0.0003586800573888092,
    "passed": true,
    "predicted_prevalence": 0.0009719672254975687,
    "prevalence_gap": 0.0006132871681087596,
    "prevalence_gap_lower": -0.00010584796946170451,
    "prevalence_gap_upper": 0.0009941588161245568,
    "status": "scored",
    "underpredicted": false,
    "verified_share": 1.0
   },
   {
    "abstain_rate": 0.03270152051655905,
    "calibration_error": 0.006442875127294457,
    "conformal_coverage": 0.9806290356175796,
    "dimension": "decade_built",
    "excludes_zero": false,
    "false_non_lead_calls": 4642.0,
    "false_non_lead_rate": 0.02003446790176648,
    "group": "1940",
    "n": 4801,
    "n_verified": 4801,
    "observed_prevalence": 0.032909810456154966,
    "passed": true,
    "predicted_prevalence": 0.031504219414177144,
    "prevalence_gap": -0.0014055910419778232,
    "prevalence_gap_lower": -0.0051509950942714615,
    "prevalence_gap_upper": 0.0027384742053514445,
    "status": "scored",
    "underpredicted": false,
    "verified_share": 1.0
   },
   {
    "abstain_rate": 0.037391887861616464,
    "calibration_error": 6.787275241934917e-05,
    "conformal_coverage": 0.9999627199522816,
    "dimension": "decade_built",
    "excludes_zero": false,
    "false_non_lead_calls": 25821.0,
    "false_non_lead_rate": 3.872816699585609e-05,
    "group": "1950",
    "n": 26824,
    "n_verified": 26824,
    "observed_prevalence": 7.456009543692216e-05,
    "passed": true,
    "predicted_prevalence": 0.00014243284785627134,
    "prevalence_gap": 6.787275241934917e-05,
    "prevalence_gap_lower": -4.540405357726002e-05,
    "prevalence_gap_upper": 0.00014327705828485302,
    "status": "scored",
    "underpredicted": false,
    "verified_share": 1.0
   },
   {
    "abstain_rate": 0.039660056657223795,
    "calibration_error": 8.734671024562898e-05,
    "conformal_coverage": 1.0,
    "dimension": "decade_built",
    "excludes_zero": true,
    "false_non_lead_calls": 31527.0,
    "false_non_lead_rate": 0.0,
    "group": "1960",
    "n": 32829,
    "n_verified": 32829,
    "observed_prevalence": 0.0,
    "passed": true,
    "predicted_prevalence": 8.734671024562898e-05,
    "prevalence_gap": 8.734671024562898e-05,
    "prevalence_gap_lower": 8.682005302238572e-05,
    "prevalence_gap_upper": 8.791644030376491e-05,
    "status": "scored",
    "underpredicted": false,
    "verified_share": 1.0
   },
   {
    "abstain_rate": 0.07232155493701939,
    "calibration_error": 5.5476875149384203e-05,
    "conformal_coverage": 0.9999842745042538,
    "dimension": "decade_built",
    "excludes_zero": true,
    "false_non_lead_calls": 58992.0,
    "false_non_lead_rate": 1.6951451044209385e-05,
    "group": "1970",
    "n": 63591,
    "n_verified": 63591,
    "observed_prevalence": 1.57254957462534e-05,
    "passed": true,
    "predicted_prevalence": 7.120237089563761e-05,
    "prevalence_gap": 5.547687514938421e-05,
    "prevalence_gap_lower": 2.3837568543983742e-05,
    "prevalence_gap_upper": 7.161150516388491e-05,
    "status": "scored",
    "underpredicted": false,
    "verified_share": 1.0
   },
   {
    "abstain_rate": 0.06303211790264811,
    "calibration_error": 1.2435431280625876e-05,
    "conformal_coverage": 0.9999412835417768,
    "dimension": "decade_built",
    "excludes_zero": false,
    "false_non_lead_calls": 31915.0,
    "false_non_lead_rate": 6.266645777847408e-05,
    "group": "1980",
    "n": 34062,
    "n_verified": 34062,
    "observed_prevalence": 5.8716458223239976e-05,
    "passed": true,
    "predicted_prevalence": 7.115188950386585e-05,
    "prevalence_gap": 1.2435431280625874e-05,
    "prevalence_gap_lower": -7.55201142621476e-05,
    "prevalence_gap_upper": 7.174712653105553e-05,
    "status": "scored",
    "underpredicted": false,
    "verified_share": 1.0
   },
   {
    "abstain_rate": 0.03705018729251362,
    "calibration_error": 2.1898661450429e-05,
    "conformal_coverage": 0.9999644943102132,
    "dimension": "decade_built",
    "excludes_zero": false,
    "false_non_lead_calls": 54242.0,
    "false_non_lead_rate": 3.6871796762656245e-05,
    "group": "1990",
    "n": 56329,
    "n_verified": 56329,
    "observed_prevalence": 3.550568978678833e-05,
    "passed": true,
    "predicted_prevalence": 5.740435123721733e-05,
    "prevalence_gap": 2.1898661450429005e-05,
    "prevalence_gap_lower": -3.1447370620050294e-05,
    "prevalence_gap_upper": 5.747327595930337e-05,
    "status": "scored",
    "underpredicted": false,
    "verified_share": 1.0
   },
   {
    "abstain_rate": 0.07091367285940339,
    "calibration_error": 3.402537129551682e-05,
    "conformal_coverage": 0.999970696829397,
    "dimension": "decade_built",
    "excludes_zero": false,
    "false_non_lead_calls": 63412.0,
    "false_non_lead_rate": 3.153977165205324e-05,
    "group": "2000",
    "n": 68252,
    "n_verified": 68252,
    "observed_prevalence": 2.930317060305925e-05,
    "passed": true,
    "predicted_prevalence": 6.332854189857608e-05,
    "prevalence_gap": 3.402537129551683e-05,
    "prevalence_gap_lower": -1.0048923377329476e-05,
    "prevalence_gap_upper": 6.377068721909942e-05,
    "status": "scored",
    "underpredicted": false,
    "verified_share": 1.0
   },
   {
    "abstain_rate": 0.07203381342521184,
    "calibration_error": 5.138362044420684e-05,
    "conformal_coverage": 1.0,
    "dimension": "decade_built",
    "excludes_zero": true,
    "false_non_lead_calls": 46325.0,
    "false_non_lead_rate": 0.0,
    "group": "2010",
    "n": 49921,
    "n_verified": 49921,
    "observed_prevalence": 0.0,
    "passed": true,
    "predicted_prevalence": 5.138362044420684e-05,
    "prevalence_gap": 5.138362044420684e-05,
    "prevalence_gap_lower": 5.113724073433221e-05,
    "prevalence_gap_upper": 5.159911216370386e-05,
    "status": "scored",
    "underpredicted": false,
    "verified_share": 1.0
   },
   {
    "abstain_rate": 0.006166810837960767,
    "calibration_error": 5.247825463558744e-05,
    "conformal_coverage": 1.0,
    "dimension": "decade_built",
    "excludes_zero": true,
    "false_non_lead_calls": 35616.0,
    "false_non_lead_rate": 0.0,
    "group": "2020",
    "n": 35837,
    "n_verified": 35837,
    "observed_prevalence": 0.0,
    "passed": true,
    "predicted_prevalence": 5.247825463558744e-05,
    "prevalence_gap": 5.247825463558744e-05,
    "prevalence_gap_lower": 5.2299455907505394e-05,
    "prevalence_gap_upper": 5.265932154815836e-05,
    "status": "scored",
    "underpredicted": false,
    "verified_share": 1.0
   }
  ],
  "min_verified": 100,
  "passes_silently": false,
  "reason": "Equity audit for decade_built. A group with fewer than 100 verified lines is insufficient_data and does not pass. 1 groups are under that floor. Share of residents of colour is an audit dimension, not a model feature. Missing dimensions are omitted.",
  "status": "audited",
  "underpredicted": []
 },
 "red_team": {
  "n": 375752,
  "n_neg": 375581,
  "n_pos": 171,
  "n_positives": 171,
  "reason": "Labels were shuffled and scored with honest cross-validation.",
  "shuffled_label_auc": 0.5313465215086408,
  "status": "scored",
  "suite": "separate",
  "suite_reason": "Named red-team cases run through craftpine.eval.redteam.run_case and tests/redteam. They are not refit inside every evaluate call."
 },
 "leakage": {
  "canary_auc": 1.0,
  "canary_detected": true,
  "feature_auc": {
   "h3": 0.7754673932284373,
   "lat": 0.5165000192387499,
   "lon": 0.5221346434304946,
   "year_built": 0.9700757696071147
  },
  "flagged_features": [],
  "id_like": [],
  "label_copies": [],
  "label_nmi": {
   "h3": 0.00019335648212179634,
   "lat": 0.00025549692730473356,
   "lon": 0.0001750318528044538,
   "year_built": 0.00040185382334528673
  },
  "lead_oof_auc": {
   "lat": 0.514613686014515,
   "lon": 0.5229750394830771,
   "year_built": 0.9694701235673054
  },
  "near_label_rules": {},
  "quality": {
   "range_violations": {
    "year_built": 0
   }
  },
  "target_encoding_outside_folds": []
 },
 "drift": {
  "reason": "Drift is not part of this report.",
  "status": "not_run"
 },
 "gate_decision": {
  "decision": "FAIL",
  "reasons": [
   "cleared share 0.9466 is not above the year-built rule's 1.0000; abstain rate 0.0534 against 0.0000",
   "false non-lead gap exceeds 0.01 for decade_built:1940",
   "release bound fails on records-only clears of lead or GRR: decade 1920 0.31% (upper 2.11%); decade 1940 2.07% (upper 3.88%)"
  ]
 },
 "versions": {
  "craftpine-eval": "0.0.0",
  "numpy": "2.3.5",
  "pandas": "3.1.0",
  "metric_source": "craftpine.eval.evaluate"
 },
 "seeds": [
  0,
  1,
  2,
  3,
  4,
  5,
  6,
  7,
  8,
  9
 ]
}