{ "schema_version": "2.1.0", "report_id": "01a1a221-406d-7ad3-87c2-d6965bddd17e", "dataset_card": { "citation": "Gebru et al., Datasheets for Datasets, arXiv:1803.09010.", "class_counts": { "GRR": 1210, "LEAD": 161, "NON_LEAD": 23249 }, "collection": "Field verifications and utility records are separate label sources. Records can disagree with the field. This card does not merge them.", "composition": { "class_counts": { "GRR": 1210, "LEAD": 161, "NON_LEAD": 23249 }, "equity_dimensions": [ "decade_built" ], "rows": 24620 }, "distribution": { "licence": "not recorded", "source_url": "not recorded" }, "maintenance": { "note": "Each EvalReport writes a new card. Hand-edited numbers are not accepted.", "versions": { "craftpine-eval": "0.0.0", "numpy": "2.3.5", "pandas": "3.1.0" } }, "motivation": "The dataset supports evaluation of a service-line material model.", "n_verified": 8793, "preprocessing": { "split": "spatial_block", "target_encoding_outside_folds": [] }, "rows": 24620, "schema": "gebru-datasheets-2021", "uses": "Evaluation, release gating, and regression checks. The dataset is not itself an inventory update.", "rows_in_snapshot": 71275 }, "model_card": { "caveats": "If the homes we score are like the homes we checked, at least 95 of every 100 will have their true material inside the range shown. The guarantee is approximate because the checked homes do not look like the rest of the system on the recorded covariates. A probability sample is what makes the guarantee firm again.", "citation": "Mitchell et al., Model Cards for Model Reporting, arXiv:1810.03993.", "ethical_considerations": "A false non-lead call can leave a lead line in service. Share of residents of colour is reported for audit and is not a training feature. The model does not change the inventory.", "evaluation_data": { "split": "spatial_block" }, "factors": { "audit_only": [ "residents_of_colour" ], "feature_violation": [], "groups": [ "decade_built" ] }, "intended_use": { "out_of_scope": [ "automatic changes to an official inventory category", "geocoding addresses", "use of race or ethnicity as a model feature" ], "primary": "Score unknown service lines and show where the score is too weak to call. A named person approves any inventory change." }, "metrics": { "calibration_ece": 0.0017146203722653815, "conformal_coverage": 0.9471770917952883, "false_non_lead_rate": 0.0015799068838277235, "overall": [ { "estimate": 0.006428043450354965, "interval": { "level": 0.95, "lower": 0.004219058226574734, "upper": 0.008479110597049637 }, "name": "brier_lead" }, { "estimate": 0.0982412821444071, "interval": { "level": 0.95, "lower": 0.08414813557542478, "upper": 0.1134832328412202 }, "name": "brier_multiclass" }, { "estimate": 1.463498115889448e-05, "interval": { "level": 0.95, "lower": 1.179872267344022e-05, "upper": 4.712486208068689e-05 }, "name": "brier_reliability" }, { "estimate": 0.00010086870389299975, "interval": { "level": 0.95, "lower": 4.620525510999451e-05, "upper": 0.00017486041324691527 }, "name": "brier_resolution" }, { "estimate": 0.006496635125227586, "interval": { "level": 0.95, "lower": 0.00422483372299577, "upper": 0.008585077430522982 }, "name": "brier_uncertainty" }, { "estimate": 77.6, "interval": { "level": 0.95, "lower": 52.575, "upper": 100.1075 }, "name": "lead_found_top_decile" }, { "estimate": 0.2065933584816279, "interval": { "level": 0.95, "lower": 0.18202270176164073, "upper": 0.23400002830298747 }, "name": "log_loss" }, { "estimate": 0.17225751219564645, "interval": { "level": 0.95, "lower": 0.1382966698906088, "upper": 0.2048691563252511 }, "name": "pr_auc_grr" }, { "estimate": 0.029684151729751595, "interval": { "level": 0.95, "lower": 0.021288104665040452, "upper": 0.039412338185790305 }, "name": "pr_auc_lead" }, { "estimate": 0.39343525463857637, "interval": { "level": 0.95, "lower": 0.3822905902584229, "upper": 0.4034738112805613 }, "name": "pr_auc_macro" }, { "estimate": 0.978364099990331, "interval": { "level": 0.95, "lower": 0.973655617975851, "upper": 0.9829068119841079 }, "name": "pr_auc_non_lead" }, { "estimate": 0.0, "interval": { "level": 0.95, "lower": 0.0, "upper": 0.0008573717948717956 }, "name": "recall_at_precision_0_8" }, { "estimate": 0.7542895492143289, "interval": { "level": 0.95, "lower": 0.7225667108250167, "upper": 0.7747527191034159 }, "name": "roc_auc_grr" }, { "estimate": 0.8392985955200982, "interval": { "level": 0.95, "lower": 0.7967273300055281, "upper": 0.8667690189757382 }, "name": "roc_auc_lead" }, { "estimate": 0.7894474869107462, "interval": { "level": 0.95, "lower": 0.7634790471727378, "upper": 0.8065594487136677 }, "name": "roc_auc_macro" }, { "estimate": 0.7747543159978113, "interval": { "level": 0.95, "lower": 0.7472854891097449, "upper": 0.7938036265849954 }, "name": "roc_auc_non_lead" } ], "shuffled_label_auc": 0.5067567756308631 }, "model_details": { "name": "lightgbm", "params": { "class_weight": null, "classes": [ "LEAD", "GRR", "NON_LEAD" ], "learning_rate": 0.08, "min_child_samples": 5, "n_estimators": 40, "n_jobs": 1, "name": "lightgbm", "num_leaves": 15, "reg_lambda": 1.0, "seed": 0 }, "seeds": [ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 ], "type": "tabular classifier over LEAD, GRR, and NON_LEAD", "versions": { "craftpine-eval": "0.0.0", "numpy": "2.3.5", "pandas": "3.1.0" } }, "quantitative_analyses": { "canary_detected": true, "flagged_features": [], "gate": "FAIL", "label_copies": [], "reasons": [ "release bound fails on field-checked clears of lead or GRR: decades 1950, 1960, 1980, 1990, 2010; overall 10.4% (upper 12.7%)" ], "underpredicted": [ { "dimension": "decade_built", "excludes_zero": true, "gap": -0.047833875774655435, "group": "1910", "lower": -0.07305723996581835, "upper": -0.022605896500002987 }, { "dimension": "decade_built", "excludes_zero": true, "gap": -0.02053795686279819, "group": "1920", "lower": -0.04313025249182712, "upper": -0.002152963450190032 }, { "dimension": "decade_built", "excludes_zero": true, "gap": -0.006661859389671749, "group": "1990", "lower": -0.012411663351436187, "upper": -0.0016205552505338696 } ] }, "schema": "mitchell-2019", "requested_algorithm": "lightgbm", "algorithm": "lightgbm", "gated_model": "lightgbm", "stored_model": null, "engine": "craftpine.eval.evaluate", "leaderboard": [ { "name": "year_built_rule", "log_loss": 0.2192622773319684, "lower": 0.19268585331389274, "upper": 0.24625937778082063, "gate": "FAIL", "label_copies": [], "log_loss_improvement": { "estimate": 0.00012556491974995975, "lower": -7.763983687220589e-05, "upper": 0.00030531833292256095, "excludes_zero": false } }, { "name": "records_face_value", "log_loss": 0.23706733827431276, "lower": 0.20794422709203547, "upper": 0.26783298647301845, "gate": "FAIL", "label_copies": [], "log_loss_improvement": { "estimate": -0.01767949602259441, "lower": -0.02208600780163707, "upper": -0.012548305074276653, "excludes_zero": true } }, { "name": "lightgbm", "log_loss": 0.2065933584816279, "lower": 0.18202270176164073, "upper": 0.23400002830298747, "gate": "FAIL", "label_copies": [], "log_loss_improvement": { "estimate": 0.01279448377009045, "lower": 0.008991597786717244, "upper": 0.016318537687242427, "excludes_zero": true } }, { "name": "xgboost", "log_loss": 0.20495966918959851, "lower": 0.17973058322543026, "upper": 0.23307976648786388, "gate": "FAIL", "label_copies": [], "log_loss_improvement": { "estimate": 0.014428173062119829, "lower": 0.010342112022011908, "upper": 0.018232339516953133, "excludes_zero": true } }, { "name": "catboost", "log_loss": 0.20929196432127722, "lower": 0.1833730843862602, "upper": 0.2388602523387079, "gate": "FAIL", "label_copies": [], "log_loss_improvement": { "estimate": 0.010095877930441099, "lower": 0.006337155700632569, "upper": 0.013993817617374128, "excludes_zero": true } } ] }, "split_strategy": "spatial_block", "metrics": [ { "scope": "GROUP", "group_key": "year_built_rule", "metrics": [ { "name": "brier_lead", "estimate": 0.006463241639334906, "interval": { "lower": 0.004218175289937172, "upper": 0.008537174548125226, "level": 0.95 } }, { "name": "brier_multiclass", "estimate": 0.1028098802111295, "interval": { "lower": 0.08825452200746829, "upper": 0.1184018107034906, "level": 0.95 } }, { "name": "brier_reliability", "estimate": 1.9187640938427072e-05, "interval": { "lower": 1.5110709132958406e-05, "upper": 5.757082906440824e-05, "level": 0.95 } }, { "name": "brier_resolution", "estimate": 4.858083321399983e-05, "interval": { "lower": 2.4930070952736765e-05, "upper": 0.00010278051484420731, "level": 0.95 } }, { "name": "brier_uncertainty", "estimate": 0.006496635125227586, "interval": { "lower": 0.00422483372299577, "upper": 0.008585077430522982, "level": 0.95 } }, { "name": "lead_found_top_decile", "estimate": 28.2, "interval": { "lower": 18.695, "upper": 38.9275, "level": 0.95 } }, { "name": "log_loss", "estimate": 0.2192622773319684, "interval": { "lower": 0.19268585331389274, "upper": 0.24625937778082063, "level": 0.95 } }, { "name": "pr_auc_grr", "estimate": 0.08028370233590669, "interval": { "lower": 0.06985702437517653, "upper": 0.09596956564897412, "level": 0.95 } }, { "name": "pr_auc_lead", "estimate": 0.01311170897201705, "interval": { "lower": 0.009930127909254674, "upper": 0.018031652366647267, "level": 0.95 } }, { "name": "pr_auc_macro", "estimate": 0.35449332124879435, "interval": { "lower": 0.35076385085268, "upper": 0.35910664144232285, "level": 0.95 } }, { "name": "pr_auc_non_lead", "estimate": 0.9700845524384594, "interval": { "lower": 0.9629821101034618, "upper": 0.9735831132530001, "level": 0.95 } }, { "name": "recall_at_precision_0_8", "estimate": 0.0, "interval": { "lower": 0.0, "upper": 0.0, "level": 0.95 } }, { "name": "roc_auc_grr", "estimate": 0.6730366570053766, "interval": { "lower": 0.645137221532944, "upper": 0.6948084402659931, "level": 0.95 } }, { "name": "roc_auc_lead", "estimate": 0.7399966962078002, "interval": { "lower": 0.6884934824215379, "upper": 0.7809841931925954, "level": 0.95 } }, { "name": "roc_auc_macro", "estimate": 0.6999837135646121, "interval": { "lower": 0.6708959253032899, "upper": 0.7212904225511639, "level": 0.95 } }, { "name": "roc_auc_non_lead", "estimate": 0.6869177874806596, "interval": { "lower": 0.6594233754649034, "upper": 0.7048300441257372, "level": 0.95 } } ] }, { "scope": "GROUP", "group_key": "records_face_value", "metrics": [ { "name": "brier_lead", "estimate": 0.006505085670307121, "interval": { "lower": 0.00423401778581592, "upper": 0.008603118586704577, "level": 0.95 } }, { "name": "brier_multiclass", "estimate": 0.10604712415721614, "interval": { "lower": 0.0905301124270173, "upper": 0.12244549539081005, "level": 0.95 } }, { "name": "brier_reliability", "estimate": 1.6994713762773603e-05, "interval": { "lower": 1.2615437841458726e-05, "upper": 4.147520354539201e-05, "level": 0.95 } }, { "name": "brier_resolution", "estimate": 8.544168683237857e-06, "interval": { "lower": 5.712525540973466e-06, "upper": 2.954802636682044e-05, "level": 0.95 } }, { "name": "brier_uncertainty", "estimate": 0.006496635125227586, "interval": { "lower": 0.00422483372299577, "upper": 0.008585077430522982, "level": 0.95 } }, { "name": "lead_found_top_decile", "estimate": 12.8, "interval": { "lower": 6.29, "upper": 19.305000000000003, "level": 0.95 } }, { "name": "log_loss", "estimate": 0.23706733827431276, "interval": { "lower": 0.20794422709203547, "upper": 0.26783298647301845, "level": 0.95 } }, { "name": "pr_auc_grr", "estimate": 0.04314975185325653, "interval": { "lower": 0.03869709548751508, "upper": 0.05339266912578545, "level": 0.95 } }, { "name": "pr_auc_lead", "estimate": 0.005555493433032162, "interval": { "lower": 0.0040891209642490555, "upper": 0.007874469446596519, "level": 0.95 } }, { "name": "pr_auc_macro", "estimate": 0.3306334155535312, "interval": { "lower": 0.328548124832521, "upper": 0.3319829290450421, "level": 0.95 } }, { "name": "pr_auc_non_lead", "estimate": 0.943195001374305, "interval": { "lower": 0.9293279363275693, "upper": 0.9500333439597075, "level": 0.95 } }, { "name": "recall_at_precision_0_8", "estimate": 0.0, "interval": { "lower": 0.0, "upper": 0.0, "level": 0.95 } }, { "name": "roc_auc_grr", "estimate": 0.46825816473146675, "interval": { "lower": 0.451776291130157, "upper": 0.48254231016823446, "level": 0.95 } }, { "name": "roc_auc_lead", "estimate": 0.43485348405329843, "interval": { "lower": 0.39546856373243283, "upper": 0.4729009959573749, "level": 0.95 } }, { "name": "roc_auc_macro", "estimate": 0.45698287092046036, "interval": { "lower": 0.4399352516017255, "upper": 0.47271271318684394, "level": 0.95 } }, { "name": "roc_auc_non_lead", "estimate": 0.46783696397661584, "interval": { "lower": 0.4489251543191047, "upper": 0.4820495787846856, "level": 0.95 } } ] }, { "scope": "GROUP", "group_key": "lightgbm", "metrics": [ { "name": "brier_lead", "estimate": 0.006428043450354965, "interval": { "lower": 0.004219058226574734, "upper": 0.008479110597049637, "level": 0.95 } }, { "name": "brier_multiclass", "estimate": 0.0982412821444071, "interval": { "lower": 0.08414813557542478, "upper": 0.1134832328412202, "level": 0.95 } }, { "name": "brier_reliability", "estimate": 1.463498115889448e-05, "interval": { "lower": 1.179872267344022e-05, "upper": 4.712486208068689e-05, "level": 0.95 } }, { "name": "brier_resolution", "estimate": 0.00010086870389299975, "interval": { "lower": 4.620525510999451e-05, "upper": 0.00017486041324691527, "level": 0.95 } }, { "name": "brier_uncertainty", "estimate": 0.006496635125227586, "interval": { "lower": 0.00422483372299577, "upper": 0.008585077430522982, "level": 0.95 } }, { "name": "lead_found_top_decile", "estimate": 77.6, "interval": { "lower": 52.575, "upper": 100.1075, "level": 0.95 } }, { "name": "log_loss", "estimate": 0.2065933584816279, "interval": { "lower": 0.18202270176164073, "upper": 0.23400002830298747, "level": 0.95 } }, { "name": "pr_auc_grr", "estimate": 0.17225751219564645, "interval": { "lower": 0.1382966698906088, "upper": 0.2048691563252511, "level": 0.95 } }, { "name": "pr_auc_lead", "estimate": 0.029684151729751595, "interval": { "lower": 0.021288104665040452, "upper": 0.039412338185790305, "level": 0.95 } }, { "name": "pr_auc_macro", "estimate": 0.39343525463857637, "interval": { "lower": 0.3822905902584229, "upper": 0.4034738112805613, "level": 0.95 } }, { "name": "pr_auc_non_lead", "estimate": 0.978364099990331, "interval": { "lower": 0.973655617975851, "upper": 0.9829068119841079, "level": 0.95 } }, { "name": "recall_at_precision_0_8", "estimate": 0.0, "interval": { "lower": 0.0, "upper": 0.0008573717948717956, "level": 0.95 } }, { "name": "roc_auc_grr", "estimate": 0.7542895492143289, "interval": { "lower": 0.7225667108250167, "upper": 0.7747527191034159, "level": 0.95 } }, { "name": "roc_auc_lead", "estimate": 0.8392985955200982, "interval": { "lower": 0.7967273300055281, "upper": 0.8667690189757382, "level": 0.95 } }, { "name": "roc_auc_macro", "estimate": 0.7894474869107462, "interval": { "lower": 0.7634790471727378, "upper": 0.8065594487136677, "level": 0.95 } }, { "name": "roc_auc_non_lead", "estimate": 0.7747543159978113, "interval": { "lower": 0.7472854891097449, "upper": 0.7938036265849954, "level": 0.95 } } ] }, { "scope": "GROUP", "group_key": "xgboost", "metrics": [ { "name": "brier_lead", "estimate": 0.00641011396349105, "interval": { "lower": 0.004212108346531322, "upper": 0.008467822616389058, "level": 0.95 } }, { "name": "brier_multiclass", "estimate": 0.09772052056462532, "interval": { "lower": 0.08367014912394932, "upper": 0.11313071170562, "level": 0.95 } }, { "name": "brier_reliability", "estimate": 1.8652820521454923e-05, "interval": { "lower": 8.685264634692238e-06, "upper": 5.82933276443536e-05, "level": 0.95 } }, { "name": "brier_resolution", "estimate": 0.00012837370105209253, "interval": { "lower": 5.660876821233948e-05, "upper": 0.00023150137884418322, "level": 0.95 } }, { "name": "brier_uncertainty", "estimate": 0.006496635125227586, "interval": { "lower": 0.00422483372299577, "upper": 0.008585077430522982, "level": 0.95 } }, { "name": "lead_found_top_decile", "estimate": 87.9, "interval": { "lower": 57.2825, "upper": 115.21000000000002, "level": 0.95 } }, { "name": "log_loss", "estimate": 0.20495966918959851, "interval": { "lower": 0.17973058322543026, "upper": 0.23307976648786388, "level": 0.95 } }, { "name": "pr_auc_grr", "estimate": 0.17620773095555947, "interval": { "lower": 0.14114801251985176, "upper": 0.2113489616544034, "level": 0.95 } }, { "name": "pr_auc_lead", "estimate": 0.030957041368540335, "interval": { "lower": 0.022403700760703894, "upper": 0.041722299930767974, "level": 0.95 } }, { "name": "pr_auc_macro", "estimate": 0.39485833879705023, "interval": { "lower": 0.3831139385456596, "upper": 0.40560784160286956, "level": 0.95 } }, { "name": "pr_auc_non_lead", "estimate": 0.9774102440670509, "interval": { "lower": 0.9725990511704125, "upper": 0.9820223418717199, "level": 0.95 } }, { "name": "recall_at_precision_0_8", "estimate": 0.0, "interval": { "lower": 0.0, "upper": 0.0008556547619047622, "level": 0.95 } }, { "name": "roc_auc_grr", "estimate": 0.7482953336322332, "interval": { "lower": 0.715143133233181, "upper": 0.7727499141348078, "level": 0.95 } }, { "name": "roc_auc_lead", "estimate": 0.847725525718156, "interval": { "lower": 0.7951441955465158, "upper": 0.8854066896671533, "level": 0.95 } }, { "name": "roc_auc_macro", "estimate": 0.7880111202881788, "interval": { "lower": 0.7569767092845195, "upper": 0.810232095277134, "level": 0.95 } }, { "name": "roc_auc_non_lead", "estimate": 0.7680125015141471, "interval": { "lower": 0.739756757662156, "upper": 0.7870408307105551, "level": 0.95 } } ] }, { "scope": "GROUP", "group_key": "catboost", "metrics": [ { "name": "brier_lead", "estimate": 0.006418852392030628, "interval": { "lower": 0.004206246188630246, "upper": 0.008495917733580953, "level": 0.95 } }, { "name": "brier_multiclass", "estimate": 0.09911073920993882, "interval": { "lower": 0.08493067292180892, "upper": 0.11454404292277905, "level": 0.95 } }, { "name": "brier_reliability", "estimate": 9.330454204145688e-06, "interval": { "lower": 5.7683995862667795e-06, "upper": 3.593942988004925e-05, "level": 0.95 } }, { "name": "brier_resolution", "estimate": 9.771434001534947e-05, "interval": { "lower": 4.841227586736246e-05, "upper": 0.00018339697289633056, "level": 0.95 } }, { "name": "brier_uncertainty", "estimate": 0.006496635125227586, "interval": { "lower": 0.00422483372299577, "upper": 0.008585077430522982, "level": 0.95 } }, { "name": "lead_found_top_decile", "estimate": 82.2, "interval": { "lower": 55.765, "upper": 111.16000000000001, "level": 0.95 } }, { "name": "log_loss", "estimate": 0.20929196432127722, "interval": { "lower": 0.1833730843862602, "upper": 0.2388602523387079, "level": 0.95 } }, { "name": "pr_auc_grr", "estimate": 0.15947747185125996, "interval": { "lower": 0.12677447354843557, "upper": 0.19043208252476887, "level": 0.95 } }, { "name": "pr_auc_lead", "estimate": 0.02904192154695051, "interval": { "lower": 0.02159171527169728, "upper": 0.039747946296521744, "level": 0.95 } }, { "name": "pr_auc_macro", "estimate": 0.3879008668870457, "interval": { "lower": 0.37708639235843416, "upper": 0.39769667642190176, "level": 0.95 } }, { "name": "pr_auc_non_lead", "estimate": 0.9751832072629268, "interval": { "lower": 0.9698512889760923, "upper": 0.9806451450900513, "level": 0.95 } }, { "name": "recall_at_precision_0_8", "estimate": 0.0, "interval": { "lower": 0.0, "upper": 0.0, "level": 0.95 } }, { "name": "roc_auc_grr", "estimate": 0.7361421056905115, "interval": { "lower": 0.701656533810796, "upper": 0.760477948470078, "level": 0.95 } }, { "name": "roc_auc_lead", "estimate": 0.8199259427425641, "interval": { "lower": 0.7649328895685599, "upper": 0.8662554486751344, "level": 0.95 } }, { "name": "roc_auc_macro", "estimate": 0.7699346641968691, "interval": { "lower": 0.7386174925278938, "upper": 0.7945139271165874, "level": 0.95 } }, { "name": "roc_auc_non_lead", "estimate": 0.7537359441575316, "interval": { "lower": 0.7248873367404053, "upper": 0.774148223823381, "level": 0.95 } } ] } ], "calibration": { "classwise_ece_equal_mass": 0.007233599425592041, "classwise_ece_equal_width": 0.006795328458649022, "counts": { "by_block": [ { "actual_lead": 0.0, "block": "892af63199bffff", "n": 1.0, "predicted_lead": 0.0022762748533749298 }, { "actual_lead": 0.0, "block": "892af631d67ffff", "n": 82.0, "predicted_lead": 0.21058681188400286 }, { "actual_lead": 0.0, "block": "892af631d77ffff", "n": 56.0, "predicted_lead": 0.07289701932759826 }, { "actual_lead": 0.0, "block": "892af63acafffff", "n": 10.0, "predicted_lead": 0.025814122255140202 }, { "actual_lead": 0.0, "block": "892af63acabffff", "n": 21.0, "predicted_lead": 0.11070960612031179 }, { "actual_lead": 0.0, "block": "892af63ac33ffff", "n": 15.0, "predicted_lead": 0.011835739532820699 }, { "actual_lead": 0.0, "block": "892af63ac07ffff", "n": 90.0, "predicted_lead": 0.11663542064139475 }, { "actual_lead": 0.0, "block": "892af631993ffff", "n": 58.0, "predicted_lead": 0.06318541927226315 }, { "actual_lead": 0.0, "block": "892af631d2bffff", "n": 65.0, "predicted_lead": 0.1530402550668543 }, { "actual_lead": 0.0, "block": "892af63ac0fffff", "n": 78.0, "predicted_lead": 0.12375510490082574 }, { "actual_lead": 0.0, "block": "892af631d2fffff", "n": 66.0, "predicted_lead": 0.28697587309798855 }, { "actual_lead": 0.0, "block": "892af631d3bffff", "n": 3.0, "predicted_lead": 0.011407266881908431 }, { "actual_lead": 0.0, "block": "892af62067bffff", "n": 9.0, "predicted_lead": 0.04385143720954059 }, { "actual_lead": 0.0, "block": "892af63ac47ffff", "n": 20.0, "predicted_lead": 0.022515985077751953 }, { "actual_lead": 0.0, "block": "892af631d23ffff", "n": 78.0, "predicted_lead": 0.3391646708178739 }, { "actual_lead": 0.0, "block": "892af630a5bffff", "n": 9.0, "predicted_lead": 0.01870766841108361 }, { "actual_lead": 0.0, "block": "892af631d27ffff", "n": 67.0, "predicted_lead": 0.1961639938321618 }, { "actual_lead": 0.0, "block": "892af620647ffff", "n": 14.0, "predicted_lead": 0.026934085359185435 }, { "actual_lead": 0.0, "block": "892af62060bffff", "n": 7.0, "predicted_lead": 0.007589402095539387 }, { "actual_lead": 0.0, "block": "892af6318a3ffff", "n": 23.0, "predicted_lead": 0.12396932404537275 }, { "actual_lead": 0.0, "block": "892af6318b7ffff", "n": 76.0, "predicted_lead": 0.16556081543749965 }, { "actual_lead": 0.0, "block": "892af63ad33ffff", "n": 16.0, "predicted_lead": 0.0911664838894729 }, { "actual_lead": 0.0, "block": "892af63adafffff", "n": 34.0, "predicted_lead": 0.046053618586732764 }, { "actual_lead": 0.0, "block": "892af631e43ffff", "n": 17.0, "predicted_lead": 0.049601365588642074 }, { "actual_lead": 0.0, "block": "892af630a53ffff", "n": 29.0, "predicted_lead": 0.04960084367876257 }, { "actual_lead": 0.0, "block": "892af631d37ffff", "n": 49.0, "predicted_lead": 0.06899382913424694 }, { "actual_lead": 0.0, "block": "892af630acbffff", "n": 153.0, "predicted_lead": 0.3015054147067694 }, { "actual_lead": 0.0, "block": "892af630acfffff", "n": 112.0, "predicted_lead": 0.14130670756658453 }, { "actual_lead": 0.0, "block": "892af630adbffff", "n": 13.0, "predicted_lead": 0.04406229683614923 }, { "actual_lead": 0.0, "block": "892af630ac3ffff", "n": 126.0, "predicted_lead": 0.12415966034661051 } ], "clipped_at_zero": false, "disagree": null, "model_based": { "expected_unknown": 54.71051783739068, "known_lead": 160.0, "level": 0.8, "lower": 205.28767462481292, "n_unknown": 15827.0, "p10": 205.28767462481292, "p50": 214.71051783739068, "p90": 224.13336104996844, "upper": 224.13336104996844, "variance": 54.061956512348345, "variance_calibration": 0.0, "variance_model": 0.0, "variance_poisson_binomial": 54.061956512348345 } }, "ece_bins": 10, "ece_confidence_equal_width": 0.011763020000945562, "ece_lead_equal_mass": 0.0033355508205943018, "ece_lead_equal_width": 0.0017146203722653815, "ece_n": 24620, "intercept": 0.4268617113397051, "lead_curve": [ 0, 1, 6, 12, 12, 16, 18, 22, 26, 33, 45, 53, 54, 56, 57, 60, 64, 69, 71, 79, 84, 87, 92, 94, 99, 107, 108, 109, 109, 112, 118, 127, 128, 131, 132, 132, 135, 136, 138, 138, 138, 138, 139, 139, 140, 140, 140, 140, 140, 140, 141, 141, 142, 142, 145, 145, 145, 146, 146, 146, 146, 148, 148, 148, 148, 148, 148, 148, 149, 149, 149, 149, 149, 149, 149, 149, 149, 149, 149, 149, 149, 149, 149, 149, 149, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 151, 151, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 154, 154, 155, 155, 155, 155, 155, 155, 155, 155, 155, 155, 155, 155, 155, 155, 155, 155, 155, 155, 155, 155, 155, 155, 155, 156, 156, 156, 156, 156, 156, 156, 156, 156, 156, 156, 157, 157, 157, 157, 157, 157, 157, 157, 157, 157, 158, 158, 158, 158, 158, 158, 158, 158, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161 ], "log_loss_improvement": { "estimate": 0.01279448377009045, "excludes_zero": true, "level": 0.95, "lower": 0.008991597786717244, "upper": 0.016318537687242427 }, "reliability_equal_mass": [ { "accuracy": 0.0002449917452097686, "confidence": 0.0007359108600920914, "lower": 0.0, "n": 2459.0, "upper": 0.0008714977259403657, "weight": 1393.8255439654922 }, { "accuracy": 0.012563389008489318, "confidence": 0.0009960505115120598, "lower": 0.0008714977259403657, "n": 2465.0, "upper": 0.0010991038846266947, "weight": 1968.130801589792 }, { "accuracy": 0.0009365957892319822, "confidence": 0.0012433992601102865, "lower": 0.0010991038846266947, "n": 2460.0, "upper": 0.0014478162776799298, "weight": 1323.8088139655488 }, { "accuracy": 0.0015772442230531031, "confidence": 0.0015977797610591186, "lower": 0.0014478162776799298, "n": 2463.0, "upper": 0.0018761230796629472, "weight": 1981.9938733680983 }, { "accuracy": 0.0020110578045672393, "confidence": 0.002389468505788535, "lower": 0.0018761230796629472, "n": 2459.0, "upper": 0.002821107755206694, "weight": 1783.8829298732 }, { "accuracy": 0.0026594962022571523, "confidence": 0.003224199351034528, "lower": 0.002821107755206694, "n": 2465.0, "upper": 0.0037790835380857027, "weight": 1405.4998690242737 }, { "accuracy": 0.002741620138563729, "confidence": 0.00450100062325542, "lower": 0.0037790835380857027, "n": 2463.0, "upper": 0.0052340193888090495, "weight": 1284.9218177716461 }, { "accuracy": 0.001515055589055604, "confidence": 0.006405066611026035, "lower": 0.0052340193888090495, "n": 2461.0, "upper": 0.007922876867086412, "weight": 1622.478982286389 }, { "accuracy": 0.023659354276381783, "confidence": 0.013489717145293964, "lower": 0.007922876867086412, "n": 2461.0, "upper": 0.02252438181911376, "weight": 1251.8283646194309 }, { "accuracy": 0.03699268358382703, "confidence": 0.03884673522561318, "lower": 0.02252438181911376, "n": 2464.0, "upper": 1.0, "weight": 1038.6780975213446 } ], "reliability_equal_width": [ { "accuracy": 0.00736906919477773, "confidence": 0.005819499176072384, "lower": 0.0, "n": 24604.0, "upper": 0.1, "weight": 15033.25528174467 }, { "accuracy": 0.0, "confidence": 0.1155654727162213, "lower": 0.1, "n": 16.0, "upper": 0.2, "weight": 21.79381224054533 } ], "slope": 1.1210890396827937, "validation": { "ban_year": 1988, "buffer_m": 200.0, "buffer_source": "config", "crossfit_folds": [ { "crossfit_buffer_m": 200.0, "crossfit_covered_share": 1.0, "crossfit_models_fit": 4, "fold": 0, "seed": 0 }, { "crossfit_buffer_m": 200.0, "crossfit_covered_share": 1.0, "crossfit_models_fit": 4, "fold": 1, "seed": 0 }, { "crossfit_buffer_m": 200.0, "crossfit_covered_share": 1.0, "crossfit_models_fit": 4, "fold": 2, "seed": 0 }, { "crossfit_buffer_m": 200.0, "crossfit_covered_share": 1.0, "crossfit_models_fit": 4, "fold": 3, "seed": 0 }, { "crossfit_buffer_m": 200.0, "crossfit_covered_share": 1.0, "crossfit_models_fit": 4, "fold": 4, "seed": 0 }, { "crossfit_buffer_m": 200.0, "crossfit_covered_share": 1.0, "crossfit_models_fit": 4, "fold": 0, "seed": 1 }, { "crossfit_buffer_m": 200.0, "crossfit_covered_share": 1.0, "crossfit_models_fit": 4, "fold": 1, "seed": 1 }, { "crossfit_buffer_m": 200.0, "crossfit_covered_share": 1.0, "crossfit_models_fit": 4, "fold": 2, "seed": 1 }, { "crossfit_buffer_m": 200.0, "crossfit_covered_share": 1.0, "crossfit_models_fit": 4, "fold": 3, "seed": 1 }, { "crossfit_buffer_m": 200.0, "crossfit_covered_share": 1.0, "crossfit_models_fit": 4, "fold": 4, "seed": 1 }, { "crossfit_buffer_m": 200.0, "crossfit_covered_share": 1.0, "crossfit_models_fit": 4, "fold": 0, "seed": 2 }, { "crossfit_buffer_m": 200.0, "crossfit_covered_share": 1.0, "crossfit_models_fit": 4, "fold": 1, "seed": 2 }, { "crossfit_buffer_m": 200.0, "crossfit_covered_share": 1.0, "crossfit_models_fit": 4, "fold": 2, "seed": 2 }, { "crossfit_buffer_m": 200.0, "crossfit_covered_share": 1.0, "crossfit_models_fit": 4, "fold": 3, "seed": 2 }, { "crossfit_buffer_m": 200.0, "crossfit_covered_share": 1.0, "crossfit_models_fit": 4, "fold": 4, "seed": 2 }, { "crossfit_buffer_m": 200.0, "crossfit_covered_share": 1.0, "crossfit_models_fit": 4, "fold": 0, "seed": 3 }, { "crossfit_buffer_m": 200.0, "crossfit_covered_share": 1.0, "crossfit_models_fit": 4, "fold": 1, "seed": 3 }, { "crossfit_buffer_m": 200.0, "crossfit_covered_share": 1.0, "crossfit_models_fit": 4, "fold": 2, "seed": 3 }, { "crossfit_buffer_m": 200.0, "crossfit_covered_share": 1.0, "crossfit_models_fit": 4, "fold": 3, "seed": 3 }, { "crossfit_buffer_m": 200.0, "crossfit_covered_share": 1.0, "crossfit_models_fit": 4, "fold": 4, "seed": 3 }, { "crossfit_buffer_m": 200.0, "crossfit_covered_share": 1.0, "crossfit_models_fit": 4, "fold": 0, "seed": 4 }, { "crossfit_buffer_m": 200.0, "crossfit_covered_share": 1.0, "crossfit_models_fit": 4, "fold": 1, "seed": 4 }, { "crossfit_buffer_m": 200.0, "crossfit_covered_share": 1.0, "crossfit_models_fit": 4, "fold": 2, "seed": 4 }, { "crossfit_buffer_m": 200.0, "crossfit_covered_share": 1.0, "crossfit_models_fit": 4, "fold": 3, "seed": 4 }, { "crossfit_buffer_m": 200.0, "crossfit_covered_share": 1.0, "crossfit_models_fit": 4, "fold": 4, "seed": 4 }, { "crossfit_buffer_m": 200.0, "crossfit_covered_share": 1.0, "crossfit_models_fit": 4, "fold": 0, "seed": 5 }, { "crossfit_buffer_m": 200.0, "crossfit_covered_share": 1.0, "crossfit_models_fit": 4, "fold": 1, "seed": 5 }, { "crossfit_buffer_m": 200.0, "crossfit_covered_share": 1.0, "crossfit_models_fit": 4, "fold": 2, "seed": 5 }, { "crossfit_buffer_m": 200.0, "crossfit_covered_share": 1.0, "crossfit_models_fit": 4, "fold": 3, "seed": 5 }, { "crossfit_buffer_m": 200.0, "crossfit_covered_share": 1.0, "crossfit_models_fit": 4, "fold": 4, "seed": 5 }, { "crossfit_buffer_m": 200.0, "crossfit_covered_share": 1.0, "crossfit_models_fit": 4, "fold": 0, "seed": 6 }, { "crossfit_buffer_m": 200.0, "crossfit_covered_share": 1.0, "crossfit_models_fit": 4, "fold": 1, "seed": 6 }, { "crossfit_buffer_m": 200.0, "crossfit_covered_share": 1.0, "crossfit_models_fit": 4, "fold": 2, "seed": 6 }, { "crossfit_buffer_m": 200.0, "crossfit_covered_share": 1.0, "crossfit_models_fit": 4, "fold": 3, "seed": 6 }, { "crossfit_buffer_m": 200.0, "crossfit_covered_share": 1.0, "crossfit_models_fit": 4, "fold": 4, "seed": 6 }, { "crossfit_buffer_m": 200.0, "crossfit_covered_share": 1.0, "crossfit_models_fit": 4, "fold": 0, "seed": 7 }, { "crossfit_buffer_m": 200.0, "crossfit_covered_share": 1.0, "crossfit_models_fit": 4, "fold": 1, "seed": 7 }, { "crossfit_buffer_m": 200.0, "crossfit_covered_share": 1.0, "crossfit_models_fit": 4, "fold": 2, "seed": 7 }, { "crossfit_buffer_m": 200.0, "crossfit_covered_share": 1.0, "crossfit_models_fit": 4, "fold": 3, "seed": 7 }, { "crossfit_buffer_m": 200.0, "crossfit_covered_share": 1.0, "crossfit_models_fit": 4, "fold": 4, "seed": 7 }, { "crossfit_buffer_m": 200.0, "crossfit_covered_share": 1.0, "crossfit_models_fit": 4, "fold": 0, "seed": 8 }, { "crossfit_buffer_m": 200.0, "crossfit_covered_share": 1.0, "crossfit_models_fit": 4, "fold": 1, "seed": 8 }, { "crossfit_buffer_m": 200.0, "crossfit_covered_share": 1.0, "crossfit_models_fit": 4, "fold": 2, "seed": 8 }, { "crossfit_buffer_m": 200.0, "crossfit_covered_share": 1.0, "crossfit_models_fit": 4, "fold": 3, "seed": 8 }, { "crossfit_buffer_m": 200.0, "crossfit_covered_share": 1.0, "crossfit_models_fit": 4, "fold": 4, "seed": 8 }, { "crossfit_buffer_m": 200.0, "crossfit_covered_share": 1.0, "crossfit_models_fit": 4, "fold": 0, "seed": 9 }, { "crossfit_buffer_m": 200.0, "crossfit_covered_share": 1.0, "crossfit_models_fit": 4, "fold": 1, "seed": 9 }, { "crossfit_buffer_m": 200.0, "crossfit_covered_share": 1.0, "crossfit_models_fit": 4, "fold": 2, "seed": 9 }, { "crossfit_buffer_m": 200.0, "crossfit_covered_share": 1.0, "crossfit_models_fit": 4, "fold": 3, "seed": 9 }, { "crossfit_buffer_m": 200.0, "crossfit_covered_share": 1.0, "crossfit_models_fit": 4, "fold": 4, "seed": 9 } ], "n_bootstrap": 200, "random_log_loss": 0.18744034106571178, "sensitivity": [ { "buffer_m": 0.0, "train_rows": 19696.0, "train_rows_kept": 19696.0 }, { "buffer_m": 100.0, "train_rows": 19696.0, "train_rows_kept": 16389.0 }, { "buffer_m": 250.0, "train_rows": 19696.0, "train_rows_kept": 8872.0 }, { "buffer_m": 500.0, "train_rows": 19696.0, "train_rows_kept": 1840.0 }, { "buffer_m": 200.0, "train_rows": 19696.0, "train_rows_kept": 11076.0 }, { "buffer_m": 1761.586594953903, "fold": 0.0, "suggested_buffer_m": 1761.586594953903, "train_rows": 19696.0, "train_rows_kept": 19696.0, "variogram": 1.0 }, { "buffer_m": 1753.5151760761366, "fold": 1.0, "suggested_buffer_m": 1753.5151760761366, "train_rows": 19696.0, "train_rows_kept": 19696.0, "variogram": 1.0 }, { "buffer_m": 1798.62883982344, "fold": 2.0, "suggested_buffer_m": 1798.62883982344, "train_rows": 19696.0, "train_rows_kept": 19696.0, "variogram": 1.0 }, { "buffer_m": 1854.9284010275578, "fold": 3.0, "suggested_buffer_m": 1854.9284010275578, "train_rows": 19696.0, "train_rows_kept": 19696.0, "variogram": 1.0 }, { "buffer_m": 1823.2571533283058, "fold": 4.0, "suggested_buffer_m": 1823.2571533283058, "train_rows": 19696.0, "train_rows_kept": 19696.0, "variogram": 1.0 } ], "split_log_loss": 0.20537719621307535 } }, "conformal": { "alpha": 0.05, "assumption": "The coverage claim needs the homes being scored to be exchangeable with the homes that were checked, or, for the weighted sets, to match them after the covariate weights. It fails when crews picked homes for a reason that is not in the data, and it fails when a group has too few checked homes for its own quantile.", "baseline_abstain": 0.2162063363119415, "baseline_non_lead_calls": 19297, "beta": 0.01, "clearance": { "baseline_share": 0.7837936636880585, "min_share": 0.05, "non_lead_calls": 20173, "scored_rows": 24620, "sentence": null, "share": 0.819374492282697, "useful": true }, "coverage": 0.9471770917952883, "coverage_by_seed": [ 0.9463850528025995, 0.9452477660438667, 0.9476441917140536, 0.945978878960195, 0.9483753046303818, 0.9465881397238017, 0.9470755483346872, 0.9477254264825345, 0.9456133225020309, 0.9511372867587328 ], "false_non_lead_by_seed": [ { "calls": 20173, "false": 33, "rate": 0.0016358498983790215, "upper": 0.00210386323289347 }, { "calls": 20384, "false": 44, "rate": 0.0021585557299843012, "upper": 0.00269323697832737 }, { "calls": 16167, "false": 26, "rate": 0.0016082142636234305, "upper": 0.0021265784541008364 }, { "calls": 20519, "false": 26, "rate": 0.0012671182806179638, "upper": 0.001675609206432934 }, { "calls": 20404, "false": 32, "rate": 0.0015683199372672025, "upper": 0.002023985394791182 }, { "calls": 16347, "false": 29, "rate": 0.0017740258151342754, "upper": 0.0023154063367662424 }, { "calls": 15946, "false": 23, "rate": 0.0014423679919729085, "upper": 0.0019367082531770744 }, { "calls": 16314, "false": 20, "rate": 0.001225940909648155, "upper": 0.0016765660565646713 }, { "calls": 20017, "false": 30, "rate": 0.0014987260828295949, "upper": 0.0019484678340439257 }, { "calls": 20371, "false": 33, "rate": 0.001619949928820382, "upper": 0.002083418004884567 } ], "false_non_lead_calls": 20173, "false_non_lead_rate": 0.0015799068838277235, "false_non_lead_seed_summary": { "failing_seeds": 0, "rate_max": 0.0021585557299843012, "rate_mean": 0.0015799068838277235, "rate_mean_ci95": [ 0.001416335757580797, 0.00174347801007465 ], "rate_min": 0.001225940909648155, "rate_sd": 0.00026390679509144765, "seeds": 10, "seeds_with_calls": 10, "upper_mean": 0.0020583839751982273 }, "false_non_lead_upper": 0.0020583839751982273, "fnl_aggregate": "seed_mean", "gate_checks": [ { "detail": "log-loss improvement over the year-built rule excludes zero", "name": "beats_year_built", "status": "pass" }, { "detail": "ECE 0.0017 is at most the adjusted bound 0.0498 (bar 0.03, B=10, N=24620)", "name": "calibration_ece", "status": "pass" }, { "detail": "coverage 0.947 is within 0.02 of 0.95", "name": "conformal_coverage", "status": "pass" }, { "detail": "every large group has coverage at least 0.90", "name": "mondrian_coverage", "status": "pass" }, { "detail": "false non-lead rate 0.0016 stays within the bound", "name": "false_non_lead", "status": "pass" }, { "detail": "cleared share 0.8194 is above the year-built rule's 0.7838; abstain rate 0.1806 against 0.2162", "name": "usefulness", "status": "pass" }, { "detail": "no judged gap has an interval wholly above 0.01", "name": "equity_gap", "status": "pass" }, { "detail": "the leakage canary was detected", "name": "canary", "status": "pass" }, { "detail": "shuffled-label AUC 0.507 is within the 3-sigma null band 0.0685 (n_pos=161, n_neg=24459)", "name": "shuffled_labels", "shuffled_min_pos": 50, "status": "pass" }, { "detail": "the three count estimators were not all available", "name": "count_agreement", "status": "skipped" }, { "detail": "no feature is a copy of the label", "name": "label_copies", "status": "pass" }, { "detail": "the named red-team suite was not supplied", "name": "red_team", "status": "skipped" }, { "detail": "no regression comparison was supplied", "name": "regression", "status": "skipped" }, { "detail": "release bound fails on field-checked clears of lead or GRR: decades 1950, 1960, 1980, 1990, 2010; overall 10.4% (upper 12.7%)", "name": "release_bound", "status": "fail" } ], "group_coverage": [ { "coverage": 1.0, "group": "1870", "n": 1.0 }, { "coverage": 1.0, "group": "1880", "n": 11.0 }, { "coverage": 0.9833333333333333, "group": "1890", "n": 60.0 }, { "coverage": 0.9473684210526315, "group": "1900", "n": 133.0 }, { "coverage": 0.95, "group": "1910", "n": 320.0 }, { "coverage": 0.9582577132486388, "group": "1920", "n": 551.0 }, { "coverage": 0.98, "group": "1930", "n": 100.0 }, { "coverage": 0.9748387096774194, "group": "1940", "n": 1550.0 }, { "coverage": 0.9771151178918169, "group": "1950", "n": 1442.0 }, { "coverage": 0.988950276243094, "group": "1960", "n": 905.0 }, { "coverage": 0.9862385321100917, "group": "1970", "n": 654.0 }, { "coverage": 0.9401330376940134, "group": "1980", "n": 3157.0 }, { "coverage": 0.9657731334242011, "group": "1990", "n": 6603.0 }, { "coverage": 0.9946181445412609, "group": "2000", "n": 3902.0 }, { "coverage": 0.9730843174951936, "group": "2010", "n": 3641.0 }, { "coverage": 0.9905660377358491, "group": "2020", "n": 1590.0 } ], "model_abstain": 0.180625507717303, "plain": { "decision": "FAIL", "headline": "The model failed the check.", "notes": [ "Of the model's not-lead calls on test lines the city had checked in the field, 10.4% turned out to be lead or galvanized pipe needing replacement; with 98.33% confidence the share is at most 12.7%. Weighted to stand for the lines nobody checked, the share is 16.2%, at most 20.2%. The model makes no call for lines built in the 1870s, the 1890s, the 1900s, the 1910s, the 1920s, the 1930s, the 1940s, the 1970s, the 2000s and the 2020s: too few checked lines there to judge it." ], "reasons": [ "Lines built in the 1950s: across 10 test runs, 287 of the model's 11,102 not-lead calls on lines the city had checked in the field (2.59%) turned out to be lead or galvanized pipe needing replacement, at most 4.64% with 98.33% confidence.", "Lines built in the 1960s: across 10 test runs, 98 of the model's 5,841 not-lead calls on lines the city had checked in the field (1.68%) turned out to be lead or galvanized pipe needing replacement, at most 3.68% with 98.33% confidence.", "Lines built in the 1980s: across 10 test runs, 1,080 of the model's 7,001 not-lead calls on lines the city had checked in the field (15.4%) turned out to be lead or galvanized pipe needing replacement, at most 22.1% with 98.33% confidence.", "Lines built in the 1990s: across 10 test runs, 1,956 of the model's 12,411 not-lead calls on lines the city had checked in the field (15.8%) turned out to be lead or galvanized pipe needing replacement, at most 19.4% with 98.33% confidence.", "Lines built in the 2010s: across 10 test runs, 865 of the model's 4,946 not-lead calls on lines the city had checked in the field (17.5%) turned out to be lead or galvanized pipe needing replacement, at most 23.6% with 98.33% confidence.", "Over the decades judged, across 10 test runs, 4,286 of the model's 41,301 not-lead calls on lines the city had checked in the field (10.4%) turned out to be lead or galvanized pipe needing replacement, at most 12.7% with 98.33% confidence.", "Weighted so that the checked lines stand for the lines nobody checked, 16.2% of the model's not-lead calls were lead or galvanized pipe needing replacement, at most 20.2%.", "The limits are 1.00% for the share of misses and 2.00% for its upper bound, in each decade and over all decades judged." ] }, "release_bound": { "alpha": 0.01666666666666668, "basis": "field_checked", "beta": 0.01, "candidates": 3, "groups": [ { "bootstrap_upper": 1.0, "cleared_calls_by_run": [ 1, 1, 0, 1, 1, 1, 1, 1, 1, 1 ], "cp_upper": 1.0, "group": "1870", "judged_calls": 0, "judged_calls_min": 0, "lead_misses": 0, "lead_point": 0.0, "lead_upper": 1.0, "misses": 0, "point": 0.0, "status": "withdrawn", "upper": 1.0 }, { "bootstrap_upper": 1.0, "cleared_calls_by_run": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ], "cp_upper": 1.0, "group": "1880", "judged_calls": 0, "judged_calls_min": 0, "lead_misses": 0, "lead_point": 0.0, "lead_upper": 1.0, "misses": 0, "point": 0.0, "status": "no calls", "upper": 1.0 }, { "bootstrap_upper": 1.0, "cleared_calls_by_run": [ 1, 0, 0, 0, 6, 0, 0, 0, 12, 0 ], "cp_upper": 1.0, "group": "1890", "judged_calls": 15, "judged_calls_min": 0, "lead_misses": 1, "lead_point": 0.06666666666666667, "lead_upper": 1.0, "misses": 2, "point": 0.13333333333333333, "status": "withdrawn", "upper": 1.0 }, { "bootstrap_upper": 1.0, "cleared_calls_by_run": [ 19, 0, 0, 0, 10, 32, 0, 1, 19, 25 ], "cp_upper": 1.0, "group": "1900", "judged_calls": 68, "judged_calls_min": 0, "lead_misses": 5, "lead_point": 0.07352941176470588, "lead_upper": 1.0, "misses": 12, "point": 0.17647058823529413, "status": "withdrawn", "upper": 1.0 }, { "bootstrap_upper": 1.0, "cleared_calls_by_run": [ 0, 0, 4, 0, 0, 58, 5, 0, 0, 0 ], "cp_upper": 1.0, "group": "1910", "judged_calls": 62, "judged_calls_min": 0, "lead_misses": 4, "lead_point": 0.06451612903225806, "lead_upper": 1.0, "misses": 11, "point": 0.1774193548387097, "status": "withdrawn", "upper": 1.0 }, { "bootstrap_upper": 1.0, "cleared_calls_by_run": [ 51, 67, 33, 4, 10, 0, 58, 11, 6, 59 ], "cp_upper": 1.0, "group": "1920", "judged_calls": 241, "judged_calls_min": 0, "lead_misses": 10, "lead_point": 0.04149377593360996, "lead_upper": 1.0, "misses": 40, "point": 0.16597510373443983, "status": "withdrawn", "upper": 1.0 }, { "bootstrap_upper": 0.16666666666666666, "cleared_calls_by_run": [ 50, 62, 79, 53, 68, 40, 62, 30, 54, 70 ], "cp_upper": 0.23162434275048416, "group": "1930", "judged_calls": 429, "judged_calls_min": 26, "lead_misses": 0, "lead_point": 0.0, "lead_upper": 0.14570165053174744, "misses": 20, "point": 0.046620046620046623, "status": "withdrawn", "upper": 0.23162434275048416 }, { "bootstrap_upper": 0.3515625, "cleared_calls_by_run": [ 83, 173, 105, 122, 201, 103, 95, 62, 35, 118 ], "cp_upper": 0.30282987206597045, "group": "1940", "judged_calls": 926, "judged_calls_min": 24, "lead_misses": 2, "lead_point": 0.0021598272138228943, "lead_upper": 0.15683928284333348, "misses": 133, "point": 0.14362850971922247, "status": "withdrawn", "upper": 0.3515625 }, { "bootstrap_upper": 0.046370967741935484, "cleared_calls_by_run": [ 1414, 1412, 1167, 1440, 1440, 1127, 1139, 1077, 1436, 1422 ], "cp_upper": 0.040139402155577035, "group": "1950", "judged_calls": 11102, "judged_calls_min": 905, "lead_misses": 10, "lead_point": 0.0009007386056566384, "lead_upper": 0.006663098262410652, "misses": 287, "point": 0.025851197982345524, "status": "fail", "upper": 0.046370967741935484 }, { "bootstrap_upper": 0.03461538461538462, "cleared_calls_by_run": [ 899, 900, 714, 896, 893, 725, 741, 721, 890, 897 ], "cp_upper": 0.036845163505480144, "group": "1960", "judged_calls": 5841, "judged_calls_min": 505, "lead_misses": 0, "lead_point": 0.0, "lead_upper": 0.008074834943998965, "misses": 98, "point": 0.016777948981338813, "status": "fail", "upper": 0.036845163505480144 }, { "bootstrap_upper": 0.2990936555891239, "cleared_calls_by_run": [ 151, 155, 125, 553, 340, 200, 163, 308, 288, 277 ], "cp_upper": 0.27719128305873, "group": "1970", "judged_calls": 1877, "judged_calls_min": 84, "lead_misses": 6, "lead_point": 0.0031965903036760787, "lead_upper": 0.051884115632993964, "misses": 286, "point": 0.1523708044752264, "status": "withdrawn", "upper": 0.2990936555891239 }, { "bootstrap_upper": 0.2209178228388474, "cleared_calls_by_run": [ 2108, 2082, 1395, 1904, 1892, 1713, 1563, 1630, 1873, 1908 ], "cp_upper": 0.20537353645454648, "group": "1980", "judged_calls": 7001, "judged_calls_min": 542, "lead_misses": 86, "lead_point": 0.012283959434366519, "lead_upper": 0.03509274824735757, "misses": 1080, "point": 0.15426367661762605, "status": "fail", "upper": 0.2209178228388474 }, { "bootstrap_upper": 0.193859649122807, "cleared_calls_by_run": [ 6588, 6541, 5129, 6574, 6593, 5281, 4972, 5478, 6603, 6591 ], "cp_upper": 0.1903266604325629, "group": "1990", "judged_calls": 12411, "judged_calls_min": 954, "lead_misses": 143, "lead_point": 0.011522036902747562, "lead_upper": 0.02273313133064544, "misses": 1956, "point": 0.15760212714527436, "status": "fail", "upper": 0.193859649122807 }, { "bootstrap_upper": 0.20388349514563106, "cleared_calls_by_run": [ 3723, 3824, 2973, 3805, 3827, 2976, 3051, 3010, 3620, 3849 ], "cp_upper": 0.1613483659896453, "group": "2000", "judged_calls": 2162, "judged_calls_min": 189, "lead_misses": 0, "lead_point": 0.0, "lead_upper": 0.02143023691730339, "misses": 204, "point": 0.09435707678075855, "status": "withdrawn", "upper": 0.20388349514563106 }, { "bootstrap_upper": 0.23588039867109634, "cleared_calls_by_run": [ 3544, 3628, 3152, 3619, 3550, 2861, 2852, 2759, 3620, 3610 ], "cp_upper": 0.23139135105692027, "group": "2010", "judged_calls": 4946, "judged_calls_min": 397, "lead_misses": 20, "lead_point": 0.0040436716538617065, "lead_upper": 0.019387859175986778, "misses": 865, "point": 0.17488879902951882, "status": "fail", "upper": 0.23588039867109634 }, { "bootstrap_upper": 0.14285714285714285, "cleared_calls_by_run": [ 1541, 1539, 1291, 1548, 1573, 1230, 1244, 1226, 1560, 1544 ], "cp_upper": 0.14548490585892096, "group": "2020", "judged_calls": 1625, "judged_calls_min": 136, "lead_misses": 0, "lead_point": 0.0, "lead_upper": 0.029656818526700546, "misses": 124, "point": 0.07630769230769231, "status": "withdrawn", "upper": 0.14548490585892096 } ], "judged_groups": [ "1950", "1960", "1980", "1990", "2010" ], "level": 0.95, "method": "Each release run's held-out calls of exactly NON_LEAD on the judged lines; a miss is a label in miss_classes; each run's bound is the larger of the exact Clopper-Pearson end and a block bootstrap over the split's blocks at one-sided level 1 - alpha; runs combine by the largest bound; decades under min_support judged calls in any run are withdrawn (ADR 0019).", "min_support": 203, "miss_classes": [ "LEAD", "GRR" ], "overall": { "bootstrap_upper": 0.12650721486459773, "calls": 41301, "calls_min": 3386, "cp_upper": 0.12105409509983661, "lead_misses": 259, "lead_point": 0.006271034599646497, "lead_upper": 0.011898131414081341, "misses": 4286, "point": 0.10377472700418876, "upper": 0.12650721486459773 }, "overall_all": { "bootstrap_upper": 0.1289228159457167, "calls": 48706, "calls_min": 4030, "cp_upper": 0.12034066441445128, "lead_misses": 287, "lead_point": 0.005892497844208105, "lead_upper": 0.010818976222866719, "misses": 5118, "point": 0.10507945632981563, "upper": 0.1289228159457167 }, "plain_reasons": [ "Lines built in the 1950s: across 10 test runs, 287 of the model's 11,102 not-lead calls on lines the city had checked in the field (2.59%) turned out to be lead or galvanized pipe needing replacement, at most 4.64% with 98.33% confidence.", "Lines built in the 1960s: across 10 test runs, 98 of the model's 5,841 not-lead calls on lines the city had checked in the field (1.68%) turned out to be lead or galvanized pipe needing replacement, at most 3.68% with 98.33% confidence.", "Lines built in the 1980s: across 10 test runs, 1,080 of the model's 7,001 not-lead calls on lines the city had checked in the field (15.4%) turned out to be lead or galvanized pipe needing replacement, at most 22.1% with 98.33% confidence.", "Lines built in the 1990s: across 10 test runs, 1,956 of the model's 12,411 not-lead calls on lines the city had checked in the field (15.8%) turned out to be lead or galvanized pipe needing replacement, at most 19.4% with 98.33% confidence.", "Lines built in the 2010s: across 10 test runs, 865 of the model's 4,946 not-lead calls on lines the city had checked in the field (17.5%) turned out to be lead or galvanized pipe needing replacement, at most 23.6% with 98.33% confidence.", "Over the decades judged, across 10 test runs, 4,286 of the model's 41,301 not-lead calls on lines the city had checked in the field (10.4%) turned out to be lead or galvanized pipe needing replacement, at most 12.7% with 98.33% confidence.", "Weighted so that the checked lines stand for the lines nobody checked, 16.2% of the model's not-lead calls were lead or galvanized pipe needing replacement, at most 20.2%.", "The limits are 1.00% for the share of misses and 2.00% for its upper bound, in each decade and over all decades judged." ], "plain_statement": "Of the model's not-lead calls on test lines the city had checked in the field, 10.4% turned out to be lead or galvanized pipe needing replacement; with 98.33% confidence the share is at most 12.7%. Weighted to stand for the lines nobody checked, the share is 16.2%, at most 20.2%. The model makes no call for lines built in the 1870s, the 1890s, the 1900s, the 1910s, the 1920s, the 1930s, the 1940s, the 1970s, the 2000s and the 2020s: too few checked lines there to judge it.", "reasons": [ "decade 1950: 287 of 11,102 judged clears were lead or GRR (2.59%), upper 4.64% at 98.33%; bars 1% and 2%", "decade 1960: 98 of 5,841 judged clears were lead or GRR (1.68%), upper 3.68% at 98.33%; bars 1% and 2%", "decade 1980: 1,080 of 7,001 judged clears were lead or GRR (15.4%), upper 22.1% at 98.33%; bars 1% and 2%", "decade 1990: 1,956 of 12,411 judged clears were lead or GRR (15.8%), upper 19.4% at 98.33%; bars 1% and 2%", "decade 2010: 865 of 4,946 judged clears were lead or GRR (17.5%), upper 23.6% at 98.33%; bars 1% and 2%", "4,286 of 41,301 judged clears were lead or GRR (10.4%), upper 12.7% at 98.33%; bars 1% and 2%", "weighted to the unchecked lines, 16.2% of clears were lead or GRR, upper 20.2%; bars 1% and 2%" ], "runs": [ { "bootstrap_upper": 0.125, "calls": 4630, "cp_upper": 0.11809341592979113, "misses": 500, "rate": 0.1079913606911447, "seed": 0, "upper": 0.125 }, { "bootstrap_upper": 0.12650721486459773, "calls": 4680, "cp_upper": 0.12105409509983661, "misses": 519, "rate": 0.11089743589743589, "seed": 1, "upper": 0.12650721486459773 }, { "bootstrap_upper": 0.11899070385126162, "calls": 3596, "cp_upper": 0.11215313255435178, "misses": 363, "rate": 0.10094549499443826, "seed": 2, "upper": 0.11899070385126162 }, { "bootstrap_upper": 0.12041309602285212, "calls": 4431, "cp_upper": 0.11327723014042776, "misses": 457, "rate": 0.10313698939291356, "seed": 3, "upper": 0.12041309602285212 }, { "bootstrap_upper": 0.11758496938990923, "calls": 4442, "cp_upper": 0.11276395989095603, "misses": 456, "rate": 0.10265646105357946, "seed": 4, "upper": 0.11758496938990923 }, { "bootstrap_upper": 0.12559618441971382, "calls": 3757, "cp_upper": 0.11988649208958924, "misses": 408, "rate": 0.1085972850678733, "seed": 5, "upper": 0.12559618441971382 }, { "bootstrap_upper": 0.1147689519782543, "calls": 3386, "cp_upper": 0.10916911389668058, "misses": 331, "rate": 0.09775546367395156, "seed": 6, "upper": 0.1147689519782543 }, { "bootstrap_upper": 0.1263565891472868, "calls": 3581, "cp_upper": 0.11845663969826624, "misses": 383, "rate": 0.10695336498184864, "seed": 7, "upper": 0.1263565891472868 }, { "bootstrap_upper": 0.11434067108356487, "calls": 4362, "cp_upper": 0.1088502524587806, "misses": 431, "rate": 0.09880788629069234, "seed": 8, "upper": 0.11434067108356487 }, { "bootstrap_upper": 0.1130242825607064, "calls": 4436, "cp_upper": 0.10868930553326561, "misses": 438, "rate": 0.09873760144274121, "seed": 9, "upper": 0.1130242825607064 } ], "statement": "Among held-out lines the model cleared that were checked in the field, the share that were lead or galvanized pipe needing replacement is 10.4%, at most 12.7% with 98.33% confidence for each release run. Only decades with at least 203 judged clears in every run are judged; the model must abstain in the others.", "status": "fail", "summary": "release bound fails on field-checked clears of lead or GRR: decades 1950, 1960, 1980, 1990, 2010; overall 10.4% (upper 12.7%)", "upper_max": 0.02, "weighted": { "kish_ess_min": 1144.7523879208677, "limit": "Weighting fixes selection on observed covariates only. If crews chose houses for reasons that are not in the data, only a probability sample fixes it.", "point": 0.16179214917685236, "status": "fail", "upper": 0.20245496081990055 }, "withdrawn_calls": 53679, "withdrawn_calls_by_run": [ 5620, 5821, 4610, 6086, 6036, 4640, 4679, 4649, 5595, 5943 ], "withdrawn_groups": [ "1870", "1890", "1900", "1910", "1920", "1930", "1940", "1970", "2000", "2020" ] }, "sentence": "If the homes we score are like the homes we checked, at least 95 of every 100 will have their true material inside the range shown. The guarantee is approximate because the checked homes do not look like the rest of the system on the recorded covariates. A probability sample is what makes the guarantee firm again.", "sufficiency": { "lead_blocks_h3_7": 14, "min_lead_blocks": 5, "min_verified_leads": 100, "verified_leads": 160, "verified_mask": "given" } }, "bias_diagnostics": { "auc": 0.8518634230973361, "balance": [ { "feature": "year_built", "flagged": true, "smd_after": 0.12269403890118133, "smd_before": -1.300633475494507 }, { "feature": "lon", "flagged": true, "smd_after": -0.2958212132718297, "smd_before": -0.2749324078740788 }, { "feature": "lat", "flagged": false, "smd_after": -0.09153925983226514, "smd_before": -0.01747449214304629 } ], "doubtful": true, "kish_ess": 2262.2620645591805, "limit": "Weighting fixes selection on observed covariates only. If crews chose houses for reasons that are not in the data, only a probability sample fixes it.", "overlap_share": 0.00016246953696181965, "status": "scored", "weight_max": 16.173469594333753 }, "equity": { "absent_dimensions": [ "income", "renter_share", "residents_of_colour", "neighbourhood" ], "allocation": { "reason": "No inspection assignment was provided.", "status": "not_run" }, "backend": "local", "bookkeeping": { "backend": "local", "by_group": [ { "group": "1870", "observed_prevalence": 0.0, "predicted_prevalence": 0.008182258652284509 }, { "group": "1880", "observed_prevalence": 0.0, "predicted_prevalence": 0.022714502299149106 }, { "group": "1890", "observed_prevalence": 0.05, "predicted_prevalence": 0.034222348394228726 }, { "group": "1900", "observed_prevalence": 0.06015037593984962, "predicted_prevalence": 0.027214962910844416 }, { "group": "1910", "observed_prevalence": 0.065625, "predicted_prevalence": 0.02667428155670539 }, { "group": "1920", "observed_prevalence": 0.038112522686025406, "predicted_prevalence": 0.02566727545567942 }, { "group": "1930", "observed_prevalence": 0.0, "predicted_prevalence": 0.01949801435390829 }, { "group": "1940", "observed_prevalence": 0.025806451612903226, "predicted_prevalence": 0.025458377152377476 }, { "group": "1950", "observed_prevalence": 0.0006934812760055479, "predicted_prevalence": 0.004477729232000154 }, { "group": "1960", "observed_prevalence": 0.0, "predicted_prevalence": 0.004042884359403816 }, { "group": "1970", "observed_prevalence": 0.0015290519877675841, "predicted_prevalence": 0.007315631995915438 }, { "group": "1980", "observed_prevalence": 0.014887551472917327, "predicted_prevalence": 0.014164191360670637 }, { "group": "1990", "observed_prevalence": 0.0025745873087990308, "predicted_prevalence": 0.004156214511439268 }, { "group": "2000", "observed_prevalence": 0.0, "predicted_prevalence": 0.0014473479049458466 }, { "group": "2010", "observed_prevalence": 0.0005492996429552321, "predicted_prevalence": 0.0019375010975159598 }, { "group": "2020", "observed_prevalence": 0.0, "predicted_prevalence": 0.0018739568115825763 } ], "difference": { "observed_prevalence": 0.065625, "predicted_prevalence": 0.03277500048928288 }, "overall": { "observed_prevalence": 0.006539398862713241, "predicted_prevalence": 0.007017109576357215 } }, "dimensions": [ "decade_built" ], "feature_violation": [], "gaps": [ { "calls": 1.0, "equity_min_calls": 50, "gap": 0.0, "group": "decade_built:1870", "group_rate": 0.0, "judged": false, "lower": -0.008132539598635082, "n": 0.0, "n_verified_positives": 0, "overall_rate": 0.006044578768417076, "status": "not judged", "upper": -0.00395661793819907 }, { "calls": 0.0, "equity_min_calls": 50, "gap": 0.0, "group": "decade_built:1880", "group_rate": 0.0, "judged": false, "lower": -0.006044578768417076, "n": 9.0, "n_verified_positives": 0, "overall_rate": 0.006044578768417076, "status": "not judged", "upper": -0.006044578768417076 }, { "calls": 1.0, "equity_min_calls": 50, "gap": 0.9939554212315829, "group": "decade_built:1890", "group_rate": 1.0, "judged": false, "lower": 0.9919001484262017, "n": 51.0, "n_verified_positives": 3, "overall_rate": 0.006044578768417076, "status": "not judged", "upper": 0.9960106940369641 }, { "calls": 14.0, "equity_min_calls": 50, "gap": 0.06538399266015435, "group": "decade_built:1900", "group_rate": 0.07142857142857142, "judged": false, "lower": -0.06917984396940222, "n": 92.0, "n_verified_positives": 8, "overall_rate": 0.006044578768417076, "status": "not judged", "upper": 0.19994782928971092 }, { "calls": 0.0, "equity_min_calls": 50, "gap": 0.0, "group": "decade_built:1910", "group_rate": 0.0, "judged": false, "lower": -0.006044578768417076, "n": 283.0, "n_verified_positives": 21, "overall_rate": 0.006044578768417076, "status": "not judged", "upper": -0.006044578768417076 }, { "calls": 41.0, "equity_min_calls": 50, "gap": 0.0671261529389, "group": "decade_built:1920", "group_rate": 0.07317073170731707, "judged": false, "lower": -0.01199367976585837, "n": 457.0, "n_verified_positives": 21, "overall_rate": 0.006044578768417076, "status": "not judged", "upper": 0.14624598564365837 }, { "calls": 36.0, "equity_min_calls": 50, "gap": 0.0, "group": "decade_built:1930", "group_rate": 0.0, "judged": false, "lower": -0.00813249732925422, "n": 75.0, "n_verified_positives": 0, "overall_rate": 0.006044578768417076, "status": "not judged", "upper": -0.003956660207579931 }, { "calls": 69.0, "equity_min_calls": 50, "gap": 0.0, "group": "decade_built:1940", "group_rate": 0.0, "judged": true, "lower": -0.008132456955793631, "n": 1437.0, "n_verified_positives": 40, "overall_rate": 0.006044578768417076, "status": "judged", "upper": -0.003956700581040521 }, { "calls": 1198.0, "equity_min_calls": 50, "gap": 0.0, "group": "decade_built:1950", "group_rate": 0.0008347245409015025, "judged": false, "lower": -0.007621866285537619, "n": 1222.0, "n_verified_positives": 1, "overall_rate": 0.006044578768417076, "status": "not judged", "upper": -0.0027978421694935265 }, { "calls": 634.0, "equity_min_calls": 50, "gap": 0.0, "group": "decade_built:1960", "group_rate": 0.0, "judged": false, "lower": -0.008131676855413933, "n": 636.0, "n_verified_positives": 0, "overall_rate": 0.006044578768417076, "status": "not judged", "upper": -0.003957480681420218 }, { "calls": 98.0, "equity_min_calls": 50, "gap": 0.0, "group": "decade_built:1970", "group_rate": 0.0, "judged": false, "lower": -0.008132421052077064, "n": 481.0, "n_verified_positives": 1, "overall_rate": 0.006044578768417076, "status": "not judged", "upper": -0.0039567364847570875 }, { "calls": 903.0, "equity_min_calls": 50, "gap": 0.002814778928149923, "group": "decade_built:1980", "group_rate": 0.008859357696566999, "judged": true, "lower": -0.0025675928848667524, "n": 1730.0, "n_verified_positives": 47, "overall_rate": 0.006044578768417076, "status": "judged", "upper": 0.008197150741166597 }, { "calls": 1357.0, "equity_min_calls": 50, "gap": 0.005746136043668407, "group": "decade_built:1990", "group_rate": 0.011790714812085483, "judged": false, "lower": 0.0012266267613560396, "n": 1358.0, "n_verified_positives": 16, "overall_rate": 0.006044578768417076, "status": "not judged", "upper": 0.010265645325980774 }, { "calls": 229.0, "equity_min_calls": 50, "gap": 0.0, "group": "decade_built:2000", "group_rate": 0.0, "judged": false, "lower": -0.008132253734839341, "n": 240.0, "n_verified_positives": 0, "overall_rate": 0.006044578768417076, "status": "not judged", "upper": -0.00395690380199481 }, { "calls": 538.0, "equity_min_calls": 50, "gap": 0.0, "group": "decade_built:2010", "group_rate": 0.0037174721189591076, "judged": false, "lower": -0.007369848934679046, "n": 542.0, "n_verified_positives": 2, "overall_rate": 0.006044578768417076, "status": "not judged", "upper": 0.00271563563576311 }, { "calls": 176.0, "equity_min_calls": 50, "gap": 0.0, "group": "decade_built:2020", "group_rate": 0.0, "judged": false, "lower": -0.008132322461362768, "n": 180.0, "n_verified_positives": 0, "overall_rate": 0.006044578768417076, "status": "not judged", "upper": -0.003956835075471383 } ], "groups": [ { "abstain_rate": 0.0, "calibration_error": 0.008182258652284509, "conformal_coverage": 1.0, "dimension": "decade_built", "excludes_zero": true, "false_non_lead_calls": 1.0, "false_non_lead_rate": 0.0, "group": "1870", "n": 1, "n_verified": 0, "observed_prevalence": 0.0, "passed": false, "predicted_prevalence": 0.008182258652284509, "prevalence_gap": 0.008182258652284509, "prevalence_gap_lower": 0.008182258652284509, "prevalence_gap_upper": 0.008182258652284509, "status": "insufficient_data", "underpredicted": false, "verified_share": 0.0 }, { "abstain_rate": 1.0, "calibration_error": 0.025978238056056188, "conformal_coverage": 1.0, "dimension": "decade_built", "excludes_zero": true, "false_non_lead_calls": 0.0, "false_non_lead_rate": 0.0, "group": "1880", "n": 11, "n_verified": 9, "observed_prevalence": 0.0, "passed": false, "predicted_prevalence": 0.025978238056056188, "prevalence_gap": 0.025978238056056188, "prevalence_gap_lower": 0.01809958701253179, "prevalence_gap_upper": 0.03472171040718953, "status": "insufficient_data", "underpredicted": false, "verified_share": 0.8181818181818182 }, { "abstain_rate": 0.9803921568627451, "calibration_error": 0.02502887837919727, "conformal_coverage": 0.9803921568627451, "dimension": "decade_built", "excludes_zero": false, "false_non_lead_calls": 1.0, "false_non_lead_rate": 1.0, "group": "1890", "n": 60, "n_verified": 51, "observed_prevalence": 0.058823529411764705, "passed": false, "predicted_prevalence": 0.03379465103256744, "prevalence_gap": -0.025028878379197272, "prevalence_gap_lower": -0.1021252579607249, "prevalence_gap_upper": 0.034164115758504754, "status": "insufficient_data", "underpredicted": false, "verified_share": 0.85 }, { "abstain_rate": 0.8478260869565217, "calibration_error": 0.06152341690187454, "conformal_coverage": 0.9239130434782609, "dimension": "decade_built", "excludes_zero": true, "false_non_lead_calls": 14.0, "false_non_lead_rate": 0.07142857142857142, "group": "1900", "n": 133, "n_verified": 92, "observed_prevalence": 0.08695652173913043, "passed": false, "predicted_prevalence": 0.025433104837255895, "prevalence_gap": -0.06152341690187455, "prevalence_gap_lower": -0.11748983994321067, "prevalence_gap_upper": -0.005781849512857188, "status": "insufficient_data", "underpredicted": false, "verified_share": 0.6917293233082706 }, { "abstain_rate": 1.0, "calibration_error": 0.047833875774655435, "conformal_coverage": 0.9434628975265018, "dimension": "decade_built", "excludes_zero": true, "false_non_lead_calls": 0.0, "false_non_lead_rate": 0.0, "group": "1910", "n": 320, "n_verified": 283, "observed_prevalence": 0.07420494699646643, "passed": false, "predicted_prevalence": 0.026371071221810996, "prevalence_gap": -0.047833875774655435, "prevalence_gap_lower": -0.07305723996581835, "prevalence_gap_upper": -0.022605896500002987, "status": "scored", "underpredicted": true, "verified_share": 0.884375 }, { "abstain_rate": 0.9102844638949672, "calibration_error": 0.02053795686279819, "conformal_coverage": 0.949671772428884, "dimension": "decade_built", "excludes_zero": true, "false_non_lead_calls": 41.0, "false_non_lead_rate": 0.07317073170731707, "group": "1920", "n": 551, "n_verified": 457, "observed_prevalence": 0.045951859956236324, "passed": false, "predicted_prevalence": 0.025413903093438134, "prevalence_gap": -0.02053795686279819, "prevalence_gap_lower": -0.04313025249182712, "prevalence_gap_upper": -0.002152963450190032, "status": "scored", "underpredicted": true, "verified_share": 0.8294010889292196 }, { "abstain_rate": 0.52, "calibration_error": 0.01803751321699205, "conformal_coverage": 0.9733333333333334, "dimension": "decade_built", "excludes_zero": true, "false_non_lead_calls": 36.0, "false_non_lead_rate": 0.0, "group": "1930", "n": 100, "n_verified": 75, "observed_prevalence": 0.0, "passed": false, "predicted_prevalence": 0.01803751321699205, "prevalence_gap": 0.01803751321699205, "prevalence_gap_lower": 0.016394358987135513, "prevalence_gap_upper": 0.019515490386852966, "status": "insufficient_data", "underpredicted": false, "verified_share": 0.75 }, { "abstain_rate": 0.9519832985386222, "calibration_error": 0.0018227169391193017, "conformal_coverage": 0.9728601252609603, "dimension": "decade_built", "excludes_zero": false, "false_non_lead_calls": 69.0, "false_non_lead_rate": 0.0, "group": "1940", "n": 1550, "n_verified": 1437, "observed_prevalence": 0.027835768963117607, "passed": true, "predicted_prevalence": 0.026013052023998305, "prevalence_gap": -0.0018227169391193015, "prevalence_gap_lower": -0.01021735088584725, "prevalence_gap_upper": 0.007085500829930232, "status": "scored", "underpredicted": false, "verified_share": 0.9270967741935484 }, { "abstain_rate": 0.019639934533551555, "calibration_error": 0.0036478676447598605, "conformal_coverage": 0.9729950900163666, "dimension": "decade_built", "excludes_zero": true, "false_non_lead_calls": 1198.0, "false_non_lead_rate": 0.0008347245409015025, "group": "1950", "n": 1442, "n_verified": 1222, "observed_prevalence": 0.0008183306055646482, "passed": true, "predicted_prevalence": 0.004466198250324509, "prevalence_gap": 0.003647867644759861, "prevalence_gap_lower": 0.0019215756201378025, "prevalence_gap_upper": 0.004600593537744156, "status": "scored", "underpredicted": false, "verified_share": 0.8474341192787794 }, { "abstain_rate": 0.0031446540880503146, "calibration_error": 0.004048390951199043, "conformal_coverage": 0.9842767295597484, "dimension": "decade_built", "excludes_zero": true, "false_non_lead_calls": 634.0, "false_non_lead_rate": 0.0, "group": "1960", "n": 905, "n_verified": 636, "observed_prevalence": 0.0, "passed": true, "predicted_prevalence": 0.004048390951199043, "prevalence_gap": 0.004048390951199043, "prevalence_gap_lower": 0.0038706672310861836, "prevalence_gap_upper": 0.004284152515023605, "status": "scored", "underpredicted": false, "verified_share": 0.7027624309392265 }, { "abstain_rate": 0.7962577962577962, "calibration_error": 0.005489533440586647, "conformal_coverage": 0.9812889812889813, "dimension": "decade_built", "excludes_zero": true, "false_non_lead_calls": 98.0, "false_non_lead_rate": 0.0, "group": "1970", "n": 654, "n_verified": 481, "observed_prevalence": 0.002079002079002079, "passed": true, "predicted_prevalence": 0.007568535519588726, "prevalence_gap": 0.005489533440586647, "prevalence_gap_lower": 0.0011551787201834883, "prevalence_gap_upper": 0.007915773273290039, "status": "scored", "underpredicted": false, "verified_share": 0.735474006116208 }, { "abstain_rate": 0.47803468208092487, "calibration_error": 0.008394148014223807, "conformal_coverage": 0.8913294797687862, "dimension": "decade_built", "excludes_zero": false, "false_non_lead_calls": 903.0, "false_non_lead_rate": 0.008859357696566999, "group": "1980", "n": 3157, "n_verified": 1730, "observed_prevalence": 0.02716763005780347, "passed": true, "predicted_prevalence": 0.020510304176887036, "prevalence_gap": -0.006657325880916434, "prevalence_gap_lower": -0.014181697804816856, "prevalence_gap_upper": 0.0005353877365705791, "status": "scored", "underpredicted": false, "verified_share": 0.5479885967690846 }, { "abstain_rate": 0.0007363770250368188, "calibration_error": 0.006661859389671749, "conformal_coverage": 0.8424153166421208, "dimension": "decade_built", "excludes_zero": true, "false_non_lead_calls": 1357.0, "false_non_lead_rate": 0.011790714812085483, "group": "1990", "n": 6603, "n_verified": 1358, "observed_prevalence": 0.011782032400589101, "passed": false, "predicted_prevalence": 0.0051201730109173525, "prevalence_gap": -0.006661859389671749, "prevalence_gap_lower": -0.012411663351436187, "prevalence_gap_upper": -0.0016205552505338696, "status": "scored", "underpredicted": true, "verified_share": 0.20566409207935787 }, { "abstain_rate": 0.04583333333333333, "calibration_error": 0.0014433218335591254, "conformal_coverage": 0.9208333333333333, "dimension": "decade_built", "excludes_zero": true, "false_non_lead_calls": 229.0, "false_non_lead_rate": 0.0, "group": "2000", "n": 3902, "n_verified": 240, "observed_prevalence": 0.0, "passed": true, "predicted_prevalence": 0.0014433218335591254, "prevalence_gap": 0.0014433218335591254, "prevalence_gap_lower": 0.0013206817613026051, "prevalence_gap_upper": 0.0015593292650327025, "status": "scored", "underpredicted": false, "verified_share": 0.06150691952844695 }, { "abstain_rate": 0.007380073800738007, "calibration_error": 0.00162011161251148, "conformal_coverage": 0.8265682656826568, "dimension": "decade_built", "excludes_zero": false, "false_non_lead_calls": 538.0, "false_non_lead_rate": 0.0037174721189591076, "group": "2010", "n": 3641, "n_verified": 542, "observed_prevalence": 0.0036900369003690036, "passed": true, "predicted_prevalence": 0.0020699252878575236, "prevalence_gap": -0.00162011161251148, "prevalence_gap_lower": -0.0071779951837892455, "prevalence_gap_upper": 0.0020770152540707573, "status": "scored", "underpredicted": false, "verified_share": 0.1488602032408679 }, { "abstain_rate": 0.022222222222222223, "calibration_error": 0.0020308467244590407, "conformal_coverage": 0.9222222222222223, "dimension": "decade_built", "excludes_zero": true, "false_non_lead_calls": 176.0, "false_non_lead_rate": 0.0, "group": "2020", "n": 1590, "n_verified": 180, "observed_prevalence": 0.0, "passed": true, "predicted_prevalence": 0.0020308467244590407, "prevalence_gap": 0.0020308467244590407, "prevalence_gap_lower": 0.0017769162186290787, "prevalence_gap_upper": 0.0022970686888803427, "status": "scored", "underpredicted": false, "verified_share": 0.11320754716981132 } ], "min_verified": 100, "passes_silently": false, "reason": "Equity audit for decade_built. A group with fewer than 100 verified lines is insufficient_data and does not pass. 5 groups are under that floor. Share of residents of colour is an audit dimension, not a model feature. Missing dimensions are omitted.", "status": "audited", "underpredicted": [ { "dimension": "decade_built", "excludes_zero": true, "gap": -0.047833875774655435, "group": "1910", "lower": -0.07305723996581835, "upper": -0.022605896500002987 }, { "dimension": "decade_built", "excludes_zero": true, "gap": -0.02053795686279819, "group": "1920", "lower": -0.04313025249182712, "upper": -0.002152963450190032 }, { "dimension": "decade_built", "excludes_zero": true, "gap": -0.006661859389671749, "group": "1990", "lower": -0.012411663351436187, "upper": -0.0016205552505338696 } ] }, "red_team": { "n": 24620, "n_neg": 24459, "n_pos": 161, "n_positives": 161, "reason": "Labels were shuffled and scored with honest cross-validation.", "shuffled_label_auc": 0.5067567756308631, "status": "scored", "suite": "separate", "suite_reason": "Named red-team cases run through craftpine.eval.redteam.run_case and tests/redteam. They are not refit inside every evaluate call." }, "leakage": { "canary_auc": 1.0, "canary_detected": true, "feature_auc": { "h3": 0.8583299541884866, "lat": 0.5445306632008746, "lon": 0.6850740620895115, "year_built": 0.8250516595935842 }, "flagged_features": [], "id_like": [], "label_copies": [], "label_nmi": { "h3": 0.010675761402974817, "lat": 0.0015164417226753288, "lon": 0.005849273778999994, "year_built": 0.014090084755347702 }, "lead_oof_auc": { "lat": 0.5580741151563309, "lon": 0.684956876750775, "year_built": 0.8268665600615963 }, "near_label_rules": {}, "quality": { "range_violations": { "year_built": 0 } }, "target_encoding_outside_folds": [] }, "drift": { "reason": "Drift is not part of this report.", "status": "not_run" }, "gate_decision": { "decision": "FAIL", "reasons": [ "release bound fails on field-checked clears of lead or GRR: decades 1950, 1960, 1980, 1990, 2010; overall 10.4% (upper 12.7%)" ] }, "versions": { "craftpine-eval": "0.0.0", "numpy": "2.3.5", "pandas": "3.1.0", "metric_source": "craftpine.eval.evaluate" }, "seeds": [ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 ] }