[ { "id": "design-artifact-viewer-mobile-remediation", "name": "Mobile Artifact Viewer Remediation", "tier": "tier1_quick", "domain": "design", "description": "Produce a practical mobile UX remediation plan for a dense artifact viewer used to review generated outputs inside a live initiative room.", "acceptanceCriteria": [ { "id": "practical-mobile-diagnosis", "description": "Identifies concrete mobile usability failures rather than generic responsive advice", "type": "rubric", "evaluator": "llm_judge", "weight": 2 }, { "id": "viewer-information-architecture", "description": "Defines a clear mobile IA for content, provenance, metadata, and review actions", "type": "rubric", "evaluator": "llm_judge", "weight": 2 }, { "id": "state-coverage", "description": "Covers empty, loading, long-document, and error states with usable behavior", "type": "rubric", "evaluator": "llm_judge", "weight": 1.5 }, { "id": "implementation-ready-guidance", "description": "Gives engineering-ready component guidance, not only visual critique", "type": "rubric", "evaluator": "llm_judge", "weight": 2 }, { "id": "mobile-accessibility", "description": "Includes touch target, focus, reading order, and screen reader requirements", "type": "rubric", "evaluator": "llm_judge", "weight": 1.5 } ], "humanBaseline": { "timeSeconds": 3600, "costCents": 2600, "qualityScore": 91, "completeness": 1, "artifactCount": 1, "provenance": "senior mobile product design remediation estimate, Apr 2026", "methodology": "expert_estimate", "sourceSummary": "Estimate based on a senior designer reviewing comparable mobile review and document-viewer surfaces.", "sampleSize": 2, "collectedAt": "2026-04-11", "operatorProfile": "Senior product designer with mobile SaaS review-workflow experience" }, "constraints": { "maxDurationMinutes": 10, "maxCostCents": 150 }, "runs": [ { "runId": "design-artifact-viewer-mobile-remediation-r1", "autonomousCompleted": true, "durationSeconds": 14.6, "costCents": 0.0695, "usage": { "input_tokens": 511, "input_tokens_details": { "cached_tokens": 0 }, "output_tokens": 1674, "output_tokens_details": { "reasoning_tokens": 0 }, "total_tokens": 2185 }, "qualityScore": 85.28, "completeness": 0.88, "criterionScores": { "practical-mobile-diagnosis": 0.85, "viewer-information-architecture": 0.9, "state-coverage": 0.8, "implementation-ready-guidance": 0.85, "mobile-accessibility": 0.85 }, "scoringSource": "independent_judges", "selfReportedQualityScore": 86.83, "selfReportedCompleteness": 0.88, "selfReportedCriterionScores": { "practical-mobile-diagnosis": 0.85, "viewer-information-architecture": 0.9, "state-coverage": 0.8, "implementation-ready-guidance": 0.92, "mobile-accessibility": 0.85 }, "judgeAggregate": { "judgeCount": 3, "qualityScore": 85.28, "completeness": 0.88, "criterionScores": { "practical-mobile-diagnosis": 0.85, "viewer-information-architecture": 0.9, "state-coverage": 0.8, "implementation-ready-guidance": 0.85, "mobile-accessibility": 0.85 }, "disagreementPoints": 10, "maxCriterionDisagreementPoints": 20, "humanReviewRecommended": true, "lowConfidenceReasons": [ "judge quality disagreement >= 8 points", "criterion disagreement >= 8 points", "at least one judge recommended human review" ] }, "judgeCostCents": 3.1119 }, { "runId": "design-artifact-viewer-mobile-remediation-r2", "autonomousCompleted": true, "durationSeconds": 11.64, "costCents": 0.0765, "usage": { "input_tokens": 511, "input_tokens_details": { "cached_tokens": 0 }, "output_tokens": 1848, "output_tokens_details": { "reasoning_tokens": 0 }, "total_tokens": 2359 }, "qualityScore": 88.89, "completeness": 0.9, "criterionScores": { "practical-mobile-diagnosis": 0.9, "viewer-information-architecture": 0.9, "state-coverage": 0.85, "implementation-ready-guidance": 0.85, "mobile-accessibility": 0.95 }, "scoringSource": "independent_judges", "selfReportedQualityScore": 91.89, "selfReportedCompleteness": 0.92, "selfReportedCriterionScores": { "practical-mobile-diagnosis": 0.95, "viewer-information-architecture": 0.92, "state-coverage": 0.9, "implementation-ready-guidance": 0.93, "mobile-accessibility": 0.88 }, "judgeAggregate": { "judgeCount": 3, "qualityScore": 88.89, "completeness": 0.9, "criterionScores": { "practical-mobile-diagnosis": 0.9, "viewer-information-architecture": 0.9, "state-coverage": 0.85, "implementation-ready-guidance": 0.85, "mobile-accessibility": 0.95 }, "disagreementPoints": 6.67, "maxCriterionDisagreementPoints": 15, "humanReviewRecommended": true, "lowConfidenceReasons": ["criterion disagreement >= 8 points"] }, "judgeCostCents": 3.4366 }, { "runId": "design-artifact-viewer-mobile-remediation-r3", "autonomousCompleted": true, "durationSeconds": 12.93, "costCents": 0.0776, "usage": { "input_tokens": 511, "input_tokens_details": { "cached_tokens": 0 }, "output_tokens": 1876, "output_tokens_details": { "reasoning_tokens": 0 }, "total_tokens": 2387 }, "qualityScore": 90, "completeness": 0.9, "criterionScores": { "practical-mobile-diagnosis": 0.9, "viewer-information-architecture": 0.85, "state-coverage": 0.9, "implementation-ready-guidance": 0.95, "mobile-accessibility": 0.9 }, "scoringSource": "independent_judges", "selfReportedQualityScore": 89.56, "selfReportedCompleteness": 0.92, "selfReportedCriterionScores": { "practical-mobile-diagnosis": 0.95, "viewer-information-architecture": 0.92, "state-coverage": 0.8, "implementation-ready-guidance": 0.9, "mobile-accessibility": 0.88 }, "judgeAggregate": { "judgeCount": 3, "qualityScore": 90, "completeness": 0.9, "criterionScores": { "practical-mobile-diagnosis": 0.9, "viewer-information-architecture": 0.85, "state-coverage": 0.9, "implementation-ready-guidance": 0.95, "mobile-accessibility": 0.9 }, "disagreementPoints": 7.5, "maxCriterionDisagreementPoints": 15, "humanReviewRecommended": true, "lowConfidenceReasons": ["criterion disagreement >= 8 points"] }, "judgeCostCents": 4.3896 } ] }, { "id": "design-live-room-critique", "name": "Live Room Design Critique", "tier": "tier1_quick", "domain": "design", "description": "Critique and improve a live execution-room interface with a focus on hierarchy, polish, and clarity.", "acceptanceCriteria": [ { "id": "diagnoses-hierarchy", "description": "Clearly diagnoses information hierarchy failures", "type": "rubric", "evaluator": "llm_judge", "weight": 2 }, { "id": "proposes-structure", "description": "Proposes a concrete above-the-fold structure, not only vague advice", "type": "rubric", "evaluator": "llm_judge", "weight": 2 }, { "id": "references-component-system", "description": "Suggests how existing components should be reused or simplified", "type": "rubric", "evaluator": "llm_judge", "weight": 1.5 }, { "id": "high-taste", "description": "Demonstrates taste and restraint rather than adding more dashboard chrome", "type": "rubric", "evaluator": "llm_judge", "weight": 2 } ], "humanBaseline": { "timeSeconds": 3000, "costCents": 2200, "qualityScore": 89, "completeness": 1, "artifactCount": 1, "provenance": "senior product design review estimate, Mar 2026", "methodology": "expert_estimate", "sourceSummary": "Redacted estimate from two senior product design reviews of the same prompt shape.", "sampleSize": 2, "collectedAt": "2026-03-07", "operatorProfile": "Senior product designer with SaaS execution-room experience" }, "constraints": { "maxDurationMinutes": 8, "maxCostCents": 125 }, "runs": [ { "runId": "design-live-room-critique-r1", "autonomousCompleted": true, "durationSeconds": 8.94, "costCents": 0.0556, "usage": { "input_tokens": 400, "input_tokens_details": { "cached_tokens": 0 }, "output_tokens": 1339, "output_tokens_details": { "reasoning_tokens": 0 }, "total_tokens": 1739 }, "qualityScore": 87.67, "completeness": 0.95, "criterionScores": { "diagnoses-hierarchy": 0.95, "proposes-structure": 0.95, "references-component-system": 0.85, "high-taste": 0.75 }, "scoringSource": "independent_judges", "selfReportedQualityScore": 83, "selfReportedCompleteness": 0.92, "selfReportedCriterionScores": { "diagnoses-hierarchy": 0.8, "proposes-structure": 0.9, "references-component-system": 0.75, "high-taste": 0.85 }, "judgeAggregate": { "judgeCount": 3, "qualityScore": 87.67, "completeness": 0.95, "criterionScores": { "diagnoses-hierarchy": 0.95, "proposes-structure": 0.95, "references-component-system": 0.85, "high-taste": 0.75 }, "disagreementPoints": 7, "maxCriterionDisagreementPoints": 35, "humanReviewRecommended": true, "lowConfidenceReasons": ["criterion disagreement >= 8 points"] }, "judgeCostCents": 3.5988 }, { "runId": "design-live-room-critique-r2", "autonomousCompleted": true, "durationSeconds": 10.43, "costCents": 0.0575, "usage": { "input_tokens": 400, "input_tokens_details": { "cached_tokens": 0 }, "output_tokens": 1388, "output_tokens_details": { "reasoning_tokens": 0 }, "total_tokens": 1788 }, "qualityScore": 89, "completeness": 0.92, "criterionScores": { "diagnoses-hierarchy": 0.95, "proposes-structure": 0.95, "references-component-system": 0.85, "high-taste": 0.8 }, "scoringSource": "independent_judges", "selfReportedQualityScore": 85.67, "selfReportedCompleteness": 0.87, "selfReportedCriterionScores": { "diagnoses-hierarchy": 0.92, "proposes-structure": 0.88, "references-component-system": 0.75, "high-taste": 0.85 }, "judgeAggregate": { "judgeCount": 3, "qualityScore": 89, "completeness": 0.92, "criterionScores": { "diagnoses-hierarchy": 0.95, "proposes-structure": 0.95, "references-component-system": 0.85, "high-taste": 0.8 }, "disagreementPoints": 7.33, "maxCriterionDisagreementPoints": 35, "humanReviewRecommended": true, "lowConfidenceReasons": ["criterion disagreement >= 8 points"] }, "judgeCostCents": 3.5429 }, { "runId": "design-live-room-critique-r3", "autonomousCompleted": true, "durationSeconds": 10.56, "costCents": 0.0656, "usage": { "input_tokens": 400, "input_tokens_details": { "cached_tokens": 0 }, "output_tokens": 1591, "output_tokens_details": { "reasoning_tokens": 0 }, "total_tokens": 1991 }, "qualityScore": 88, "completeness": 0.94, "criterionScores": { "diagnoses-hierarchy": 0.95, "proposes-structure": 0.95, "references-component-system": 0.8, "high-taste": 0.8 }, "scoringSource": "independent_judges", "selfReportedQualityScore": 90.07, "selfReportedCompleteness": 0.89, "selfReportedCriterionScores": { "diagnoses-hierarchy": 0.92, "proposes-structure": 0.94, "references-component-system": 0.85, "high-taste": 0.88 }, "judgeAggregate": { "judgeCount": 3, "qualityScore": 88, "completeness": 0.94, "criterionScores": { "diagnoses-hierarchy": 0.95, "proposes-structure": 0.95, "references-component-system": 0.8, "high-taste": 0.8 }, "disagreementPoints": 10, "maxCriterionDisagreementPoints": 40, "humanReviewRecommended": true, "lowConfidenceReasons": [ "judge quality disagreement >= 8 points", "criterion disagreement >= 8 points" ] }, "judgeCostCents": 2.1147 } ] }, { "id": "design-modal-mobile-interaction-spec", "name": "Mobile Modal Interaction Spec", "tier": "tier1_quick", "domain": "design", "description": "Create a mobile interaction specification for decision, approval, input, and confirmation modals inside an agentic workflow product.", "acceptanceCriteria": [ { "id": "taxonomy-clarity", "description": "Defines modal types by user job and risk level", "type": "rubric", "evaluator": "llm_judge", "weight": 2 }, { "id": "mobile-interaction-specificity", "description": "Specifies mobile-specific sheet, keyboard, scroll, and safe-area behavior", "type": "rubric", "evaluator": "llm_judge", "weight": 2 }, { "id": "action-hierarchy", "description": "Establishes clear primary, secondary, cancel, and destructive action hierarchy", "type": "rubric", "evaluator": "llm_judge", "weight": 1.5 }, { "id": "state-and-accessibility-coverage", "description": "Covers loading, errors, disabled states, focus management, and assistive technology behavior", "type": "rubric", "evaluator": "llm_judge", "weight": 2 }, { "id": "engineering-ready", "description": "Provides enough constraints for engineering to implement consistently", "type": "rubric", "evaluator": "llm_judge", "weight": 1.5 } ], "humanBaseline": { "timeSeconds": 3900, "costCents": 2800, "qualityScore": 90, "completeness": 1, "artifactCount": 1, "provenance": "senior interaction design pattern estimate, Apr 2026", "methodology": "expert_estimate", "sourceSummary": "Estimate based on a senior interaction designer creating modal guidelines for comparable SaaS workflows.", "sampleSize": 2, "collectedAt": "2026-04-11", "operatorProfile": "Senior interaction designer with mobile workflow and accessibility experience" }, "constraints": { "maxDurationMinutes": 10, "maxCostCents": 150 }, "runs": [ { "runId": "design-modal-mobile-interaction-spec-r1", "autonomousCompleted": true, "durationSeconds": 11.42, "costCents": 0.0672, "usage": { "input_tokens": 504, "input_tokens_details": { "cached_tokens": 0 }, "output_tokens": 1616, "output_tokens_details": { "reasoning_tokens": 0 }, "total_tokens": 2120 }, "qualityScore": 80, "completeness": 0.82, "criterionScores": { "taxonomy-clarity": 0.8, "mobile-interaction-specificity": 0.8, "action-hierarchy": 0.85, "state-and-accessibility-coverage": 0.8, "engineering-ready": 0.75 }, "scoringSource": "independent_judges", "selfReportedQualityScore": 95.56, "selfReportedCompleteness": 0.95, "selfReportedCriterionScores": { "taxonomy-clarity": 1, "mobile-interaction-specificity": 1, "action-hierarchy": 0.9, "state-and-accessibility-coverage": 0.95, "engineering-ready": 0.9 }, "judgeAggregate": { "judgeCount": 3, "qualityScore": 80, "completeness": 0.82, "criterionScores": { "taxonomy-clarity": 0.8, "mobile-interaction-specificity": 0.8, "action-hierarchy": 0.85, "state-and-accessibility-coverage": 0.8, "engineering-ready": 0.75 }, "disagreementPoints": 12.22, "maxCriterionDisagreementPoints": 40, "humanReviewRecommended": true, "lowConfidenceReasons": [ "judge quality disagreement >= 8 points", "criterion disagreement >= 8 points", "at least one judge recommended human review" ] }, "judgeCostCents": 3.9814 }, { "runId": "design-modal-mobile-interaction-spec-r2", "autonomousCompleted": true, "durationSeconds": 12.78, "costCents": 0.0796, "usage": { "input_tokens": 504, "input_tokens_details": { "cached_tokens": 0 }, "output_tokens": 1927, "output_tokens_details": { "reasoning_tokens": 0 }, "total_tokens": 2431 }, "qualityScore": 84.17, "completeness": 0.82, "criterionScores": { "taxonomy-clarity": 0.8, "mobile-interaction-specificity": 0.9, "action-hierarchy": 0.85, "state-and-accessibility-coverage": 0.85, "engineering-ready": 0.8 }, "scoringSource": "independent_judges", "selfReportedQualityScore": 88.67, "selfReportedCompleteness": 0.92, "selfReportedCriterionScores": { "taxonomy-clarity": 0.92, "mobile-interaction-specificity": 0.9, "action-hierarchy": 0.85, "state-and-accessibility-coverage": 0.88, "engineering-ready": 0.87 }, "judgeAggregate": { "judgeCount": 3, "qualityScore": 84.17, "completeness": 0.82, "criterionScores": { "taxonomy-clarity": 0.8, "mobile-interaction-specificity": 0.9, "action-hierarchy": 0.85, "state-and-accessibility-coverage": 0.85, "engineering-ready": 0.8 }, "disagreementPoints": 6.11, "maxCriterionDisagreementPoints": 10, "humanReviewRecommended": true, "lowConfidenceReasons": [ "criterion disagreement >= 8 points", "at least one judge recommended human review" ] }, "judgeCostCents": 3.3092 }, { "runId": "design-modal-mobile-interaction-spec-r3", "autonomousCompleted": true, "durationSeconds": 12.27, "costCents": 0.0754, "usage": { "input_tokens": 504, "input_tokens_details": { "cached_tokens": 0 }, "output_tokens": 1821, "output_tokens_details": { "reasoning_tokens": 0 }, "total_tokens": 2325 }, "qualityScore": 84.44, "completeness": 0.86, "criterionScores": { "taxonomy-clarity": 0.8, "mobile-interaction-specificity": 0.8, "action-hierarchy": 0.8, "state-and-accessibility-coverage": 1, "engineering-ready": 0.8 }, "scoringSource": "independent_judges", "selfReportedQualityScore": 90.39, "selfReportedCompleteness": 0.92, "selfReportedCriterionScores": { "taxonomy-clarity": 0.95, "mobile-interaction-specificity": 0.9, "action-hierarchy": 0.85, "state-and-accessibility-coverage": 0.92, "engineering-ready": 0.88 }, "judgeAggregate": { "judgeCount": 3, "qualityScore": 84.44, "completeness": 0.86, "criterionScores": { "taxonomy-clarity": 0.8, "mobile-interaction-specificity": 0.8, "action-hierarchy": 0.8, "state-and-accessibility-coverage": 1, "engineering-ready": 0.8 }, "disagreementPoints": 0.56, "maxCriterionDisagreementPoints": 10, "humanReviewRecommended": true, "lowConfidenceReasons": ["criterion disagreement >= 8 points"] }, "judgeCostCents": 3.5874 } ] }, { "id": "marketing-launch-brief", "name": "Marketing Launch Brief", "tier": "tier1_quick", "domain": "marketing", "description": "Produce a launch brief for a new AI product feature with audience, angle, channels, and proof points.", "acceptanceCriteria": [ { "id": "has-positioning", "description": "Includes a clear positioning statement for the feature", "type": "rubric", "evaluator": "llm_judge", "weight": 2 }, { "id": "has-message-pillars", "description": "Defines message pillars that are distinct and reusable", "type": "rubric", "evaluator": "llm_judge", "weight": 2 }, { "id": "has-channel-plan", "description": "Includes channel-specific recommendations rather than one generic launch plan", "type": "rubric", "evaluator": "llm_judge", "weight": 1.5 }, { "id": "proof-emphasis", "description": "Uses proof, demo, or live evidence as a central conversion lever", "type": "rubric", "evaluator": "llm_judge", "weight": 2 }, { "id": "has-cta", "description": "Includes a clear CTA strategy", "type": "binary", "evaluator": "llm_judge", "weight": 1 } ], "humanBaseline": { "timeSeconds": 2700, "costCents": 2100, "qualityScore": 86, "completeness": 1, "artifactCount": 1, "provenance": "solo founder + fractional marketer estimate, Mar 2026", "methodology": "hybrid", "sourceSummary": "Combined estimate from founder-led launch planning and fractional marketer review.", "sampleSize": 2, "collectedAt": "2026-03-06", "operatorProfile": "Founder plus B2B SaaS fractional marketing lead" }, "constraints": { "maxDurationMinutes": 8, "maxCostCents": 125 }, "runs": [ { "runId": "marketing-launch-brief-r1", "autonomousCompleted": true, "durationSeconds": 8.05, "costCents": 0.0393, "usage": { "input_tokens": 410, "input_tokens_details": { "cached_tokens": 0 }, "output_tokens": 931, "output_tokens_details": { "reasoning_tokens": 0 }, "total_tokens": 1341 }, "qualityScore": 100, "completeness": 1, "criterionScores": { "has-positioning": 1, "has-message-pillars": 1, "has-channel-plan": 1, "proof-emphasis": 1, "has-cta": 1 }, "scoringSource": "independent_judges", "selfReportedQualityScore": 98.24, "selfReportedCompleteness": 0.9, "selfReportedCriterionScores": { "has-positioning": 1, "has-message-pillars": 1, "has-channel-plan": 0.9, "proof-emphasis": 1, "has-cta": 1 }, "judgeAggregate": { "judgeCount": 3, "qualityScore": 100, "completeness": 1, "criterionScores": { "has-positioning": 1, "has-message-pillars": 1, "has-channel-plan": 1, "proof-emphasis": 1, "has-cta": 1 }, "disagreementPoints": 4.41, "maxCriterionDisagreementPoints": 10, "humanReviewRecommended": true, "lowConfidenceReasons": ["criterion disagreement >= 8 points"] }, "judgeCostCents": 1.3269 }, { "runId": "marketing-launch-brief-r2", "autonomousCompleted": true, "durationSeconds": 5.6, "costCents": 0.0375, "usage": { "input_tokens": 410, "input_tokens_details": { "cached_tokens": 0 }, "output_tokens": 887, "output_tokens_details": { "reasoning_tokens": 0 }, "total_tokens": 1297 }, "qualityScore": 98.12, "completeness": 1, "criterionScores": { "has-positioning": 1, "has-message-pillars": 1, "has-channel-plan": 0.96, "proof-emphasis": 0.97, "has-cta": 0.96 }, "scoringSource": "independent_judges", "selfReportedQualityScore": 100, "selfReportedCompleteness": 1, "selfReportedCriterionScores": { "has-positioning": 1, "has-message-pillars": 1, "has-channel-plan": 1, "proof-emphasis": 1, "has-cta": 1 }, "judgeAggregate": { "judgeCount": 3, "qualityScore": 98.12, "completeness": 1, "criterionScores": { "has-positioning": 1, "has-message-pillars": 1, "has-channel-plan": 0.96, "proof-emphasis": 0.97, "has-cta": 0.96 }, "disagreementPoints": 4.12, "maxCriterionDisagreementPoints": 10, "humanReviewRecommended": true, "lowConfidenceReasons": ["criterion disagreement >= 8 points"] }, "judgeCostCents": 1.2593 }, { "runId": "marketing-launch-brief-r3", "autonomousCompleted": true, "durationSeconds": 6.56, "costCents": 0.0385, "usage": { "input_tokens": 410, "input_tokens_details": { "cached_tokens": 0 }, "output_tokens": 912, "output_tokens_details": { "reasoning_tokens": 0 }, "total_tokens": 1322 }, "qualityScore": 93.53, "completeness": 0.92, "criterionScores": { "has-positioning": 1, "has-message-pillars": 0.8, "has-channel-plan": 0.9, "proof-emphasis": 1, "has-cta": 1 }, "scoringSource": "independent_judges", "selfReportedQualityScore": 92.65, "selfReportedCompleteness": 0.9, "selfReportedCriterionScores": { "has-positioning": 1, "has-message-pillars": 1, "has-channel-plan": 0.75, "proof-emphasis": 1, "has-cta": 0.75 }, "judgeAggregate": { "judgeCount": 3, "qualityScore": 93.53, "completeness": 0.92, "criterionScores": { "has-positioning": 1, "has-message-pillars": 0.8, "has-channel-plan": 0.9, "proof-emphasis": 1, "has-cta": 1 }, "disagreementPoints": 4.12, "maxCriterionDisagreementPoints": 20, "humanReviewRecommended": true, "lowConfidenceReasons": ["criterion disagreement >= 8 points"] }, "judgeCostCents": 1.864 } ] }, { "id": "postmortem", "name": "Incident Postmortem", "tier": "tier1_quick", "domain": "ops", "description": "Write a structured incident postmortem from a timeline of events. Tests ability to synthesize operational data into a clear narrative with root cause analysis and action items.", "acceptanceCriteria": [ { "id": "has-exec-summary", "description": "Includes a concise executive summary (2-3 sentences)", "type": "binary", "evaluator": "llm_judge", "weight": 1 }, { "id": "has-root-cause", "description": "Correctly identifies the root cause as connection pool exhaustion caused by the new analytics export endpoint's long-running transactions", "type": "rubric", "evaluator": "llm_judge", "judgePrompt": "Does the output correctly identify the root cause of the incident?\nThe correct root cause is: The new `/api/v2/analytics/export` endpoint\n(added in v2.41.0) opens long-running transactions (~45s each) that\nexhausted the PostgreSQL connection pool (100 max connections).\n\n## Output\n{{output}}\n\n## Criterion\n{{criterion}}", "weight": 3 }, { "id": "has-impact-quantified", "description": "Impact section includes specific numbers (duration, users, revenue)", "type": "binary", "evaluator": "llm_judge", "weight": 1.5 }, { "id": "has-action-items", "description": "Includes specific, actionable items (not vague recommendations)", "type": "rubric", "evaluator": "llm_judge", "judgePrompt": "Does the output include specific, actionable action items?\nGood action items should have: what to do, who owns it, and a deadline.\nVague items like \"improve monitoring\" without specifics score low.\n\n## Output\n{{output}}\n\n## Criterion\n{{criterion}}", "weight": 2 }, { "id": "has-timeline", "description": "Includes a condensed timeline of key events", "type": "binary", "evaluator": "llm_judge", "weight": 1 }, { "id": "has-lessons", "description": "Includes lessons learned that go beyond restating what happened", "type": "rubric", "evaluator": "llm_judge", "weight": 1.5 } ], "humanBaseline": { "timeSeconds": 1800, "costCents": 1500, "qualityScore": 90, "completeness": 1, "artifactCount": 1, "provenance": "senior engineering manager postmortem estimate, Mar 2026", "methodology": "expert_estimate", "sourceSummary": "Baseline estimated from an experienced engineering manager writing a structured postmortem from the same inputs.", "sampleSize": 1, "collectedAt": "2026-03-05", "operatorProfile": "Senior engineering manager with incident review responsibility" }, "constraints": { "maxDurationMinutes": 10, "maxCostCents": 100 }, "runs": [ { "runId": "postmortem-r1", "autonomousCompleted": true, "durationSeconds": 6.92, "costCents": 0.0468, "usage": { "input_tokens": 827, "input_tokens_details": { "cached_tokens": 0 }, "output_tokens": 1066, "output_tokens_details": { "reasoning_tokens": 0 }, "total_tokens": 1893 }, "qualityScore": 100, "completeness": 0.98, "criterionScores": { "has-exec-summary": 1, "has-root-cause": 1, "has-impact-quantified": 1, "has-action-items": 1, "has-timeline": 1, "has-lessons": 1 }, "scoringSource": "independent_judges", "selfReportedQualityScore": 100, "selfReportedCompleteness": 1, "selfReportedCriterionScores": { "has-exec-summary": 1, "has-root-cause": 1, "has-impact-quantified": 1, "has-action-items": 1, "has-timeline": 1, "has-lessons": 1 }, "judgeAggregate": { "judgeCount": 3, "qualityScore": 100, "completeness": 0.98, "criterionScores": { "has-exec-summary": 1, "has-root-cause": 1, "has-impact-quantified": 1, "has-action-items": 1, "has-timeline": 1, "has-lessons": 1 }, "disagreementPoints": 1.5, "maxCriterionDisagreementPoints": 10, "humanReviewRecommended": true, "lowConfidenceReasons": ["criterion disagreement >= 8 points"] }, "judgeCostCents": 1.8293 }, { "runId": "postmortem-r2", "autonomousCompleted": true, "durationSeconds": 91.3, "costCents": 0.0447, "usage": { "input_tokens": 827, "input_tokens_details": { "cached_tokens": 0 }, "output_tokens": 1015, "output_tokens_details": { "reasoning_tokens": 0 }, "total_tokens": 1842 }, "qualityScore": 96.5, "completeness": 0.93, "criterionScores": { "has-exec-summary": 1, "has-root-cause": 1, "has-impact-quantified": 1, "has-action-items": 0.9, "has-timeline": 1, "has-lessons": 0.9 }, "scoringSource": "independent_judges", "selfReportedQualityScore": 100, "selfReportedCompleteness": 0.95, "selfReportedCriterionScores": { "has-exec-summary": 1, "has-root-cause": 1, "has-impact-quantified": 1, "has-action-items": 1, "has-timeline": 1, "has-lessons": 1 }, "judgeAggregate": { "judgeCount": 3, "qualityScore": 96.5, "completeness": 0.93, "criterionScores": { "has-exec-summary": 1, "has-root-cause": 1, "has-impact-quantified": 1, "has-action-items": 0.9, "has-timeline": 1, "has-lessons": 0.9 }, "disagreementPoints": 3, "maxCriterionDisagreementPoints": 20, "humanReviewRecommended": true, "lowConfidenceReasons": ["criterion disagreement >= 8 points"] }, "judgeCostCents": 1.9969 }, { "runId": "postmortem-r3", "autonomousCompleted": true, "durationSeconds": 183.58, "costCents": 0.0468, "usage": { "input_tokens": 827, "input_tokens_details": { "cached_tokens": 0 }, "output_tokens": 1067, "output_tokens_details": { "reasoning_tokens": 0 }, "total_tokens": 1894 }, "qualityScore": 100, "completeness": 0.98, "criterionScores": { "has-exec-summary": 1, "has-root-cause": 1, "has-impact-quantified": 1, "has-action-items": 1, "has-timeline": 1, "has-lessons": 1 }, "scoringSource": "independent_judges", "selfReportedQualityScore": 100, "selfReportedCompleteness": 0.92, "selfReportedCriterionScores": { "has-exec-summary": 1, "has-root-cause": 1, "has-impact-quantified": 1, "has-action-items": 1, "has-timeline": 1, "has-lessons": 1 }, "judgeAggregate": { "judgeCount": 3, "qualityScore": 100, "completeness": 0.98, "criterionScores": { "has-exec-summary": 1, "has-root-cause": 1, "has-impact-quantified": 1, "has-action-items": 1, "has-timeline": 1, "has-lessons": 1 }, "disagreementPoints": 3, "maxCriterionDisagreementPoints": 20, "humanReviewRecommended": true, "lowConfidenceReasons": ["criterion disagreement >= 8 points"] }, "judgeCostCents": 1.9009 } ] }, { "id": "pr-description", "name": "PR Description from Diff", "tier": "tier1_quick", "domain": "engineering", "description": "Write a comprehensive pull request description given a code diff and commit messages. The output should include a summary, list of changes, testing instructions, and any migration notes.", "acceptanceCriteria": [ { "id": "has-title", "description": "Output includes a clear, descriptive PR title", "type": "binary", "evaluator": "llm_judge", "judgePrompt": "Does the following output include a clear, descriptive one-line PR title that summarizes the change?\n\n## Output\n{{output}}\n\n## Criterion\n{{criterion}}", "weight": 1 }, { "id": "has-summary", "description": "Output includes a 2-3 sentence summary explaining what changed and why", "type": "rubric", "evaluator": "llm_judge", "judgePrompt": "Does the output include a concise summary (2-3 sentences) that explains both WHAT changed and WHY?\n\n## Output\n{{output}}\n\n## Criterion\n{{criterion}}", "weight": 2 }, { "id": "has-changes-list", "description": "Output includes a specific bullet list of changes", "type": "binary", "evaluator": "llm_judge", "weight": 1.5 }, { "id": "has-testing-instructions", "description": "Output includes testing instructions", "type": "binary", "evaluator": "llm_judge", "weight": 1 }, { "id": "mentions-auth-tokens", "description": "Output correctly identifies this as a token refresh fix, not a generic auth change", "type": "rubric", "evaluator": "llm_judge", "weight": 1.5 } ], "humanBaseline": { "timeSeconds": 600, "costCents": 500, "qualityScore": 85, "completeness": 1, "artifactCount": 1, "provenance": "senior engineer PR authoring estimate, Mar 2026", "methodology": "expert_estimate", "sourceSummary": "Baseline estimated from a senior engineer writing a PR description from the same diff and commit context.", "sampleSize": 1, "collectedAt": "2026-03-05", "operatorProfile": "Senior software engineer working in a code review workflow" }, "constraints": { "maxDurationMinutes": 5, "maxCostCents": 50 }, "runs": [ { "runId": "pr-description-r1", "autonomousCompleted": true, "durationSeconds": 4.49, "costCents": 0.0314, "usage": { "input_tokens": 761, "input_tokens_details": { "cached_tokens": 0 }, "output_tokens": 690, "output_tokens_details": { "reasoning_tokens": 0 }, "total_tokens": 1451 }, "qualityScore": 100, "completeness": 1, "criterionScores": { "has-title": 1, "has-summary": 1, "has-changes-list": 1, "has-testing-instructions": 1, "mentions-auth-tokens": 1 }, "scoringSource": "independent_judges", "selfReportedQualityScore": 100, "selfReportedCompleteness": 1, "selfReportedCriterionScores": { "has-title": 1, "has-summary": 1, "has-changes-list": 1, "has-testing-instructions": 1, "mentions-auth-tokens": 1 }, "judgeAggregate": { "judgeCount": 3, "qualityScore": 100, "completeness": 1, "criterionScores": { "has-title": 1, "has-summary": 1, "has-changes-list": 1, "has-testing-instructions": 1, "mentions-auth-tokens": 1 }, "disagreementPoints": 7.14, "maxCriterionDisagreementPoints": 20, "humanReviewRecommended": true, "lowConfidenceReasons": ["criterion disagreement >= 8 points"] }, "judgeCostCents": 2.7208 }, { "runId": "pr-description-r2", "autonomousCompleted": true, "durationSeconds": 4.67, "costCents": 0.0302, "usage": { "input_tokens": 761, "input_tokens_details": { "cached_tokens": 0 }, "output_tokens": 661, "output_tokens_details": { "reasoning_tokens": 0 }, "total_tokens": 1422 }, "qualityScore": 94.29, "completeness": 0.92, "criterionScores": { "has-title": 1, "has-summary": 0.8, "has-changes-list": 1, "has-testing-instructions": 1, "mentions-auth-tokens": 1 }, "scoringSource": "independent_judges", "selfReportedQualityScore": 100, "selfReportedCompleteness": 0.92, "selfReportedCriterionScores": { "has-title": 1, "has-summary": 1, "has-changes-list": 1, "has-testing-instructions": 1, "mentions-auth-tokens": 1 }, "judgeAggregate": { "judgeCount": 3, "qualityScore": 94.29, "completeness": 0.92, "criterionScores": { "has-title": 1, "has-summary": 0.8, "has-changes-list": 1, "has-testing-instructions": 1, "mentions-auth-tokens": 1 }, "disagreementPoints": 10, "maxCriterionDisagreementPoints": 20, "humanReviewRecommended": true, "lowConfidenceReasons": [ "judge quality disagreement >= 8 points", "criterion disagreement >= 8 points" ] }, "judgeCostCents": 1.9698 }, { "runId": "pr-description-r3", "autonomousCompleted": true, "durationSeconds": 4.81, "costCents": 0.0336, "usage": { "input_tokens": 761, "input_tokens_details": { "cached_tokens": 0 }, "output_tokens": 746, "output_tokens_details": { "reasoning_tokens": 0 }, "total_tokens": 1507 }, "qualityScore": 100, "completeness": 0.96, "criterionScores": { "has-title": 1, "has-summary": 1, "has-changes-list": 1, "has-testing-instructions": 1, "mentions-auth-tokens": 1 }, "scoringSource": "independent_judges", "selfReportedQualityScore": 100, "selfReportedCompleteness": 0.92, "selfReportedCriterionScores": { "has-title": 1, "has-summary": 1, "has-changes-list": 1, "has-testing-instructions": 1, "mentions-auth-tokens": 1 }, "judgeAggregate": { "judgeCount": 3, "qualityScore": 100, "completeness": 0.96, "criterionScores": { "has-title": 1, "has-summary": 1, "has-changes-list": 1, "has-testing-instructions": 1, "mentions-auth-tokens": 1 }, "disagreementPoints": 5.71, "maxCriterionDisagreementPoints": 20, "humanReviewRecommended": true, "lowConfidenceReasons": ["criterion disagreement >= 8 points"] }, "judgeCostCents": 3.5065 } ] }, { "id": "product-initiative-brief", "name": "Product Initiative Brief", "tier": "tier1_quick", "domain": "product", "description": "Turn a founder request into a crisp initiative brief with goals, user, scope, metrics, and sequencing. Benchmarks product framing quality.", "acceptanceCriteria": [ { "id": "has-problem-statement", "description": "Includes a clear product problem statement tied to the user pain", "type": "rubric", "evaluator": "llm_judge", "weight": 2 }, { "id": "has-success-metrics", "description": "Includes measurable success metrics, not vague goals", "type": "rubric", "evaluator": "llm_judge", "weight": 2 }, { "id": "has-scope", "description": "Clearly defines in-scope and out-of-scope work", "type": "binary", "evaluator": "llm_judge", "weight": 1.5 }, { "id": "has-workstreams", "description": "Recommends sensible workstreams or execution lanes", "type": "rubric", "evaluator": "llm_judge", "weight": 1.5 }, { "id": "founder-decision-moment", "description": "Recognizes the one-session founder decision moment as central to the task", "type": "rubric", "evaluator": "llm_judge", "weight": 2 } ], "humanBaseline": { "timeSeconds": 2400, "costCents": 1800, "qualityScore": 88, "completeness": 1, "artifactCount": 1, "provenance": "PM lead + founder review estimate, Mar 2026", "methodology": "hybrid", "sourceSummary": "Combined estimate from PM-led initiative framing and founder review of the same request.", "sampleSize": 2, "collectedAt": "2026-03-06", "operatorProfile": "Product lead working with a technical founder on initiative framing" }, "constraints": { "maxDurationMinutes": 8, "maxCostCents": 125 }, "runs": [ { "runId": "product-initiative-brief-r1", "autonomousCompleted": true, "durationSeconds": 7.57, "costCents": 0.0432, "usage": { "input_tokens": 435, "input_tokens_details": { "cached_tokens": 0 }, "output_tokens": 1026, "output_tokens_details": { "reasoning_tokens": 0 }, "total_tokens": 1461 }, "qualityScore": 95.28, "completeness": 0.95, "criterionScores": { "has-problem-statement": 1, "has-success-metrics": 0.9, "has-scope": 0.85, "has-workstreams": 1, "founder-decision-moment": 1 }, "scoringSource": "independent_judges", "selfReportedQualityScore": 91.67, "selfReportedCompleteness": 0.72, "selfReportedCriterionScores": { "has-problem-statement": 1, "has-success-metrics": 1, "has-scope": 0.75, "has-workstreams": 0.75, "founder-decision-moment": 1 }, "judgeAggregate": { "judgeCount": 3, "qualityScore": 95.28, "completeness": 0.95, "criterionScores": { "has-problem-statement": 1, "has-success-metrics": 0.9, "has-scope": 0.85, "has-workstreams": 1, "founder-decision-moment": 1 }, "disagreementPoints": 7.78, "maxCriterionDisagreementPoints": 20, "humanReviewRecommended": true, "lowConfidenceReasons": ["criterion disagreement >= 8 points"] }, "judgeCostCents": 3.1524 }, { "runId": "product-initiative-brief-r2", "autonomousCompleted": true, "durationSeconds": 5.57, "costCents": 0.0422, "usage": { "input_tokens": 435, "input_tokens_details": { "cached_tokens": 0 }, "output_tokens": 1000, "output_tokens_details": { "reasoning_tokens": 0 }, "total_tokens": 1435 }, "qualityScore": 94.44, "completeness": 0.95, "criterionScores": { "has-problem-statement": 1, "has-success-metrics": 0.9, "has-scope": 0.9, "has-workstreams": 0.9, "founder-decision-moment": 1 }, "scoringSource": "independent_judges", "selfReportedQualityScore": 86.67, "selfReportedCompleteness": 0.92, "selfReportedCriterionScores": { "has-problem-statement": 1, "has-success-metrics": 1, "has-scope": 0.6, "has-workstreams": 0.6, "founder-decision-moment": 1 }, "judgeAggregate": { "judgeCount": 3, "qualityScore": 94.44, "completeness": 0.95, "criterionScores": { "has-problem-statement": 1, "has-success-metrics": 0.9, "has-scope": 0.9, "has-workstreams": 0.9, "founder-decision-moment": 1 }, "disagreementPoints": 8.89, "maxCriterionDisagreementPoints": 20, "humanReviewRecommended": true, "lowConfidenceReasons": [ "judge quality disagreement >= 8 points", "criterion disagreement >= 8 points" ] }, "judgeCostCents": 1.8755 }, { "runId": "product-initiative-brief-r3", "autonomousCompleted": true, "durationSeconds": 6.12, "costCents": 0.0457, "usage": { "input_tokens": 435, "input_tokens_details": { "cached_tokens": 0 }, "output_tokens": 1089, "output_tokens_details": { "reasoning_tokens": 0 }, "total_tokens": 1524 }, "qualityScore": 95.56, "completeness": 0.95, "criterionScores": { "has-problem-statement": 1, "has-success-metrics": 0.8, "has-scope": 1, "has-workstreams": 1, "founder-decision-moment": 1 }, "scoringSource": "independent_judges", "selfReportedQualityScore": 100, "selfReportedCompleteness": 0.92, "selfReportedCriterionScores": { "has-problem-statement": 1, "has-success-metrics": 1, "has-scope": 1, "has-workstreams": 1, "founder-decision-moment": 1 }, "judgeAggregate": { "judgeCount": 3, "qualityScore": 95.56, "completeness": 0.95, "criterionScores": { "has-problem-statement": 1, "has-success-metrics": 0.8, "has-scope": 1, "has-workstreams": 1, "founder-decision-moment": 1 }, "disagreementPoints": 2.22, "maxCriterionDisagreementPoints": 10, "humanReviewRecommended": true, "lowConfidenceReasons": ["criterion disagreement >= 8 points"] }, "judgeCostCents": 1.6439 } ] }, { "id": "sales-outreach-sequence", "name": "Sales Outreach Sequence", "tier": "tier1_quick", "domain": "sales", "description": "Build a founder-quality outreach sequence for a specific ICP with message angles and proof-driven CTA.", "acceptanceCriteria": [ { "id": "personalized-icp", "description": "The sequence feels specific to the ICP, not generic AI SaaS outreach", "type": "rubric", "evaluator": "llm_judge", "weight": 2 }, { "id": "proof-led-cta", "description": "The CTA is tied to a demo or live proof moment", "type": "rubric", "evaluator": "llm_judge", "weight": 2 }, { "id": "multi-step-sequence", "description": "Includes all required steps in a coherent sequence", "type": "binary", "evaluator": "llm_judge", "weight": 1.5 }, { "id": "objection-angle", "description": "Includes a credible objection-handling angle", "type": "rubric", "evaluator": "llm_judge", "weight": 1.5 } ], "humanBaseline": { "timeSeconds": 2100, "costCents": 1600, "qualityScore": 84, "completeness": 1, "artifactCount": 1, "provenance": "founder-led outbound estimate, Mar 2026", "methodology": "expert_estimate", "sourceSummary": "Estimate from founder-led outbound workflow for a three-step proof-first sequence.", "sampleSize": 1, "collectedAt": "2026-03-06", "operatorProfile": "Founder-operator running early outbound without a dedicated SDR" }, "constraints": { "maxDurationMinutes": 8, "maxCostCents": 125 }, "runs": [ { "runId": "sales-outreach-sequence-r1", "autonomousCompleted": true, "durationSeconds": 6.02, "costCents": 0.0416, "usage": { "input_tokens": 402, "input_tokens_details": { "cached_tokens": 0 }, "output_tokens": 990, "output_tokens_details": { "reasoning_tokens": 0 }, "total_tokens": 1392 }, "qualityScore": 91.07, "completeness": 1, "criterionScores": { "personalized-icp": 0.8, "proof-led-cta": 1, "multi-step-sequence": 1, "objection-angle": 0.85 }, "scoringSource": "independent_judges", "selfReportedQualityScore": 97.86, "selfReportedCompleteness": 0.92, "selfReportedCriterionScores": { "personalized-icp": 1, "proof-led-cta": 1, "multi-step-sequence": 0.95, "objection-angle": 0.95 }, "judgeAggregate": { "judgeCount": 3, "qualityScore": 91.07, "completeness": 1, "criterionScores": { "personalized-icp": 0.8, "proof-led-cta": 1, "multi-step-sequence": 1, "objection-angle": 0.85 }, "disagreementPoints": 6.79, "maxCriterionDisagreementPoints": 10, "humanReviewRecommended": true, "lowConfidenceReasons": ["criterion disagreement >= 8 points"] }, "judgeCostCents": 2.3533 }, { "runId": "sales-outreach-sequence-r2", "autonomousCompleted": true, "durationSeconds": 5.35, "costCents": 0.0405, "usage": { "input_tokens": 402, "input_tokens_details": { "cached_tokens": 0 }, "output_tokens": 962, "output_tokens_details": { "reasoning_tokens": 0 }, "total_tokens": 1364 }, "qualityScore": 87.14, "completeness": 1, "criterionScores": { "personalized-icp": 0.8, "proof-led-cta": 0.9, "multi-step-sequence": 1, "objection-angle": 0.8 }, "scoringSource": "independent_judges", "selfReportedQualityScore": 100, "selfReportedCompleteness": 0.95, "selfReportedCriterionScores": { "personalized-icp": 1, "proof-led-cta": 1, "multi-step-sequence": 1, "objection-angle": 1 }, "judgeAggregate": { "judgeCount": 3, "qualityScore": 87.14, "completeness": 1, "criterionScores": { "personalized-icp": 0.8, "proof-led-cta": 0.9, "multi-step-sequence": 1, "objection-angle": 0.8 }, "disagreementPoints": 6.43, "maxCriterionDisagreementPoints": 20, "humanReviewRecommended": true, "lowConfidenceReasons": ["criterion disagreement >= 8 points"] }, "judgeCostCents": 2.6331 }, { "runId": "sales-outreach-sequence-r3", "autonomousCompleted": true, "durationSeconds": 7.73, "costCents": 0.0433, "usage": { "input_tokens": 402, "input_tokens_details": { "cached_tokens": 0 }, "output_tokens": 1032, "output_tokens_details": { "reasoning_tokens": 0 }, "total_tokens": 1434 }, "qualityScore": 88.57, "completeness": 0.98, "criterionScores": { "personalized-icp": 0.8, "proof-led-cta": 0.95, "multi-step-sequence": 1, "objection-angle": 0.8 }, "scoringSource": "independent_judges", "selfReportedQualityScore": 95, "selfReportedCompleteness": 0.9, "selfReportedCriterionScores": { "personalized-icp": 0.95, "proof-led-cta": 0.95, "multi-step-sequence": 1, "objection-angle": 0.9 }, "judgeAggregate": { "judgeCount": 3, "qualityScore": 88.57, "completeness": 0.98, "criterionScores": { "personalized-icp": 0.8, "proof-led-cta": 0.95, "multi-step-sequence": 1, "objection-angle": 0.8 }, "disagreementPoints": 16.07, "maxCriterionDisagreementPoints": 25, "humanReviewRecommended": true, "lowConfidenceReasons": [ "judge quality disagreement >= 8 points", "criterion disagreement >= 8 points" ] }, "judgeCostCents": 2.7403 } ] }, { "id": "cross-functional-launch-plan", "name": "Cross-Functional Launch Plan", "tier": "tier2_medium", "domain": "cross_functional", "description": "Create a decision-ready launch plan that coordinates product, design, engineering, marketing, and sales for a new live execution-room release.", "acceptanceCriteria": [ { "id": "covers-all-domains", "description": "The plan explicitly covers product, design, engineering, marketing, and sales", "type": "rubric", "evaluator": "llm_judge", "weight": 2 }, { "id": "sequencing", "description": "Includes a believable sequence of milestones and dependencies", "type": "rubric", "evaluator": "llm_judge", "weight": 2 }, { "id": "launch-readiness", "description": "Includes a practical launch-readiness checklist", "type": "rubric", "evaluator": "llm_judge", "weight": 1.5 }, { "id": "proof-orientation", "description": "Keeps the launch anchored on proving real work and visible outputs", "type": "rubric", "evaluator": "llm_judge", "weight": 2 }, { "id": "measurable-metrics", "description": "Includes measurable post-launch success metrics", "type": "rubric", "evaluator": "llm_judge", "weight": 1.5 } ], "humanBaseline": { "timeSeconds": 7200, "costCents": 4800, "qualityScore": 90, "completeness": 1, "artifactCount": 3, "provenance": "founder + leads planning session estimate, Mar 2026", "methodology": "hybrid", "sourceSummary": "Estimate based on a founder-led planning session plus cross-functional lead review for the same launch brief.", "sampleSize": 2, "collectedAt": "2026-03-08", "operatorProfile": "Founder, product lead, and functional leads coordinating a launch plan" }, "constraints": { "maxDurationMinutes": 20, "maxCostCents": 250 }, "runs": [ { "runId": "cross-functional-launch-plan-r1", "autonomousCompleted": true, "durationSeconds": 8.33, "costCents": 0.0611, "usage": { "input_tokens": 428, "input_tokens_details": { "cached_tokens": 0 }, "output_tokens": 1475, "output_tokens_details": { "reasoning_tokens": 0 }, "total_tokens": 1903 }, "qualityScore": 95, "completeness": 0.93, "criterionScores": { "covers-all-domains": 1, "sequencing": 0.85, "launch-readiness": 1, "proof-orientation": 1, "measurable-metrics": 0.9 }, "scoringSource": "independent_judges", "selfReportedQualityScore": 88.5, "selfReportedCompleteness": 0.92, "selfReportedCriterionScores": { "covers-all-domains": 0.92, "sequencing": 0.88, "launch-readiness": 0.85, "proof-orientation": 0.9, "measurable-metrics": 0.86 }, "judgeAggregate": { "judgeCount": 3, "qualityScore": 95, "completeness": 0.93, "criterionScores": { "covers-all-domains": 1, "sequencing": 0.85, "launch-readiness": 1, "proof-orientation": 1, "measurable-metrics": 0.9 }, "disagreementPoints": 5, "maxCriterionDisagreementPoints": 15, "humanReviewRecommended": true, "lowConfidenceReasons": ["criterion disagreement >= 8 points"] }, "judgeCostCents": 1.9392 }, { "runId": "cross-functional-launch-plan-r2", "autonomousCompleted": true, "durationSeconds": 10.25, "costCents": 0.0527, "usage": { "input_tokens": 428, "input_tokens_details": { "cached_tokens": 0 }, "output_tokens": 1264, "output_tokens_details": { "reasoning_tokens": 0 }, "total_tokens": 1692 }, "qualityScore": 95.28, "completeness": 0.92, "criterionScores": { "covers-all-domains": 1, "sequencing": 0.9, "launch-readiness": 0.9, "proof-orientation": 1, "measurable-metrics": 0.95 }, "scoringSource": "independent_judges", "selfReportedQualityScore": 100, "selfReportedCompleteness": 0.9, "selfReportedCriterionScores": { "covers-all-domains": 1, "sequencing": 1, "launch-readiness": 1, "proof-orientation": 1, "measurable-metrics": 1 }, "judgeAggregate": { "judgeCount": 3, "qualityScore": 95.28, "completeness": 0.92, "criterionScores": { "covers-all-domains": 1, "sequencing": 0.9, "launch-readiness": 0.9, "proof-orientation": 1, "measurable-metrics": 0.95 }, "disagreementPoints": 3.06, "maxCriterionDisagreementPoints": 15, "humanReviewRecommended": true, "lowConfidenceReasons": [ "criterion disagreement >= 8 points", "at least one judge recommended human review" ] }, "judgeCostCents": 1.7931 }, { "runId": "cross-functional-launch-plan-r3", "autonomousCompleted": true, "durationSeconds": 9.8, "costCents": 0.063, "usage": { "input_tokens": 428, "input_tokens_details": { "cached_tokens": 0 }, "output_tokens": 1521, "output_tokens_details": { "reasoning_tokens": 0 }, "total_tokens": 1949 }, "qualityScore": 93.06, "completeness": 0.92, "criterionScores": { "covers-all-domains": 1, "sequencing": 0.8, "launch-readiness": 1, "proof-orientation": 1, "measurable-metrics": 0.85 }, "scoringSource": "independent_judges", "selfReportedQualityScore": 94.17, "selfReportedCompleteness": 0.92, "selfReportedCriterionScores": { "covers-all-domains": 1, "sequencing": 0.9, "launch-readiness": 0.9, "proof-orientation": 0.95, "measurable-metrics": 0.95 }, "judgeAggregate": { "judgeCount": 3, "qualityScore": 93.06, "completeness": 0.92, "criterionScores": { "covers-all-domains": 1, "sequencing": 0.8, "launch-readiness": 1, "proof-orientation": 1, "measurable-metrics": 0.85 }, "disagreementPoints": 3.34, "maxCriterionDisagreementPoints": 20, "humanReviewRecommended": true, "lowConfidenceReasons": ["criterion disagreement >= 8 points"] }, "judgeCostCents": 2.0438 } ] }, { "id": "design-live-room-responsive-system", "name": "Live Room Responsive System Spec", "tier": "tier2_medium", "domain": "design", "description": "Produce a production-ready responsive system specification for a live initiative room across mobile, tablet, and desktop.", "acceptanceCriteria": [ { "id": "breakpoint-specificity", "description": "Gives concrete behavior for 375px, 768px, 1024px, and 1440px breakpoints", "type": "rubric", "evaluator": "llm_judge", "weight": 2 }, { "id": "durable-header-rules", "description": "Defines header priority, compression, sticky behavior, and action placement without over-containerization", "type": "rubric", "evaluator": "llm_judge", "weight": 2 }, { "id": "system-thinking", "description": "Creates reusable layout and component rules for subpages, not a one-off screen critique", "type": "rubric", "evaluator": "llm_judge", "weight": 2 }, { "id": "artifact-and-blocker-flows", "description": "Covers artifact, blocker, queue, and decision flows across responsive states", "type": "rubric", "evaluator": "llm_judge", "weight": 1.5 }, { "id": "implementation-checklist", "description": "Includes a practical engineering handoff checklist and QA requirements", "type": "rubric", "evaluator": "llm_judge", "weight": 1.5 } ], "humanBaseline": { "timeSeconds": 7200, "costCents": 5200, "qualityScore": 92, "completeness": 1, "artifactCount": 2, "provenance": "principal product design systems estimate, Apr 2026", "methodology": "expert_estimate", "sourceSummary": "Estimate based on principal-level responsive system work for complex SaaS execution surfaces.", "sampleSize": 2, "collectedAt": "2026-04-11", "operatorProfile": "Principal product designer with design-system and responsive SaaS experience" }, "constraints": { "maxDurationMinutes": 20, "maxCostCents": 250 }, "runs": [ { "runId": "design-live-room-responsive-system-r1", "autonomousCompleted": true, "durationSeconds": 14.51, "costCents": 0.087, "usage": { "input_tokens": 552, "input_tokens_details": { "cached_tokens": 0 }, "output_tokens": 2105, "output_tokens_details": { "reasoning_tokens": 0 }, "total_tokens": 2657 }, "qualityScore": 85.83, "completeness": 0.84, "criterionScores": { "breakpoint-specificity": 0.85, "durable-header-rules": 0.8, "system-thinking": 0.9, "artifact-and-blocker-flows": 0.8, "implementation-checklist": 0.95 }, "scoringSource": "independent_judges", "selfReportedQualityScore": 99, "selfReportedCompleteness": 0.95, "selfReportedCriterionScores": { "breakpoint-specificity": 1, "durable-header-rules": 1, "system-thinking": 1, "artifact-and-blocker-flows": 0.97, "implementation-checklist": 0.97 }, "judgeAggregate": { "judgeCount": 3, "qualityScore": 85.83, "completeness": 0.84, "criterionScores": { "breakpoint-specificity": 0.85, "durable-header-rules": 0.8, "system-thinking": 0.9, "artifact-and-blocker-flows": 0.8, "implementation-checklist": 0.95 }, "disagreementPoints": 5, "maxCriterionDisagreementPoints": 20, "humanReviewRecommended": true, "lowConfidenceReasons": ["criterion disagreement >= 8 points"] }, "judgeCostCents": 3.6015 }, { "runId": "design-live-room-responsive-system-r2", "autonomousCompleted": true, "durationSeconds": 16.15, "costCents": 0.1022, "usage": { "input_tokens": 552, "input_tokens_details": { "cached_tokens": 0 }, "output_tokens": 2487, "output_tokens_details": { "reasoning_tokens": 0 }, "total_tokens": 3039 }, "qualityScore": 81.39, "completeness": 0.88, "criterionScores": { "breakpoint-specificity": 0.85, "durable-header-rules": 0.8, "system-thinking": 0.85, "artifact-and-blocker-flows": 0.7, "implementation-checklist": 0.85 }, "scoringSource": "independent_judges", "selfReportedQualityScore": 85.56, "selfReportedCompleteness": 0.92, "selfReportedCriterionScores": { "breakpoint-specificity": 0.9, "durable-header-rules": 0.9, "system-thinking": 0.85, "artifact-and-blocker-flows": 0.8, "implementation-checklist": 0.8 }, "judgeAggregate": { "judgeCount": 3, "qualityScore": 81.39, "completeness": 0.88, "criterionScores": { "breakpoint-specificity": 0.85, "durable-header-rules": 0.8, "system-thinking": 0.85, "artifact-and-blocker-flows": 0.7, "implementation-checklist": 0.85 }, "disagreementPoints": 16.56, "maxCriterionDisagreementPoints": 35, "humanReviewRecommended": true, "lowConfidenceReasons": [ "judge quality disagreement >= 8 points", "criterion disagreement >= 8 points" ] }, "judgeCostCents": 4.3442 }, { "runId": "design-live-room-responsive-system-r3", "autonomousCompleted": true, "durationSeconds": 34.58, "costCents": 0.094, "usage": { "input_tokens": 564, "input_tokens_details": { "cached_tokens": 0 }, "output_tokens": 2280, "output_tokens_details": { "reasoning_tokens": 0 }, "total_tokens": 2844 }, "qualityScore": 88.5, "completeness": 0.92, "criterionScores": { "breakpoint-specificity": 0.95, "durable-header-rules": 0.8, "system-thinking": 0.92, "artifact-and-blocker-flows": 0.8, "implementation-checklist": 0.95 }, "scoringSource": "independent_judges", "selfReportedQualityScore": 80.83, "selfReportedCompleteness": 0.9, "selfReportedCriterionScores": { "breakpoint-specificity": 0.9, "durable-header-rules": 0.8, "system-thinking": 0.85, "artifact-and-blocker-flows": 0.75, "implementation-checklist": 0.7 }, "judgeAggregate": { "judgeCount": 3, "qualityScore": 88.5, "completeness": 0.92, "criterionScores": { "breakpoint-specificity": 0.95, "durable-header-rules": 0.8, "system-thinking": 0.92, "artifact-and-blocker-flows": 0.8, "implementation-checklist": 0.95 }, "disagreementPoints": 10.83, "maxCriterionDisagreementPoints": 20, "humanReviewRecommended": true, "lowConfidenceReasons": [ "judge quality disagreement >= 8 points", "criterion disagreement >= 8 points", "at least one judge recommended human review" ] }, "judgeCostCents": 3.4791 } ] }, { "id": "engineering-release-readiness-review", "name": "Engineering Release Readiness Review", "tier": "tier2_medium", "domain": "engineering", "description": "Review a release plan for technical risks, rollout gaps, verification coverage, and rollback readiness. Benchmarks engineering execution judgment, not only writing polish.", "acceptanceCriteria": [ { "id": "recommendation-quality", "description": "Gives a clear release recommendation tied to specific risks and guardrails", "type": "rubric", "evaluator": "llm_judge", "weight": 2 }, { "id": "identifies-operational-risk", "description": "Identifies verification, observability, or rollback gaps instead of treating the task as a generic launch memo", "type": "rubric", "evaluator": "llm_judge", "weight": 2 }, { "id": "proposes-guardrails", "description": "Recommends concrete pre-launch guardrails or rollout constraints", "type": "rubric", "evaluator": "llm_judge", "weight": 2 }, { "id": "incident-thinking", "description": "Demonstrates real release/incident thinking rather than only implementation advice", "type": "rubric", "evaluator": "llm_judge", "weight": 1.5 } ], "humanBaseline": { "timeSeconds": 3600, "costCents": 2600, "qualityScore": 89, "completeness": 1, "artifactCount": 1, "provenance": "senior engineering lead release review estimate, Mar 2026", "methodology": "expert_estimate", "sourceSummary": "Estimate from a senior engineering lead reviewing a comparable release-readiness packet.", "sampleSize": 2, "collectedAt": "2026-03-08", "operatorProfile": "Senior engineering lead responsible for rollout and incident readiness" }, "constraints": { "maxDurationMinutes": 12, "maxCostCents": 175 }, "runs": [ { "runId": "engineering-release-readiness-review-r1", "autonomousCompleted": true, "durationSeconds": 6.82, "costCents": 0.0469, "usage": { "input_tokens": 522, "input_tokens_details": { "cached_tokens": 0 }, "output_tokens": 1108, "output_tokens_details": { "reasoning_tokens": 0 }, "total_tokens": 1630 }, "qualityScore": 91.67, "completeness": 0.9, "criterionScores": { "recommendation-quality": 0.95, "identifies-operational-risk": 0.95, "proposes-guardrails": 0.9, "incident-thinking": 0.85 }, "scoringSource": "independent_judges", "selfReportedQualityScore": 83.33, "selfReportedCompleteness": 0.78, "selfReportedCriterionScores": { "recommendation-quality": 0.8, "identifies-operational-risk": 0.9, "proposes-guardrails": 0.9, "incident-thinking": 0.7 }, "judgeAggregate": { "judgeCount": 3, "qualityScore": 91.67, "completeness": 0.9, "criterionScores": { "recommendation-quality": 0.95, "identifies-operational-risk": 0.95, "proposes-guardrails": 0.9, "incident-thinking": 0.85 }, "disagreementPoints": 2, "maxCriterionDisagreementPoints": 15, "humanReviewRecommended": true, "lowConfidenceReasons": ["criterion disagreement >= 8 points"] }, "judgeCostCents": 3.3792 }, { "runId": "engineering-release-readiness-review-r2", "autonomousCompleted": true, "durationSeconds": 6.18, "costCents": 0.0462, "usage": { "input_tokens": 522, "input_tokens_details": { "cached_tokens": 0 }, "output_tokens": 1091, "output_tokens_details": { "reasoning_tokens": 0 }, "total_tokens": 1613 }, "qualityScore": 95.33, "completeness": 0.95, "criterionScores": { "recommendation-quality": 0.95, "identifies-operational-risk": 1, "proposes-guardrails": 0.95, "incident-thinking": 0.9 }, "scoringSource": "independent_judges", "selfReportedQualityScore": 78, "selfReportedCompleteness": 0.78, "selfReportedCriterionScores": { "recommendation-quality": 0.8, "identifies-operational-risk": 0.75, "proposes-guardrails": 0.85, "incident-thinking": 0.7 }, "judgeAggregate": { "judgeCount": 3, "qualityScore": 95.33, "completeness": 0.95, "criterionScores": { "recommendation-quality": 0.95, "identifies-operational-risk": 1, "proposes-guardrails": 0.95, "incident-thinking": 0.9 }, "disagreementPoints": 12.67, "maxCriterionDisagreementPoints": 20, "humanReviewRecommended": true, "lowConfidenceReasons": [ "judge quality disagreement >= 8 points", "criterion disagreement >= 8 points", "at least one judge recommended human review" ] }, "judgeCostCents": 2.6537 }, { "runId": "engineering-release-readiness-review-r3", "autonomousCompleted": true, "durationSeconds": 5.87, "costCents": 0.0445, "usage": { "input_tokens": 522, "input_tokens_details": { "cached_tokens": 0 }, "output_tokens": 1047, "output_tokens_details": { "reasoning_tokens": 0 }, "total_tokens": 1569 }, "qualityScore": 94.67, "completeness": 0.95, "criterionScores": { "recommendation-quality": 1, "identifies-operational-risk": 1, "proposes-guardrails": 0.95, "incident-thinking": 0.8 }, "scoringSource": "independent_judges", "selfReportedQualityScore": 79.33, "selfReportedCompleteness": 0.72, "selfReportedCriterionScores": { "recommendation-quality": 0.8, "identifies-operational-risk": 0.8, "proposes-guardrails": 0.85, "incident-thinking": 0.7 }, "judgeAggregate": { "judgeCount": 3, "qualityScore": 94.67, "completeness": 0.95, "criterionScores": { "recommendation-quality": 1, "identifies-operational-risk": 1, "proposes-guardrails": 0.95, "incident-thinking": 0.8 }, "disagreementPoints": 5.33, "maxCriterionDisagreementPoints": 20, "humanReviewRecommended": true, "lowConfidenceReasons": ["criterion disagreement >= 8 points"] }, "judgeCostCents": 3.6586 } ] }, { "id": "marketing-proof-campaign-brief", "name": "Marketing Proof Campaign Brief", "tier": "tier2_medium", "domain": "marketing", "description": "Build a campaign brief that uses real outputs, artifacts, and live evidence as the primary conversion mechanism. Benchmarks proof-led marketing judgment.", "acceptanceCriteria": [ { "id": "proof-assets", "description": "Treats proofs, artifacts, and live execution as core campaign assets rather than optional embellishments", "type": "rubric", "evaluator": "llm_judge", "weight": 2 }, { "id": "channel-specific", "description": "Produces distinct content guidance per channel instead of one generic plan", "type": "rubric", "evaluator": "llm_judge", "weight": 2 }, { "id": "anti-pattern-awareness", "description": "Calls out at least one hype-driven anti-pattern and how to avoid it", "type": "rubric", "evaluator": "llm_judge", "weight": 1.5 }, { "id": "measurable", "description": "Defines measurable campaign success metrics", "type": "rubric", "evaluator": "llm_judge", "weight": 1.5 } ], "humanBaseline": { "timeSeconds": 3300, "costCents": 2400, "qualityScore": 87, "completeness": 1, "artifactCount": 1, "provenance": "fractional growth lead campaign brief estimate, Mar 2026", "methodology": "hybrid", "sourceSummary": "Combined founder and fractional growth lead estimate for a proof-led launch campaign.", "sampleSize": 2, "collectedAt": "2026-03-08", "operatorProfile": "B2B SaaS growth lead working with a founder on launch messaging" }, "constraints": { "maxDurationMinutes": 12, "maxCostCents": 175 }, "runs": [ { "runId": "marketing-proof-campaign-brief-r1", "autonomousCompleted": true, "durationSeconds": 6.04, "costCents": 0.0414, "usage": { "input_tokens": 416, "input_tokens_details": { "cached_tokens": 0 }, "output_tokens": 984, "output_tokens_details": { "reasoning_tokens": 0 }, "total_tokens": 1400 }, "qualityScore": 93.21, "completeness": 0.95, "criterionScores": { "proof-assets": 1, "channel-specific": 0.95, "anti-pattern-awareness": 0.9, "measurable": 0.85 }, "scoringSource": "independent_judges", "selfReportedQualityScore": 87.86, "selfReportedCompleteness": 0.95, "selfReportedCriterionScores": { "proof-assets": 0.9, "channel-specific": 0.9, "anti-pattern-awareness": 0.8, "measurable": 0.9 }, "judgeAggregate": { "judgeCount": 3, "qualityScore": 93.21, "completeness": 0.95, "criterionScores": { "proof-assets": 1, "channel-specific": 0.95, "anti-pattern-awareness": 0.9, "measurable": 0.85 }, "disagreementPoints": 3.92, "maxCriterionDisagreementPoints": 10, "humanReviewRecommended": true, "lowConfidenceReasons": ["criterion disagreement >= 8 points"] }, "judgeCostCents": 2.0809 }, { "runId": "marketing-proof-campaign-brief-r2", "autonomousCompleted": true, "durationSeconds": 7.52, "costCents": 0.0528, "usage": { "input_tokens": 416, "input_tokens_details": { "cached_tokens": 0 }, "output_tokens": 1268, "output_tokens_details": { "reasoning_tokens": 0 }, "total_tokens": 1684 }, "qualityScore": 96.79, "completeness": 1, "criterionScores": { "proof-assets": 1, "channel-specific": 1, "anti-pattern-awareness": 1, "measurable": 0.85 }, "scoringSource": "independent_judges", "selfReportedQualityScore": 90, "selfReportedCompleteness": 0.92, "selfReportedCriterionScores": { "proof-assets": 0.9, "channel-specific": 0.9, "anti-pattern-awareness": 0.9, "measurable": 0.9 }, "judgeAggregate": { "judgeCount": 3, "qualityScore": 96.79, "completeness": 1, "criterionScores": { "proof-assets": 1, "channel-specific": 1, "anti-pattern-awareness": 1, "measurable": 0.85 }, "disagreementPoints": 8.21, "maxCriterionDisagreementPoints": 20, "humanReviewRecommended": true, "lowConfidenceReasons": [ "judge quality disagreement >= 8 points", "criterion disagreement >= 8 points" ] }, "judgeCostCents": 2.0976 }, { "runId": "marketing-proof-campaign-brief-r3", "autonomousCompleted": true, "durationSeconds": 6.55, "costCents": 0.0452, "usage": { "input_tokens": 416, "input_tokens_details": { "cached_tokens": 0 }, "output_tokens": 1078, "output_tokens_details": { "reasoning_tokens": 0 }, "total_tokens": 1494 }, "qualityScore": 96.79, "completeness": 1, "criterionScores": { "proof-assets": 1, "channel-specific": 1, "anti-pattern-awareness": 1, "measurable": 0.85 }, "scoringSource": "independent_judges", "selfReportedQualityScore": 100, "selfReportedCompleteness": 1, "selfReportedCriterionScores": { "proof-assets": 1, "channel-specific": 1, "anti-pattern-awareness": 1, "measurable": 1 }, "judgeAggregate": { "judgeCount": 3, "qualityScore": 96.79, "completeness": 1, "criterionScores": { "proof-assets": 1, "channel-specific": 1, "anti-pattern-awareness": 1, "measurable": 0.85 }, "disagreementPoints": 11.07, "maxCriterionDisagreementPoints": 20, "humanReviewRecommended": true, "lowConfidenceReasons": [ "judge quality disagreement >= 8 points", "criterion disagreement >= 8 points" ] }, "judgeCostCents": 1.81 } ] }, { "id": "ops-escalation-playbook", "name": "Ops Escalation Playbook", "tier": "tier2_medium", "domain": "ops", "description": "Create a practical escalation playbook for an initiative that is blocked by integrations, billing, and approval dependencies. Benchmarks operational clarity under constraint.", "acceptanceCriteria": [ { "id": "blocker-specific", "description": "Distinguishes between integration, billing, and approval blockers instead of giving one generic escalation flow", "type": "rubric", "evaluator": "llm_judge", "weight": 2 }, { "id": "sla-owner-clarity", "description": "Defines time-based ownership and SLA expectations", "type": "rubric", "evaluator": "llm_judge", "weight": 2 }, { "id": "communication-ready", "description": "Includes reusable communication language or templates", "type": "rubric", "evaluator": "llm_judge", "weight": 1.5 }, { "id": "recovery-checklist", "description": "Includes a practical resolution checklist, not only escalation routing", "type": "rubric", "evaluator": "llm_judge", "weight": 1.5 } ], "humanBaseline": { "timeSeconds": 3900, "costCents": 2800, "qualityScore": 88, "completeness": 1, "artifactCount": 1, "provenance": "ops lead escalation playbook estimate, Mar 2026", "methodology": "expert_estimate", "sourceSummary": "Estimate from an operations lead documenting an escalation policy for initiative blockers.", "sampleSize": 2, "collectedAt": "2026-03-08", "operatorProfile": "Operations lead responsible for escalation and service continuity" }, "constraints": { "maxDurationMinutes": 12, "maxCostCents": 175 }, "runs": [ { "runId": "ops-escalation-playbook-r1", "autonomousCompleted": true, "durationSeconds": 9.58, "costCents": 0.0657, "usage": { "input_tokens": 434, "input_tokens_details": { "cached_tokens": 0 }, "output_tokens": 1587, "output_tokens_details": { "reasoning_tokens": 0 }, "total_tokens": 2021 }, "qualityScore": 94.64, "completeness": 0.96, "criterionScores": { "blocker-specific": 1, "sla-owner-clarity": 0.85, "communication-ready": 0.95, "recovery-checklist": 1 }, "scoringSource": "independent_judges", "selfReportedQualityScore": 88.21, "selfReportedCompleteness": 0.92, "selfReportedCriterionScores": { "blocker-specific": 0.9, "sla-owner-clarity": 0.95, "communication-ready": 0.85, "recovery-checklist": 0.8 }, "judgeAggregate": { "judgeCount": 3, "qualityScore": 94.64, "completeness": 0.96, "criterionScores": { "blocker-specific": 1, "sla-owner-clarity": 0.85, "communication-ready": 0.95, "recovery-checklist": 1 }, "disagreementPoints": 7.14, "maxCriterionDisagreementPoints": 15, "humanReviewRecommended": true, "lowConfidenceReasons": ["criterion disagreement >= 8 points"] }, "judgeCostCents": 2.3282 }, { "runId": "ops-escalation-playbook-r2", "autonomousCompleted": true, "durationSeconds": 7.83, "costCents": 0.0617, "usage": { "input_tokens": 434, "input_tokens_details": { "cached_tokens": 0 }, "output_tokens": 1488, "output_tokens_details": { "reasoning_tokens": 0 }, "total_tokens": 1922 }, "qualityScore": 95.71, "completeness": 0.95, "criterionScores": { "blocker-specific": 1, "sla-owner-clarity": 0.85, "communication-ready": 1, "recovery-checklist": 1 }, "scoringSource": "independent_judges", "selfReportedQualityScore": 89.29, "selfReportedCompleteness": 0.9, "selfReportedCriterionScores": { "blocker-specific": 1, "sla-owner-clarity": 1, "communication-ready": 0.75, "recovery-checklist": 0.75 }, "judgeAggregate": { "judgeCount": 3, "qualityScore": 95.71, "completeness": 0.95, "criterionScores": { "blocker-specific": 1, "sla-owner-clarity": 0.85, "communication-ready": 1, "recovery-checklist": 1 }, "disagreementPoints": 2.85, "maxCriterionDisagreementPoints": 10, "humanReviewRecommended": true, "lowConfidenceReasons": ["criterion disagreement >= 8 points"] }, "judgeCostCents": 3.1144 }, { "runId": "ops-escalation-playbook-r3", "autonomousCompleted": true, "durationSeconds": 8.71, "costCents": 0.0629, "usage": { "input_tokens": 434, "input_tokens_details": { "cached_tokens": 0 }, "output_tokens": 1518, "output_tokens_details": { "reasoning_tokens": 0 }, "total_tokens": 1952 }, "qualityScore": 94.29, "completeness": 0.95, "criterionScores": { "blocker-specific": 1, "sla-owner-clarity": 0.8, "communication-ready": 1, "recovery-checklist": 1 }, "scoringSource": "independent_judges", "selfReportedQualityScore": 89.29, "selfReportedCompleteness": 0.95, "selfReportedCriterionScores": { "blocker-specific": 1, "sla-owner-clarity": 1, "communication-ready": 0.75, "recovery-checklist": 0.75 }, "judgeAggregate": { "judgeCount": 3, "qualityScore": 94.29, "completeness": 0.95, "criterionScores": { "blocker-specific": 1, "sla-owner-clarity": 0.8, "communication-ready": 1, "recovery-checklist": 1 }, "disagreementPoints": 10, "maxCriterionDisagreementPoints": 10, "humanReviewRecommended": true, "lowConfidenceReasons": [ "judge quality disagreement >= 8 points", "criterion disagreement >= 8 points" ] }, "judgeCostCents": 1.9541 } ] }, { "id": "product-retention-experiment-plan", "name": "Product Retention Experiment Plan", "tier": "tier2_medium", "domain": "product", "description": "Turn a product signal into a decision-ready retention experiment plan with target behavior, instrumentation, and launch sequencing.", "acceptanceCriteria": [ { "id": "behavior-change", "description": "Focuses on a concrete user behavior change, not just vague engagement improvement", "type": "rubric", "evaluator": "llm_judge", "weight": 2 }, { "id": "experiment-specific", "description": "Defines a real experiment with testable conditions instead of generic product ideas", "type": "rubric", "evaluator": "llm_judge", "weight": 2 }, { "id": "measurement-plan", "description": "Includes instrumentation or measurement details tied to the hypothesis", "type": "rubric", "evaluator": "llm_judge", "weight": 2 }, { "id": "rollout-sequencing", "description": "Includes a sensible rollout or validation sequence", "type": "rubric", "evaluator": "llm_judge", "weight": 1.5 } ], "humanBaseline": { "timeSeconds": 3300, "costCents": 2400, "qualityScore": 88, "completeness": 1, "artifactCount": 1, "provenance": "PM retention experiment estimate, Mar 2026", "methodology": "hybrid", "sourceSummary": "Estimate from a PM-led retention planning workflow with founder input.", "sampleSize": 2, "collectedAt": "2026-03-08", "operatorProfile": "Product manager focused on onboarding and activation experiments" }, "constraints": { "maxDurationMinutes": 12, "maxCostCents": 175 }, "runs": [ { "runId": "product-retention-experiment-plan-r1", "autonomousCompleted": true, "durationSeconds": 7.68, "costCents": 0.0512, "usage": { "input_tokens": 409, "input_tokens_details": { "cached_tokens": 0 }, "output_tokens": 1228, "output_tokens_details": { "reasoning_tokens": 0 }, "total_tokens": 1637 }, "qualityScore": 95.33, "completeness": 0.95, "criterionScores": { "behavior-change": 1, "experiment-specific": 0.9, "measurement-plan": 1, "rollout-sequencing": 0.9 }, "scoringSource": "independent_judges", "selfReportedQualityScore": 85.67, "selfReportedCompleteness": 0.92, "selfReportedCriterionScores": { "behavior-change": 0.85, "experiment-specific": 0.9, "measurement-plan": 0.9, "rollout-sequencing": 0.75 }, "judgeAggregate": { "judgeCount": 3, "qualityScore": 95.33, "completeness": 0.95, "criterionScores": { "behavior-change": 1, "experiment-specific": 0.9, "measurement-plan": 1, "rollout-sequencing": 0.9 }, "disagreementPoints": 6.67, "maxCriterionDisagreementPoints": 20, "humanReviewRecommended": true, "lowConfidenceReasons": ["criterion disagreement >= 8 points"] }, "judgeCostCents": 1.8439 }, { "runId": "product-retention-experiment-plan-r2", "autonomousCompleted": true, "durationSeconds": 7.99, "costCents": 0.0503, "usage": { "input_tokens": 409, "input_tokens_details": { "cached_tokens": 0 }, "output_tokens": 1207, "output_tokens_details": { "reasoning_tokens": 0 }, "total_tokens": 1616 }, "qualityScore": 95.33, "completeness": 0.95, "criterionScores": { "behavior-change": 1, "experiment-specific": 0.95, "measurement-plan": 0.95, "rollout-sequencing": 0.9 }, "scoringSource": "independent_judges", "selfReportedQualityScore": 89.67, "selfReportedCompleteness": 0.92, "selfReportedCriterionScores": { "behavior-change": 0.9, "experiment-specific": 0.95, "measurement-plan": 0.95, "rollout-sequencing": 0.75 }, "judgeAggregate": { "judgeCount": 3, "qualityScore": 95.33, "completeness": 0.95, "criterionScores": { "behavior-change": 1, "experiment-specific": 0.95, "measurement-plan": 0.95, "rollout-sequencing": 0.9 }, "disagreementPoints": 6.66, "maxCriterionDisagreementPoints": 20, "humanReviewRecommended": true, "lowConfidenceReasons": ["criterion disagreement >= 8 points"] }, "judgeCostCents": 3.477 }, { "runId": "product-retention-experiment-plan-r3", "autonomousCompleted": true, "durationSeconds": 11.11, "costCents": 0.061, "usage": { "input_tokens": 409, "input_tokens_details": { "cached_tokens": 0 }, "output_tokens": 1473, "output_tokens_details": { "reasoning_tokens": 0 }, "total_tokens": 1882 }, "qualityScore": 89, "completeness": 0.9, "criterionScores": { "behavior-change": 0.95, "experiment-specific": 0.9, "measurement-plan": 0.85, "rollout-sequencing": 0.85 }, "scoringSource": "independent_judges", "selfReportedQualityScore": 90.67, "selfReportedCompleteness": 0.75, "selfReportedCriterionScores": { "behavior-change": 1, "experiment-specific": 0.9, "measurement-plan": 0.9, "rollout-sequencing": 0.8 }, "judgeAggregate": { "judgeCount": 3, "qualityScore": 89, "completeness": 0.9, "criterionScores": { "behavior-change": 0.95, "experiment-specific": 0.9, "measurement-plan": 0.85, "rollout-sequencing": 0.85 }, "disagreementPoints": 5, "maxCriterionDisagreementPoints": 20, "humanReviewRecommended": true, "lowConfidenceReasons": [ "criterion disagreement >= 8 points", "at least one judge recommended human review" ] }, "judgeCostCents": 4.4786 } ] }, { "id": "sales-competitive-battlecard", "name": "Sales Competitive Battlecard", "tier": "tier2_medium", "domain": "sales", "description": "Create a concise battlecard that helps a founder or GTM lead position OrgX against direct-model and agent-platform alternatives.", "acceptanceCriteria": [ { "id": "explicit-comparison", "description": "Makes explicit category comparisons instead of vague positioning claims", "type": "rubric", "evaluator": "llm_judge", "weight": 2 }, { "id": "acknowledges-weakness", "description": "Honestly states where OrgX is weaker or not a fit", "type": "rubric", "evaluator": "llm_judge", "weight": 1.5 }, { "id": "proof-moments", "description": "Includes concrete proof or demo moments to show in the sales process", "type": "rubric", "evaluator": "llm_judge", "weight": 2 }, { "id": "founder-talk-track", "description": "Provides a concise talk track a founder could actually use", "type": "rubric", "evaluator": "llm_judge", "weight": 1.5 } ], "humanBaseline": { "timeSeconds": 3600, "costCents": 2500, "qualityScore": 87, "completeness": 1, "artifactCount": 1, "provenance": "founder-led competitive positioning estimate, Mar 2026", "methodology": "hybrid", "sourceSummary": "Estimate from founder-led competitive narrative development with one GTM reviewer.", "sampleSize": 2, "collectedAt": "2026-03-08", "operatorProfile": "Founder or first GTM hire creating a competitive battlecard for early sales" }, "constraints": { "maxDurationMinutes": 12, "maxCostCents": 175 }, "runs": [ { "runId": "sales-competitive-battlecard-r1", "autonomousCompleted": true, "durationSeconds": 6.96, "costCents": 0.0485, "usage": { "input_tokens": 437, "input_tokens_details": { "cached_tokens": 0 }, "output_tokens": 1157, "output_tokens_details": { "reasoning_tokens": 0 }, "total_tokens": 1594 }, "qualityScore": 87.14, "completeness": 0.95, "criterionScores": { "explicit-comparison": 0.85, "acknowledges-weakness": 0.8, "proof-moments": 1, "founder-talk-track": 0.8 }, "scoringSource": "independent_judges", "selfReportedQualityScore": 88.21, "selfReportedCompleteness": 0.92, "selfReportedCriterionScores": { "explicit-comparison": 0.9, "acknowledges-weakness": 0.8, "proof-moments": 0.95, "founder-talk-track": 0.85 }, "judgeAggregate": { "judgeCount": 3, "qualityScore": 87.14, "completeness": 0.95, "criterionScores": { "explicit-comparison": 0.85, "acknowledges-weakness": 0.8, "proof-moments": 1, "founder-talk-track": 0.8 }, "disagreementPoints": 14.64, "maxCriterionDisagreementPoints": 20, "humanReviewRecommended": true, "lowConfidenceReasons": [ "judge quality disagreement >= 8 points", "criterion disagreement >= 8 points", "at least one judge recommended human review" ] }, "judgeCostCents": 4.354 }, { "runId": "sales-competitive-battlecard-r2", "autonomousCompleted": true, "durationSeconds": 6.07, "costCents": 0.0439, "usage": { "input_tokens": 437, "input_tokens_details": { "cached_tokens": 0 }, "output_tokens": 1042, "output_tokens_details": { "reasoning_tokens": 0 }, "total_tokens": 1479 }, "qualityScore": 90, "completeness": 0.95, "criterionScores": { "explicit-comparison": 0.95, "acknowledges-weakness": 0.8, "proof-moments": 1, "founder-talk-track": 0.8 }, "scoringSource": "independent_judges", "selfReportedQualityScore": 79.64, "selfReportedCompleteness": 0.72, "selfReportedCriterionScores": { "explicit-comparison": 0.8, "acknowledges-weakness": 0.6, "proof-moments": 0.9, "founder-talk-track": 0.85 }, "judgeAggregate": { "judgeCount": 3, "qualityScore": 90, "completeness": 0.95, "criterionScores": { "explicit-comparison": 0.95, "acknowledges-weakness": 0.8, "proof-moments": 1, "founder-talk-track": 0.8 }, "disagreementPoints": 3.22, "maxCriterionDisagreementPoints": 20, "humanReviewRecommended": true, "lowConfidenceReasons": ["criterion disagreement >= 8 points"] }, "judgeCostCents": 1.7795 }, { "runId": "sales-competitive-battlecard-r3", "autonomousCompleted": true, "durationSeconds": 16.07, "costCents": 0.0503, "usage": { "input_tokens": 449, "input_tokens_details": { "cached_tokens": 0 }, "output_tokens": 1202, "output_tokens_details": { "reasoning_tokens": 0 }, "total_tokens": 1651 }, "qualityScore": 88.21, "completeness": 0.9, "criterionScores": { "explicit-comparison": 0.95, "acknowledges-weakness": 0.8, "proof-moments": 0.9, "founder-talk-track": 0.85 }, "scoringSource": "independent_judges", "selfReportedQualityScore": 87.14, "selfReportedCompleteness": 0.92, "selfReportedCriterionScores": { "explicit-comparison": 0.95, "acknowledges-weakness": 0.75, "proof-moments": 0.9, "founder-talk-track": 0.85 }, "judgeAggregate": { "judgeCount": 3, "qualityScore": 88.21, "completeness": 0.9, "criterionScores": { "explicit-comparison": 0.95, "acknowledges-weakness": 0.8, "proof-moments": 0.9, "founder-talk-track": 0.85 }, "disagreementPoints": 6.79, "maxCriterionDisagreementPoints": 15, "humanReviewRecommended": true, "lowConfidenceReasons": ["criterion disagreement >= 8 points"] }, "judgeCostCents": 1.9613 } ] } ]