{
  "protocol": "docs/reports/state-of-ai-facilitation-2026/BENCHMARK-PROTOCOL.md (frozen 2026-09-24, amendments 2026-09-25)",
  "generated": "2026-10-07",
  "criteria": {
    "1": "Timing arithmetic",
    "2": "Method provenance",
    "3": "When not to use respected",
    "4": "Group-size fit",
    "5": "Breakout logistics",
    "6": "Participant-safe language",
    "7": "Brief fidelity",
    "8": "Clarifying behaviour (B3)"
  },
  "agreement": {
    "compared": 40,
    "agreed": 26,
    "share": 65,
    "threshold": 70,
    "cells": [
      "X01",
      "X07",
      "X10",
      "X03",
      "X06",
      "X04"
    ],
    "disagreements": [
      {
        "cell": "X01",
        "system": "claude",
        "criterion": 3,
        "primary": 2,
        "spotcheck": 1
      },
      {
        "cell": "X01",
        "system": "claude",
        "criterion": 6,
        "primary": 1,
        "spotcheck": 2
      },
      {
        "cell": "X01",
        "system": "claude",
        "criterion": 7,
        "primary": 1,
        "spotcheck": 2
      },
      {
        "cell": "X07",
        "system": "chatgpt",
        "criterion": 3,
        "primary": 2,
        "spotcheck": 1
      },
      {
        "cell": "X10",
        "system": "perplexity",
        "criterion": 4,
        "primary": 2,
        "spotcheck": 1
      },
      {
        "cell": "X10",
        "system": "perplexity",
        "criterion": 5,
        "primary": 2,
        "spotcheck": 1
      },
      {
        "cell": "X10",
        "system": "perplexity",
        "criterion": 6,
        "primary": 1,
        "spotcheck": 2
      },
      {
        "cell": "X06",
        "system": "perplexity",
        "criterion": 3,
        "primary": 2,
        "spotcheck": 1
      },
      {
        "cell": "X06",
        "system": "perplexity",
        "criterion": 4,
        "primary": 2,
        "spotcheck": 1
      },
      {
        "cell": "X06",
        "system": "perplexity",
        "criterion": 6,
        "primary": 1,
        "spotcheck": 2
      },
      {
        "cell": "X06",
        "system": "perplexity",
        "criterion": 7,
        "primary": 1,
        "spotcheck": 2
      },
      {
        "cell": "X04",
        "system": "claude",
        "criterion": 3,
        "primary": 2,
        "spotcheck": 1
      },
      {
        "cell": "X04",
        "system": "claude",
        "criterion": 4,
        "primary": 2,
        "spotcheck": 1
      },
      {
        "cell": "X04",
        "system": "claude",
        "criterion": 6,
        "primary": 1,
        "spotcheck": 2
      }
    ],
    "resolutions": [
      {
        "cell": "X01",
        "criterion": 3,
        "score": 1,
        "note": "re-read: dots decide the shortlist with the GF in the room and the warning is not stated; 'borderline fit, no warning given' = 1"
      },
      {
        "cell": "X01",
        "criterion": 6,
        "score": 2,
        "note": "re-read: rubric asks for clear separation and participant lines a participant could read without jargon; third-person phrasing is not a defect = 2"
      },
      {
        "cell": "X01",
        "criterion": 7,
        "score": 2,
        "note": "re-read: 'fixed elements' are lunch, guest slot, end time, register; the materials list names what is available, not a fixed element = 2"
      },
      {
        "cell": "X07",
        "criterion": 3,
        "score": 1,
        "note": "re-read: affinity clustering, naming and voting in 20 min after a 45-min wall runs against the affinity-map warning on rushed clustering; not flagged = 1"
      },
      {
        "cell": "X10",
        "criterion": 4,
        "score": 1,
        "note": "re-read: rubric wants size handled explicitly in each block; only the breakout block is sized = 1"
      },
      {
        "cell": "X10",
        "criterion": 5,
        "score": 1,
        "note": "re-read: rubric 2 needs groups, split rule, timing, report-back and materials all stated; no split rule, no internal timing = 1"
      },
      {
        "cell": "X10",
        "criterion": 6,
        "score": 2,
        "note": "re-read: participant column is separated and jargon-free = 2"
      },
      {
        "cell": "X03",
        "criterion": 4,
        "score": 1,
        "note": "consistency with X10 re-reading: size explicit only in the breakout block = 1"
      },
      {
        "cell": "X06",
        "criterion": 6,
        "score": 2,
        "note": "consistency with X01 re-reading: separated, Sie-form, jargon-free participant column = 2"
      },
      {
        "cell": "X06",
        "criterion": 7,
        "score": 2,
        "note": "consistency with X01 re-reading: extra materials are not a fixed element = 2"
      },
      {
        "cell": "X06",
        "criterion": 3,
        "score": 1,
        "note": "re-read after second pass: dot vote decides the binding initiative choice under the Führungsteam's eyes, unflagged; same reading as X01/X03/X10 = 1"
      },
      {
        "cell": "X06",
        "criterion": 4,
        "score": 1,
        "note": "re-read after second pass: two of the three group blocks sized, plenary silent, three groups from ten unsplit; size not handled in each block = 1"
      },
      {
        "cell": "X04",
        "criterion": 3,
        "score": 1,
        "note": "re-read after second pass: six-minute silent affinity clustering against the affinity-map warning on rushed clustering, and a one-word round on a group that needs to process; neither flagged; same reading as X07 = 1"
      },
      {
        "cell": "X04",
        "criterion": 4,
        "score": 1,
        "note": "re-read after second pass: the 5 Whys groups are sized (2–4) but no other block is; size not handled in each block = 1"
      },
      {
        "cell": "X04",
        "criterion": 6,
        "score": 2,
        "note": "re-read after second pass: participant lines are plain verbs a participant could read (write, post, vote, volunteer); 'Prime Directive' appears only on the facilitator side = 2"
      },
      {
        "cell": "X03",
        "criterion": 4,
        "score": 2,
        "note": "re-read after second pass: tables of four are set in the check-in and carried through, votes per person and the ROTI round are sized; the X10 reading (only the breakout sized) does not apply here = 2, reversing the earlier consistency change"
      },
      {
        "cell": "X11",
        "criterion": 3,
        "score": 1,
        "note": "consistency with X01 re-reading: dots narrow the initiative shortlist with the Führungsteam in the room and the warning is not stated, whatever decision step follows; 'borderline fit, no warning given' = 1"
      }
    ]
  },
  "per_criterion": {
    "c1": {
      "name": "Timing arithmetic",
      "n": 9,
      "mean": 1.78,
      "scored_2": 8,
      "scored_0": 1,
      "by_system": {
        "chatgpt": 2,
        "claude": 2,
        "perplexity": 1.33
      }
    },
    "c2": {
      "name": "Method provenance",
      "n": 9,
      "mean": 1,
      "scored_2": 0,
      "scored_0": 0,
      "by_system": {
        "chatgpt": 1,
        "claude": 1,
        "perplexity": 1
      }
    },
    "c3": {
      "name": "When not to use respected",
      "n": 9,
      "mean": 1.22,
      "scored_2": 2,
      "scored_0": 0,
      "by_system": {
        "chatgpt": 1,
        "claude": 1.33,
        "perplexity": 1.33
      }
    },
    "c4": {
      "name": "Group-size fit",
      "n": 9,
      "mean": 1.44,
      "scored_2": 4,
      "scored_0": 0,
      "by_system": {
        "chatgpt": 1.67,
        "claude": 1.67,
        "perplexity": 1
      }
    },
    "c5": {
      "name": "Breakout logistics",
      "n": 3,
      "mean": 1.67,
      "scored_2": 2,
      "scored_0": 0,
      "by_system": {
        "chatgpt": 2,
        "claude": 2,
        "perplexity": 1
      }
    },
    "c6": {
      "name": "Participant-safe language",
      "n": 9,
      "mean": 1.56,
      "scored_2": 5,
      "scored_0": 0,
      "by_system": {
        "chatgpt": 1,
        "claude": 2,
        "perplexity": 1.67
      }
    },
    "c7": {
      "name": "Brief fidelity",
      "n": 9,
      "mean": 1.89,
      "scored_2": 8,
      "scored_0": 0,
      "by_system": {
        "chatgpt": 2,
        "claude": 2,
        "perplexity": 1.67
      }
    },
    "c8": {
      "name": "Clarifying behaviour (B3)",
      "n": 3,
      "mean": 1,
      "scored_2": 0,
      "scored_0": 0,
      "by_system": {
        "chatgpt": 1,
        "claude": 1,
        "perplexity": 1
      }
    }
  },
  "per_system": {
    "chatgpt": {
      "cells": 3,
      "gaps": 0,
      "full_marks": 9,
      "zeros": 0
    },
    "claude": {
      "cells": 3,
      "gaps": 0,
      "full_marks": 13,
      "zeros": 0
    },
    "perplexity": {
      "cells": 3,
      "gaps": 0,
      "full_marks": 7,
      "zeros": 1
    }
  },
  "cells": [
    {
      "blind": "X01",
      "cell": "claude__B2",
      "system": "claude",
      "brief": "B2",
      "gap": false,
      "spotchecked": true,
      "c1": 2,
      "c2": 1,
      "c3": 1,
      "c4": 2,
      "c5": null,
      "c6": 2,
      "c7": 2,
      "notes": {
        "c1": "17 blocks 09:00–17:00 contiguous, breaks included, sums exactly",
        "c2": "Cover Story, 6-3-5, Widerstandsabfrage, Konsent named; no origin or source given",
        "c3": "dot voting only as a step before a resistance check; 6-3-5 with 10 people in range; matrix in range | resolved: re-read: dots decide the shortlist with the GF in the room and the warning is not stated; 'borderline fit, no warning given' = 1",
        "c4": "3–4 per group stated in every small-group block; plenary explicit",
        "c5": "B2 requests no breakouts",
        "c6": "Teilnehmende lines separated but descriptive third person, not a hand-out | resolved: re-read: rubric asks for clear separation and participant lines a participant could read without jargon; third-person phrasing is not a defect = 2",
        "c7": "lunch, guest slot, end, two facilitators, Sie all kept; uses flip charts, dots, laptop not in the brief's material list | resolved: re-read: 'fixed elements' are lunch, guest slot, end time, register; the materials list names what is available, not a fixed element = 2"
      }
    },
    {
      "blind": "X02",
      "cell": "perplexity__B3",
      "system": "perplexity",
      "brief": "B3",
      "gap": false,
      "spotchecked": false,
      "c1": 0,
      "c2": 1,
      "c3": 2,
      "c4": 1,
      "c5": null,
      "c6": 1,
      "c7": 1,
      "c8": 1,
      "notes": {
        "c1": "assumption says 4-hour, agenda runs 9:00–12:00 = 3 h; blocks do not sum to the stated length",
        "c2": "'silent write then share', 'grouping + dot vote', 'action planning': generic labels",
        "c3": "nothing misapplied for a 6–12 group",
        "c4": "6–12 assumed once; no block handles size",
        "c5": "B3 requests no breakouts",
        "c6": "table columns separated; participant column carries citation markers and jargon",
        "c7": "half-day kept in spirit; internal 4 h vs 3 h contradiction",
        "c8": "states an assumption in one line; no question, no variants (offers alternatives only after the fact)"
      }
    },
    {
      "blind": "X03",
      "cell": "chatgpt__B1",
      "system": "chatgpt",
      "brief": "B1",
      "gap": false,
      "spotchecked": true,
      "c1": 2,
      "c2": 1,
      "c3": 1,
      "c4": 2,
      "c5": 2,
      "c6": 1,
      "c7": 2,
      "notes": {
        "c1": "10+10+25+20+20+5 = 90, contiguous",
        "c2": "Impact/Effort 2×2, multi-voting, Fist-to-Five, ROTI named; no origin",
        "c3": "multi-vote (dot vote) used for the binding Q4 decision, which the method's literature warns against; softened by a consent check but not flagged",
        "c4": "tables of 4 assigned; plenary explicit | resolved: consistency with X10 re-reading: size explicit only in the breakout block = 1 | resolved: re-read after second pass: tables of four are set in the check-in and carried through, votes per person and the ROTI round are sized; the X10 reading (only the breakout sized) does not apply here = 2, reversing the earlier consistency change",
        "c5": "3 tables × 4, two items each, timeboxed, pitch template, 2-min report-outs, materials",
        "c6": "'Participants do' lines readable but carry DRI, 2×2, ROTI jargon",
        "c7": "10:00–11:30, six blocks, two facilitators, listed materials, no block >20 min except breakouts"
      }
    },
    {
      "blind": "X04",
      "cell": "claude__B3",
      "system": "claude",
      "brief": "B3",
      "gap": false,
      "spotchecked": true,
      "c1": 2,
      "c2": 1,
      "c3": 1,
      "c4": 1,
      "c5": null,
      "c6": 2,
      "c7": 2,
      "c8": 1,
      "notes": {
        "c1": "09:00–13:00 contiguous, 11 blocks incl. break, sums to 4 h",
        "c2": "Prime Directive, Safety Check, Timeline, 5 Whys, 15% Solutions, Impact/Effort, Fist-to-Five correctly named; no origin or source",
        "c3": "dot voting for theme selection not a binding decision; 5 Whys in 2–4; 15% Solutions fits | resolved: re-read after second pass: six-minute silent affinity clustering against the affinity-map warning on rushed clustering, and a one-word round on a group that needs to process; neither flagged; same reading as X07 = 1",
        "c4": "6–10 assumed and stated; groups of 2–4 explicit | resolved: re-read after second pass: the 5 Whys groups are sized (2–4) but no other block is; size not handled in each block = 1",
        "c5": "B3 requests no breakouts",
        "c6": "Facilitator/Participants separated; participant lines short but 'Prime Directive', '15%' unexplained | resolved: re-read after second pass: participant lines are plain verbs a participant could read (write, post, vote, volunteer); 'Prime Directive' appears only on the facilitator side = 2",
        "c7": "half-day = 4 h with assumption stated up front",
        "c8": "states assumptions in one line; asks nothing, no variants"
      }
    },
    {
      "blind": "X06",
      "cell": "perplexity__B2",
      "system": "perplexity",
      "brief": "B2",
      "gap": false,
      "spotchecked": true,
      "c1": 2,
      "c2": 1,
      "c3": 1,
      "c4": 1,
      "c5": null,
      "c6": 2,
      "c7": 2,
      "notes": {
        "c1": "16 rows 09:00–17:00 contiguous, sums exactly",
        "c2": "SWOT, Brainwriting, Bewertungsmatrix, Klebepunkte named; no origin",
        "c3": "brainwriting with 10 in range; dot voting for shortlist, decision by rule | resolved: re-read after second pass: dot vote decides the binding initiative choice under the Führungsteam's eyes, unflagged; same reading as X01/X03/X10 = 1",
        "c4": "'zwei Gruppen à fünf', three groups stated | resolved: re-read after second pass: two of the three group blocks sized, plenary silent, three groups from ten unsplit; size not handled in each block = 1",
        "c5": "B2 requests no breakouts",
        "c6": "table columns separated; participant column in Sie but descriptive | resolved: consistency with X01 re-reading: separated, Sie-form, jargon-free participant column = 2",
        "c7": "lunch, guest, end, Sie kept; adds Klebepunkte, Arbeitsvorlagen, Timer beyond the brief's materials | resolved: consistency with X01 re-reading: extra materials are not a fixed element = 2"
      }
    },
    {
      "blind": "X07",
      "cell": "chatgpt__B3",
      "system": "chatgpt",
      "brief": "B3",
      "gap": false,
      "spotchecked": true,
      "c1": 2,
      "c2": 1,
      "c3": 1,
      "c4": 1,
      "c5": null,
      "c6": 1,
      "c7": 2,
      "c8": 1,
      "notes": {
        "c1": "09:00–13:00 contiguous, 12 blocks incl. two breaks, sums to 4 h",
        "c2": "Prime Directive, ESVP, Timeline, Affinity, Fishbone/5 Whys, Impact/Effort, SMART, Fist-to-Five named; no origin",
        "c3": "nothing misapplied | resolved: re-read: affinity clustering, naming and voting in 20 min after a 45-min wall runs against the affinity-map warning on rushed clustering; not flagged = 1",
        "c4": "~8–12 assumed; 'two small groups' without sizes",
        "c5": "B3 requests no breakouts",
        "c6": "participant lines short; ESVP, WIP limit, +/Δ unexplained",
        "c7": "half-day = 4 h with assumption stated",
        "c8": "states assumptions in one line; no question"
      }
    },
    {
      "blind": "X10",
      "cell": "perplexity__B1",
      "system": "perplexity",
      "brief": "B1",
      "gap": false,
      "spotchecked": true,
      "c1": 2,
      "c2": 1,
      "c3": 1,
      "c4": 1,
      "c5": 1,
      "c6": 2,
      "c7": 2,
      "notes": {
        "c1": "8+12+30+15+17+8 = 90, contiguous",
        "c2": "one-word round, gallery walk, dot vote, commitment check: generic labels",
        "c3": "dot vote/show of hands for the binding decision; criteria and trade-off talk added, mismatch not flagged",
        "c4": "three groups of four stated | resolved: re-read: rubric wants size handled explicitly in each block; only the breakout block is sized = 1",
        "c5": "one priority per group, same five questions, facilitator allocation, 3-min presentations + 2-min questions, materials | resolved: re-read: rubric 2 needs groups, split rule, timing, report-back and materials all stated; no split rule, no internal timing = 1",
        "c6": "table columns separated; participant column descriptive | resolved: re-read: participant column is separated and jargon-free = 2",
        "c7": "times, six blocks, two facilitators, materials; no block >20 except breakouts (17, 15)"
      }
    },
    {
      "blind": "X11",
      "cell": "chatgpt__B2",
      "system": "chatgpt",
      "brief": "B2",
      "gap": false,
      "spotchecked": false,
      "c1": 2,
      "c2": 1,
      "c3": 1,
      "c4": 2,
      "c5": null,
      "c6": 1,
      "c7": 2,
      "notes": {
        "c1": "17 blocks 09:00–17:00 contiguous, sums exactly",
        "c2": "Silent Writing, Cover Story, Fist-to-Five, Brainwriting, 2×2, Gallery Walk, Blitzlicht named; no origin",
        "c3": "nothing misapplied; dot voting for shortlist, decision by consent/thumbs | resolved: consistency with X01 re-reading: dots narrow the initiative shortlist with the Führungsteam in the room and the warning is not stated, whatever decision step follows; 'borderline fit, no warning given' = 1",
        "c4": "2 groups, 3 groups stated per block",
        "c5": "B2 requests no breakouts",
        "c6": "Teilnehmende lines separated but descriptive",
        "c7": "lunch, guest, end, Sie kept; extra materials declared as the one-line assumption up front"
      }
    },
    {
      "blind": "X12",
      "cell": "claude__B1",
      "system": "claude",
      "brief": "B1",
      "gap": false,
      "spotchecked": false,
      "c1": 2,
      "c2": 1,
      "c3": 2,
      "c4": 2,
      "c5": 2,
      "c6": 2,
      "c7": 2,
      "notes": {
        "c1": "10+15+25+20+15+5 = 90 with an explicit timing check",
        "c2": "round-robin, silent sticky reaction, gallery report-out, Fist-to-five consent named; no origin",
        "c3": "consent round for the binding decision; no dot voting",
        "c4": "groups of four pre-allocated; roles per facilitator",
        "c5": "pre-allocated cross-functional groups, six-line template, 10/5-min warnings, spokesperson, 3×3-min report-out",
        "c6": "Participants lines are plain instructions a participant could read; fist-to-five explained in place",
        "c7": "all fixed elements; recap 15, consolidation 20, decision 15; assumption stated"
      }
    }
  ]
}