[
  {
    "id": "REL-01",
    "area": "Release",
    "action": "Confirm merge and active beta deployment",
    "expected": "PR 602 is merged; deployed frontend commit contains it.",
    "priority": "P0",
    "status": "Pass",
    "evidence": "PR 602 merged 26 Oct 2026 at 16:11 UTC. DigitalOcean deployment ae248d2c is ACTIVE; frontend commit and backend image tag are f305ddf3. Git ancestry confirms that commit contains merge 67290b88."
  },
  {
    "id": "CFG-01",
    "area": "Instructor",
    "action": "Open a ready video lesson",
    "expected": "Study Buddy card shows off, no transcript, and an upload control.",
    "priority": "P0",
    "status": "Pass",
    "evidence": "Beta lesson editor shows the off/empty card and upload control."
  },
  {
    "id": "GEN-01",
    "area": "Transcription",
    "action": "Turn on a ready video without a transcript",
    "expected": "Pending state appears; exactly one job is requested.",
    "priority": "P0",
    "status": "Partial",
    "evidence": "Real beta UI changed to generating after one switch action. Job-count accounting is not verified. One UI toggle request was sent. Exact job and billed-call counts were not read from the ledger."
  },
  {
    "id": "GEN-02",
    "area": "Transcription",
    "action": "Wait for real Gemini transcription",
    "expected": "Ready state shows machine origin, language and nonempty segments.",
    "priority": "P0",
    "status": "Pass",
    "evidence": "Real transcription completed in 18.07s; origin=machine_generated, segments=17, language=en-US."
  },
  {
    "id": "GEN-03",
    "area": "Transcription",
    "action": "Enable again with a saved transcript",
    "expected": "Existing transcript is reused; no new generation is pending.",
    "priority": "P0",
    "status": "Pass",
    "evidence": "Repeated enable reuses transcript: HTTP 200"
  },
  {
    "id": "UP-01",
    "area": "Transcript upload",
    "action": "Upload a valid SRT",
    "expected": "Instructor origin replaces machine origin; timings and cue count match.",
    "priority": "P0",
    "status": "Pass",
    "evidence": "SRT uploaded through the beta lesson editor; 17 cues, instructor origin, en-US. The file reuses machine text for format testing, not independent review."
  },
  {
    "id": "UP-02",
    "area": "Transcript upload",
    "action": "Upload a valid VTT",
    "expected": "VTT is parsed; instructor origin and cue count are correct.",
    "priority": "P1",
    "status": "Pass",
    "evidence": "Valid VTT upload returned HTTP 200, 17 cues and instructor origin."
  },
  {
    "id": "UP-03",
    "area": "Transcript upload",
    "action": "Upload reversed cue timing",
    "expected": "400 with a stable error; previous transcript stays available.",
    "priority": "P0",
    "status": "Pass",
    "evidence": "Reverse cue: HTTP 400; saved transcript unchanged=True"
  },
  {
    "id": "UP-04",
    "area": "Transcript upload",
    "action": "Upload a file over 2 MB",
    "expected": "413; previous transcript stays available.",
    "priority": "P1",
    "status": "Pass",
    "evidence": "Oversized file: HTTP 413; saved transcript unchanged=True"
  },
  {
    "id": "UP-05",
    "area": "Transcript upload",
    "action": "Upload text with an unsupported extension",
    "expected": "400; previous transcript stays available.",
    "priority": "P1",
    "status": "Pass",
    "evidence": "Unsupported extension: HTTP 400; saved transcript unchanged=True"
  },
  {
    "id": "UP-06",
    "area": "Transcript upload",
    "action": "Upload a cue after the video ends",
    "expected": "400; previous transcript stays available.",
    "priority": "P1",
    "status": "Pass",
    "evidence": "Past video duration: HTTP 400; saved transcript unchanged=True"
  },
  {
    "id": "VIS-01",
    "area": "Learner",
    "action": "Open the lesson while enabled and ready",
    "expected": "Enrolled learner sees attribution, question input and suggestions.",
    "priority": "P0",
    "status": "Pass",
    "evidence": "Enrolled synthetic learner sees input, scope and instructor attribution on beta."
  },
  {
    "id": "VIS-02",
    "area": "Learner",
    "action": "Turn off, then reload as learner",
    "expected": "Panel is hidden; video still plays; saved transcript remains.",
    "priority": "P0",
    "status": "Pass",
    "evidence": "After disabling and reload, panel is absent, saved transcript remains, and Bunny video element remains. Playback decoding is checked separately. Panel hidden while actual HTML5 playback advanced 0.01s to 1.78s, readyState 4, no playback error."
  },
  {
    "id": "VIS-03",
    "area": "Learner",
    "action": "Turn on again, then reload",
    "expected": "Panel returns without regeneration.",
    "priority": "P0",
    "status": "Pass",
    "evidence": "Panel returns after re-enable; saved transcript reused, generation state none."
  },
  {
    "id": "ANS-01",
    "area": "Answers",
    "action": "Ask for the three review intervals",
    "expected": "Answer includes one, three and seven days and a valid source.",
    "priority": "P0",
    "status": "Pass",
    "evidence": "Answer checked against cue 5: review after one, three and seven days."
  },
  {
    "id": "ANS-02",
    "area": "Answers",
    "action": "Ask a follow-up about active recall",
    "expected": "Answer uses earlier context and the correct lesson evidence.",
    "priority": "P0",
    "status": "Pass",
    "evidence": "Follow-up returned a validated final answer about retrieval from memory with lesson citations. Manual spot check is recorded in the report."
  },
  {
    "id": "ANS-03",
    "area": "Answers",
    "action": "Ask an unrelated question",
    "expected": "Answer refuses or clearly says the lesson does not support it.",
    "priority": "P0",
    "status": "Pass",
    "evidence": "Unrelated stock-price question received a lesson-scope refusal without a price."
  },
  {
    "id": "STR-01",
    "area": "Streaming",
    "action": "Observe the real beta reply stream",
    "expected": "Ordered version-1 start/status/text/final events arrive; text precedes final.",
    "priority": "P0",
    "status": "Pass",
    "evidence": "Beta endpoint returned application/x-ndjson; 9 ordered events. First event 0.5533s, first text 7.2262s, final 8.405s."
  },
  {
    "id": "STR-02",
    "area": "Streaming",
    "action": "Observe learner answer during delivery",
    "expected": "Provisional text is visible before final; final answer enables source controls.",
    "priority": "P0",
    "status": "Pass",
    "evidence": "Provisional answer observed before final feedback controls. First observed text=5.554s; final=6.349s. Browser sampling includes screenshot overhead."
  },
  {
    "id": "CIT-01",
    "area": "Citations",
    "action": "Click a timestamp in a validated answer",
    "expected": "Bunny player seeks to the cited time; evidence supports the answer.",
    "priority": "P0",
    "status": "Pass",
    "evidence": "Clicked Jump to Spaced repetition and active recall at [00:17]; player time changed 14.75s → 19.93s; target 17s."
  },
  {
    "id": "CAN-01",
    "area": "Recovery",
    "action": "Leave the lesson during a reply",
    "expected": "Request cancels; no stale answer appears in the next lesson.",
    "priority": "P1",
    "status": "Pass",
    "evidence": "Used the course sidebar to change lessons during an answer. Stream failure observed: ['net::ERR_ABORTED']; original question remains on destination=False. Backend provider cancellation was not traced."
  },
  {
    "id": "REC-01",
    "area": "Recovery",
    "action": "Interrupt the browser network during a reply",
    "expected": "Clear retry state; retry uses the original question; no automatic paid resend.",
    "priority": "P1",
    "status": "Partial",
    "evidence": "Network interruption was injected; full retry check did not complete: TimeoutError"
  },
  {
    "id": "SEC-01",
    "area": "Access",
    "action": "Read config without authentication",
    "expected": "401 before any sensitive config is returned.",
    "priority": "P0",
    "status": "Fail",
    "evidence": "Anonymous config: HTTP 403 Access was safely denied with Not authenticated. The mismatch is 403 versus the planned 401, not an authentication bypass."
  },
  {
    "id": "SEC-02",
    "area": "Access",
    "action": "Read and change config as a student",
    "expected": "403; student cannot manage transcript or toggle.",
    "priority": "P0",
    "status": "Pass",
    "evidence": "Student config: HTTP 403 Student toggle: HTTP 403"
  },
  {
    "id": "SEC-03",
    "area": "Access",
    "action": "Ask as a student without course enrolment",
    "expected": "403 before stream headers and provider work.",
    "priority": "P0",
    "status": "Pass",
    "evidence": "Unenrolled student context: HTTP 403 Unenrolled student stream: HTTP 403"
  },
  {
    "id": "SEC-04",
    "area": "Access",
    "action": "Use the QA resource under another tenant host",
    "expected": "Request fails and returns no QA lesson or transcript.",
    "priority": "P0",
    "status": "Pass",
    "evidence": "Actual other-tenant hostname returned HTTP 403. Request was denied by the other tenant entitlement. This does not isolate every ownership or membership branch. A header-only probe on the default host was discarded because the actual Host takes precedence."
  },
  {
    "id": "SEC-05",
    "area": "Access",
    "action": "Request a nonexistent resource",
    "expected": "404 without leaking another resource.",
    "priority": "P1",
    "status": "Pass",
    "evidence": "Unknown video: HTTP 404"
  },
  {
    "id": "VAL-01",
    "area": "Validation",
    "action": "Send an empty question",
    "expected": "422 before a provider call.",
    "priority": "P1",
    "status": "Pass",
    "evidence": "Blank question: HTTP 422"
  },
  {
    "id": "SRC-01",
    "area": "Source priority",
    "action": "Try JSON replacement after instructor upload",
    "expected": "409; instructor transcript remains the source.",
    "priority": "P1",
    "status": "Pass",
    "evidence": "JSON/API transcript replacement after instructor upload returned HTTP 409."
  },
  {
    "id": "CAP-01",
    "area": "Captions",
    "action": "Inspect Bunny captions after saving transcript",
    "expected": "Caption track is attached in transcript language and selectable.",
    "priority": "P1",
    "status": "Partial",
    "evidence": "Bunny exposes caption tracks [{'language': 'en', 'label': 'English'}]; HTML5 readiness=4, duration=75.264s. English caption file fetched with HTTP 200. Player caption-menu selection was not exercised."
  },
  {
    "id": "LANG-01",
    "area": "Language",
    "action": "Change course language after an upload",
    "expected": "Existing transcript stays unchanged; instructor sees a mismatch warning.",
    "priority": "P1",
    "status": "Pass",
    "evidence": "Changed only the QA course to Vietnamese. Existing en-US transcript and update timestamp stayed unchanged; beta instructor card displayed the language warning. Restored course language to en-US afterward."
  },
  {
    "id": "LEC-01",
    "area": "Lecture video",
    "action": "Create a fresh lecture-video lesson in the QA course",
    "expected": "Narration transcript is saved automatically with lecture_script origin.",
    "priority": "P0",
    "status": "Pass",
    "evidence": "Fresh beta lecture job completed; origin=lecture_script, segments=7, generation_state=none. Uploaded XLSX/PPTX through presigned R2 URLs; no transcript was manually supplied."
  },
  {
    "id": "ERR-01",
    "area": "Failure states",
    "action": "Use a QA-only failed transcription fixture, then retry",
    "expected": "Readable error; Retry succeeds; video remains playable.",
    "priority": "P1",
    "status": "Partial",
    "evidence": "Zero-credit toggle created a failed transcription with credit_used_up and a readable message. Restored QA allowance, called Retry, then real Gemini transcription became none, ready=True in 16.1s. This follow-up used API evidence; the failed-state UI was not captured. Readable failure and successful API retry are proven. Playback in the failed-state UI was not independently exercised."
  },
  {
    "id": "QUO-01",
    "area": "Quotas",
    "action": "Exhaust credit on a separate QA tenant",
    "expected": "429 AI_ALLOWANCE_USED before stream headers; no provider work.",
    "priority": "P0",
    "status": "Pass",
    "evidence": "Isolated tenant text allowance set to 0. Stream returned HTTP 429, application/json, AI_ALLOWANCE_USED; ledger rows 0 -> 0. No provider row created."
  },
  {
    "id": "KEY-01",
    "area": "Provider modes",
    "action": "Use a separate own-key tenant without Gemini",
    "expected": "Upload remains available; automatic transcription explains the missing key.",
    "priority": "P1",
    "status": "Pass",
    "evidence": "Isolated own-key OpenAI tenant reported no_gemini_key. A VTT upload succeeded. Provider ledger rows remained 1. The placeholder key was never used in an OpenAI call."
  },
  {
    "id": "RATE-01",
    "area": "Quotas",
    "action": "Reach the configured request rate on a separate QA tenant",
    "expected": "429; no second billed request is started.",
    "priority": "P1",
    "status": "Pass",
    "evidence": "11 explicit QA requests in 6.7s. Statuses [504, 504, 504, 504, 504, 504, 504, 504, 504, 504, 429]. Invalid QA Gemini credentials prevented successful provider calls; the final request was rate-limited. Ledger grew by 10 rows."
  },
  {
    "id": "RACE-01",
    "area": "Concurrency",
    "action": "Upload while a QA transcription is pending",
    "expected": "Instructor upload wins; late machine result cannot overwrite it.",
    "priority": "P1",
    "status": "Pass",
    "evidence": "Fresh fixture upload raced with a provider ledger operation. Observed pending rows: 1; terminal statuses: ['completed']. Final origin: instructor_upload. All short quality jobs were already terminal; the long quality operation existed in the baseline. The only new transcription row belonged to this fresh fixture. Instructor source survived its completed late result."
  },
  {
    "id": "PERF-01",
    "area": "Performance",
    "action": "Measure first event, first text and final completion",
    "expected": "Record actual timings; do not claim the deferred five-second target.",
    "priority": "P1",
    "status": "Pass",
    "evidence": "Measured warm API stream: first text 7.2262s; final 8.405s. No five-second performance target is claimed."
  },
  {
    "id": "BILL-01",
    "area": "Accounting",
    "action": "Compare QA operation usage with ledger and provider invoice",
    "expected": "Call count, token usage and charged amount reconcile.",
    "priority": "P1",
    "status": "Partial",
    "evidence": "Inspected beta tenant ledger and signed-in AI Studio usage/spend/billing. Completed transcription and failed own-key attempts are recorded. AI Studio exposed aggregate Skill Pixel usage, but the accessible billing account linked a different project. No beta invoice or per-request provider reconciliation was available. Private financial values are excluded."
  },
  {
    "id": "QUAL-01",
    "area": "Quality",
    "action": "Grade Vietnamese, noisy and long audio with an independent reviewer",
    "expected": "Use declared thresholds and report every failure; no broad quality claim from this short clip.",
    "priority": "P1",
    "status": "Fail",
    "evidence": "Fresh beta transcription scored against licensed reference text: 4/5 clips passed the declared 15% normalized word-error threshold and timestamp bounds. Independent human review and absolute timestamp alignment remain unverified."
  },
  {
    "id": "FB-01",
    "area": "Feedback",
    "action": "Vote on one validated answer in a tagged QA session",
    "expected": "Expected rating/answer/lesson/request identifiers arrive; no conversation text.",
    "priority": "P1",
    "status": "Partial",
    "evidence": "Clicked Helpful on a freshly validated beta answer in a QA browser session tagged is_test/test_run. The button became aria-pressed=true and a screenshot was saved. PostHog SDK/flags loaded successfully, but no matching event appeared in project 417708 after repeated queries. No synthetic analytics event was sent to replace the real UI action. Delivery, identifier payload and absence of conversation text remain unverified."
  }
]