{"templateId":"markdown","sharedDataIds":{"sidebar":"sidebar-sidebars.yaml"},"props":{"metadata":{"markdoc":{"tagList":["img"]},"type":"markdown"},"seo":{"title":"Test of Humanity debrief","llmstxt":{"hide":true,"excludeFiles":[]}},"dynamicMarkdocComponents":[],"compilationErrors":[],"ast":{"$$mdtype":"Tag","name":"article","attributes":{},"children":[{"$$mdtype":"Tag","name":"Heading","attributes":{"level":1,"id":"test-of-humanity-debrief","__idx":0},"children":["Test of Humanity debrief"]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":2,"id":"detection-accuracy","__idx":1},"children":["Detection accuracy"]},{"$$mdtype":"Tag","name":"ul","attributes":{},"children":[{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"p","attributes":{},"children":["How many of your intentional problems did the AI reviewer catch?"]},{"$$mdtype":"Tag","name":"ul","attributes":{},"children":[{"$$mdtype":"Tag","name":"li","attributes":{},"children":["All problems except 1 were catched"]}]}]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"p","attributes":{},"children":["How many did it miss? Do the misses share a pattern (e.g., all subtle, all in a specific file type)?"]},{"$$mdtype":"Tag","name":"ul","attributes":{},"children":[{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"p","attributes":{},"children":["Missed a problem in a file that was moved and had a wrong change:"]},{"$$mdtype":"Tag","name":"Image","attributes":{"src":"/assets/image-copy.630ecd5163e96688108e23e9258145db83abb502e51e276c562664b71bdaca63.8bed52ad.png","alt":"image copy.png","framed":false,"withLightbox":true,"width":"","height":""},"children":[]}]}]}]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"p","attributes":{},"children":["Were there any false positives - comments about things that weren't actually problems?"]},{"$$mdtype":"Tag","name":"ul","attributes":{},"children":[{"$$mdtype":"Tag","name":"li","attributes":{},"children":["No"]}]}]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"p","attributes":{},"children":["Did the severity levels match how serious the problems actually were?"]},{"$$mdtype":"Tag","name":"ul","attributes":{},"children":[{"$$mdtype":"Tag","name":"li","attributes":{},"children":["Yes"]}]}]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"p","attributes":{},"children":["How actionable were the comments? Could you fix the issue based on the comment alone?"]},{"$$mdtype":"Tag","name":"ul","attributes":{},"children":[{"$$mdtype":"Tag","name":"li","attributes":{},"children":["Yes"," ","Would be good that when clicking on file from review, user will be redirected to the Editor to fix this issue"]}]}]}]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":2,"id":"verification-accuracy","__idx":2},"children":["Verification accuracy"]},{"$$mdtype":"Tag","name":"ul","attributes":{},"children":[{"$$mdtype":"Tag","name":"li","attributes":{},"children":["Did the AI correctly identify which issues you fixed?",{"$$mdtype":"Tag","name":"ul","attributes":{},"children":[{"$$mdtype":"Tag","name":"li","attributes":{},"children":["Yes"]}]}]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":["Did it incorrectly resolve any issues you left broken?",{"$$mdtype":"Tag","name":"ul","attributes":{},"children":[{"$$mdtype":"Tag","name":"li","attributes":{},"children":["No"]}]}]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":["Did it miss any fixes you made?",{"$$mdtype":"Tag","name":"ul","attributes":{},"children":[{"$$mdtype":"Tag","name":"li","attributes":{},"children":["No"]}]}]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":["How confident are you in the verification after this test?",{"$$mdtype":"Tag","name":"ul","attributes":{},"children":[{"$$mdtype":"Tag","name":"li","attributes":{},"children":["Very confident"]}]}]}]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":2,"id":"analytics-page","__idx":3},"children":["Analytics page"]},{"$$mdtype":"Tag","name":"ul","attributes":{},"children":[{"$$mdtype":"Tag","name":"li","attributes":{},"children":["Did the numbers on the analytics page match your experience?"," ","Yes"]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":["What would you add or change about the page?",{"$$mdtype":"Tag","name":"ul","attributes":{},"children":[{"$$mdtype":"Tag","name":"li","attributes":{},"children":["Add a refresh button or keep state after refresh (deep linking for selected project?)"]}]}]}]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":2,"id":"overall-experience","__idx":4},"children":["Overall experience"]},{"$$mdtype":"Tag","name":"ul","attributes":{},"children":[{"$$mdtype":"Tag","name":"li","attributes":{},"children":["Was it clear whether the AI review was in progress or complete? - Yes"]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":["How long did the AI review take? - ~8 minutes (maybe because we moved a lot of files, though with minimal changes)"]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":["Did the summary comment posted by the AI reviewer make sense? - Yes"]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":["If you added custom AI rules, were they accounted for in the review? - We didn't add custom AI rules"]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":["Would you trust this AI reviewer on real PRs after this test? - Yes, with caution (as with others)"]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":["What made you smile?",{"$$mdtype":"Tag","name":"ul","attributes":{},"children":[{"$$mdtype":"Tag","name":"li","attributes":{},"children":["The reviewer was aware that we are testing it :D",{"$$mdtype":"Tag","name":"Image","attributes":{"src":"/assets/image.27f31e0894fb6a8da10c98ec7d9a9ec4043d648343dc2a245d39dab63d11e0e5.8bed52ad.png","alt":"image.png","framed":false,"withLightbox":true,"width":"","height":""},"children":[]}]}]}]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":["What felt frustrating or broken?",{"$$mdtype":"Tag","name":"ul","attributes":{},"children":[{"$$mdtype":"Tag","name":"li","attributes":{},"children":["When AI reviewer flags issues, you can't navigate directly to the files in the editor (as with the link checker, for example)"]}]}]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":["On a scale of 1 to 10, how would you rate the overall quality of the AI reviewer?"]}]},{"$$mdtype":"Tag","name":"ul","attributes":{},"children":[{"$$mdtype":"Tag","name":"li","attributes":{},"children":["9/10"]}]},{"$$mdtype":"Tag","name":"ul","attributes":{},"children":[{"$$mdtype":"Tag","name":"li","attributes":{},"children":["Are there any additional notes you would like to share?",{"$$mdtype":"Tag","name":"ul","attributes":{},"children":[{"$$mdtype":"Tag","name":"li","attributes":{},"children":["N/A"]}]}]}]}]},"headings":[{"value":"Test of Humanity debrief","id":"test-of-humanity-debrief","depth":1},{"value":"Detection accuracy","id":"detection-accuracy","depth":2},{"value":"Verification accuracy","id":"verification-accuracy","depth":2},{"value":"Analytics page","id":"analytics-page","depth":2},{"value":"Overall experience","id":"overall-experience","depth":2}],"frontmatter":{"seo":{"title":"Test of Humanity debrief"}},"lastModified":"2026-07-01T11:26:31.000Z","pagePropGetterError":{"message":"","name":""}},"slug":"/debrief/project-66","userData":{"isAuthenticated":false,"teams":["anonymous"]},"isPublic":true}