feat: Capture errors thrown by stories and report them for simpler de…

…bugging (#22)

runtime-logs-server/serve.ts

@@ -25,8 +25,9 @@ let testStatus: TestStatus | undefined = undefined
 interface StoryLog {
     status: Status
     log: string
+    linkRoot: string
 }
-type Status = 'start' | 'waiting' | 'success' | 'failure' | 'skipped' | 'unknown'
+type Status = 'start' | 'waiting' | 'success' | 'failure' | 'thrown' | 'skipped' | 'unknown'
 const stories = new Map<string, StoryLog>()
 let isComplete = false
 const frames = ['⠋', '⠙', '⠹', '⠸', '⠼', '⠴', '⠦', '⠧', '⠇', '⠏'];
@@ -56,17 +57,31 @@ const createStatusString = (): string[] => {
 }
 
 const failedStoryStringWithLinks = (storyEntry: [string, StoryLog]): string[] => {
-    const [story, { log }] = storyEntry
-    const localDir = log
+    const [story, { status, log, linkRoot }] = storyEntry
+    const reason = []
+
+    if (status === 'failure') {
+        if (log) {
+            reason.push(`          ${chalk.grey(log)}`)
+        }
+
+        const actualLink = `          ${chalk.grey('Actual:')} ${chalk.grey(`${linkRoot}/actual.jpeg`)}`
+        const expectedLink = `          ${chalk.grey('Expected:')} ${chalk.grey(`${linkRoot}/expected.jpeg`)}`
+        const diffLink = `          ${chalk.grey('Diff:')} ${chalk.grey(`${linkRoot}/diff.jpeg`)}`
+
+        reason.push(actualLink, expectedLink, diffLink)
+
+    } else if (status === 'thrown') {
+        const padded = log.split('\n').map((line) => `          ${line}`).join('\n')
+        const screenshot = `          ${chalk.grey('Screen:')} ${chalk.grey(`${linkRoot}/error.jpeg`)}`
+        const consoleLogs = `          ${chalk.grey('Logs:')} ${chalk.grey(`${linkRoot}/error.log`)}`
+
+        reason.push(chalk.grey(padded), screenshot, consoleLogs)
+    }
 
-    const actualLink = `          ${chalk.grey('Actual:')} ${chalk.grey(`${localDir}/actual.jpeg`)}`
-    const expectedLink = `          ${chalk.grey('Expected:')} ${chalk.grey(`${localDir}/expected.jpeg`)}`
-    const diffLink = `          ${chalk.grey('Diff:')} ${chalk.grey(`${localDir}/diff.jpeg`)}`
     return [
         `      ${chalk.red(story)}`,
-        actualLink,
-        expectedLink,
-        diffLink
+        ...reason
     ]
 }
 
@@ -128,6 +143,8 @@ setInterval(() => {
                 return `  ✅ ${story}${appendedLog}`
             case 'failure':
                 return `  ❌ ${story}`
+            case 'thrown':
+                return `  ❌ ${story} (threw error)`
             case 'unknown':
                 return `  ❓ ${story}${appendedLog}`
             case 'skipped':
@@ -147,13 +164,15 @@ setInterval(() => {
 interface LogBody {
     story: string
     log: string
-    status: Status
+    status: Status,
+    linkRoot: string
 }
 app.post('/log', (req, res) => {
     const body = req.body as LogBody
     stories.set(body.story, {
         status: body.status,
-        log: body.log
+        log: body.log,
+        linkRoot: body.linkRoot,
     })
     res.sendStatus(200)
 })

runtime/storyshots.testStory.ts

@@ -14,52 +14,73 @@ export async function testStory(page: Page, { storyshot, url, ignore, title }: S
     const {
         start,
         failure,
+        thrown,
         success,
         waiting,
         skipped,
         unknown
-    } = startLogging(storyshot)
+    } = startLogging(storyshot, `${localResultsDir}/${title}`)
     await start()
 
+    const consoleErrors = captureConsoleErrors(page)
+
     if (ignore) {
         await skipped("skipped by config")
         return true
     }
 
-    await page.goto(url, { waitUntil: 'domcontentloaded' })
-    await waitForStoryReady(page)
+    try {
+        await page.goto(url, { waitUntil: 'domcontentloaded' })
+        await waitForStoryReady(page)
 
-    if (waitForStableMillis > 0) {
-        await waitForStable(page, waitForStableMillis, waiting)
-    }
+        if (waitForStableMillis > 0) {
+            await waitForStable(page, waitForStableMillis, waiting)
+        }
 
-    const {
-        storyPassed,
-        snapshotNotExistant,
-        storyPassedButUpdated,
-    } = await assertMatchingSnapshot(page, storyshot, unknown)
+        const {
+            storyPassed,
+            snapshotNotExistant,
+            storyPassedButUpdated,
+        } = await assertMatchingSnapshot(page, storyshot, unknown)
 
-    if (storyPassed) {
-        if (snapshotNotExistant) {
-            await unknown("no baseline found")
-        } else if (!storyPassedButUpdated) {
-            await success()
+        if (storyPassed) {
+            if (snapshotNotExistant) {
+                await unknown("no baseline found")
+            } else if (!storyPassedButUpdated) {
+                await success()
+            } else {
+                await unknown("baseline updated")
+            }
         } else {
-            await unknown("baseline updated")
+            await shortenArtifactNames(title)
+
+            await failure()
         }
-    } else {
-        await shortenArtifactNames(title)
 
-        await failure(
-            `${localResultsDir}/${title}`
-        )
+        return storyPassed
+    } catch (err) {
+        const image = await page.screenshot({
+            type: 'jpeg',
+            animations: 'disabled',
+            fullPage: true,
+        })
+
+        fs.mkdirSync(`/storyshots/test-results/${title}`, { recursive: true })
+        fs.writeFileSync(`/storyshots/test-results/${title}/error.jpeg`, image)
+        fs.writeFileSync(`/storyshots/test-results/${title}/error.log`, consoleErrors.dump())
+
+        await thrown(err.message ?? err)
+        return false
     }
-    return storyPassed
 }
 
 async function waitForStoryReady(page: Page) {
-    await page.locator('#storybook-root > *:first-child')
-        .waitFor({ state: 'attached', timeout: 5000 })
+    try {
+        await page.locator('#storybook-root > *:first-child')
+            .waitFor({ state: 'attached', timeout: 5000 })
+    } catch (err) {
+        throw new Error('Story did not mount in time.')
+    }
 }
 
 // continually take screenshots until the page is stable (i.e. no more animations)
@@ -147,3 +168,17 @@ async function shortenArtifactNames(title: string) {
         `/storyshots/test-results/${title}/diff.jpeg`,
     )
 }
+
+function captureConsoleErrors(page: Page) {
+    const errors: string[] = []
+
+    page.on('console', async (msg) => {
+        if (msg.type() === 'error') {
+            errors.push(msg.text())
+        }
+    })
+
+    return {
+        dump: () => errors.join('\n'),
+    }
+}

runtime/updateLog.ts

@@ -8,7 +8,8 @@ interface Log {
     start: () => Promise<void>
     waiting: (log?: string) => Promise<void>
     success: () => Promise<void>
-    failure: (storyLink: string) => Promise<void>
+    failure: (log?: string) => Promise<void>
+    thrown: (reason: string) => Promise<void>
     skipped: (log?: string) => Promise<void>
     unknown: (log?: string) => Promise<void>
 }
@@ -38,13 +39,14 @@ export const completeStoryshots = async () => {
 }
 
 
-export const startLogging = (story: string): Log => {
+export const startLogging = (story: string, linkRoot: string): Log => {
 
-    const send = async (status: 'start' | 'waiting' | 'success' | 'unknown' | 'failure' | 'skipped', log: string) => {
-        axios.post('http://localhost:3000/log', {
+    const send = async (status: 'start' | 'waiting' | 'success' | 'unknown' | 'failure' | 'thrown' | 'skipped', log: string) => {
+        return axios.post('http://localhost:3000/log', {
             story,
             status,
-            log
+            log,
+            linkRoot,
         })
     }
 
@@ -64,8 +66,12 @@ export const startLogging = (story: string): Log => {
         await send('unknown', log ?? '')
     }
 
-    const failure = async (storyLink: string) => {
-        await send('failure', storyLink)
+    const failure = async (log?: string) => {
+        await send('failure', log ?? '')
+    }
+
+    const thrown = async (reason: string) => {
+        await send('thrown', reason)
     }
 
     const skipped = async (log?: string) => {
@@ -77,6 +83,7 @@ export const startLogging = (story: string): Log => {
         waiting,
         success,
         failure,
+        thrown,
         unknown,
         skipped
     }

sample-storybook/src/stories/FailSwitch.stories.tsx

@@ -0,0 +1,15 @@
+import React from 'react'
+import { Meta, StoryObj } from '@storybook/react'
+
+import { FailSwitch } from './FailSwitch'
+
+export default {
+  title: 'Example/FailSwitch',
+  component: FailSwitch,
+} as Meta
+
+type Story = StoryObj<typeof FailSwitch>
+
+export const Example: Story = {
+  render: () => <FailSwitch />,
+}

sample-storybook/src/stories/FailSwitch.tsx

@@ -0,0 +1,17 @@
+import React from 'react'
+
+interface FailSwitchProps {
+  fail?: boolean
+}
+
+export const FailSwitch = ({
+  fail = false,
+}: FailSwitchProps) => {
+  if (fail) {
+    throw new Error('FailSwitch was forced to fail.')
+  }
+
+  return (
+    <p>Hello</p>
+  )
+}

spec/bundle_spec.sh

@@ -19,10 +19,14 @@ Describe 'the bundle'
   reset_button_stories() {
     git checkout sample-storybook/src/stories/Button.stories.tsx > /dev/null 2>&1
   }
+  reset_failswitch_stories() {
+    git checkout sample-storybook/src/stories/FailSwitch.stories.tsx > /dev/null 2>&1
+  }
 
   reset_repo() {
     reset_box_stories
     reset_button_stories
+    reset_failswitch_stories
   }
 
   run_batect() {
@@ -64,7 +68,7 @@ Describe 'the bundle'
 
     It 'can list stories to test'
       When run storyshots_list
-      The output should include '13 stories to test'
+      The output should include '14 stories to test'
 
       The output should include '🔘 example-button--primary'
       The output should include '🔘 example-button--secondary'
@@ -123,27 +127,54 @@ Describe 'the bundle'
 
   Describe 'failure cases'
 
-    setup_new_diffs() { 
-      git apply spec/ensure_failure.patch
-      pnpm -F sample build-storybook > /dev/null 2>&1
-    }
-    BeforeAll 'setup_new_diffs'
+    Describe 'baseline deviated'
+      setup_new_diffs() {
+        git apply spec/ensure_failure.patch
+        pnpm -F sample build-storybook > /dev/null 2>&1
+      }
+      BeforeAll 'setup_new_diffs'
 
-    It 'can detect when the baseline has deviated and show the actual vs expected'
-      When run storyshots_with_ignore
-      The output should include '❌ bigbox--a-big-box'
-      The output should include '✅ example-page--logged-out'
-      The stderr should match pattern '*'
-      The path ${TEST_RESULTS}/bigbox--a-big-box/actual.jpeg should be file
-      The path ${TEST_RESULTS}/bigbox--a-big-box/expected.jpeg should be file
-      The status should be failure
+      It 'can detect when the baseline has deviated and show the actual vs expected'
+        When run storyshots_with_ignore
+        The output should include '❌ bigbox--a-big-box'
+        The output should include '✅ example-page--logged-out'
+        The stderr should match pattern '*'
+        The path ${TEST_RESULTS}/bigbox--a-big-box/actual.jpeg should be file
+        The path ${TEST_RESULTS}/bigbox--a-big-box/expected.jpeg should be file
+        The status should be failure
+      End
+
+      It 'can update baselines'
+        When run storyshots_update
+        The output should include '❓ bigbox--a-big-box (baseline updated)'
+        The stderr should match pattern '*'
+        The status should be success
+      End
     End
 
-    It 'can update baselines'
-      When run storyshots_update
-      The output should include '❓ bigbox--a-big-box (baseline updated)'
-      The stderr should match pattern '*'
-      The status should be success
+    Describe 'story has failure'
+      setup_new_diffs() {
+        git apply spec/ensure_throw.patch
+        pnpm -F sample build-storybook > /dev/null 2>&1
+      }
+      BeforeAll 'setup_new_diffs'
+
+      It 'can detect when the story has thrown an error'
+        When run storyshots_with_ignore
+        The output should include '❌ example-failswitch--example'
+        The output should include '✅ example-page--logged-out'
+        The stderr should match pattern '*'
+        The path ${TEST_RESULTS}/example-failswitch--example/error.jpeg should be file
+        The path ${TEST_RESULTS}/example-failswitch--example/error.log should be file
+        The status should be failure
+      End
+
+      It 'cannot update baselines'
+        When run storyshots_update
+        The output should include '❌ example-failswitch--example'
+        The stderr should match pattern '*'
+        The status should be failure
+      End
     End
 
   End

spec/ensure_throw.patch

@@ -0,0 +1,11 @@
+diff --git a/sample-storybook/src/stories/FailSwitch.stories.tsx b/sample-storybook/src/stories/FailSwitch.stories.tsx
+index 1d5103f..2b4d847 100644
+--- a/sample-storybook/src/stories/FailSwitch.stories.tsx
++++ b/sample-storybook/src/stories/FailSwitch.stories.tsx
+@@ -11,5 +11,5 @@ export default {
+ type Story = StoryObj<typeof FailSwitch>
+
+ export const Example: Story = {
+-  render: () => <FailSwitch />,
++  render: () => <FailSwitch fail />,
+ }