From dafc70b0471885f015f21f11b9fc6d2070dd1b07 Mon Sep 17 00:00:00 2001 From: jenstandstad Date: Tue, 18 Aug 2026 19:51:44 +0200 Subject: [PATCH] Sound: per-node scene music + synthesized SFX (zero-dep) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - migration 023: story_nodes.music_asset_id (references the asset store). - src/audio.ts: a tiny audio manager — one looping scene-music track (fade, dedup, gesture-primed autoplay) and synthesized one-shot SFX; global mute. - Runtime: RuntimeNode.musicUrl; music follows the current node (null inherits, a finished playthrough stops). SFX blips on dialogue advance/choice (not in the editor preview). Floating mute toggle during play. - Graph inspector: a "Scene music" picker sourced from uploaded audio assets. No external library (we did not adopt react-winamp — it is a React 16 CRA app, not an installable package). Bundle grew ~2 KB. Co-Authored-By: Claude Opus 4.8 --- migrations/023_story_node_music.sql | 4 ++ server/migrations.integration.test.ts | 6 +-- server/narrativeRepository.ts | 13 ++--- server/storyGraphRepository.ts | 11 ++-- src/App.tsx | 15 +++++- src/audio.ts | 76 +++++++++++++++++++++++++++ src/mysteryGraph.tsx | 22 ++++++-- src/narrative.tsx | 5 +- src/styles.css | 5 ++ 9 files changed, 135 insertions(+), 22 deletions(-) create mode 100644 migrations/023_story_node_music.sql create mode 100644 src/audio.ts diff --git a/migrations/023_story_node_music.sql b/migrations/023_story_node_music.sql new file mode 100644 index 0000000..ba4fe6c --- /dev/null +++ b/migrations/023_story_node_music.sql @@ -0,0 +1,4 @@ +-- Optional scene music per story node. The runtime plays/loops it while the node +-- is active; a NULL value inherits whatever is already playing (so a track set on +-- one node carries through the region until another node changes it). +ALTER TABLE osint.story_nodes ADD COLUMN music_asset_id UUID REFERENCES osint.assets(id); diff --git a/server/migrations.integration.test.ts b/server/migrations.integration.test.ts index ae3c97e..e677182 100644 --- a/server/migrations.integration.test.ts +++ b/server/migrations.integration.test.ts @@ -33,7 +33,7 @@ suite('PostgreSQL migrations', () => { const migrationsDir = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..', 'migrations') const firstRun: string[] = [] await runMigrations(testDatabaseUrl, migrationsDir, message => firstRun.push(message)) - expect(firstRun.filter(message => message.startsWith('apply '))).toHaveLength(22) + expect(firstRun.filter(message => message.startsWith('apply '))).toHaveLength(23) const client = new Client({ connectionString: testDatabaseUrl }) await client.connect() @@ -49,7 +49,7 @@ suite('PostgreSQL migrations', () => { ])) expect(tableNames).not.toEqual(expect.arrayContaining(['cases', 'widgets', 'widget_relations', 'cutscenes', 'dialogue_steps', 'mystery_chapters', 'seen_dialogue'])) const ledger = await client.query<{ count: string }>('SELECT COUNT(*)::text AS count FROM osint.schema_migrations') - expect(ledger.rows[0].count).toBe('22') + expect(ledger.rows[0].count).toBe('23') const connectionColumns = await client.query<{ column_name: string }>(`SELECT column_name FROM information_schema.columns WHERE table_schema='osint' AND table_name='exhibit_connections'`) expect(connectionColumns.rows.map(row => row.column_name)).toEqual(expect.arrayContaining(['label', 'tightness', 'tag_style', 'tag_position_percent', 'tag_lateral_offset'])) const eventOccurrence = await client.query<{ is_nullable: string }>(`SELECT is_nullable FROM information_schema.columns WHERE table_schema='osint' AND table_name='event_exhibits' AND column_name='occurred_at'`) @@ -58,7 +58,7 @@ suite('PostgreSQL migrations', () => { const secondRun: string[] = [] await runMigrations(testDatabaseUrl, migrationsDir, message => secondRun.push(message)) - expect(secondRun.filter(message => message.startsWith('skip '))).toHaveLength(22) + expect(secondRun.filter(message => message.startsWith('skip '))).toHaveLength(23) expect(secondRun.some(message => message.startsWith('apply '))).toBe(false) }) }) diff --git a/server/narrativeRepository.ts b/server/narrativeRepository.ts index 99b8d7b..96c3533 100644 --- a/server/narrativeRepository.ts +++ b/server/narrativeRepository.ts @@ -16,7 +16,7 @@ export type RuntimeUtterance = { } export type RuntimeNode = { id: string; kind: 'cutscene' | 'dialogue' | 'level'; label: string - componentKey?: string | null; levelSlug?: string | null + componentKey?: string | null; levelSlug?: string | null; musicUrl?: string | null utterances?: RuntimeUtterance[]; rootId?: string | null } export type PlaythroughState = { playthrough: PlaythroughSummary; node: RuntimeNode | null } @@ -60,7 +60,7 @@ export interface NarrativeRepository { deletePose(npcId: string, poseKey: string): Promise } -type GraphNodeRow = { id: string; node_type: string; label: string; component_key: string | null; level_template_version_id: string | null } +type GraphNodeRow = { id: string; node_type: string; label: string; component_key: string | null; level_template_version_id: string | null; music_asset_id: string | null } export function createNarrativeRepository(pool: Pool, objectStorage: ObjectStorage): NarrativeRepository { // ---- Runtime: walking the story graph ------------------------------------- @@ -97,11 +97,12 @@ export function createNarrativeRepository(pool: Pool, objectStorage: ObjectStora } async function resolveNodeForPlay(nodeId: string, levelSlug: string | null): Promise { - const node = (await pool.query('SELECT id,node_type,label,component_key,level_template_version_id FROM osint.story_nodes WHERE id=$1', [nodeId])).rows[0] + const node = (await pool.query('SELECT id,node_type,label,component_key,level_template_version_id,music_asset_id FROM osint.story_nodes WHERE id=$1', [nodeId])).rows[0] if (!node) return null - if (node.node_type === 'cutscene') return { id: node.id, kind: 'cutscene', label: node.label, componentKey: node.component_key } - if (node.node_type === 'level') return { id: node.id, kind: 'level', label: node.label, levelSlug } - if (node.node_type === 'dialogue') return { id: node.id, kind: 'dialogue', label: node.label, ...(await resolveDialogueGraph(node.id)) } + const musicUrl = node.music_asset_id ? `/api/assets/${node.music_asset_id}` : null + if (node.node_type === 'cutscene') return { id: node.id, kind: 'cutscene', label: node.label, componentKey: node.component_key, musicUrl } + if (node.node_type === 'level') return { id: node.id, kind: 'level', label: node.label, levelSlug, musicUrl } + if (node.node_type === 'dialogue') return { id: node.id, kind: 'dialogue', label: node.label, musicUrl, ...(await resolveDialogueGraph(node.id)) } return null // gates are auto-resolved during advance and never surfaced } diff --git a/server/storyGraphRepository.ts b/server/storyGraphRepository.ts index 413b928..9603b79 100644 --- a/server/storyGraphRepository.ts +++ b/server/storyGraphRepository.ts @@ -13,7 +13,7 @@ export type StoryNodeType = 'cutscene' | 'dialogue' | 'level' | 'det_gate' | 'll export type TerminalDto = { id: string; terminalKey: string; label: string; toNodeId: string | null; sortOrder: number } export type StoryNodeDto = { id: string; nodeType: StoryNodeType; label: string; hasUtterances: boolean - xpos: number; ypos: number; levelTemplateVersionId: string | null; componentKey: string | null + xpos: number; ypos: number; levelTemplateVersionId: string | null; componentKey: string | null; musicAssetId: string | null terminals: TerminalDto[] } export type StoryGraphDto = { mysteryId: string; entryNodeId: string | null; nodes: StoryNodeDto[] } @@ -37,7 +37,7 @@ const DEFAULT_TERMINALS: Record export interface StoryGraphRepository { getGraph(mysteryId: string): Promise createNode(mysteryId: string, input: { nodeType: StoryNodeType; xpos: number; ypos: number; label?: string }): Promise - updateNode(nodeId: string, input: Partial<{ label: string; xpos: number; ypos: number; hasUtterances: boolean; componentKey: string | null; levelTemplateVersionId: string | null }>): Promise + updateNode(nodeId: string, input: Partial<{ label: string; xpos: number; ypos: number; hasUtterances: boolean; componentKey: string | null; levelTemplateVersionId: string | null; musicAssetId: string | null }>): Promise deleteNode(nodeId: string): Promise addTerminal(nodeId: string, input: { terminalKey: string; label?: string }): Promise updateTerminal(terminalId: string, input: Partial<{ label: string; sortOrder: number; toNodeId: string | null }>): Promise<{ ok: boolean; error?: string }> @@ -61,8 +61,8 @@ export function createStoryGraphRepository(pool: Pool): StoryGraphRepository { const mystery = await pool.query<{ id: string; entry_node_id: string | null }>('SELECT id,entry_node_id FROM osint.mysteries WHERE id=$1', [mysteryId]) if (!mystery.rows[0]) return null const [nodes, terminals] = await Promise.all([ - pool.query<{ id: string; node_type: StoryNodeType; label: string; has_utterances: boolean; xpos: number; ypos: number; level_template_version_id: string | null; component_key: string | null }>( - 'SELECT id,node_type,label,has_utterances,xpos,ypos,level_template_version_id,component_key FROM osint.story_nodes WHERE mystery_id=$1 ORDER BY created_at', [mysteryId]), + pool.query<{ id: string; node_type: StoryNodeType; label: string; has_utterances: boolean; xpos: number; ypos: number; level_template_version_id: string | null; component_key: string | null; music_asset_id: string | null }>( + 'SELECT id,node_type,label,has_utterances,xpos,ypos,level_template_version_id,component_key,music_asset_id FROM osint.story_nodes WHERE mystery_id=$1 ORDER BY created_at', [mysteryId]), pool.query<{ id: string; parent_node_id: string; terminal_key: string; label: string; to_node_id: string | null; sort_order: number }>( `SELECT t.id,t.parent_node_id,t.terminal_key,t.label,t.to_node_id,t.sort_order FROM osint.story_node_terminals t JOIN osint.story_nodes n ON n.id=t.parent_node_id WHERE n.mystery_id=$1 ORDER BY t.sort_order,t.terminal_key`, [mysteryId]), @@ -77,7 +77,7 @@ export function createStoryGraphRepository(pool: Pool): StoryGraphRepository { mysteryId, entryNodeId: mystery.rows[0].entry_node_id, nodes: nodes.rows.map(row => ({ id: row.id, nodeType: row.node_type, label: row.label, hasUtterances: row.has_utterances, - xpos: row.xpos, ypos: row.ypos, levelTemplateVersionId: row.level_template_version_id, componentKey: row.component_key, + xpos: row.xpos, ypos: row.ypos, levelTemplateVersionId: row.level_template_version_id, componentKey: row.component_key, musicAssetId: row.music_asset_id, terminals: byNode.get(row.id) || [], })), } @@ -117,6 +117,7 @@ export function createStoryGraphRepository(pool: Pool): StoryGraphRepository { if (input.hasUtterances !== undefined) set('has_utterances', input.hasUtterances) if (input.componentKey !== undefined) set('component_key', input.componentKey || null) if (input.levelTemplateVersionId !== undefined) set('level_template_version_id', input.levelTemplateVersionId || null) + if (input.musicAssetId !== undefined) set('music_asset_id', input.musicAssetId || null) if (sets.length) await pool.query(`UPDATE osint.story_nodes SET ${sets.join(',')} WHERE id=$1`, values) const graph = await loadGraph(mysteryId) return graph?.nodes.find(node => node.id === nodeId) ?? null diff --git a/src/App.tsx b/src/App.tsx index d3473f2..3ce944c 100644 --- a/src/App.tsx +++ b/src/App.tsx @@ -3,6 +3,7 @@ import { BookOpen, Building2, CalendarClock, ChevronRight, CircleHelp, FileText, import type { BriefConcept, CaseDocument, CaseState, Connection, EventExhibit, Evidence, Exhibit, ExhibitRelation, FolderExhibit, LevelBrief, OrganizationKind, PartyExhibit, PartyKind, SourceFileType, TimelineRange, TimelineView } from './types' import { CutsceneHost, DialoguePlayer, SplashScreen, type PlaythroughState, type PlaythroughSummary, type RuntimeNode } from './narrative' import { AdminPanel } from './admin' +import { audio } from './audio' import { clampBoardZoom, containedIds, dateValue, discardExhibit, folderIsOpen, moveBoardPoint, nextOpenBoardPosition, normalizeCase, panViewport, projectThreadTag, threadCurve, threadTagLateralLimit, threadTagPlacement, timelinePositionPercent, timelineRange, zoomFromPinch, zoomFromWheel, zoomViewportAt } from './boardDomain' import { documentExhibits, documentWidget, evidenceExhibits, exhibitWidget, type ExhibitWidgetContext, type WidgetCommand } from './exhibitRegistry' @@ -66,6 +67,7 @@ export function App() { const [splashBusy, setSplashBusy] = useState(false) const [playthrough, setPlaythrough] = useState(null) const [runtimeNode, setRuntimeNode] = useState(null) + const [muted, setMuted] = useState(audio.isMuted()) const saveTimer = useRef(undefined) const boardRef = useRef(null) const fileInputRef = useRef(null) @@ -151,6 +153,12 @@ export function App() { } catch { setStatus('COULD NOT ADVANCE') } }, [playthrough, applyState]) + // Scene music follows the current node (null inherits; a finished playthrough stops). + useEffect(() => { + if (runtimeNode?.musicUrl) audio.setMusic(runtimeNode.musicUrl) + else if (playthrough?.status === 'finished') audio.setMusic(null) + }, [runtimeNode, playthrough]) + useEffect(() => { if (!adminMenuOpen) return const close = (event: PointerEvent) => { if (!adminMenuRef.current?.contains(event.target as Node)) setAdminMenuOpen(false) } @@ -383,11 +391,13 @@ export function App() { } } + const audioToggle = playthrough ? : null + if (adminRoute) return if (splashOpen) return setSplashOpen(false)} /> // Story-graph runtime: cutscene and dialogue nodes play full-screen (no board). - if (runtimeNode?.kind === 'cutscene') return advance()} /> - if (runtimeNode?.kind === 'dialogue') return + if (runtimeNode?.kind === 'cutscene') return <>{audioToggle} advance()} /> + if (runtimeNode?.kind === 'dialogue') return <>{audioToggle} if (noLevels) return { window.location.assign(`?level=${encodeURIComponent(level.id)}&edit=1`) }} /> if (!caseState) return
GU

GLITCH UNIVERSITY NETWORK TERMINAL

{status}
@@ -411,6 +421,7 @@ export function App() { }) const timelineView = caseState.views.find((view): view is TimelineView => view.type === 'timeline') return
+ {audioToggle}
GUOSINT BOARD / {requestedEditMode && caseState.editingAllowed ? 'LEVEL EDITOR' : 'CASE TERMINAL'}