diff --git a/docs/TODO.md b/docs/TODO.md index 880269d..91a5eec 100644 --- a/docs/TODO.md +++ b/docs/TODO.md @@ -28,12 +28,12 @@ This is the ordered implementation roadmap following the accepted exhibit model. ## Milestone 2: exhibit-schema cutover -- [ ] Add boards, exhibits, exhibit types, subtype tables, immutable template versions, and cloned mutable levels. -- [ ] Migrate transitional `widgets`, `widget_relations`, and `playthrough_*` data with equivalence checks. +- [x] Add boards, exhibits, exhibit types, subtype tables, immutable template versions, and mutable levels. +- [x] Cut over the explicitly disposable POC database directly; no transitional data existed to backfill or compare. - [ ] Implement template instantiation and “save level as template” as transactional clone operations. - [x] Introduce a server-side repository/service boundary so SQL and cloning transactions do not live in Express route handlers. - [ ] Move the frontend to an exhibit/widget registry backed by the normalized API. -- [ ] Remove transitional tables only after automated data-equivalence and behavior checks pass. +- [x] Remove transitional `widgets`, `widget_relations`, and `playthrough_*` tables in the canonical cutover migration. ## Milestone 3: events and parties diff --git a/docs/exhibit-data-model.md b/docs/exhibit-data-model.md index 2eca99d..e143a7e 100644 --- a/docs/exhibit-data-model.md +++ b/docs/exhibit-data-model.md @@ -1,6 +1,6 @@ # GUPI OSINT Board: canonical exhibit data model -Status: accepted design foundation. +Status: accepted design foundation; core schema implemented by migration 006. Template clone operations, parties, and the frontend exhibit registry remain roadmap work. ## Vocabulary diff --git a/e2e/board.smoke.spec.ts b/e2e/board.smoke.spec.ts index 855b7b6..1a56dd6 100644 --- a/e2e/board.smoke.spec.ts +++ b/e2e/board.smoke.spec.ts @@ -28,8 +28,8 @@ test('move, folder expansion, hand pan, pinch zoom, and reload persistence', asy await page.goto('/?level=e2e-level&edit=1') await expect(page.getByRole('heading', { name: 'Browser Safety Test' })).toBeVisible() - const folder = page.locator('[data-temporal-id="widget:e2e-folder"]') - const file = page.locator('[data-temporal-id="file:e2e-membership"]') + const folder = page.locator('[data-temporal-id="widget:11111111-1111-4111-8111-111111111111"]') + const file = page.locator('[data-temporal-id="file:contains:11111111-1111-4111-8111-111111111111:22222222-2222-4222-8222-222222222222"]') const board = page.locator('.board') const boardViewport = page.locator('.board-viewport') const containmentBand = page.locator('.folder-bands line') diff --git a/migrations/006_normalized_exhibit_schema.sql b/migrations/006_normalized_exhibit_schema.sql new file mode 100644 index 0000000..019035a --- /dev/null +++ b/migrations/006_normalized_exhibit_schema.sql @@ -0,0 +1,261 @@ +-- Canonical exhibit model. The POC data predating this migration is disposable: +-- there is deliberately no compatibility view, backfill, or playthrough overlay. +DROP TABLE IF EXISTS osint.playthrough_widget_relation_state CASCADE; +DROP TABLE IF EXISTS osint.playthrough_widget_relations CASCADE; +DROP TABLE IF EXISTS osint.playthrough_widget_state CASCADE; +DROP TABLE IF EXISTS osint.playthrough_connections CASCADE; +DROP TABLE IF EXISTS osint.playthrough_widgets CASCADE; +DROP TABLE IF EXISTS osint.playthroughs CASCADE; +DROP TABLE IF EXISTS osint.level_connections CASCADE; +DROP TABLE IF EXISTS osint.widget_regions CASCADE; +DROP TABLE IF EXISTS osint.widget_relations CASCADE; +DROP TABLE IF EXISTS osint.widgets CASCADE; +DROP TABLE IF EXISTS osint.assets CASCADE; +DROP TABLE IF EXISTS osint.levels CASCADE; +DROP TABLE IF EXISTS osint.cases CASCADE; + +CREATE TABLE osint.boards ( + id UUID PRIMARY KEY, + board_kind TEXT NOT NULL CHECK (board_kind IN ('level', 'template_version')), + revision BIGINT NOT NULL DEFAULT 0 CHECK (revision >= 0), + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); + +CREATE TABLE osint.assets ( + id UUID PRIMARY KEY, + original_name TEXT NOT NULL, + mime_type TEXT NOT NULL, + byte_size BIGINT NOT NULL CHECK (byte_size >= 0), + content BYTEA NOT NULL, + checksum_sha256 TEXT NOT NULL CHECK (checksum_sha256 ~ '^[0-9a-f]{64}$'), + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + UNIQUE (checksum_sha256, byte_size) +); + +CREATE TABLE osint.level_templates ( + id UUID PRIMARY KEY, + slug TEXT NOT NULL UNIQUE, + name TEXT NOT NULL, + current_version_id UUID, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); + +CREATE TABLE osint.level_template_versions ( + id UUID PRIMARY KEY, + template_id UUID NOT NULL REFERENCES osint.level_templates(id) ON DELETE CASCADE, + version INTEGER NOT NULL CHECK (version > 0), + board_id UUID NOT NULL UNIQUE REFERENCES osint.boards(id) ON DELETE RESTRICT, + title TEXT NOT NULL, + subtitle TEXT NOT NULL DEFAULT '', + created_from_level_id UUID, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + UNIQUE (template_id, version) +); + +CREATE TABLE osint.levels ( + id UUID PRIMARY KEY, + slug TEXT NOT NULL UNIQUE, + board_id UUID NOT NULL UNIQUE REFERENCES osint.boards(id) ON DELETE CASCADE, + source_template_version_id UUID REFERENCES osint.level_template_versions(id) ON DELETE RESTRICT, + title TEXT NOT NULL, + subtitle TEXT NOT NULL DEFAULT '', + status TEXT NOT NULL DEFAULT 'draft' CHECK (status IN ('draft', 'active', 'complete', 'archived')), + viewport_x DOUBLE PRECISION NOT NULL DEFAULT 0, + viewport_y DOUBLE PRECISION NOT NULL DEFAULT 28, + viewport_zoom DOUBLE PRECISION NOT NULL DEFAULT 0.7 CHECK (viewport_zoom > 0), + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); + +ALTER TABLE osint.level_templates + ADD CONSTRAINT level_templates_current_version_fk + FOREIGN KEY (current_version_id) REFERENCES osint.level_template_versions(id) ON DELETE SET NULL; +ALTER TABLE osint.level_template_versions + ADD CONSTRAINT level_template_versions_source_level_fk + FOREIGN KEY (created_from_level_id) REFERENCES osint.levels(id) ON DELETE SET NULL; + +CREATE TABLE osint.exhibit_types ( + id TEXT PRIMARY KEY, + name TEXT NOT NULL UNIQUE, + is_spatial BOOLEAN NOT NULL DEFAULT TRUE +); +INSERT INTO osint.exhibit_types (id, name) VALUES + ('folder', 'Folder'), ('document', 'Document'), ('note', 'Note'), ('event', 'Event'); + +CREATE TABLE osint.exhibits ( + id UUID PRIMARY KEY, + board_id UUID NOT NULL REFERENCES osint.boards(id) ON DELETE CASCADE, + exhibit_type_id TEXT NOT NULL REFERENCES osint.exhibit_types(id), + origin_exhibit_id UUID REFERENCES osint.exhibits(id) ON DELETE SET NULL, + xpos DOUBLE PRECISION NOT NULL DEFAULT 100, + ypos DOUBLE PRECISION NOT NULL DEFAULT 100, + width DOUBLE PRECISION NOT NULL DEFAULT 240 CHECK (width > 0), + height DOUBLE PRECISION NOT NULL DEFAULT 160 CHECK (height > 0), + rotation DOUBLE PRECISION NOT NULL DEFAULT 0, + z_index INTEGER NOT NULL DEFAULT 0, + hidden BOOLEAN NOT NULL DEFAULT FALSE, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + UNIQUE (board_id, id) +); +CREATE INDEX exhibits_board_idx ON osint.exhibits (board_id, z_index, created_at); + +CREATE TABLE osint.folder_exhibits ( + exhibit_id UUID PRIMARY KEY REFERENCES osint.exhibits(id) ON DELETE CASCADE, + title TEXT NOT NULL, + label_text TEXT NOT NULL DEFAULT '', + is_open BOOLEAN NOT NULL DEFAULT FALSE +); + +CREATE TABLE osint.document_types ( + id TEXT PRIMARY KEY, + name TEXT NOT NULL UNIQUE +); +INSERT INTO osint.document_types (id, name) VALUES + ('image', 'Image'), ('pdf', 'PDF'), ('web_capture', 'Web capture'), + ('email', 'Email'), ('article', 'Article'), ('filing', 'Filing'), + ('price_list', 'Price list'), ('text', 'Text'), ('file', 'Generic file'); + +CREATE TABLE osint.document_exhibits ( + exhibit_id UUID PRIMARY KEY REFERENCES osint.exhibits(id) ON DELETE CASCADE, + document_type_id TEXT NOT NULL REFERENCES osint.document_types(id), + asset_id UUID REFERENCES osint.assets(id) ON DELETE RESTRICT, + title TEXT NOT NULL, + published_at TIMESTAMPTZ, + captured_at TIMESTAMPTZ, + source_uri TEXT +); +CREATE INDEX document_exhibits_published_idx ON osint.document_exhibits (published_at); + +CREATE TABLE osint.image_documents ( + exhibit_id UUID PRIMARY KEY REFERENCES osint.document_exhibits(exhibit_id) ON DELETE CASCADE, + pixel_width INTEGER CHECK (pixel_width > 0), + pixel_height INTEGER CHECK (pixel_height > 0), + alt_text TEXT NOT NULL DEFAULT '' +); + +CREATE TABLE osint.note_exhibits ( + exhibit_id UUID PRIMARY KEY REFERENCES osint.exhibits(id) ON DELETE CASCADE, + title TEXT NOT NULL, + note_text TEXT NOT NULL DEFAULT '' +); + +CREATE TABLE osint.event_exhibits ( + exhibit_id UUID PRIMARY KEY REFERENCES osint.exhibits(id) ON DELETE CASCADE, + title TEXT NOT NULL, + narrative_text TEXT NOT NULL DEFAULT '', + occurred_at TIMESTAMPTZ NOT NULL +); + +CREATE TABLE osint.document_content_blocks ( + id UUID PRIMARY KEY, + document_exhibit_id UUID NOT NULL REFERENCES osint.document_exhibits(exhibit_id) ON DELETE CASCADE, + sort_order INTEGER NOT NULL CHECK (sort_order >= 0), + content TEXT NOT NULL, + UNIQUE (document_exhibit_id, sort_order) +); + +CREATE TABLE osint.document_regions ( + id UUID PRIMARY KEY, + document_exhibit_id UUID NOT NULL REFERENCES osint.document_exhibits(exhibit_id) ON DELETE CASCADE, + region_key TEXT NOT NULL, + label TEXT NOT NULL, + excerpt TEXT NOT NULL, + occurred_at TIMESTAMPTZ, + sort_order INTEGER NOT NULL DEFAULT 0 CHECK (sort_order >= 0), + UNIQUE (document_exhibit_id, region_key) +); + +CREATE TABLE osint.folder_memberships ( + board_id UUID NOT NULL REFERENCES osint.boards(id) ON DELETE CASCADE, + folder_exhibit_id UUID NOT NULL REFERENCES osint.folder_exhibits(exhibit_id) ON DELETE CASCADE, + child_exhibit_id UUID NOT NULL REFERENCES osint.exhibits(id) ON DELETE CASCADE, + sort_order INTEGER NOT NULL DEFAULT 0 CHECK (sort_order >= 0), + PRIMARY KEY (folder_exhibit_id, child_exhibit_id), + UNIQUE (board_id, child_exhibit_id), + CHECK (folder_exhibit_id <> child_exhibit_id) +); + +CREATE TABLE osint.event_evidence ( + board_id UUID NOT NULL REFERENCES osint.boards(id) ON DELETE CASCADE, + event_exhibit_id UUID NOT NULL REFERENCES osint.event_exhibits(exhibit_id) ON DELETE CASCADE, + evidence_exhibit_id UUID NOT NULL REFERENCES osint.exhibits(id) ON DELETE CASCADE, + sort_order INTEGER NOT NULL DEFAULT 0 CHECK (sort_order >= 0), + note TEXT, + PRIMARY KEY (event_exhibit_id, evidence_exhibit_id), + CHECK (event_exhibit_id <> evidence_exhibit_id) +); + +CREATE TABLE osint.connection_types ( + id TEXT PRIMARY KEY, + name TEXT NOT NULL UNIQUE, + directed BOOLEAN NOT NULL DEFAULT FALSE +); +INSERT INTO osint.connection_types (id, name) VALUES ('thread', 'Red thread'); + +CREATE TABLE osint.exhibit_connections ( + id UUID PRIMARY KEY, + board_id UUID NOT NULL REFERENCES osint.boards(id) ON DELETE CASCADE, + connection_type_id TEXT NOT NULL REFERENCES osint.connection_types(id), + from_exhibit_id UUID NOT NULL REFERENCES osint.exhibits(id) ON DELETE CASCADE, + to_exhibit_id UUID NOT NULL REFERENCES osint.exhibits(id) ON DELETE CASCADE, + label TEXT, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + CHECK (from_exhibit_id <> to_exhibit_id) +); + +CREATE TABLE osint.exhibit_sources ( + exhibit_id UUID PRIMARY KEY REFERENCES osint.exhibits(id) ON DELETE CASCADE, + source_document_exhibit_id UUID NOT NULL REFERENCES osint.document_exhibits(exhibit_id) ON DELETE CASCADE, + source_region_id UUID REFERENCES osint.document_regions(id) ON DELETE SET NULL +); + +CREATE TABLE osint.metadata_fields ( + id UUID PRIMARY KEY, + board_id UUID NOT NULL REFERENCES osint.boards(id) ON DELETE CASCADE, + field_key TEXT NOT NULL, + label TEXT NOT NULL, + value_type TEXT NOT NULL CHECK (value_type IN ('text', 'timestamp', 'number', 'boolean')), + UNIQUE (board_id, field_key) +); +CREATE TABLE osint.exhibit_metadata_text_values ( + exhibit_id UUID NOT NULL REFERENCES osint.exhibits(id) ON DELETE CASCADE, + field_id UUID NOT NULL REFERENCES osint.metadata_fields(id) ON DELETE CASCADE, + value TEXT NOT NULL, + PRIMARY KEY (exhibit_id, field_id) +); +CREATE TABLE osint.exhibit_metadata_timestamp_values ( + exhibit_id UUID NOT NULL REFERENCES osint.exhibits(id) ON DELETE CASCADE, + field_id UUID NOT NULL REFERENCES osint.metadata_fields(id) ON DELETE CASCADE, + value TIMESTAMPTZ NOT NULL, + PRIMARY KEY (exhibit_id, field_id) +); +CREATE TABLE osint.exhibit_metadata_number_values ( + exhibit_id UUID NOT NULL REFERENCES osint.exhibits(id) ON DELETE CASCADE, + field_id UUID NOT NULL REFERENCES osint.metadata_fields(id) ON DELETE CASCADE, + value NUMERIC NOT NULL, + PRIMARY KEY (exhibit_id, field_id) +); +CREATE TABLE osint.exhibit_metadata_boolean_values ( + exhibit_id UUID NOT NULL REFERENCES osint.exhibits(id) ON DELETE CASCADE, + field_id UUID NOT NULL REFERENCES osint.metadata_fields(id) ON DELETE CASCADE, + value BOOLEAN NOT NULL, + PRIMARY KEY (exhibit_id, field_id) +); + +-- Relationships carry board_id so cross-board references can be rejected by FKs. +ALTER TABLE osint.folder_memberships + ADD CONSTRAINT folder_membership_folder_board_fk FOREIGN KEY (board_id, folder_exhibit_id) REFERENCES osint.exhibits(board_id, id) ON DELETE CASCADE, + ADD CONSTRAINT folder_membership_child_board_fk FOREIGN KEY (board_id, child_exhibit_id) REFERENCES osint.exhibits(board_id, id) ON DELETE CASCADE; +ALTER TABLE osint.event_evidence + ADD CONSTRAINT event_evidence_event_board_fk FOREIGN KEY (board_id, event_exhibit_id) REFERENCES osint.exhibits(board_id, id) ON DELETE CASCADE, + ADD CONSTRAINT event_evidence_evidence_board_fk FOREIGN KEY (board_id, evidence_exhibit_id) REFERENCES osint.exhibits(board_id, id) ON DELETE CASCADE; +ALTER TABLE osint.exhibit_connections + ADD CONSTRAINT exhibit_connections_from_board_fk FOREIGN KEY (board_id, from_exhibit_id) REFERENCES osint.exhibits(board_id, id) ON DELETE CASCADE, + ADD CONSTRAINT exhibit_connections_to_board_fk FOREIGN KEY (board_id, to_exhibit_id) REFERENCES osint.exhibits(board_id, id) ON DELETE CASCADE; + +COMMENT ON TABLE osint.exhibits IS 'Canonical domain objects; frontend widgets are projections selected by exhibit_type_id.'; +COMMENT ON TABLE osint.level_template_versions IS 'Immutable template snapshots. Application code must clone, never update, their boards.'; +COMMENT ON TABLE osint.assets IS 'Immutable shared binary content referenced by document exhibits.'; diff --git a/server/api.integration.test.ts b/server/api.integration.test.ts index 48fd7cc..cc99e68 100644 --- a/server/api.integration.test.ts +++ b/server/api.integration.test.ts @@ -1,4 +1,5 @@ import { createServer } from 'node:net' +import { randomUUID } from 'node:crypto' import path from 'node:path' import { fileURLToPath } from 'node:url' import pg from 'pg' @@ -58,7 +59,7 @@ suite('level persistence API', () => { await adminClient.end() }) - it('persists authoring, uploads, player state, and reset behavior', async () => { + it('persists one normalized level across authoring and play views', async () => { const createResponse = await fetch(`${baseUrl}/api/levels`, { method: 'POST', headers: { 'content-type': 'application/json' }, @@ -67,9 +68,11 @@ suite('level persistence API', () => { expect(createResponse.status).toBe(201) const state = await createResponse.json() as CaseState state.viewport = { x: 91, y: -42, zoom: 0.85 } - state.documents = [{ id: 'doc-1', title: 'Evidence', kind: 'IMAGE', date: '2021-04-17', publishedAt: '2021-04-17T12:00:00Z', body: [], regions: [], fileType: 'image', metadata: {} }] - state.evidence = [{ id: 'folder-1', type: 'folder', title: 'Folder', content: 'Evidence folder', x: 685, y: 417, width: 260, config: { open: true }, containedDocumentIds: ['doc-1'] }] - state.relations = [{ id: 'membership-1', fromWidgetId: 'folder-1', toWidgetId: 'doc-1', type: 'contains', sortOrder: 0, config: { x: 1051, y: 417 } }] + const documentId = randomUUID() + const folderId = randomUUID() + state.documents = [{ id: documentId, title: 'Evidence', kind: 'IMAGE', date: '2021-04-17', publishedAt: '2021-04-17T12:00:00Z', body: [], regions: [], fileType: 'image', metadata: {} }] + state.evidence = [{ id: folderId, type: 'folder', title: 'Folder', content: 'Evidence folder', x: 685, y: 417, width: 260, config: { open: true }, containedDocumentIds: [documentId] }] + state.relations = [{ id: `contains:${folderId}:${documentId}`, fromWidgetId: folderId, toWidgetId: documentId, type: 'contains', sortOrder: 0, config: { x: 1051, y: 417 } }] const saveResponse = await fetch(`${baseUrl}/api/levels/${state.id}?edit=1`, { method: 'PUT', @@ -80,15 +83,15 @@ suite('level persistence API', () => { const loaded = await (await fetch(`${baseUrl}/api/levels/${state.id}?edit=1`)).json() as CaseState expect(loaded.viewport).toEqual(state.viewport) - expect(loaded.evidence[0]).toMatchObject({ id: 'folder-1', x: 685, y: 417, config: { open: true } }) - expect(loaded.relations[0]).toMatchObject({ id: 'membership-1', config: { x: 1051, y: 417 } }) + expect(loaded.evidence[0]).toMatchObject({ id: folderId, x: 685, y: 417, config: { open: true } }) + expect(loaded.relations[0]).toMatchObject({ id: `contains:${folderId}:${documentId}`, config: { x: 1051, y: 417 } }) const upload = new FormData() upload.append('file', new Blob(['OSINT smoke evidence'], { type: 'text/plain' }), 'smoke-evidence.txt') const uploadResponse = await fetch(`${baseUrl}/api/levels/${state.id}/documents?edit=1`, { method: 'POST', body: upload }) expect(uploadResponse.status).toBe(201) const uploaded = await uploadResponse.json() as CaseState['documents'][number] - expect(uploaded).toMatchObject({ title: 'smoke-evidence.txt', fileName: 'smoke-evidence.txt', mimeType: 'text/plain', fileType: 'file' }) + expect(uploaded).toMatchObject({ title: 'smoke-evidence.txt', fileName: 'smoke-evidence.txt', mimeType: 'text/plain', fileType: 'text' }) expect(uploaded.assetId).toBeTruthy() expect(await (await fetch(`${baseUrl}/api/assets/${uploaded.assetId}`)).text()).toBe('OSINT smoke evidence') @@ -121,12 +124,23 @@ suite('level persistence API', () => { expect(await playerSave.json()).toEqual({ ok: true, mode: 'play' }) const savedPlayerState = await (await fetch(`${baseUrl}/api/levels/${state.id}`)).json() as CaseState expect(savedPlayerState.viewport).toEqual(playerState.viewport) - expect(savedPlayerState.evidence[0]).toMatchObject({ id: 'folder-1', x: 812, y: 533 }) + expect(savedPlayerState.evidence[0]).toMatchObject({ id: folderId, x: 812, y: 533 }) + const sameLevelInEditView = await (await fetch(`${baseUrl}/api/levels/${state.id}?edit=1`)).json() as CaseState + expect(sameLevelInEditView.viewport).toEqual(playerState.viewport) + expect(sameLevelInEditView.evidence[0]).toMatchObject({ id: folderId, x: 812, y: 533 }) + + const normalized = await appPool.query<{ exhibits: string; documents: string; folders: string; memberships: string; metadata: string }>(`SELECT + (SELECT COUNT(*) FROM osint.exhibits)::text AS exhibits, + (SELECT COUNT(*) FROM osint.document_exhibits)::text AS documents, + (SELECT COUNT(*) FROM osint.folder_exhibits)::text AS folders, + (SELECT COUNT(*) FROM osint.folder_memberships)::text AS memberships, + (SELECT COUNT(*) FROM osint.exhibit_metadata_text_values)::text AS metadata`) + expect(normalized.rows[0]).toEqual({ exhibits: '3', documents: '2', folders: '1', memberships: '1', metadata: '2' }) const resetResponse = await fetch(`${baseUrl}/api/levels/${state.id}/reset`, { method: 'POST' }) expect(resetResponse.ok).toBe(true) const resetState = await resetResponse.json() as CaseState - expect(resetState.viewport).toEqual({ x: 0, y: 28, zoom: 0.7 }) - expect(resetState.evidence[0]).toMatchObject({ id: 'folder-1', x: 685, y: 417 }) + expect(resetState.viewport).toEqual(playerState.viewport) + expect(resetState.evidence[0]).toMatchObject({ id: folderId, x: 812, y: 533 }) }) }) diff --git a/server/e2eHarness.ts b/server/e2eHarness.ts index 364bf95..ca951ff 100644 --- a/server/e2eHarness.ts +++ b/server/e2eHarness.ts @@ -30,6 +30,8 @@ process.env.OSINT_MANAGED_SERVER = 'true' const { server, pool } = await import('./index.js') if (!server.listening) await once(server, 'listening') const baseUrl = `http://127.0.0.1:${port}` +const documentId = '22222222-2222-4222-8222-222222222222' +const folderId = '11111111-1111-4111-8111-111111111111' const created = await fetch(`${baseUrl}/api/levels`, { method: 'POST', @@ -39,15 +41,15 @@ const created = await fetch(`${baseUrl}/api/levels`, { if (!created.ok) throw new Error(`Could not create browser test level: ${created.status}`) const state = await created.json() as CaseState state.documents = [{ - id: 'e2e-document', title: 'Dated source image', kind: 'IMAGE', date: '2021-04-17', publishedAt: '2021-04-17T12:00:00.000Z', + id: documentId, title: 'Dated source image', kind: 'IMAGE', date: '2021-04-17', publishedAt: '2021-04-17T12:00:00.000Z', body: [], regions: [], fileType: 'image', metadata: {}, }] state.evidence = [{ - id: 'e2e-folder', type: 'folder', title: 'BROWSER TEST FOLDER', content: 'Disposable evidence', - x: 600, y: 360, width: 260, config: { open: false }, containedDocumentIds: ['e2e-document'], + id: folderId, type: 'folder', title: 'BROWSER TEST FOLDER', content: 'Disposable evidence', + x: 600, y: 360, width: 260, config: { open: false }, containedDocumentIds: [documentId], }] state.relations = [{ - id: 'e2e-membership', fromWidgetId: 'e2e-folder', toWidgetId: 'e2e-document', type: 'contains', sortOrder: 0, + id: `contains:${folderId}:${documentId}`, fromWidgetId: folderId, toWidgetId: documentId, type: 'contains', sortOrder: 0, config: { x: 980, y: 360 }, }] state.connections = [] diff --git a/server/index.ts b/server/index.ts index 2c14c51..4b49b4d 100644 --- a/server/index.ts +++ b/server/index.ts @@ -7,7 +7,7 @@ import { fileURLToPath } from 'node:url' import multer from 'multer' import pg from 'pg' import type { CaseState } from '../src/types.js' -import { createLegacyLevelRepository } from './levelRepository.js' +import { createLevelRepository } from './levelRepository.js' const { Pool } = pg const databaseUrl = process.env.DATABASE_URL @@ -18,7 +18,7 @@ if (!databaseUrl) { export const pool = new Pool({ connectionString: databaseUrl }) const editingEnabled = process.env.LEVEL_EDITING_ENABLED === 'true' -const levels = createLegacyLevelRepository(pool, editingEnabled) +const levels = createLevelRepository(pool, editingEnabled) function wantsEdit(req: express.Request) { return editingEnabled && req.query.edit === '1' diff --git a/server/levelRepository.ts b/server/levelRepository.ts index bf040cd..2d98b7f 100644 --- a/server/levelRepository.ts +++ b/server/levelRepository.ts @@ -1,20 +1,9 @@ import { createHash, randomUUID } from 'node:crypto' import type { Pool, PoolClient } from 'pg' -import type { CaseDocument, CaseState, Connection, Evidence, WidgetRelation } from '../src/types.js' +import type { CaseDocument, CaseState, Evidence, SourceFileType, WidgetRelation } from '../src/types.js' -export type UploadedDocument = { - buffer: Buffer - originalname: string - mimetype: string - size: number -} - -export type AssetRecord = { - original_name: string - mime_type: string - byte_size: string - content: Buffer -} +export type UploadedDocument = { buffer: Buffer; originalname: string; mimetype: string; size: number } +export type AssetRecord = { original_name: string; mime_type: string; byte_size: string; content: Buffer } export interface LevelRepository { listLevels(): Promise @@ -26,242 +15,272 @@ export interface LevelRepository { uploadDocument(levelId: string, file: UploadedDocument): Promise } -type WidgetRow = { - id: string; widget_type: 'document' | Evidence['type']; title: string; content: string - config: { kind?: string; date?: string; body?: string[]; fileType?: CaseDocument['fileType']; metadata?: Record; [key: string]: unknown }; source_widget_id?: string; source_region_key?: string - event_date?: string; published_at?: string; x?: number; y?: number; width?: number; sort_order: number; asset_id?: string - original_name?: string; mime_type?: string; byte_size?: number +type LevelRow = { + id: string; slug: string; board_id: string; title: string; subtitle: string; status: string + viewport_x: number; viewport_y: number; viewport_zoom: number; updated_at: Date + source_template_version_id: string | null +} +type ExhibitRow = { + id: string; exhibit_type_id: 'folder' | 'document' | 'note' | 'event'; xpos: number; ypos: number; width: number; hidden: boolean + title: string; content: string; is_open: boolean | null; document_type_id: SourceFileType | null + asset_id: string | null; published_at: Date | null; occurred_at: Date | null + original_name: string | null; mime_type: string | null; byte_size: string | null + source_document_id: string | null; source_region_key: string | null } -function isoTimestamp(value: unknown) { - if (!value) return undefined - const parsed = new Date(String(value)) - return Number.isFinite(parsed.getTime()) ? parsed.toISOString() : undefined +const uuidPattern = /^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i +function requireUuid(value: string, label: string) { + if (!uuidPattern.test(value)) throw new Error(`${label} must be a UUID`) + return value +} +function timestamp(value: string | undefined) { + if (!value) return null + const date = new Date(value) + return Number.isFinite(date.getTime()) ? date.toISOString() : null +} +function documentType(document: CaseDocument): SourceFileType { + const allowed: SourceFileType[] = ['image', 'pdf', 'web_capture', 'email', 'article', 'filing', 'price_list', 'text', 'file'] + return allowed.includes(document.fileType) ? document.fileType : 'file' +} +function documentKind(type: SourceFileType) { + return type === 'web_capture' ? 'WEB CAPTURE' : type.toUpperCase() } -export function createLegacyLevelRepository(pool: Pool, editingEnabled: boolean): LevelRepository { - async function assembleLevel(levelId: string, playthroughId = `default:${levelId}`, authorMode = false): Promise { - const levelResult = await pool.query<{ id: string; title: string; subtitle: string; status: string }>( - 'SELECT id, title, subtitle, status FROM osint.levels WHERE id = $1', [levelId], - ) - const level = levelResult.rows[0] +export function createLevelRepository(pool: Pool, editingEnabled: boolean): LevelRepository { + async function findLevel(client: Pool | PoolClient, slug: string, lock = false) { + const result = await client.query(`SELECT id, slug, board_id, title, subtitle, status, + viewport_x, viewport_y, viewport_zoom, updated_at, source_template_version_id + FROM osint.levels WHERE slug = $1${lock ? ' FOR UPDATE' : ''}`, [slug]) + return result.rows[0] || null + } + + async function assembleLevel(slug: string): Promise { + const level = await findLevel(pool, slug) if (!level) return null - - await pool.query( - `INSERT INTO osint.playthroughs (id, level_id) VALUES ($1, $2) ON CONFLICT (id) DO NOTHING`, - [playthroughId, levelId], - ) - const [widgetsResult, regionsResult, authoredConnections, authoredRelations, playthroughResult, generatedResult, playerConnections, playerRelations, stateResult, relationStateResult] = await Promise.all([ - pool.query(`SELECT w.id, w.widget_type, w.title, w.content, w.config, w.source_widget_id, w.source_region_key, - w.event_date::text, w.published_at::text, w.x, w.y, w.width, w.sort_order, w.asset_id, a.original_name, a.mime_type, a.byte_size - FROM osint.widgets w LEFT JOIN osint.assets a ON a.id = w.asset_id - WHERE w.level_id = $1 ORDER BY w.sort_order, w.id`, [levelId]), - pool.query<{ document_widget_id: string; region_key: string; label: string; excerpt: string; event_date?: string }>( - `SELECT r.document_widget_id, r.region_key, r.label, r.excerpt, r.event_date::text - FROM osint.widget_regions r JOIN osint.widgets w ON w.id = r.document_widget_id - WHERE w.level_id = $1 ORDER BY r.sort_order, r.id`, [levelId]), - pool.query<{ id: string; from_widget_id: string; to_widget_id: string }>( - 'SELECT id, from_widget_id, to_widget_id FROM osint.level_connections WHERE level_id = $1', [levelId]), - pool.query<{ id: string; from_widget_id: string; to_widget_id: string; relation_type: string; sort_order: number; config: Record }>( - `SELECT id, from_widget_id, to_widget_id, relation_type, sort_order, config FROM osint.widget_relations - WHERE level_id = $1 ORDER BY sort_order, id`, [levelId]), - pool.query<{ viewport: CaseState['viewport']; updated_at: Date }>( - 'SELECT viewport, updated_at FROM osint.playthroughs WHERE id = $1', [playthroughId]), - pool.query(`SELECT id, widget_type, title, content, config, source_widget_id, - source_region_key, event_date::text, x, y, width, 0 AS sort_order - FROM osint.playthrough_widgets WHERE playthrough_id = $1 ORDER BY created_at, id`, [playthroughId]), - pool.query<{ id: string; from_widget_id: string; to_widget_id: string }>( - 'SELECT id, from_widget_id, to_widget_id FROM osint.playthrough_connections WHERE playthrough_id = $1', [playthroughId]), - pool.query<{ id: string; from_widget_id: string; to_widget_id: string; relation_type: string; sort_order: number; config: Record }>( - `SELECT id, from_widget_id, to_widget_id, relation_type, sort_order, config FROM osint.playthrough_widget_relations - WHERE playthrough_id = $1 ORDER BY sort_order, id`, [playthroughId]), - pool.query<{ widget_id: string; x: number; y: number; width: number; hidden: boolean; config: Record }>( - 'SELECT widget_id, x, y, width, hidden, config FROM osint.playthrough_widget_state WHERE playthrough_id = $1', [playthroughId]), - pool.query<{ relation_id: string; config: Record }>( - 'SELECT relation_id, config FROM osint.playthrough_widget_relation_state WHERE playthrough_id = $1', [playthroughId]), + const [exhibitsResult, blocksResult, regionsResult, membershipsResult, connectionsResult, metadataResult] = await Promise.all([ + pool.query(`SELECT e.id, e.exhibit_type_id, e.xpos, e.ypos, e.width, e.hidden, + COALESCE(f.title, d.title, n.title, ev.title, '') AS title, + COALESCE(f.label_text, n.note_text, ev.narrative_text, '') AS content, + f.is_open, d.document_type_id, d.asset_id, d.published_at, ev.occurred_at, + a.original_name, a.mime_type, a.byte_size, + s.source_document_exhibit_id AS source_document_id, sr.region_key AS source_region_key + FROM osint.exhibits e + LEFT JOIN osint.folder_exhibits f ON f.exhibit_id = e.id + LEFT JOIN osint.document_exhibits d ON d.exhibit_id = e.id + LEFT JOIN osint.note_exhibits n ON n.exhibit_id = e.id + LEFT JOIN osint.event_exhibits ev ON ev.exhibit_id = e.id + LEFT JOIN osint.assets a ON a.id = d.asset_id + LEFT JOIN osint.exhibit_sources s ON s.exhibit_id = e.id + LEFT JOIN osint.document_regions sr ON sr.id = s.source_region_id + WHERE e.board_id = $1 ORDER BY e.z_index, e.created_at, e.id`, [level.board_id]), + pool.query<{ document_exhibit_id: string; content: string }>( + `SELECT b.document_exhibit_id, b.content FROM osint.document_content_blocks b + JOIN osint.exhibits e ON e.id = b.document_exhibit_id WHERE e.board_id = $1 ORDER BY b.document_exhibit_id, b.sort_order`, [level.board_id]), + pool.query<{ document_exhibit_id: string; region_key: string; label: string; excerpt: string; occurred_at: Date | null }>( + `SELECT r.document_exhibit_id, r.region_key, r.label, r.excerpt, r.occurred_at FROM osint.document_regions r + JOIN osint.exhibits e ON e.id = r.document_exhibit_id WHERE e.board_id = $1 ORDER BY r.document_exhibit_id, r.sort_order`, [level.board_id]), + pool.query<{ folder_exhibit_id: string; child_exhibit_id: string; sort_order: number; xpos: number; ypos: number }>( + `SELECT m.folder_exhibit_id, m.child_exhibit_id, m.sort_order, child.xpos, child.ypos + FROM osint.folder_memberships m JOIN osint.exhibits child ON child.id = m.child_exhibit_id + WHERE m.board_id = $1 ORDER BY m.sort_order, m.child_exhibit_id`, [level.board_id]), + pool.query<{ id: string; from_exhibit_id: string; to_exhibit_id: string }>( + `SELECT id, from_exhibit_id, to_exhibit_id FROM osint.exhibit_connections WHERE board_id = $1 ORDER BY created_at, id`, [level.board_id]), + pool.query<{ exhibit_id: string; field_key: string; value: string }>( + `SELECT v.exhibit_id, f.field_key, v.value FROM osint.exhibit_metadata_text_values v + JOIN osint.metadata_fields f ON f.id = v.field_id WHERE f.board_id = $1 ORDER BY f.field_key`, [level.board_id]), ]) - const stateByWidget = new Map(authorMode ? [] : stateResult.rows.map(row => [row.widget_id, row])) - const documents: CaseDocument[] = widgetsResult.rows.filter(w => w.widget_type === 'document').map(w => { const override = stateByWidget.get(w.id)?.config || {}; const publishedAt = isoTimestamp(override.publishedAt || w.published_at); return ({ - id: w.id, title: String(override.title || w.title), kind: w.config.kind || 'DOCUMENT', date: publishedAt?.slice(0, 10) || w.config.date || '', publishedAt, - body: w.config.body || [], fileType: (override.fileType || w.config.fileType || (w.mime_type?.startsWith('image/') ? 'image' : 'file')) as CaseDocument['fileType'], metadata: (override.metadata || w.config.metadata || {}) as Record, - assetId: w.asset_id, fileName: w.original_name, mimeType: w.mime_type, fileSize: w.byte_size, - regions: regionsResult.rows.filter(r => r.document_widget_id === w.id).map(r => ({ id: r.region_key, label: r.label, excerpt: r.excerpt, date: r.event_date })), - }) }) - const relationState = new Map(authorMode ? [] : relationStateResult.rows.map(row => [row.relation_id, row.config])) - const containedByFolder = new Map() - for (const relation of [...authoredRelations.rows, ...(authorMode ? [] : playerRelations.rows)]) { - if (relation.relation_type !== 'contains') continue - containedByFolder.set(relation.from_widget_id, [...(containedByFolder.get(relation.from_widget_id) || []), relation.to_widget_id]) - } - const toEvidence = (w: WidgetRow): Evidence => { - const override = stateByWidget.get(w.id) - const runtimeConfig = { ...w.config, ...(override?.config || {}) } - return { id: w.id, type: w.widget_type as Evidence['type'], title: String(override?.config?.title || w.title), content: String(override?.config?.content ?? w.content), config: runtimeConfig, - sourceDocumentId: w.source_widget_id, sourceRegionId: w.source_region_key, eventDate: w.widget_type === 'event' ? w.event_date : undefined, - containedDocumentIds: containedByFolder.get(w.id) || (w.source_widget_id ? [w.source_widget_id] : []), - x: override?.x ?? w.x ?? 100, y: override?.y ?? w.y ?? 100, width: override?.width ?? w.width ?? 240 } - } - const authoredEvidence = widgetsResult.rows.filter(w => w.widget_type !== 'document' && !stateByWidget.get(w.id)?.hidden).map(toEvidence) - const evidence = [...authoredEvidence, ...(authorMode ? [] : generatedResult.rows.map(toEvidence))] - const connections: Connection[] = [...authoredConnections.rows, ...(authorMode ? [] : playerConnections.rows)].map(c => ({ - id: c.id, fromEvidenceId: c.from_widget_id, toEvidenceId: c.to_widget_id, + const blocks = new Map() + for (const row of blocksResult.rows) blocks.set(row.document_exhibit_id, [...(blocks.get(row.document_exhibit_id) || []), row.content]) + const regions = new Map() + for (const row of regionsResult.rows) regions.set(row.document_exhibit_id, [...(regions.get(row.document_exhibit_id) || []), { + id: row.region_key, label: row.label, excerpt: row.excerpt, date: row.occurred_at?.toISOString(), + }]) + const metadata = new Map>() + for (const row of metadataResult.rows) metadata.set(row.exhibit_id, { ...(metadata.get(row.exhibit_id) || {}), [row.field_key]: row.value }) + const contained = new Map() + const relations: WidgetRelation[] = membershipsResult.rows.map(row => { + contained.set(row.folder_exhibit_id, [...(contained.get(row.folder_exhibit_id) || []), row.child_exhibit_id]) + return { id: `contains:${row.folder_exhibit_id}:${row.child_exhibit_id}`, fromWidgetId: row.folder_exhibit_id, + toWidgetId: row.child_exhibit_id, type: 'contains', sortOrder: row.sort_order, config: { x: row.xpos, y: row.ypos } } + }) + const documents: CaseDocument[] = exhibitsResult.rows.filter(row => row.exhibit_type_id === 'document').map(row => { + const type = row.document_type_id || 'file' + const publishedAt = row.published_at?.toISOString() + return { id: row.id, title: row.title, kind: documentKind(type), date: publishedAt?.slice(0, 10) || '', publishedAt, + body: blocks.get(row.id) || [], regions: regions.get(row.id) || [], assetId: row.asset_id || undefined, + fileName: row.original_name || undefined, mimeType: row.mime_type || undefined, + fileSize: row.byte_size === null ? undefined : Number(row.byte_size), fileType: type, metadata: metadata.get(row.id) || {} } + }) + const evidence: Evidence[] = exhibitsResult.rows.filter(row => row.exhibit_type_id !== 'document' && !row.hidden).map(row => ({ + id: row.id, type: row.exhibit_type_id as Evidence['type'], title: row.title, content: row.content, + sourceDocumentId: row.source_document_id || undefined, sourceRegionId: row.source_region_key || undefined, + eventDate: row.occurred_at?.toISOString(), x: row.xpos, y: row.ypos, width: row.width, + config: row.exhibit_type_id === 'folder' ? { open: Boolean(row.is_open) } : {}, containedDocumentIds: contained.get(row.id) || [], })) - const relations: WidgetRelation[] = [...authoredRelations.rows, ...(authorMode ? [] : playerRelations.rows)].map(relation => ({ - id: relation.id, fromWidgetId: relation.from_widget_id, toWidgetId: relation.to_widget_id, type: relation.relation_type, - sortOrder: relation.sort_order, config: { ...relation.config, ...(relationState.get(relation.id) || {}) }, - })) - const playthrough = playthroughResult.rows[0] - return { id: level.id, title: level.title, subtitle: level.subtitle, documents, evidence, relations, connections, - viewport: playthrough.viewport, updatedAt: playthrough.updated_at.toISOString(), levelStatus: level.status, editingAllowed: editingEnabled } + return { id: level.slug, title: level.title, subtitle: level.subtitle, documents, evidence, relations, + connections: connectionsResult.rows.map(row => ({ id: row.id, fromEvidenceId: row.from_exhibit_id, toEvidenceId: row.to_exhibit_id })), + viewport: { x: level.viewport_x, y: level.viewport_y, zoom: level.viewport_zoom }, updatedAt: level.updated_at.toISOString(), + levelStatus: level.status, editingAllowed: editingEnabled } } - async function savePlaythrough(client: PoolClient, state: CaseState) { - const playthroughId = `default:${state.id}` - const authored = await client.query<{ id: string }>('SELECT id FROM osint.widgets WHERE level_id = $1', [state.id]) - const authoredIds = new Set(authored.rows.map(row => row.id)) - const authoredRelations = await client.query<{ id: string }>('SELECT id FROM osint.widget_relations WHERE level_id = $1', [state.id]) - const authoredRelationIds = new Set(authoredRelations.rows.map(row => row.id)) - await client.query('UPDATE osint.playthroughs SET viewport = $2::jsonb, updated_at = NOW() WHERE id = $1', [playthroughId, JSON.stringify(state.viewport)]) - await client.query('DELETE FROM osint.playthrough_widget_state WHERE playthrough_id = $1', [playthroughId]) - await client.query('DELETE FROM osint.playthrough_widget_relation_state WHERE playthrough_id = $1', [playthroughId]) - await client.query('DELETE FROM osint.playthrough_widget_relations WHERE playthrough_id = $1', [playthroughId]) - await client.query('DELETE FROM osint.playthrough_widgets WHERE playthrough_id = $1', [playthroughId]) - for (const document of state.documents) { - await client.query(`INSERT INTO osint.playthrough_widget_state (playthrough_id, widget_id, x, y, width, config) - VALUES ($1,$2,0,0,0,$3::jsonb)`, [playthroughId, document.id, JSON.stringify({ title: document.title, publishedAt: document.publishedAt || null, fileType: document.fileType, metadata: document.metadata })]) - } - for (const widget of state.evidence) { - if (authoredIds.has(widget.id)) { - await client.query(`INSERT INTO osint.playthrough_widget_state (playthrough_id, widget_id, x, y, width, config) - VALUES ($1, $2, $3, $4, $5, $6::jsonb)`, [playthroughId, widget.id, widget.x, widget.y, widget.width, JSON.stringify({ ...(widget.config || {}), title: widget.title, content: widget.content })]) - } else { - await client.query(`INSERT INTO osint.playthrough_widgets - (id, playthrough_id, widget_type, title, content, config, source_widget_id, source_region_key, event_date, x, y, width) - VALUES ($1,$2,$3,$4,$5,$6::jsonb,$7,$8,$9,$10,$11,$12)`, [widget.id, playthroughId, widget.type, widget.title, widget.content, JSON.stringify(widget.config || {}), - widget.sourceDocumentId || null, widget.sourceRegionId || null, widget.eventDate || null, widget.x, widget.y, widget.width]) - } - } - const fallbackRelations: WidgetRelation[] = state.evidence.flatMap(widget => (widget.containedDocumentIds || []).map((documentId, index) => ({ id: `contains:${widget.id}:${documentId}`, fromWidgetId: widget.id, toWidgetId: documentId, type: 'contains', sortOrder: index }))) - for (const relation of (state.relations || fallbackRelations).filter(relation => authoredRelationIds.has(relation.id))) { - await client.query(`INSERT INTO osint.playthrough_widget_relation_state (playthrough_id, relation_id, config) - VALUES ($1,$2,$3::jsonb)`, [playthroughId, relation.id, JSON.stringify(relation.config || {})]) - } - for (const relation of (state.relations || fallbackRelations).filter(relation => !authoredRelationIds.has(relation.id))) { - await client.query(`INSERT INTO osint.playthrough_widget_relations - (id, playthrough_id, from_widget_id, to_widget_id, relation_type, sort_order, config) - VALUES ($1,$2,$3,$4,$5,$6,$7::jsonb)`, - [relation.id, playthroughId, relation.fromWidgetId, relation.toWidgetId, relation.type, relation.sortOrder || 0, JSON.stringify(relation.config || {})]) - } - await client.query('DELETE FROM osint.playthrough_connections WHERE playthrough_id = $1', [playthroughId]) - const authoredConnections = await client.query<{ id: string }>('SELECT id FROM osint.level_connections WHERE level_id = $1', [state.id]) - const authoredConnectionIds = new Set(authoredConnections.rows.map(row => row.id)) - for (const connection of state.connections.filter(c => !authoredConnectionIds.has(c.id))) { - await client.query(`INSERT INTO osint.playthrough_connections (id, playthrough_id, from_widget_id, to_widget_id) - VALUES ($1, $2, $3, $4)`, [connection.id, playthroughId, connection.fromEvidenceId, connection.toEvidenceId]) - } - } + async function replaceBoard(client: PoolClient, level: LevelRow, state: CaseState) { + const documentIds = new Set(state.documents.map(document => requireUuid(document.id, 'Document id'))) + const evidenceIds = new Set(state.evidence.map(exhibit => requireUuid(exhibit.id, 'Exhibit id'))) + const allIds = [...documentIds, ...evidenceIds] + if (new Set(allIds).size !== allIds.length) throw new Error('An id cannot identify both a document and another exhibit') - async function saveAuthoredLevel(client: PoolClient, state: CaseState) { - await client.query('UPDATE osint.levels SET title = $2, subtitle = $3, updated_at = NOW() WHERE id = $1', [state.id, state.title, state.subtitle]) - await client.query(`INSERT INTO osint.playthroughs (id, level_id, viewport, updated_at) - VALUES ($1, $2, $3::jsonb, NOW()) - ON CONFLICT (id) DO UPDATE SET viewport = EXCLUDED.viewport, updated_at = NOW()`, - [`default:${state.id}`, state.id, JSON.stringify(state.viewport)]) - await client.query('DELETE FROM osint.level_connections WHERE level_id = $1', [state.id]) - await client.query('DELETE FROM osint.widget_relations WHERE level_id = $1', [state.id]) - await client.query('DELETE FROM osint.widgets WHERE level_id = $1', [state.id]) - for (const [index, doc] of state.documents.entries()) { - await client.query(`INSERT INTO osint.widgets (id, level_id, widget_type, title, config, published_at, asset_id, sort_order) - VALUES ($1,$2,'document',$3,$4::jsonb,$5,$6,$7)`, [doc.id, state.id, doc.title, JSON.stringify({ kind: doc.kind, body: doc.body, fileType: doc.fileType, metadata: doc.metadata }), doc.publishedAt || doc.date || null, doc.assetId || null, index]) - for (const [regionIndex, region] of doc.regions.entries()) { - await client.query(`INSERT INTO osint.widget_regions (id, document_widget_id, region_key, label, excerpt, event_date, sort_order) - VALUES ($1,$2,$3,$4,$5,$6,$7)`, [`${doc.id}:${region.id}`, doc.id, region.id, region.label, region.excerpt, region.date || null, regionIndex]) - } + const relationList = state.relations || state.evidence.flatMap(exhibit => (exhibit.containedDocumentIds || []).map((documentId, index) => ({ + id: `contains:${exhibit.id}:${documentId}`, fromWidgetId: exhibit.id, toWidgetId: documentId, type: 'contains', sortOrder: index, + }))) + const positions = new Map() + for (const relation of relationList.filter(item => item.type === 'contains')) { + positions.set(relation.toWidgetId, { x: Number(relation.config?.x ?? 100), y: Number(relation.config?.y ?? 100) }) } - for (const [index, widget] of state.evidence.entries()) { - await client.query(`INSERT INTO osint.widgets - (id, level_id, widget_type, title, content, config, source_widget_id, source_region_key, event_date, x, y, width, sort_order) - VALUES ($1,$2,$3,$4,$5,$6::jsonb,$7,$8,$9,$10,$11,$12,$13)`, [widget.id, state.id, widget.type, widget.title, widget.content, JSON.stringify(widget.config || {}), - widget.sourceDocumentId || null, widget.sourceRegionId || null, widget.eventDate || null, widget.x, widget.y, widget.width, index]) + const existing = await client.query<{ id: string; xpos: number; ypos: number }>('SELECT id, xpos, ypos FROM osint.exhibits WHERE board_id = $1', [level.board_id]) + for (const row of existing.rows) if (!positions.has(row.id)) positions.set(row.id, { x: row.xpos, y: row.ypos }) + + await client.query(`UPDATE osint.levels SET title=$2, subtitle=$3, viewport_x=$4, viewport_y=$5, viewport_zoom=$6, + updated_at=NOW() WHERE id=$1`, [level.id, state.title, state.subtitle, state.viewport.x, state.viewport.y, state.viewport.zoom]) + await client.query('UPDATE osint.boards SET revision=revision+1, updated_at=NOW() WHERE id=$1', [level.board_id]) + await client.query('DELETE FROM osint.exhibit_connections WHERE board_id=$1', [level.board_id]) + await client.query('DELETE FROM osint.folder_memberships WHERE board_id=$1', [level.board_id]) + await client.query('DELETE FROM osint.event_evidence WHERE board_id=$1', [level.board_id]) + await client.query('DELETE FROM osint.exhibit_sources WHERE exhibit_id IN (SELECT id FROM osint.exhibits WHERE board_id=$1)', [level.board_id]) + await client.query('DELETE FROM osint.metadata_fields WHERE board_id=$1', [level.board_id]) + for (const table of ['folder_exhibits', 'image_documents', 'note_exhibits', 'event_exhibits', 'document_exhibits']) { + await client.query(`DELETE FROM osint.${table} WHERE exhibit_id IN (SELECT id FROM osint.exhibits WHERE board_id=$1)`, [level.board_id]) } - const fallbackRelations: WidgetRelation[] = state.evidence.flatMap(widget => (widget.containedDocumentIds || []).map((documentId, index) => ({ id: `contains:${widget.id}:${documentId}`, fromWidgetId: widget.id, toWidgetId: documentId, type: 'contains', sortOrder: index }))) - for (const relation of state.relations || fallbackRelations) { - await client.query(`INSERT INTO osint.widget_relations - (id, level_id, from_widget_id, to_widget_id, relation_type, sort_order, config) - VALUES ($1,$2,$3,$4,$5,$6,$7::jsonb)`, - [relation.id, state.id, relation.fromWidgetId, relation.toWidgetId, relation.type, relation.sortOrder || 0, JSON.stringify(relation.config || {})]) + if (allIds.length) await client.query('DELETE FROM osint.exhibits WHERE board_id=$1 AND NOT (id = ANY($2::uuid[]))', [level.board_id, allIds]) + else await client.query('DELETE FROM osint.exhibits WHERE board_id=$1', [level.board_id]) + + for (const [index, document] of state.documents.entries()) { + const position = positions.get(document.id) || { x: 100, y: 100 } + await client.query(`INSERT INTO osint.exhibits (id,board_id,exhibit_type_id,xpos,ypos,width,height,z_index,hidden) + VALUES ($1,$2,'document',$3,$4,174,145,$5,FALSE) + ON CONFLICT (id) DO UPDATE SET exhibit_type_id='document',xpos=$3,ypos=$4,width=174,height=145,z_index=$5,hidden=FALSE,updated_at=NOW()`, + [document.id, level.board_id, position.x, position.y, index]) + await client.query(`INSERT INTO osint.document_exhibits (exhibit_id,document_type_id,asset_id,title,published_at) + VALUES ($1,$2,$3,$4,$5)`, [document.id, documentType(document), document.assetId || null, document.title, timestamp(document.publishedAt || document.date)]) + if (document.fileType === 'image') await client.query('INSERT INTO osint.image_documents (exhibit_id) VALUES ($1)', [document.id]) + for (const [sortOrder, content] of document.body.entries()) await client.query( + 'INSERT INTO osint.document_content_blocks (id,document_exhibit_id,sort_order,content) VALUES ($1,$2,$3,$4)', [randomUUID(), document.id, sortOrder, content]) + for (const [sortOrder, region] of document.regions.entries()) await client.query( + `INSERT INTO osint.document_regions (id,document_exhibit_id,region_key,label,excerpt,occurred_at,sort_order) + VALUES ($1,$2,$3,$4,$5,$6,$7)`, [randomUUID(), document.id, region.id, region.label, region.excerpt, timestamp(region.date), sortOrder]) + } + for (const [index, exhibit] of state.evidence.entries()) { + const type = exhibit.type === 'evidence' ? 'folder' : exhibit.type + const canonicalType = type === 'folder' || type === 'note' || type === 'event' ? type : 'note' + await client.query(`INSERT INTO osint.exhibits (id,board_id,exhibit_type_id,xpos,ypos,width,height,z_index,hidden) + VALUES ($1,$2,$3,$4,$5,$6,160,$7,FALSE) + ON CONFLICT (id) DO UPDATE SET exhibit_type_id=$3,xpos=$4,ypos=$5,width=$6,height=160,z_index=$7,hidden=FALSE,updated_at=NOW()`, + [exhibit.id, level.board_id, canonicalType, exhibit.x, exhibit.y, exhibit.width, state.documents.length + index]) + if (canonicalType === 'folder') await client.query( + 'INSERT INTO osint.folder_exhibits (exhibit_id,title,label_text,is_open) VALUES ($1,$2,$3,$4)', + [exhibit.id, exhibit.title, exhibit.content, Boolean(exhibit.config?.open)]) + if (canonicalType === 'note') await client.query('INSERT INTO osint.note_exhibits (exhibit_id,title,note_text) VALUES ($1,$2,$3)', [exhibit.id, exhibit.title, exhibit.content]) + if (canonicalType === 'event') await client.query( + 'INSERT INTO osint.event_exhibits (exhibit_id,title,narrative_text,occurred_at) VALUES ($1,$2,$3,$4)', + [exhibit.id, exhibit.title, exhibit.content, timestamp(exhibit.eventDate) || new Date().toISOString()]) + } + + for (const relation of relationList.filter(item => item.type === 'contains')) { + if (!evidenceIds.has(relation.fromWidgetId) || !allIds.includes(relation.toWidgetId)) throw new Error('Folder membership references an unknown exhibit') + await client.query(`INSERT INTO osint.folder_memberships (board_id,folder_exhibit_id,child_exhibit_id,sort_order) + VALUES ($1,$2,$3,$4)`, [level.board_id, relation.fromWidgetId, relation.toWidgetId, relation.sortOrder || 0]) } for (const connection of state.connections) { - await client.query(`INSERT INTO osint.level_connections (id, level_id, from_widget_id, to_widget_id) - VALUES ($1,$2,$3,$4)`, [connection.id, state.id, connection.fromEvidenceId, connection.toEvidenceId]) + requireUuid(connection.id, 'Connection id') + if (!evidenceIds.has(connection.fromEvidenceId) || !evidenceIds.has(connection.toEvidenceId)) throw new Error('Connection references an unknown exhibit') + await client.query(`INSERT INTO osint.exhibit_connections (id,board_id,connection_type_id,from_exhibit_id,to_exhibit_id) + VALUES ($1,$2,'thread',$3,$4)`, [connection.id, level.board_id, connection.fromEvidenceId, connection.toEvidenceId]) + } + for (const exhibit of state.evidence.filter(item => item.sourceDocumentId)) { + if (!documentIds.has(exhibit.sourceDocumentId!)) throw new Error('Exhibit source references an unknown document') + let regionId: string | null = null + if (exhibit.sourceRegionId) { + const region = await client.query<{ id: string }>( + 'SELECT id FROM osint.document_regions WHERE document_exhibit_id=$1 AND region_key=$2', [exhibit.sourceDocumentId, exhibit.sourceRegionId]) + regionId = region.rows[0]?.id || null + } + await client.query('INSERT INTO osint.exhibit_sources (exhibit_id,source_document_exhibit_id,source_region_id) VALUES ($1,$2,$3)', + [exhibit.id, exhibit.sourceDocumentId, regionId]) + } + const fields = new Map() + for (const document of state.documents) for (const key of Object.keys(document.metadata || {})) { + if (!fields.has(key)) { + const fieldId = randomUUID(); fields.set(key, fieldId) + await client.query(`INSERT INTO osint.metadata_fields (id,board_id,field_key,label,value_type) VALUES ($1,$2,$3,$3,'text')`, [fieldId, level.board_id, key]) + } + await client.query('INSERT INTO osint.exhibit_metadata_text_values (exhibit_id,field_id,value) VALUES ($1,$2,$3)', + [document.id, fields.get(key), document.metadata[key]]) } } return { async listLevels() { - const result = await pool.query('SELECT id, title, subtitle, status, updated_at AS "updatedAt" FROM osint.levels ORDER BY updated_at DESC') + const result = await pool.query(`SELECT slug AS id, title, subtitle, status, updated_at AS "updatedAt" + FROM osint.levels ORDER BY updated_at DESC`) return result.rows }, async createLevel(input) { - await pool.query('INSERT INTO osint.levels (id, title, subtitle) VALUES ($1, $2, $3)', [input.id, input.title, input.subtitle]) - return (await assembleLevel(input.id, `default:${input.id}`, true))! - }, - getLevel(levelId, authorMode = false) { - return assembleLevel(levelId, `default:${levelId}`, authorMode) - }, - async saveLevel(state, authorMode) { const client = await pool.connect() try { await client.query('BEGIN') - if (authorMode) await saveAuthoredLevel(client, state) - else await savePlaythrough(client, state) + const boardId = randomUUID(); const levelId = randomUUID() + await client.query(`INSERT INTO osint.boards (id,board_kind) VALUES ($1,'level')`, [boardId]) + await client.query(`INSERT INTO osint.levels (id,slug,board_id,title,subtitle) VALUES ($1,$2,$3,$4,$5)`, + [levelId, input.id, boardId, input.title, input.subtitle]) await client.query('COMMIT') - } catch (error) { - await client.query('ROLLBACK') - throw error - } finally { client.release() } + } catch (error) { await client.query('ROLLBACK'); throw error } finally { client.release() } + return (await assembleLevel(input.id))! + }, + getLevel(levelId) { return assembleLevel(levelId) }, + async saveLevel(state) { + const client = await pool.connect() + try { + await client.query('BEGIN') + const level = await findLevel(client, state.id, true) + if (!level) throw new Error('Level not found') + await replaceBoard(client, level, state) + await client.query('COMMIT') + } catch (error) { await client.query('ROLLBACK'); throw error } finally { client.release() } }, async resetLevel(levelId) { - const client = await pool.connect() - try { - await client.query('BEGIN') - await client.query('DELETE FROM osint.playthroughs WHERE id = $1', [`default:${levelId}`]) - await client.query('COMMIT') - } catch (error) { - await client.query('ROLLBACK') - throw error - } finally { client.release() } + // A level without a source template has no earlier canonical state to restore. + // Template cloning will replace this branch when template lifecycle endpoints land. return assembleLevel(levelId) }, async getAsset(assetId) { - const result = await pool.query('SELECT original_name, mime_type, byte_size, content FROM osint.assets WHERE id = $1', [assetId]) + if (!uuidPattern.test(assetId)) return null + const result = await pool.query('SELECT original_name,mime_type,byte_size,content FROM osint.assets WHERE id=$1', [assetId]) return result.rows[0] || null }, async uploadDocument(levelId, file) { const client = await pool.connect() try { - const level = await client.query('SELECT id FROM osint.levels WHERE id = $1', [levelId]) - if (!level.rows[0]) return null - const assetId = `asset-${randomUUID()}` - const widgetId = `document-${randomUUID()}` - const checksum = createHash('sha256').update(file.buffer).digest('hex') - const kind = file.mimetype === 'application/pdf' ? 'PDF' : file.mimetype.startsWith('image/') ? 'IMAGE' : 'FILE' - const fileType: CaseDocument['fileType'] = file.mimetype.startsWith('image/') ? 'image' : file.mimetype === 'application/pdf' ? 'pdf' : 'file' await client.query('BEGIN') - await client.query(`INSERT INTO osint.assets (id, level_id, original_name, mime_type, byte_size, content, checksum_sha256) - VALUES ($1,$2,$3,$4,$5,$6,$7)`, [assetId, levelId, file.originalname, file.mimetype || 'application/octet-stream', file.size, file.buffer, checksum]) - await client.query(`INSERT INTO osint.widgets (id, level_id, widget_type, title, config, asset_id, sort_order) - VALUES ($1,$2,'document',$3,$4::jsonb,$5,(SELECT COALESCE(MAX(sort_order),-1)+1 FROM osint.widgets WHERE level_id=$2 AND widget_type='document'))`, - [widgetId, levelId, file.originalname, JSON.stringify({ kind, body: [], fileType, metadata: {} }), assetId]) - await client.query('UPDATE osint.levels SET updated_at = NOW() WHERE id = $1', [levelId]) + const level = await findLevel(client, levelId, true) + if (!level) { await client.query('ROLLBACK'); return null } + const candidateAssetId = randomUUID(); const exhibitId = randomUUID() + const checksum = createHash('sha256').update(file.buffer).digest('hex') + const asset = await client.query<{ id: string }>(`INSERT INTO osint.assets + (id,original_name,mime_type,byte_size,content,checksum_sha256) VALUES ($1,$2,$3,$4,$5,$6) + ON CONFLICT (checksum_sha256,byte_size) DO UPDATE SET checksum_sha256=EXCLUDED.checksum_sha256 RETURNING id`, + [candidateAssetId, file.originalname, file.mimetype || 'application/octet-stream', file.size, file.buffer, checksum]) + const fileType: SourceFileType = file.mimetype.startsWith('image/') ? 'image' : file.mimetype === 'application/pdf' ? 'pdf' : file.mimetype.startsWith('text/') ? 'text' : 'file' + await client.query(`INSERT INTO osint.exhibits (id,board_id,exhibit_type_id,xpos,ypos,width,height,z_index,hidden) + VALUES ($1,$2,'document',100,100,174,145,(SELECT COUNT(*) FROM osint.exhibits WHERE board_id=$2),FALSE)`, [exhibitId, level.board_id]) + await client.query(`INSERT INTO osint.document_exhibits (exhibit_id,document_type_id,asset_id,title) VALUES ($1,$2,$3,$4)`, + [exhibitId, fileType, asset.rows[0].id, file.originalname]) + if (fileType === 'image') await client.query('INSERT INTO osint.image_documents (exhibit_id) VALUES ($1)', [exhibitId]) + await client.query('UPDATE osint.levels SET updated_at=NOW() WHERE id=$1', [level.id]) + await client.query('UPDATE osint.boards SET revision=revision+1,updated_at=NOW() WHERE id=$1', [level.board_id]) await client.query('COMMIT') - return { id: widgetId, title: file.originalname, kind, fileType, metadata: {}, date: '', body: [], regions: [], assetId, - fileName: file.originalname, mimeType: file.mimetype, fileSize: file.size } - } catch (error) { - await client.query('ROLLBACK') - throw error - } finally { client.release() } + return { id: exhibitId, title: file.originalname, kind: documentKind(fileType), fileType, metadata: {}, date: '', body: [], regions: [], + assetId: asset.rows[0].id, fileName: file.originalname, mimeType: file.mimetype, fileSize: file.size } + } catch (error) { await client.query('ROLLBACK'); throw error } finally { client.release() } }, } } diff --git a/server/migrations.integration.test.ts b/server/migrations.integration.test.ts index f82f356..f63dc20 100644 --- a/server/migrations.integration.test.ts +++ b/server/migrations.integration.test.ts @@ -33,20 +33,24 @@ suite('PostgreSQL migrations', () => { const migrationsDir = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..', 'migrations') const firstRun: string[] = [] await runMigrations(testDatabaseUrl, migrationsDir, message => firstRun.push(message)) - expect(firstRun.filter(message => message.startsWith('apply '))).toHaveLength(5) + expect(firstRun.filter(message => message.startsWith('apply '))).toHaveLength(6) const client = new Client({ connectionString: testDatabaseUrl }) await client.connect() const tables = await client.query<{ table_name: string }>(`SELECT table_name FROM information_schema.tables WHERE table_schema = 'osint'`) const tableNames = tables.rows.map(row => row.table_name) - expect(tableNames).toEqual(expect.arrayContaining(['levels', 'widgets', 'widget_relations', 'playthroughs', 'assets', 'schema_migrations'])) + expect(tableNames).toEqual(expect.arrayContaining([ + 'boards', 'levels', 'level_templates', 'level_template_versions', 'exhibits', 'folder_exhibits', + 'document_exhibits', 'folder_memberships', 'exhibit_connections', 'metadata_fields', 'assets', 'schema_migrations', + ])) + expect(tableNames).not.toEqual(expect.arrayContaining(['cases', 'widgets', 'widget_relations', 'playthroughs'])) const ledger = await client.query<{ count: string }>('SELECT COUNT(*)::text AS count FROM osint.schema_migrations') - expect(ledger.rows[0].count).toBe('5') + expect(ledger.rows[0].count).toBe('6') await client.end() const secondRun: string[] = [] await runMigrations(testDatabaseUrl, migrationsDir, message => secondRun.push(message)) - expect(secondRun.filter(message => message.startsWith('skip '))).toHaveLength(5) + expect(secondRun.filter(message => message.startsWith('skip '))).toHaveLength(6) expect(secondRun.some(message => message.startsWith('apply '))).toBe(false) }) }) diff --git a/src/App.tsx b/src/App.tsx index f7b9719..355dc7c 100644 --- a/src/App.tsx +++ b/src/App.tsx @@ -11,7 +11,7 @@ const SOURCE_FILE_TYPES: { value: SourceFileType; label: string }[] = [ { value: 'price_list', label: 'Price list' }, { value: 'text', label: 'Text document' }, { value: 'file', label: 'Generic file' }, ] -function uid(prefix: string) { return `${prefix}-${Date.now()}-${Math.random().toString(36).slice(2, 7)}` } +function uid(_prefix: string) { return crypto.randomUUID() } function connectionPoint(item: Evidence) { return item.type === 'note' ? { x: item.x + 54, y: item.y + 12 } : { x: item.x + item.width / 2, y: item.y + 68 } }