diff --git a/.gitignore b/.gitignore index 7f3f076b..2b4f8c30 100644 --- a/.gitignore +++ b/.gitignore @@ -22,4 +22,7 @@ # dist build output target/ +# The runtime capture config is resolved from the repository root. +server/capture_config.json + # CodeChat Editor lexer: python. See TODO. diff --git a/capture_config.json b/capture_config.json new file mode 100644 index 00000000..574f1477 --- /dev/null +++ b/capture_config.json @@ -0,0 +1,9 @@ +{ + "host": "3.146.138.182", + "port": 5432, + "dbname": "CodeChatCaptureDB", + "user": "CodeChatCaptureUser", + "password": "OB3yc8Hk9SuVjzXMdUDr0C7w4PqLQisn", + "max_connections": 5, + "timeout_seconds": 30 +} diff --git a/extensions/VSCode/.gitignore b/extensions/VSCode/.gitignore index 8c5160c5..3780ba9f 100644 --- a/extensions/VSCode/.gitignore +++ b/extensions/VSCode/.gitignore @@ -33,5 +33,5 @@ src/index.d.ts src/index.js src/codechat-editor-client.win32-x64-msvc.node .windows/ - +*.log # CodeChat Editor lexer: python. See TODO. diff --git a/extensions/VSCode/src/extension.ts b/extensions/VSCode/src/extension.ts index fe592075..03c053bf 100644 --- a/extensions/VSCode/src/extension.ts +++ b/extensions/VSCode/src/extension.ts @@ -3,7 +3,8 @@ // This file is part of the CodeChat Editor. The CodeChat Editor is free // software: you can redistribute it and/or modify it under the terms of the GNU // General Public License as published by the Free Software Foundation, either -// version 3 of the License, or (at your option) any later version. +// version 3 of the License, or (at your option) any later version of the GNU +// General Public License. // // The CodeChat Editor is distributed in the hope that it will be useful, but // WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or @@ -52,6 +53,9 @@ import { MAX_MESSAGE_LENGTH, } from "../../../client/src/debug_enabled.mjs"; import { ResultErrTypes } from "../../../client/src/rust-types/ResultErrTypes.js"; +import * as os from "os"; + +import * as crypto from "crypto"; // Globals // ------- @@ -60,6 +64,9 @@ enum CodeChatEditorClientLocation { browser, } +// Create a unique session ID for logging +const CAPTURE_SESSION_ID = crypto.randomUUID(); + // True on Windows, false on OS X / Linux. const is_windows = process.platform === "win32"; @@ -111,12 +118,250 @@ let codeChatEditorServer: CodeChatEditorServer | undefined; initServer(ext.extensionPath); } +// --- +// +// CAPTURE (Dissertation instrumentation) +// -------------------------------------- + +function isInMarkdownCodeFence( + doc: vscode.TextDocument, + line: number, +): boolean { + // Very simple fence tracker: toggles when encountering \`\`\` or ~~~ at + // start of line. Good enough for dissertation instrumentation; refine later + // if needed. + let inFence = false; + for (let i = 0; i <= line; i++) { + const t = doc.lineAt(i).text.trim(); + if (t.startsWith("```") || t.startsWith("~~~")) { + inFence = !inFence; + } + } + return inFence; +} + +function isInRstCodeBlock(doc: vscode.TextDocument, line: number): boolean { + // Heuristic: find the most recent ".. code-block::" (or "::") and see if + // we're in its indented region. This won’t be perfect, but it’s far better + // than file-level classification. + let blockLine = -1; + for (let i = line; i >= 0; i--) { + const t = doc.lineAt(i).text; + const tt = t.trim(); + if (tt.startsWith(".. code-block::") || tt === "::") { + blockLine = i; + break; + } + // If we hit a non-indented line after searching upward too far, keep + // going; rst blocks can be separated by blank lines. + } + if (blockLine < 0) return false; + + // RST code block content usually begins after optional blank line(s), + // indented. Determine whether current line is indented relative to block + // directive line. + const cur = doc.lineAt(line).text; + if (cur.trim().length === 0) return false; + + // If it's indented at least one space/tab, treat it as inside block. + return /^\s+/.test(cur); +} + +function classifyAtPosition( + doc: vscode.TextDocument, + pos: vscode.Position, +): ActivityKind { + if (DOC_LANG_IDS.has(doc.languageId)) { + if (doc.languageId === "markdown") { + return isInMarkdownCodeFence(doc, pos.line) ? "code" : "doc"; + } + if (doc.languageId === "restructuredtext") { + return isInRstCodeBlock(doc, pos.line) ? "code" : "doc"; + } + // Other doc types: default to doc + return "doc"; + } + return "code"; +} + +// Types for sending capture events to the Rust server. This mirrors +// `CaptureEventWire` in webserver.rs. +type CaptureEventData = Record; + +interface CaptureEventPayload { + user_id: string; + assignment_id?: string; + group_id?: string; + file_path?: string; + event_type: string; + client_timestamp_ms?: number; + client_tz_offset_min?: number; + data: CaptureEventData; +} + +// TODO: replace these with something real (e.g., VS Code settings) For now, we +// hard-code to prove that the pipeline works end-to-end. +const CAPTURE_USER_ID: string = (() => { + try { + const u = os.userInfo().username; + if (u && u.trim().length > 0) { + return u.trim(); + } + } catch (_) { + // fall through + } + + // Fallbacks (should rarely be needed) + return process.env["USERNAME"] || process.env["USER"] || "unknown-user"; +})(); + +const CAPTURE_ASSIGNMENT_ID = "demo-assignment"; +const CAPTURE_GROUP_ID = "demo-group"; + +let capture_output_channel: vscode.OutputChannel | undefined; +let captureFailureLogged = false; +let captureTransportReady = false; +let extensionCaptureSessionStarted = false; + +// Simple classification of what the user is currently doing. +type ActivityKind = "doc" | "code" | "other"; + +// Language IDs that we treat as "documentation" for the dissertation metrics. +// You can refine this later if you want. +const DOC_LANG_IDS = new Set([ + "markdown", + "plaintext", + "latex", + "restructuredtext", +]); + +// Track the last activity kind and when a reflective-writing (doc) session +// started. +let lastActivityKind: ActivityKind = "other"; +let docSessionStart: number | null = null; + +// Helper to send a capture event to the Rust server. +async function sendCaptureEvent( + eventType: string, + filePath?: string, + data: CaptureEventData = {}, +): Promise { + const payload: CaptureEventPayload = { + user_id: CAPTURE_USER_ID, + assignment_id: CAPTURE_ASSIGNMENT_ID, + group_id: CAPTURE_GROUP_ID, + file_path: filePath, + event_type: eventType, + client_timestamp_ms: Date.now(), + client_tz_offset_min: new Date().getTimezoneOffset(), + data: { + ...data, + session_id: CAPTURE_SESSION_ID, + }, + }; + + if (codeChatEditorServer === undefined) { + reportCaptureFailure("CodeChat server is not running"); + return; + } + if (!captureTransportReady) { + capture_output_channel?.appendLine( + `${new Date().toISOString()} capture skipped before server handshake: ${JSON.stringify(payload)}`, + ); + return; + } + + logCaptureEvent(payload); + + try { + await codeChatEditorServer.sendCaptureEvent(JSON.stringify(payload)); + captureFailureLogged = false; + } catch (err) { + reportCaptureFailure(err instanceof Error ? err.message : String(err)); + } +} + +function logCaptureEvent(payload: CaptureEventPayload) { + capture_output_channel?.appendLine( + `${new Date().toISOString()} ${JSON.stringify(payload)}`, + ); +} + +function reportCaptureFailure(message: string) { + capture_output_channel?.appendLine( + `${new Date().toISOString()} capture send failed: ${message}`, + ); + if (captureFailureLogged) { + return; + } + captureFailureLogged = true; + console.warn(`CodeChat capture event was not queued: ${message}`); +} + +async function startExtensionCaptureSession(filePath?: string) { + if (extensionCaptureSessionStarted) { + return; + } + extensionCaptureSessionStarted = true; + await sendCaptureEvent("session_start", filePath, { + mode: "vscode_extension", + }); +} + +// Update activity state, emit switch + doc\_session events as needed. +function noteActivity(kind: ActivityKind, filePath?: string) { + const now = Date.now(); + + // Handle entering / leaving a "doc" session. + if (kind === "doc") { + if (docSessionStart === null) { + // Starting a new reflective-writing session. + docSessionStart = now; + void sendCaptureEvent("session_start", filePath, { + mode: "doc", + }); + } + } else { + if (docSessionStart !== null) { + // Ending a reflective-writing session. + const durationMs = now - docSessionStart; + docSessionStart = null; + void sendCaptureEvent("doc_session", filePath, { + duration_ms: durationMs, + duration_seconds: durationMs / 1000.0, + }); + void sendCaptureEvent("session_end", filePath, { + mode: "doc", + }); + } + } + + // If we switched between doc and code, log a switch\_pane event. + const docOrCode = (k: ActivityKind) => k === "doc" || k === "code"; + if ( + docOrCode(lastActivityKind) && + docOrCode(kind) && + kind !== lastActivityKind + ) { + void sendCaptureEvent("switch_pane", filePath, { + from: lastActivityKind, + to: kind, + }); + } + + lastActivityKind = kind; +} + // Activation/deactivation // ----------------------- // // This is invoked when the extension is activated. It either creates a new // CodeChat Editor Server instance or reveals the currently running one. export const activate = (context: vscode.ExtensionContext) => { + capture_output_channel = + vscode.window.createOutputChannel("CodeChat Capture"); + context.subscriptions.push(capture_output_channel); + context.subscriptions.push( vscode.commands.registerCommand( "extension.codeChatEditorDeactivate", @@ -148,6 +393,20 @@ export const activate = (context: vscode.ExtensionContext) => { event.reason }, ${format_struct(event.contentChanges)}.`, ); + + // CAPTURE: update session/switch state. The server + // classifies write_* events after parsing. + const doc = event.document; + const firstChange = event.contentChanges[0]; + const pos = firstChange.range.start; + const kind = classifyAtPosition(doc, pos); + + const filePath = doc.fileName; + + // Update our notion of current activity + doc + // session. + noteActivity(kind, filePath); + send_update(true); }), ); @@ -172,24 +431,112 @@ export const activate = (context: vscode.ExtensionContext) => { ) { return; } + + // CAPTURE: update activity + possible + // switch\_pane/doc\_session. + const doc = event.document; + const pos = + event.selection?.active ?? + new vscode.Position(0, 0); + const kind = classifyAtPosition(doc, pos); + + const filePath = doc.fileName; + noteActivity(kind, filePath); + send_update(true); }), ); context.subscriptions.push( vscode.window.onDidChangeTextEditorSelection( - (_event) => { + (event) => { if (ignore_selection_change) { ignore_selection_change = false; return; } + console_log( "CodeChat Editor extension: sending updated cursor/scroll position.", ); + + // CAPTURE: treat a selection change as "activity" + // in this document. + const doc = event.textEditor.document; + const pos = + event.selections?.[0]?.active ?? + event.textEditor.selection.active; + const kind = classifyAtPosition(doc, pos); + const filePath = doc.fileName; + noteActivity(kind, filePath); + send_update(false); }, ), ); + + // CAPTURE: end of a debug/run session. + context.subscriptions.push( + vscode.debug.onDidTerminateDebugSession((session) => { + const active = vscode.window.activeTextEditor; + const filePath = active?.document.fileName; + void sendCaptureEvent("run_end", filePath, { + sessionName: session.name, + sessionType: session.type, + }); + }), + ); + + // CAPTURE: compile/build end events via VS Code tasks. + context.subscriptions.push( + vscode.tasks.onDidEndTaskProcess((e) => { + const active = vscode.window.activeTextEditor; + const filePath = active?.document.fileName; + const task = e.execution.task; + void sendCaptureEvent("compile_end", filePath, { + taskName: task.name, + taskSource: task.source, + exitCode: e.exitCode, + }); + }), + ); + + // CAPTURE: listen for file saves. + context.subscriptions.push( + vscode.workspace.onDidSaveTextDocument((doc) => { + void sendCaptureEvent("save", doc.fileName, { + reason: "manual_save", + languageId: doc.languageId, + lineCount: doc.lineCount, + }); + }), + ); + + // CAPTURE: start of a debug/run session. + context.subscriptions.push( + vscode.debug.onDidStartDebugSession((session) => { + const active = vscode.window.activeTextEditor; + const filePath = active?.document.fileName; + void sendCaptureEvent("run", filePath, { + sessionName: session.name, + sessionType: session.type, + }); + }), + ); + + // CAPTURE: compile/build events via VS Code tasks. + context.subscriptions.push( + vscode.tasks.onDidStartTaskProcess((e) => { + const active = vscode.window.activeTextEditor; + const filePath = active?.document.fileName; + const task = e.execution.task; + void sendCaptureEvent("compile", filePath, { + taskName: task.name, + taskSource: task.source, + definition: task.definition, + processId: e.processId, + }); + }), + ); } // Get the CodeChat Client's location from the VSCode @@ -220,40 +567,24 @@ export const activate = (context: vscode.ExtensionContext) => { CodeChatEditorClientLocation.html ) { if (webview_panel !== undefined) { - // As below, don't take the focus when revealing. webview_panel.reveal(undefined, true); } else { - // Create a webview panel. webview_panel = vscode.window.createWebviewPanel( "CodeChat Editor", "CodeChat Editor", { - // Without this, the focus becomes this webview; - // setting this allows the code window open - // before this command was executed to retain - // the focus and be immediately rendered. preserveFocus: true, - // Put this in the a column beside the current - // column. viewColumn: vscode.ViewColumn.Beside, }, - // See - // [WebViewOptions](https://code.visualstudio.com/api/references/vscode-api#WebviewOptions). { enableScripts: true, - // Without this, the websocket connection is - // dropped when the panel is hidden. retainContextWhenHidden: true, }, ); webview_panel.onDidDispose(async () => { - // Shut down the render client when the webview - // panel closes. console_log( "CodeChat Editor extension: shut down webview.", ); - // Closing the webview abruptly closes the Client, - // which produces an error. Don't report it. quiet_next_error = true; webview_panel = undefined; await stop_client(); @@ -261,11 +592,9 @@ export const activate = (context: vscode.ExtensionContext) => { } } - // Provide a simple status display while the CodeChat Editor - // Server is starting up. + // Provide a simple status display while the server is starting + // up. if (webview_panel !== undefined) { - // If we have an ID, then the GUI is already running; don't - // replace it. webview_panel.webview.html = "

CodeChat Editor

Loading...

"; } else { @@ -277,6 +606,9 @@ export const activate = (context: vscode.ExtensionContext) => { // Start the server. console_log("CodeChat Editor extension: starting server."); codeChatEditorServer = new CodeChatEditorServer(); + captureFailureLogged = false; + captureTransportReady = false; + extensionCaptureSessionStarted = false; const hosted_in_ide = codechat_client_location === @@ -285,13 +617,16 @@ export const activate = (context: vscode.ExtensionContext) => { `CodeChat Editor extension: sending message Opened(${hosted_in_ide}).`, ); await codeChatEditorServer.sendMessageOpened(hosted_in_ide); - // For the external browser, we can immediately send the - // `CurrentFile` message. For the WebView, we must first wait to - // receive the HTML for the WebView (the `ClientHtml` message). + if ( codechat_client_location === CodeChatEditorClientLocation.browser ) { + captureTransportReady = true; + const active = vscode.window.activeTextEditor; + void startExtensionCaptureSession( + active?.document.fileName, + ); send_update(false); } @@ -301,7 +636,7 @@ export const activate = (context: vscode.ExtensionContext) => { console_log("CodeChat Editor extension: queue closed."); break; } - // Parse the data into a message. + const { id, message } = JSON.parse( message_raw, ) as EditorMessage; @@ -321,7 +656,6 @@ export const activate = (context: vscode.ExtensionContext) => { keys[0] as KeysOfRustEnum; const value = Object.values(message)[0]; - // Process this message. switch (key) { case "Update": { const current_update = @@ -335,17 +669,12 @@ export const activate = (context: vscode.ExtensionContext) => { } if (current_update.contents !== undefined) { const source = current_update.contents.source; - // This will produce a change event, which we'll - // ignore. The change may also produce a - // selection change, which should also be - // ignored. + ignore_text_document_change = true; ignore_selection_change = true; - // Use a workspace edit, since calls to - // `TextEditor.edit` must be made to the active - // editor only. + const wse = new vscode.WorkspaceEdit(); - // Is this plain text, or a diff? + if ("Plain" in source) { wse.replace( doc.uri, @@ -361,8 +690,7 @@ export const activate = (context: vscode.ExtensionContext) => { ); } else { assert("Diff" in source); - // If this diff was not made against the - // text we currently have, reject it. + if (source.Diff.version !== version) { await sendResult(id, { OutOfSync: [ @@ -380,20 +708,14 @@ export const activate = (context: vscode.ExtensionContext) => { } const diffs = source.Diff.doc; for (const diff of diffs) { - // Convert from character offsets from the - // beginning of the document to a - // `Position` (line, then offset on that - // line) needed by VSCode. const from = doc.positionAt(diff.from); if (diff.to === undefined) { - // This is an insert. wse.insert( doc.uri, from, diff.insert, ); } else { - // This is a replace or delete. const to = doc.positionAt(diff.to); wse.replace( doc.uri, @@ -407,19 +729,16 @@ export const activate = (context: vscode.ExtensionContext) => { ignore_text_document_change = false; ignore_selection_change = false; - // Now that we've updated our text, update the - // associated version as well. version = current_update.contents.version; } - // Update the cursor and scroll position if - // provided. const editor = get_text_editor(doc); + const scroll_line = current_update.scroll_position; if (scroll_line !== undefined && editor) { - // Don't set `ignore_scroll_position` here, - // since `revealRange` doesn't change the - // editor's text selection. + // Don't set `ignore_selection_change` here: + // `revealRange` doesn't change the editor's + // text selection. const scroll_position = new vscode.Position( // The VSCode line is zero-based; the // CodeMirror line is one-based. @@ -431,8 +750,6 @@ export const activate = (context: vscode.ExtensionContext) => { scroll_position, scroll_position, ), - // This is still not the top of the - // viewport, but a bit below it. TextEditorRevealType.AtTop, ); } @@ -441,8 +758,6 @@ export const activate = (context: vscode.ExtensionContext) => { if (cursor_line !== undefined && editor) { ignore_selection_change = true; const cursor_position = new vscode.Position( - // The VSCode line is zero-based; the - // CodeMirror line is one-based. cursor_line - 1, 0, ); @@ -505,19 +820,13 @@ export const activate = (context: vscode.ExtensionContext) => { .executeCommand( "vscode.open", vscode.Uri.file(current_file), - { - viewColumn: - current_editor?.viewColumn, - }, + { viewColumn: current_editor?.viewColumn }, ) .then( async () => await sendResult(id), async (reason) => await sendResult(id, { - OpenFileFailed: [ - current_file, - reason, - ], + OpenFileFailed: [current_file, reason], }), ); */ @@ -527,7 +836,6 @@ export const activate = (context: vscode.ExtensionContext) => { } case "Result": { - // Report if this was an error. const result_contents = value as MessageResult; if ("Err" in result_contents) { const err = result_contents["Err"]; @@ -551,7 +859,7 @@ export const activate = (context: vscode.ExtensionContext) => { } case "LoadFile": { - const [load_file, is_client_current] = value as [ + const [load_file, is_current] = value as [ string, boolean, ]; @@ -561,9 +869,7 @@ export const activate = (context: vscode.ExtensionContext) => { // If we have this file and the request is for the // current file to edit/view in the Client, assign a // version. - const is_current_ide = - doc !== undefined && is_client_current; - if (is_current_ide) { + if (doc !== undefined && is_current) { version = rand(); } const load_file_result: null | [string, number] = @@ -577,12 +883,6 @@ export const activate = (context: vscode.ExtensionContext) => { id, load_file_result, ); - // If this is the currently active file in VSCode, - // send its cursor location that VSCode - // automatically restores. - if (is_current_ide) { - send_update(false); - } break; } @@ -591,17 +891,18 @@ export const activate = (context: vscode.ExtensionContext) => { assert(webview_panel !== undefined); webview_panel.webview.html = client_html; await sendResult(id); - // Now that the Client is loaded, send the editor's - // current file to the server. + captureTransportReady = true; + const active = vscode.window.activeTextEditor; + void startExtensionCaptureSession( + active?.document.fileName, + ); send_update(false); break; } default: console.error( - `Unhandled message ${key}(${format_struct( - value, - )}`, + `Unhandled message ${key}(${format_struct(value)}`, ); break; } @@ -614,6 +915,34 @@ export const activate = (context: vscode.ExtensionContext) => { // On deactivation, close everything down. export const deactivate = async () => { console_log("CodeChat Editor extension: deactivating."); + + // CAPTURE: if we were in a doc session, close it out so duration is + // recorded. + if (docSessionStart !== null) { + const now = Date.now(); + const durationMs = now - docSessionStart; + docSessionStart = null; + const active = vscode.window.activeTextEditor; + const filePath = active?.document.fileName; + + await sendCaptureEvent("doc_session", filePath, { + duration_ms: durationMs, + duration_seconds: durationMs / 1000.0, + closed_by: "extension_deactivate", + }); + await sendCaptureEvent("session_end", filePath, { + mode: "doc", + closed_by: "extension_deactivate", + }); + } + + // CAPTURE: mark the end of an editor session. + const active = vscode.window.activeTextEditor; + const endFilePath = active?.document.fileName; + await sendCaptureEvent("session_end", endFilePath, { + mode: "vscode_extension", + }); + await stop_client(); webview_panel?.dispose(); console_log("CodeChat Editor extension: deactivated."); @@ -636,7 +965,9 @@ const format_struct = (complex_data_structure: any): string => const sendResult = async (id: number, result?: ResultErrTypes) => { assert(codeChatEditorServer); console_log( - `CodeChat Editor extension: sending Result(id = ${id}, ${format_struct(result)}).`, + `CodeChat Editor extension: sending Result(id = ${id}, ${format_struct( + result, + )}).`, ); try { await codeChatEditorServer.sendResult( @@ -654,18 +985,13 @@ const sendResult = async (id: number, result?: ResultErrTypes) => { const send_update = (this_is_dirty: boolean) => { is_dirty ||= this_is_dirty; if (can_render()) { - // Render after some inactivity: cancel any existing timer, then ... if (idle_timer !== undefined) { clearTimeout(idle_timer); } - // ... schedule a render after an autosave timeout. idle_timer = setTimeout(async () => { if (can_render()) { const ate = vscode.window.activeTextEditor; if (ate !== undefined && ate !== current_editor) { - // Send a new current file after a short delay; this allows - // the user to rapidly cycle through several editors without - // needing to reload the Client with each cycle. current_editor = ate; const current_file = ate.document.fileName; console_log( @@ -678,32 +1004,25 @@ const send_update = (this_is_dirty: boolean) => { } catch (e) { show_error(`Error sending CurrentFile message: ${e}.`); } - // Since we just requested a new file, the contents are - // clean by definition. is_dirty = false; - // Don't send an updated cursor position until this file is - // loaded. return; } - // The - // [Position](https://code.visualstudio.com/api/references/vscode-api#Position) - // encodes the line as a zero-based value. In contrast, - // CodeMirror - // [Text.line](https://codemirror.net/docs/ref/#state.Text.line) - // is 1-based. const cursor_position = current_editor!.selection.active.line + 1; const scroll_position = current_editor!.visibleRanges[0].start.line + 1; const file_path = current_editor!.document.fileName; - // Send contents only if necessary. + const option_contents: null | [string, number] = is_dirty ? [current_editor!.document.getText(), (version = rand())] : null; is_dirty = false; + console_log( - `CodeChat Editor extension: sending Update(${file_path}, ${cursor_position}, ${scroll_position}, ${format_struct(option_contents)})`, + `CodeChat Editor extension: sending Update(${file_path}, ${cursor_position}, ${scroll_position}, ${format_struct( + option_contents, + )})`, ); await codeChatEditorServer!.sendMessageUpdatePlain( file_path, @@ -725,9 +1044,8 @@ const stop_client = async () => { await codeChatEditorServer.stopServer(); codeChatEditorServer = undefined; } + captureTransportReady = false; - // Shut the timer down after the client is undefined, to ensure it can't be - // started again by a call to `start_render()`. if (idle_timer !== undefined) { clearTimeout(idle_timer); idle_timer = undefined; @@ -744,7 +1062,6 @@ const show_error = (message: string) => { } console.error(`CodeChat Editor extension: ${message}`); if (webview_panel !== undefined) { - // If the panel was displaying other content, reset it for errors. if ( !webview_panel.webview.html.startsWith("

CodeChat Editor

") ) { @@ -767,31 +1084,13 @@ const can_render = () => { (vscode.window.activeTextEditor !== undefined || current_editor !== undefined) && codeChatEditorServer !== undefined && - // TODO: I don't think these matter -- the Server is in charge of - // sending output to the Client. (codechat_client_location === CodeChatEditorClientLocation.browser || webview_panel !== undefined) ); }; const get_document = (file_path: string) => { - // Look through all open documents to see if we have the requested file. for (const doc of vscode.workspace.textDocuments) { - // Make the possibly incorrect assumption that only Windows filesystems - // are case-insensitive; I don't know how to easily determine the - // case-sensitivity of the current filesystem without extra probing code - // (write a file in mixed case, try to open it in another mixed case.) - // Per - // [How to Work with Different Filesystems](https://nodejs.org/en/learn/manipulating-files/working-with-different-filesystems#filesystem-behavior), - // "Be wary of inferring filesystem behavior from `process.platform`. - // For example, do not assume that because your program is running on - // Darwin that you are therefore working on a case-insensitive - // filesystem (HFS+), as the user may be using a case-sensitive - // filesystem (HFSX)." - // - // The same article - // [recommends](https://nodejs.org/en/learn/manipulating-files/working-with-different-filesystems#be-prepared-for-slight-differences-in-comparison-functions) - // using `toUpperCase` for case-insensitive filename comparisons. if ( (!is_windows && doc.fileName === file_path) || (is_windows && diff --git a/extensions/VSCode/src/lib.rs b/extensions/VSCode/src/lib.rs index ceec586a..e94472c3 100644 --- a/extensions/VSCode/src/lib.rs +++ b/extensions/VSCode/src/lib.rs @@ -80,6 +80,13 @@ impl CodeChatEditorServer { self.0.send_message_opened(hosted_in_ide).await } + #[napi] + pub async fn send_capture_event(&self, capture_event_json: String) -> std::io::Result { + let capture_event = serde_json::from_str(&capture_event_json) + .map_err(|err| std::io::Error::other(err.to_string()))?; + self.0.send_capture_event(capture_event).await + } + #[napi] pub async fn send_message_current_file(&self, url: String) -> std::io::Result { self.0.send_message_current_file(url).await diff --git a/server/Cargo.lock b/server/Cargo.lock index f93653c3..dd33246d 100644 --- a/server/Cargo.lock +++ b/server/Cargo.lock @@ -3368,9 +3368,9 @@ checksum = "f87165f0995f63a9fbeea62b64d10b4d9d8e78ec6d7d51fb2125fda7bb36788f" [[package]] name = "rustls-webpki" -version = "0.103.12" +version = "0.103.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8279bb85272c9f10811ae6a6c547ff594d6a7f3c6c6b02ee9726d1d0dcfcdd06" +checksum = "61c429a8649f110dddef65e2a5ad240f747e85f7758a6bccc7e5777bd33f756e" dependencies = [ "aws-lc-rs", "ring", diff --git a/server/log4rs.yml b/server/log4rs.yml index 544068f2..d534ba2a 100644 --- a/server/log4rs.yml +++ b/server/log4rs.yml @@ -40,7 +40,7 @@ loggers: level: warn root: - level: info + level: debug appenders: - console_appender - file_appender \ No newline at end of file diff --git a/server/src/capture.rs b/server/src/capture.rs index 3f8f7c15..c8904a75 100644 --- a/server/src/capture.rs +++ b/server/src/capture.rs @@ -13,227 +13,601 @@ // You should have received a copy of the GNU General Public License along with // the CodeChat Editor. If not, see // [http://www.gnu.org/licenses](http://www.gnu.org/licenses). -/// # `Capture.rs` -- Capture CodeChat Editor Events -// ## Submodules + +// `capture.rs` -- Capture CodeChat Editor Events +// ============================================================================ +// +// This module provides an asynchronous event capture facility backed by a +// PostgreSQL database. It is designed to support the dissertation study by +// recording process-level data such as: +// +// * Frequency and timing of writing entries +// * Edits to documentation and code +// * Switches between documentation and coding activity +// * Duration of engagement with reflective writing +// * Save, compile, and run events +// +// Events are sent from the client (browser and/or VS Code extension) to the +// server as JSON. The server enqueues events into an asynchronous worker which +// performs batched inserts into the `events` table. +// +// Database schema +// ---------------------------------------------------------------------------- // -// ## Imports +// The following SQL statement creates the `events` table used by this module: // -// Standard library -use indoc::indoc; -use std::fs; -use std::io; -use std::path::Path; -use std::sync::Arc; - -// Third-party -use chrono::Local; -use log::{error, info}; +// ```sql +// CREATE TABLE events ( +// id SERIAL PRIMARY KEY, +// user_id TEXT NOT NULL, +// assignment_id TEXT, +// group_id TEXT, +// file_path TEXT, +// event_type TEXT NOT NULL, +// timestamp TEXT NOT NULL, +// data TEXT +// ); +// ``` +// +// * `user_id` – participant identifier (student id, pseudonym, etc.). +// * `assignment_id` – logical assignment / lab identifier. +// * `group_id` – optional grouping (treatment / comparison, section). +// * `file_path` – logical path of the file being edited. +// * `event_type` – coarse event type (see `event_type` constants below). +// * `timestamp` – RFC3339 timestamp (in UTC). +// * `data` – JSON payload with event-specific details. + +use std::{io, thread}; + +use chrono::{DateTime, Utc}; +use log::{debug, error, info, warn}; use serde::{Deserialize, Serialize}; -use tokio::sync::Mutex; +use std::error::Error; +use tokio::sync::mpsc; use tokio_postgres::{Client, NoTls}; -// Local - -/* ## The Event Structure: - - The `Event` struct represents an event to be stored in the database. - - Fields: - `user_id`: The ID of the user associated with the event. - - `event_type`: The type of event (e.g., "keystroke", "file_open"). - `data`: - Optional additional data associated with the event. +/// Canonical event type strings. Keep these stable for analysis. +pub mod event_types { + pub const WRITE_DOC: &str = "write_doc"; + pub const WRITE_CODE: &str = "write_code"; + pub const SWITCH_PANE: &str = "switch_pane"; + pub const DOC_SESSION: &str = "doc_session"; // duration of reflective writing + pub const SAVE: &str = "save"; + pub const COMPILE: &str = "compile"; + pub const RUN: &str = "run"; + pub const SESSION_START: &str = "session_start"; + pub const SESSION_END: &str = "session_end"; + pub const COMPILE_END: &str = "compile_end"; + pub const RUN_END: &str = "run_end"; +} - ### Example +/// Configuration used to construct the PostgreSQL connection string. +/// +/// You can populate this from a JSON file or environment variables in +/// `main.rs`; this module stays agnostic. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct CaptureConfig { + pub host: String, + pub user: String, + pub password: String, + pub dbname: String, + /// Optional: application-level identifier for this deployment (e.g., course + /// code or semester). Not stored in the DB directly; callers can embed this + /// in `data` if desired. + #[serde(default)] + pub app_id: Option, +} - let event = Event { user_id: "user123".to_string(), event_type: - "keystroke".to_string(), data: Some("Pressed key A".to_string()), }; -*/ +impl CaptureConfig { + /// Build a libpq-style connection string. + pub fn to_conn_str(&self) -> String { + format!( + "host={} user={} password={} dbname={}", + self.host, self.user, self.password, self.dbname + ) + } +} -#[derive(Deserialize, Debug)] -pub struct Event { +/// The in-memory representation of a single capture event. +#[derive(Debug, Clone)] +pub struct CaptureEvent { pub user_id: String, + pub assignment_id: Option, + pub group_id: Option, + pub file_path: Option, pub event_type: String, - pub data: Option, + /// When the event occurred, in UTC. + pub timestamp: DateTime, + /// Event-specific payload, stored as JSON text in the DB. + pub data: serde_json::Value, } -/* - ## The Config Structure: - - The `Config` struct represents the database connection parameters read from - `config.json`. - - Fields: - `db_host`: The hostname or IP address of the database server. - - `db_user`: The username for the database connection. - `db_password`: The - password for the database connection. - `db_name`: The name of the database. - - let config = Config { db_host: "localhost".to_string(), db_user: - "your_db_user".to_string(), db_password: "your_db_password".to_string(), - db_name: "your_db_name".to_string(), }; -*/ +impl CaptureEvent { + /// Convenience constructor when the caller already has a timestamp. + pub fn new( + user_id: String, + assignment_id: Option, + group_id: Option, + file_path: Option, + event_type: impl Into, + timestamp: DateTime, + data: serde_json::Value, + ) -> Self { + Self { + user_id, + assignment_id, + group_id, + file_path, + event_type: event_type.into(), + timestamp, + data, + } + } -#[derive(Deserialize, Serialize, Debug)] -pub struct Config { - pub db_ip: String, - pub db_user: String, - pub db_password: String, - pub db_name: String, + /// Convenience constructor which uses the current time. + pub fn now( + user_id: String, + assignment_id: Option, + group_id: Option, + file_path: Option, + event_type: impl Into, + data: serde_json::Value, + ) -> Self { + Self::new( + user_id, + assignment_id, + group_id, + file_path, + event_type, + Utc::now(), + data, + ) + } } -/* +/// Internal worker message. Identical to `CaptureEvent`, but separated in case +/// we later want to add batching / flush control signals. +type WorkerMsg = CaptureEvent; - ## The EventCapture Structure: - - The `EventCapture` struct provides methods to interact with the database. It -holds a `tokio_postgres::Client` for database operations. +/// Handle used by the rest of the server to record events. +/// +/// Cloning this handle is cheap: it only clones an `mpsc::UnboundedSender`. +#[derive(Clone)] +pub struct EventCapture { + tx: mpsc::UnboundedSender, +} -### Usage Example +impl EventCapture { + /// Create a new `EventCapture` instance and spawn a background worker which + /// consumes events and inserts them into PostgreSQL. + /// + /// This function is synchronous so it can be called from non-async server + /// setup code. It spawns an async task internally which performs the + /// database connection and event processing. + pub fn new(config: CaptureConfig) -> Result { + let conn_str = config.to_conn_str(); + + // High-level DB connection details (no password). + info!( + "Capture: preparing PostgreSQL connection (host={}, dbname={}, user={}, app_id={:?})", + config.host, config.dbname, config.user, config.app_id + ); + debug!("Capture: raw PostgreSQL connection string: {}", conn_str); + + let (tx, mut rx) = mpsc::unbounded_channel::(); + + // Create a dedicated runtime so capture can be started from sync code + // before the Actix/Tokio server runtime exists. + thread::Builder::new() + .name("codechat-capture".to_string()) + .spawn(move || { + let runtime = tokio::runtime::Builder::new_multi_thread() + .worker_threads(1) + .enable_all() + .build() + .expect("Capture: failed to build Tokio runtime"); + + runtime.block_on(async move { + info!("Capture: attempting to connect to PostgreSQL…"); + + match tokio_postgres::connect(&conn_str, NoTls).await { + Ok((client, connection)) => { + info!("Capture: successfully connected to PostgreSQL."); + + // Drive the connection in its own task. + tokio::spawn(async move { + if let Err(err) = connection.await { + error!("Capture PostgreSQL connection error: {err}"); + } + }); + + // Main event loop: pull events off the channel and insert + // them into the database. + while let Some(event) = rx.recv().await { + debug!( + "Capture: inserting event: type={}, user_id={}, assignment_id={:?}, group_id={:?}, file_path={:?}", + event.event_type, + event.user_id, + event.assignment_id, + event.group_id, + event.file_path + ); + + if let Err(err) = insert_event(&client, &event).await { + error!( + "Capture: FAILED to insert event (type={}, user_id={}): {err}", + event.event_type, event.user_id + ); + } else { + debug!("Capture: event insert successful."); + } + } + + info!("Capture: event channel closed; background worker exiting."); + } + + Err(err) => { + let ctx = format!( + "Capture: FAILED to connect to PostgreSQL (host={}, dbname={}, user={})", + config.host, config.dbname, config.user + ); + + log_pg_connect_error(&ctx, &err); + + // Drain and drop any events so we don't hold the sender. + warn!("Capture: draining pending events after failed DB connection."); + while rx.recv().await.is_some() {} + warn!("Capture: all pending events dropped due to connection failure."); + } + } + }); + }) + .map_err(|err| { + io::Error::other(format!("Capture: failed to start worker thread: {err}")) + })?; + + Ok(Self { tx }) + } -#\[tokio::main\] async fn main() -> Result<(), Box> { + /// Enqueue an event for insertion. This is non-blocking. + pub fn log(&self, event: CaptureEvent) { + debug!( + "Capture: queueing event: type={}, user_id={}, assignment_id={:?}, group_id={:?}, file_path={:?}", + event.event_type, event.user_id, event.assignment_id, event.group_id, event.file_path + ); -``` - // Create an instance of EventCapture using the configuration file - let event_capture = EventCapture::new("config.json").await?; + if let Err(err) = self.tx.send(event) { + error!("Capture: FAILED to enqueue capture event: {err}"); + } + } +} - // Create an event - let event = Event { - user_id: "user123".to_string(), - event_type: "keystroke".to_string(), - data: Some("Pressed key A".to_string()), - }; +fn log_pg_connect_error(context: &str, err: &tokio_postgres::Error) { + // If Postgres returned a structured DbError, log it ONCE and bail. + if let Some(db) = err.as_db_error() { + // Example: 28P01 = invalid\_password + error!( + "{context}: PostgreSQL {} (SQLSTATE {})", + db.message(), + db.code().code() + ); - // Insert the event into the database - event_capture.insert_event(event).await?; + if let Some(detail) = db.detail() { + error!("{context}: detail: {detail}"); + } + if let Some(hint) = db.hint() { + error!("{context}: hint: {hint}"); + } + return; + } - Ok(()) -``` -} */ + // Otherwise, try to find an underlying std::io::Error (refused, timed out, + // DNS, etc.) + let mut current: &(dyn Error + 'static) = err; + while let Some(source) = current.source() { + if let Some(ioe) = source.downcast_ref::() { + error!( + "{context}: I/O error kind={:?} raw_os_error={:?} msg={}", + ioe.kind(), + ioe.raw_os_error(), + ioe + ); + return; + } + current = source; + } -pub struct EventCapture { - db_client: Arc>, + // Fallback: log once (Display) + error!("{context}: {err}"); } -/* - ## The EventCapture Implementation -*/ - -impl EventCapture { - /* - Creates a new `EventCapture` instance by reading the database connection parameters from the `config.json` file and connecting to the PostgreSQL database. - # Arguments - - config_path: The file path to the config.json file. +/// Insert a single event into the `events` table. +async fn insert_event(client: &Client, event: &CaptureEvent) -> Result { + let timestamp = event.timestamp.to_rfc3339(); + let data_text = event.data.to_string(); + + debug!( + "Capture: executing INSERT for user_id={}, event_type={}, timestamp={}", + event.user_id, event.event_type, timestamp + ); + + client + .execute( + "INSERT INTO events \ + (user_id, assignment_id, group_id, file_path, event_type, timestamp, data) \ + VALUES ($1, $2, $3, $4, $5, $6, $7)", + &[ + &event.user_id, + &event.assignment_id, + &event.group_id, + &event.file_path, + &event.event_type, + ×tamp, + &data_text, + ], + ) + .await +} - # Returns +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + #[test] + fn capture_config_to_conn_str_is_well_formed() { + let cfg = CaptureConfig { + host: "localhost".to_string(), + user: "alice".to_string(), + password: "secret".to_string(), + dbname: "codechat_capture".to_string(), + app_id: Some("spring25-study".to_string()), + }; + + let conn = cfg.to_conn_str(); + // Very simple checks: we don't care about ordering beyond what we + // format. + assert!(conn.contains("host=localhost")); + assert!(conn.contains("user=alice")); + assert!(conn.contains("password=secret")); + assert!(conn.contains("dbname=codechat_capture")); + } - A `Result` containing an `EventCapture` instance - */ + #[test] + fn capture_event_new_sets_all_fields() { + let ts = Utc::now(); + + let ev = CaptureEvent::new( + "user123".to_string(), + Some("lab1".to_string()), + Some("groupA".to_string()), + Some("/path/to/file.rs".to_string()), + "write_doc", + ts, + json!({ "chars_typed": 42 }), + ); - pub async fn new>(config_path: P) -> Result { - // Read the configuration file - let config_content = fs::read_to_string(config_path).map_err(io::Error::other)?; - let config: Config = serde_json::from_str(&config_content) - .map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e))?; + assert_eq!(ev.user_id, "user123"); + assert_eq!(ev.assignment_id.as_deref(), Some("lab1")); + assert_eq!(ev.group_id.as_deref(), Some("groupA")); + assert_eq!(ev.file_path.as_deref(), Some("/path/to/file.rs")); + assert_eq!(ev.event_type, "write_doc"); + assert_eq!(ev.timestamp, ts); + assert_eq!(ev.data, json!({ "chars_typed": 42 })); + } - // Build the connection string for the PostgreSQL database - let conn_str = format!( - "host={} user={} password={} dbname={}", - config.db_ip, config.db_user, config.db_password, config.db_name + #[test] + fn capture_event_now_uses_current_time_and_fields() { + let before = Utc::now(); + let ev = CaptureEvent::now( + "user123".to_string(), + None, + None, + None, + "save", + json!({ "reason": "manual" }), ); + let after = Utc::now(); + + assert_eq!(ev.user_id, "user123"); + assert!(ev.assignment_id.is_none()); + assert!(ev.group_id.is_none()); + assert!(ev.file_path.is_none()); + assert_eq!(ev.event_type, "save"); + assert_eq!(ev.data, json!({ "reason": "manual" })); + + // Timestamp sanity check: it should be between before and after + assert!(ev.timestamp >= before); + assert!(ev.timestamp <= after); + } - info!( - "Attempting Capture Database Connection. IP:[{}] Username:[{}] Database Name:[{}]", - config.db_ip, config.db_user, config.db_name + #[test] + fn capture_config_json_round_trip() { + let json_text = r#" + { + "host": "db.example.com", + "user": "bob", + "password": "hunter2", + "dbname": "cc_events", + "app_id": "fall25" + } + "#; + + let cfg: CaptureConfig = serde_json::from_str(json_text).expect("JSON should parse"); + assert_eq!(cfg.host, "db.example.com"); + assert_eq!(cfg.user, "bob"); + assert_eq!(cfg.password, "hunter2"); + assert_eq!(cfg.dbname, "cc_events"); + assert_eq!(cfg.app_id.as_deref(), Some("fall25")); + + // And it should serialize back to JSON without error + let _back = serde_json::to_string(&cfg).expect("Should serialize"); + } + + use std::fs; + //use tokio::time::{sleep, Duration}; + + /// Integration-style test: verify that EventCapture actually inserts into + /// the DB. + /// + /// Reads connection parameters from `capture_config.json` in the current + /// working directory. Logs the config and connection details via log4rs so + /// you can confirm what is used. + /// + /// Run this test with: cargo test event\_capture\_inserts\_event\_into\_db + /// -- --ignored --nocapture + /// + /// You must have a PostgreSQL database and a `capture_config.json` file + /// such as: { "host": "localhost", "user": "codechat\_test\_user", + /// "password": "codechat\_test\_password", "dbname": + /// "codechat\_capture\_test", "app\_id": "integration-test" } + #[tokio::test] + #[ignore] + async fn event_capture_inserts_event_into_db() -> Result<(), Box> { + // Initialize logging for this test, using the same log4rs.yml as the + // server. If logging is already initialized, this will just return an + // error which we ignore. + let _ = log4rs::init_file("log4rs.yml", Default::default()); + + // 1. Load the capture configuration from file. + let cfg_text = fs::read_to_string("capture_config.json") + .expect("capture_config.json must exist in project root for this test"); + let cfg: CaptureConfig = + serde_json::from_str(&cfg_text).expect("capture_config.json must be valid JSON"); + + log::info!( + "TEST: Loaded DB config from capture_config.json: host={}, user={}, dbname={}, app_id={:?}", + cfg.host, + cfg.user, + cfg.dbname, + cfg.app_id ); - // Connect to the database asynchronously - let (client, connection) = tokio_postgres::connect(&conn_str, NoTls) - .await - .map_err(|e| io::Error::new(io::ErrorKind::ConnectionRefused, e))?; + // 2. Connect directly for setup + verification. + let conn_str = cfg.to_conn_str(); + log::info!("TEST: Attempting direct tokio_postgres connection for verification."); - // Spawn a task to manage the database connection in the background + let (client, connection) = tokio_postgres::connect(&conn_str, NoTls).await?; tokio::spawn(async move { if let Err(e) = connection.await { - error!("Database connection error: [{e}]"); + log::error!("TEST: direct connection error: {e}"); } }); - info!( - "Connected to Database [{}] as User [{}]", - config.db_name, config.db_user - ); - - Ok(EventCapture { - db_client: Arc::new(Mutex::new(client)), - }) - } - - /* - Inserts an event into the database. - - # Arguments - - `event`: An `Event` instance containing the event data to insert. - - # Returns - A `Result` indicating success or containing a `tokio_postgres::Error`. - - # Example - #[tokio::main] - async fn main() -> Result<(), Box> { - let event_capture = EventCapture::new("config.json").await?; - - let event = Event { - user_id: "user123".to_string(), - event_type: "keystroke".to_string(), - data: Some("Pressed key A".to_string()), - }; - - event_capture.insert_event(event).await?; - Ok(()) - } - */ - - pub async fn insert_event(&self, event: Event) -> Result<(), io::Error> { - let current_time = Local::now(); - let formatted_time = current_time.to_rfc3339(); + // Verify the events table already exists + let row = client + .query_one( + r#" + SELECT EXISTS ( + SELECT 1 + FROM information_schema.tables + WHERE table_schema = 'public' + AND table_name = 'events' + ) AS exists + "#, + &[], + ) + .await?; - // SQL statement to insert the event into the 'events' table - let stmt = indoc! {" - INSERT INTO events (user_id, event_type, timestamp, data) - VALUES ($1, $2, $3, $4) - "}; + let exists: bool = row.get("exists"); + assert!( + exists, + "TEST SETUP ERROR: public.events table does not exist. \ + It must be created by a migration or admin step." + ); - // Acquire a lock on the database client for thread-safe access - let client = self.db_client.lock().await; + // Insert a single test row (this is what the app really needs) + let test_user_id = format!( + "TEST_USER_{}", + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_millis() + ); - // Execute the SQL statement with the event data - client - .execute( - stmt, + let insert_row = client + .query_one( + r#" + INSERT INTO public.events + (user_id, assignment_id, group_id, file_path, event_type, timestamp, data) + VALUES + ($1, NULL, NULL, NULL, 'test_event', $2, '{"test":true}') + RETURNING id + "#, &[ - &event.user_id, - &event.event_type, - &formatted_time, - &event.data, + &test_user_id, + &format!("{:?}", std::time::SystemTime::now()), ], ) - .await - .map_err(io::Error::other)?; - - info!("Event inserted into database: {event:?}"); + .await?; + + let inserted_id: i32 = insert_row.get("id"); + info!("TEST: inserted event id={}", inserted_id); + + // 4. Start the EventCapture worker using the loaded config. + let capture = EventCapture::new(cfg.clone())?; + log::info!("TEST: EventCapture worker started."); + + // 5. Log a test event. + let expected_data = json!({ "chars_typed": 123 }); + let event = CaptureEvent::now( + "test-user".to_string(), + Some("hw1".to_string()), + Some("groupA".to_string()), + Some("/tmp/test.rs".to_string()), + event_types::WRITE_DOC, + expected_data.clone(), + ); + log::info!("TEST: logging a test capture event."); + capture.log(event); + + // 6. Wait (deterministically) for the background worker to insert the event, + // then fetch THAT row (instead of "latest row in the table"). + use tokio::time::{Duration, Instant, sleep}; + + let deadline = Instant::now() + Duration::from_secs(2); + + let row = loop { + match client + .query_one( + r#" + SELECT user_id, assignment_id, group_id, file_path, event_type, data + FROM events + WHERE user_id = $1 AND event_type = $2 + ORDER BY id DESC + LIMIT 1 + "#, + &[&"test-user", &event_types::WRITE_DOC], + ) + .await + { + Ok(row) => break row, // found it + Err(_) => { + if Instant::now() >= deadline { + return Err("Timed out waiting for EventCapture insert".into()); + } + sleep(Duration::from_millis(50)).await; + } + } + }; + + let user_id: String = row.get(0); + let assignment_id: Option = row.get(1); + let group_id: Option = row.get(2); + let file_path: Option = row.get(3); + let event_type: String = row.get(4); + let data_text: String = row.get(5); + let data_value: serde_json::Value = serde_json::from_str(&data_text)?; + + assert_eq!(user_id, "test-user"); + assert_eq!(assignment_id.as_deref(), Some("hw1")); + assert_eq!(group_id.as_deref(), Some("groupA")); + assert_eq!(file_path.as_deref(), Some("/tmp/test.rs")); + assert_eq!(event_type, event_types::WRITE_DOC); + assert_eq!(data_value, expected_data); + + log::info!("✅ TEST: EventCapture integration test succeeded and wrote to database."); Ok(()) } } - -/* Database Schema (SQL DDL) - -The following SQL statement creates the `events` table used by this library: - -CREATE TABLE events ( id SERIAL PRIMARY KEY, user_id TEXT NOT NULL, -event_type TEXT NOT NULL, timestamp TEXT NOT NULL, data TEXT ); - -- **`id SERIAL PRIMARY KEY`**: Auto-incrementing primary key. -- **`user_id TEXT NOT NULL`**: The ID of the user associated with the event. -- **`event_type TEXT NOT NULL`**: The type of event. -- **`timestamp TEXT NOT NULL`**: The timestamp of the event. -- **`data TEXT`**: Optional additional data associated with the event. - **Note:** Ensure this table exists in your PostgreSQL database before using - the library. */ diff --git a/server/src/ide.rs b/server/src/ide.rs index b609b441..181bde33 100644 --- a/server/src/ide.rs +++ b/server/src/ide.rs @@ -251,6 +251,14 @@ impl CodeChatEditorServer { .await } + pub async fn send_capture_event( + &self, + capture_event: webserver::CaptureEventWire, + ) -> std::io::Result { + self.send_message_timeout(EditorMessageContents::Capture(capture_event)) + .await + } + // Send a `CurrentFile` message. The other parameter (true if text/false if // binary/None if ignored) is ignored by the server, so it's always sent as // `None`. diff --git a/server/src/ide/filewatcher.rs b/server/src/ide/filewatcher.rs index 72abcf8b..1906c404 100644 --- a/server/src/ide/filewatcher.rs +++ b/server/src/ide/filewatcher.rs @@ -672,6 +672,7 @@ async fn processing_task( EditorMessageContents::Opened(_) | EditorMessageContents::OpenUrl(_) | + EditorMessageContents::Capture(_) | EditorMessageContents::ClientHtml(_) | EditorMessageContents::RequestClose => { let err = ResultErrTypes::ClientIllegalMessage; diff --git a/server/src/translation.rs b/server/src/translation.rs index 5bbf5908..f1116ccf 100644 --- a/server/src/translation.rs +++ b/server/src/translation.rs @@ -221,6 +221,7 @@ use tokio::{ // ### Local use crate::{ + capture::event_types, lexer::supported_languages::MARKDOWN_MODE, processing::{ CodeChatForWeb, CodeMirror, CodeMirrorDiff, CodeMirrorDiffable, CodeMirrorDocBlock, @@ -230,11 +231,11 @@ use crate::{ }, queue_send, queue_send_func, webserver::{ - EditorMessage, EditorMessageContents, INITIAL_MESSAGE_ID, MESSAGE_ID_INCREMENT, - ProcessingTaskHttpRequest, ProcessingTaskHttpRequestFlags, ResultErrTypes, ResultOkTypes, - SimpleHttpResponse, SimpleHttpResponseError, UpdateMessageContents, WebAppState, - WebsocketQueues, file_to_response, path_to_url, send_response, try_canonicalize, - try_read_as_text, url_to_path, + CaptureEventWire, EditorMessage, EditorMessageContents, INITIAL_MESSAGE_ID, + MESSAGE_ID_INCREMENT, ProcessingTaskHttpRequest, ProcessingTaskHttpRequestFlags, + ResultErrTypes, ResultOkTypes, SimpleHttpResponse, SimpleHttpResponseError, + UpdateMessageContents, WebAppState, WebsocketQueues, file_to_response, log_capture_event, + path_to_url, send_response, try_canonicalize, try_read_as_text, url_to_path, }, }; @@ -385,6 +386,7 @@ pub fn create_translation_queues( /// allows factoring out lengthy contents in the loop into subfunctions. struct TranslationTask { // These parameters are passed to us. + app_state: WebAppState, connection_id_raw: String, prefix: &'static [&'static str], allow_source_diffs: bool, @@ -433,6 +435,69 @@ struct TranslationTask { /// Has the full (non-diff) version of the current file been sent? Don't /// send diffs until this is sent. sent_full: bool, + capture_context: CaptureContext, +} + +#[derive(Clone, Debug, Default)] +struct CaptureContext { + user_id: Option, + assignment_id: Option, + group_id: Option, + session_id: Option, + client_tz_offset_min: Option, +} + +impl CaptureContext { + fn update_from_wire(&mut self, wire: &CaptureEventWire) { + if !wire.user_id.trim().is_empty() { + self.user_id = Some(wire.user_id.clone()); + } + if let Some(assignment_id) = &wire.assignment_id { + self.assignment_id = Some(assignment_id.clone()); + } + if let Some(group_id) = &wire.group_id { + self.group_id = Some(group_id.clone()); + } + if let Some(client_tz_offset_min) = wire.client_tz_offset_min { + self.client_tz_offset_min = Some(client_tz_offset_min); + } + if let Some(serde_json::Value::Object(data)) = &wire.data + && let Some(session_id) = data.get("session_id").and_then(serde_json::Value::as_str) + { + self.session_id = Some(session_id.to_string()); + } + } + + fn capture_event( + &self, + event_type: &str, + file_path: Option, + data: serde_json::Value, + ) -> Option { + let mut data = match data { + serde_json::Value::Object(map) => map, + other => { + let mut map = serde_json::Map::new(); + map.insert("value".to_string(), other); + map + } + }; + if let Some(session_id) = &self.session_id { + data.entry("session_id".to_string()) + .or_insert_with(|| serde_json::json!(session_id)); + } + + Some(CaptureEventWire { + user_id: self.user_id.clone()?, + assignment_id: self.assignment_id.clone(), + group_id: self.group_id.clone(), + file_path, + event_type: event_type.to_string(), + client_timestamp_ms: None, + client_tz_offset_min: self.client_tz_offset_min, + data: Some(serde_json::Value::Object(data)), + }) + } } /// This is the processing task for the Visual Studio Code IDE. It handles all @@ -464,6 +529,7 @@ pub async fn translation_task( let mut continue_loop = true; let mut tt = TranslationTask { + app_state: app_state.clone(), connection_id_raw, prefix, allow_source_diffs, @@ -487,6 +553,7 @@ pub async fn translation_task( version: 0.0, // Don't send diffs until this is sent. sent_full: false, + capture_context: CaptureContext::default(), }; while continue_loop { select! { @@ -513,6 +580,11 @@ pub async fn translation_task( EditorMessageContents::Result(_) => continue_loop = tt.ide_result(ide_message).await, EditorMessageContents::Update(_) => continue_loop = tt.ide_update(ide_message).await, + EditorMessageContents::Capture(capture_event) => { + tt.capture_context.update_from_wire(&capture_event); + log_capture_event(&app_state, capture_event); + send_response(&tt.to_ide_tx, ide_message.id, Ok(ResultOkTypes::Void)).await; + }, // Update the current file; translate it to a URL then // pass it to the Client. @@ -608,6 +680,11 @@ pub async fn translation_task( }, EditorMessageContents::Update(_) => continue_loop = tt.client_update(client_message).await, + EditorMessageContents::Capture(capture_event) => { + tt.capture_context.update_from_wire(&capture_event); + log_capture_event(&app_state, capture_event); + send_response(&tt.to_client_tx, client_message.id, Ok(ResultOkTypes::Void)).await; + }, // Update the current file; translate it to a URL then // pass it to the IDE. @@ -698,6 +775,103 @@ pub async fn translation_task( // These provide translation for messages passing through the Server. impl TranslationTask { + fn capture_file_path(file_path: &std::path::Path) -> Option { + file_path.to_str().map(str::to_string) + } + + fn log_server_capture_event( + &self, + event_type: &str, + file_path: &std::path::Path, + data: serde_json::Value, + ) { + let Some(capture_event) = self.capture_context.capture_event( + event_type, + Self::capture_file_path(file_path), + data, + ) else { + debug!("Skipping server-classified capture event; capture identity is not known yet."); + return; + }; + log_capture_event(&self.app_state, capture_event); + } + + fn log_raw_write_event(&self, file_path: &std::path::Path, before: &str, after: &str) { + if before == after { + return; + } + self.log_server_capture_event( + event_types::WRITE_CODE, + file_path, + serde_json::json!({ + "source": "server_translation", + "classification_basis": "raw_text", + "diff": diff_str(before, after), + }), + ); + } + + fn log_code_mirror_write_events( + &self, + file_path: &std::path::Path, + metadata: &SourceFileMetadata, + before_doc: &str, + before_doc_blocks: Option<&CodeMirrorDocBlockVec>, + after: &CodeMirror, + source: &str, + ) { + if metadata.mode == MARKDOWN_MODE { + if !compare_html(before_doc, &after.doc) { + self.log_server_capture_event( + event_types::WRITE_DOC, + file_path, + serde_json::json!({ + "source": source, + "classification_basis": "markdown_document", + "mode": metadata.mode, + "diff": diff_str(before_doc, &after.doc), + }), + ); + } + return; + } + + if before_doc != after.doc { + self.log_server_capture_event( + event_types::WRITE_CODE, + file_path, + serde_json::json!({ + "source": source, + "classification_basis": "codemirror_code_text", + "mode": metadata.mode, + "diff": diff_str(before_doc, &after.doc), + }), + ); + } + + let doc_blocks_changed = match before_doc_blocks { + Some(before) => !doc_block_compare(before, &after.doc_blocks), + None => !after.doc_blocks.is_empty(), + }; + if doc_blocks_changed { + let doc_block_diff = before_doc_blocks.map(|before| { + serde_json::json!(diff_code_mirror_doc_blocks(before, &after.doc_blocks)) + }); + self.log_server_capture_event( + event_types::WRITE_DOC, + file_path, + serde_json::json!({ + "source": source, + "classification_basis": "codemirror_doc_blocks", + "mode": metadata.mode, + "doc_block_count_before": before_doc_blocks.map_or(0, Vec::len), + "doc_block_count_after": after.doc_blocks.len(), + "doc_block_diff": doc_block_diff, + }), + ); + } + } + // Pass a `Result` message to the Client, unless it's a `LoadFile` result. async fn ide_result(&mut self, ide_message: EditorMessage) -> bool { let EditorMessageContents::Result(ref result) = ide_message.message else { @@ -893,6 +1067,16 @@ impl TranslationTask { else { panic!("Unexpected diff value."); }; + if self.sent_full { + self.log_code_mirror_write_events( + &clean_file_path, + &ccfw.metadata, + &self.code_mirror_doc, + self.code_mirror_doc_blocks.as_ref(), + code_mirror_translated, + "ide", + ); + } // Send a diff if possible. let client_contents = if self.sent_full { self.diff_code_mirror( @@ -938,6 +1122,13 @@ impl TranslationTask { Err(ResultErrTypes::TodoBinarySupport) } TranslationResultsString::Unknown => { + if self.sent_full { + self.log_raw_write_event( + &clean_file_path, + &self.source_code, + &code_mirror.doc, + ); + } // Send the new raw contents. debug!("Sending translated contents to Client."); queue_send_func!(self.to_client_tx.send(EditorMessage { @@ -954,13 +1145,16 @@ impl TranslationTask { mode: "".to_string(), }, source: CodeMirrorDiffable::Plain(CodeMirror { - doc: code_mirror.doc, + doc: code_mirror.doc.clone(), doc_blocks: vec![] }), version: contents.version }), }), })); + self.source_code = code_mirror.doc; + self.code_mirror_doc = self.source_code.clone(); + self.code_mirror_doc_blocks = Some(vec![]); Ok(ResultOkTypes::Void) } TranslationResultsString::Toc(_) => { @@ -1043,12 +1237,22 @@ impl TranslationTask { // what we just received. This must be updated // before we can translate back to check for changes // (the next step). - let CodeMirrorDiffable::Plain(code_mirror) = cfw.source else { + let CodeMirrorDiffable::Plain(ref code_mirror) = cfw.source else { // TODO: support diffable! panic!("Diff not supported."); }; - self.code_mirror_doc = code_mirror.doc; - self.code_mirror_doc_blocks = Some(code_mirror.doc_blocks); + if self.sent_full { + self.log_code_mirror_write_events( + &clean_file_path, + &cfw.metadata, + &self.code_mirror_doc, + self.code_mirror_doc_blocks.as_ref(), + code_mirror, + "client", + ); + } + self.code_mirror_doc = code_mirror.doc.clone(); + self.code_mirror_doc_blocks = Some(code_mirror.doc_blocks.clone()); // We may need to change this version if we send a // diff back to the Client. let mut cfw_version = cfw.version; diff --git a/server/src/webserver.rs b/server/src/webserver.rs index b4fb389e..fbeb86e6 100644 --- a/server/src/webserver.rs +++ b/server/src/webserver.rs @@ -38,15 +38,17 @@ use std::{ // ### Third-party use actix_files; + use actix_web::{ App, HttpRequest, HttpResponse, HttpServer, dev::{Server, ServerHandle, ServiceFactory, ServiceRequest}, error::Error, get, http::header::{ContentType, DispositionType}, - middleware, + middleware, post, web::{self, Data}, }; + use actix_web_httpauth::{extractors::basic::BasicAuth, middleware::HttpAuthentication}; use actix_ws::AggregatedMessage; use bytes::Bytes; @@ -95,6 +97,10 @@ use crate::{ }, }; +use crate::capture::{CaptureConfig, CaptureEvent, EventCapture}; + +use chrono::Utc; + // Data structures // --------------- // @@ -201,6 +207,8 @@ pub enum EditorMessageContents { // Server will determine the value if needed. Option, ), + /// Record an instrumentation event. Valid destinations: Server. + Capture(CaptureEventWire), // #### These messages may only be sent by the IDE. /// This is the first message sent when the IDE starts up. It may only be @@ -381,6 +389,8 @@ pub struct AppState { pub connection_id: Mutex>, /// The auth credentials if authentication is used. credentials: Option, + // Added to support capture - JDS - 11/2025 + pub capture: Option, } pub type WebAppState = web::Data; @@ -391,6 +401,35 @@ pub struct Credentials { pub password: String, } +/// JSON payload received from clients for capture events. +/// +/// The server will supply the timestamp; clients do not need to send it. +#[derive(Debug, Serialize, Deserialize, PartialEq, TS)] +#[ts(export, optional_fields)] +pub struct CaptureEventWire { + pub user_id: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub assignment_id: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub group_id: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub file_path: Option, + pub event_type: String, + + /// Optional client-side timestamp (milliseconds since Unix epoch). + #[serde(skip_serializing_if = "Option::is_none")] + pub client_timestamp_ms: Option, + + /// Optional client timezone offset in minutes (JS Date().getTimezoneOffset()). + #[serde(skip_serializing_if = "Option::is_none")] + pub client_tz_offset_min: Option, + + /// Arbitrary event-specific data stored as JSON (optional). + #[serde(skip_serializing_if = "Option::is_none")] + #[ts(type = "unknown")] + pub data: Option, +} + // Macros // ------ /// Create a macro to report an error when enqueueing an item. @@ -579,6 +618,52 @@ async fn stop(app_state: WebAppState) -> HttpResponse { HttpResponse::NoContent().finish() } +#[post("/capture")] +async fn capture_endpoint( + app_state: WebAppState, + payload: web::Json, +) -> HttpResponse { + log_capture_event(&app_state, payload.into_inner()); + HttpResponse::Ok().finish() +} + +/// Log a capture event if capture is enabled. +pub fn log_capture_event(app_state: &WebAppState, wire: CaptureEventWire) { + if let Some(capture) = &app_state.capture { + // Default missing data to empty object + let mut data = wire.data.unwrap_or_else(|| serde_json::json!({})); + + // Ensure data is an object so we can attach fields + if !data.is_object() { + data = serde_json::json!({ "value": data }); + } + + // Add client timestamp fields if present (even if extension also sends them; + // overwriting is fine and consistent). + if let serde_json::Value::Object(map) = &mut data { + if let Some(ms) = wire.client_timestamp_ms { + map.insert("client_timestamp_ms".to_string(), serde_json::json!(ms)); + } + if let Some(tz) = wire.client_tz_offset_min { + map.insert("client_tz_offset_min".to_string(), serde_json::json!(tz)); + } + } + + let event = CaptureEvent { + user_id: wire.user_id, + assignment_id: wire.assignment_id, + group_id: wire.group_id, + file_path: wire.file_path, + event_type: wire.event_type, + // Server decides when the event is recorded. + timestamp: Utc::now(), + data, + }; + + capture.log(event); + } +} + // Get the `mode` query parameter to determine `is_test_mode`; default to // `false`. pub fn get_test_mode(req: &HttpRequest) -> bool { @@ -1427,9 +1512,6 @@ pub fn setup_server( addr: &SocketAddr, credentials: Option, ) -> std::io::Result<(Server, Data)> { - // Connect to the Capture Database - //let _event_capture = EventCapture::new("config.json").await?; - // Pre-load the bundled files before starting the webserver. let _ = &*BUNDLED_FILES_MAP; let app_data = make_app_data(credentials); @@ -1505,6 +1587,38 @@ pub fn configure_logger(level: LevelFilter) -> Result<(), Box) -> WebAppState { + // Initialize event capture from a config file (optional). + let capture: Option = { + // Build path: /capture_config.json + let mut config_path = ROOT_PATH.lock().unwrap().clone(); + config_path.push("capture_config.json"); + + match fs::read_to_string(&config_path) { + Ok(json) => match serde_json::from_str::(&json) { + Ok(cfg) => match EventCapture::new(cfg) { + Ok(ec) => { + eprintln!("Capture: enabled (config file: {config_path:?})"); + Some(ec) + } + Err(err) => { + eprintln!("Capture: failed to initialize from {config_path:?}: {err}"); + None + } + }, + Err(err) => { + eprintln!("Capture: invalid JSON in {config_path:?}: {err}"); + None + } + }, + Err(err) => { + eprintln!( + "Capture: disabled (config file not found or unreadable: {config_path:?}: {err})" + ); + None + } + } + }; + web::Data::new(AppState { server_handle: Mutex::new(None), filewatcher_next_connection_id: Mutex::new(0), @@ -1515,6 +1629,7 @@ pub fn make_app_data(credentials: Option) -> WebAppState { client_queues: Arc::new(Mutex::new(HashMap::new())), connection_id: Mutex::new(HashSet::new()), credentials, + capture, }) } @@ -1544,6 +1659,7 @@ where .service(vscode_client_framework) .service(ping) .service(stop) + .service(capture_endpoint) // Reroute to the filewatcher filesystem for typical user-requested // URLs. .route("/", web::get().to(filewatcher_root_fs_redirect))