LocalMind / index.html
NakliTechie
Sync from GitHub 2026-04-25T17:36:12Z
a923b4b
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<meta name="author" content="Chirag Patnaik">
<meta name="version" content="2.0.0">
<meta name="description" content="Private AI chatbot running Gemma entirely in your browser. No server, no API keys, no data leaves your device.">
<title>LocalMind β€” Private AI Chat in Your Browser</title>
<link rel="icon" href="data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 100 100'%3E%3Ctext y='.9em' font-size='90'%3E%F0%9F%A7%A0%3C/text%3E%3C/svg%3E">
<!-- KaTeX for inline math ($…$) and display math ($$…$$) in assistant
responses. Loaded from jsdelivr with defer to avoid blocking parse.
TODO: add SRI hashes alongside the transformers@4 SRI work. -->
<link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.16.11/dist/katex.min.css" crossorigin="anonymous">
<script defer src="https://cdn.jsdelivr.net/npm/katex@0.16.11/dist/katex.min.js" crossorigin="anonymous"></script>
<style>
/* Rangrez Β· westafrica-01 Β· BAOBAB β€” Senegalese baobab bark, savanna pale dawn */
:root {
--gray-900: #1a1408; /* INK */
--gray-800: #2a2014;
--gray-700: #4a3818;
--gray-600: #7a5a28;
--gray-400: #b89a68;
--gray-200: #ddd6c0; /* line */
--gray-100: #f6f3ec; /* BODY */
--gray-50: #ece8de; /* row */
--white: #fdfbf4; /* PANEL */
--indigo-500: #08184a; /* ACT β€” savanna near-black navy */
--indigo-600: #050f30;
--indigo-100: #d8dceb;
--indigo-focus-ring: rgba(8, 24, 74, 0.20);
--success-bg: #dceadb;
--success-border: #2a8a3a;
--success-text: #0a3014;
--loading-bg: #fde6c4;
--loading-border: #c08018;
--loading-text: #744210;
--error-bg: #f8d4dc;
--error-border: #b81848; /* BRAND β€” hot rose */
--error-text: #6a0a30;
}
* { box-sizing: border-box; margin: 0; padding: 0; }
body {
font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif;
background: var(--gray-100);
color: var(--gray-800);
line-height: 1.6;
padding: 24px 16px 60px;
}
.app {
max-width: 780px;
margin: 0 auto;
display: flex;
flex-direction: column;
height: calc(100vh - 108px);
position: relative;
overflow: hidden;
}
/* Header */
.header {
display: flex;
justify-content: space-between;
align-items: center;
margin-bottom: 16px;
flex-shrink: 0;
}
.header h1 {
font-size: 1.5rem;
font-weight: 700;
color: var(--gray-900);
}
.header-right {
display: flex;
align-items: center;
gap: 10px;
}
/* Help button */
.help-wrap { position: relative; }
.help-btn {
width: 26px;
height: 26px;
border-radius: 50%;
border: 1.5px solid var(--gray-400);
background: var(--white);
color: var(--gray-600);
font-size: 0.8rem;
font-weight: 600;
cursor: pointer;
display: flex;
align-items: center;
justify-content: center;
transition: border-color 0.15s, color 0.15s;
}
.help-btn:hover {
border-color: var(--indigo-500);
color: var(--indigo-500);
}
.help-popover {
display: none;
position: absolute;
top: calc(100% + 10px);
right: 0;
width: 340px;
background: var(--white);
border-radius: 12px;
box-shadow: 0 4px 24px rgba(0,0,0,0.12);
font-size: 0.82rem;
color: var(--gray-700);
line-height: 1.55;
z-index: 100;
overflow: hidden;
}
.help-popover::before {
content: '';
position: absolute;
top: -6px;
right: 10px;
width: 12px;
height: 12px;
background: var(--white);
transform: rotate(45deg);
box-shadow: -2px -2px 4px rgba(0,0,0,0.04);
z-index: 1;
}
.help-popover.open { display: block; }
.help-tabs {
display: flex;
border-bottom: 1px solid var(--gray-200);
background: var(--gray-50);
}
.help-tab {
flex: 1;
padding: 10px 6px;
font-size: 0.72rem;
font-weight: 600;
color: var(--gray-500);
text-align: center;
cursor: pointer;
border: none;
background: none;
border-bottom: 2px solid transparent;
transition: color 0.15s, border-color 0.15s;
}
.help-tab:hover { color: var(--gray-700); }
.help-tab.active {
color: var(--indigo-600);
border-bottom-color: var(--indigo-500);
}
.help-tab-content {
display: none;
padding: 14px 16px;
max-height: 320px;
overflow-y: auto;
}
.help-tab-content.active { display: block; }
.help-tab-content h4 {
font-size: 0.82rem;
font-weight: 600;
color: var(--gray-900);
margin: 10px 0 4px;
}
.help-tab-content h4:first-child { margin-top: 0; }
.help-tab-content p { margin-bottom: 8px; }
.help-tab-content ul {
padding-left: 18px;
margin-bottom: 8px;
}
.help-tab-content li { margin-bottom: 4px; }
.help-tab-content .help-note {
font-size: 0.75rem;
color: var(--gray-400);
border-top: 1px solid var(--gray-200);
padding-top: 8px;
margin-top: 8px;
}
.try-prompt {
display: block;
background: var(--gray-50);
border: 1px solid var(--gray-200);
border-radius: 8px;
padding: 7px 10px;
margin-bottom: 6px;
font-size: 0.78rem;
color: var(--gray-700);
font-style: italic;
cursor: pointer;
transition: background 0.15s, border-color 0.15s;
}
.try-prompt:hover {
background: rgba(102, 126, 234, 0.06);
border-color: var(--indigo-300);
}
/* Status badge */
.status-badge {
display: flex;
align-items: center;
gap: 8px;
padding: 4px 12px;
border-radius: 20px;
font-size: 0.75rem;
font-weight: 500;
transition: background 0.4s, border-color 0.4s, color 0.4s;
}
.status-badge.loading {
background: var(--loading-bg);
border: 1px solid var(--loading-border);
color: var(--loading-text);
}
.status-badge.ready {
background: var(--success-bg);
border: 1px solid var(--success-border);
color: var(--success-text);
}
.status-badge.error {
background: var(--error-bg);
border: 1px solid var(--error-border);
color: var(--error-text);
}
.spinner {
width: 10px;
height: 10px;
border: 2px solid currentColor;
border-top-color: transparent;
border-radius: 50%;
animation: spin 0.75s linear infinite;
}
@keyframes spin { to { transform: rotate(360deg); } }
/* Card */
.card {
background: var(--white);
border-radius: 16px;
box-shadow: 0 2px 12px rgba(0,0,0,0.08);
overflow: hidden;
display: flex;
flex-direction: column;
flex: 1;
min-height: 0;
}
/* Chat area */
.chat-area {
flex: 1;
overflow-y: auto;
padding: 20px;
display: flex;
flex-direction: column;
gap: 16px;
min-height: 0;
}
.chat-area::-webkit-scrollbar { width: 6px; }
.chat-area::-webkit-scrollbar-track { background: transparent; }
.chat-area::-webkit-scrollbar-thumb { background: var(--gray-200); border-radius: 3px; }
/* Welcome message */
.welcome {
text-align: center;
color: var(--gray-400);
font-size: 0.88rem;
margin: auto 0;
padding: 40px 20px;
}
.welcome-icon {
font-size: 2.5rem;
margin-bottom: 12px;
}
.welcome h2 {
font-size: 1.1rem;
font-weight: 600;
color: var(--gray-600);
margin-bottom: 6px;
}
/* Messages */
.msg {
display: flex;
flex-direction: column;
max-width: 85%;
animation: fadeIn 0.2s ease;
}
@keyframes fadeIn { from { opacity: 0; transform: translateY(6px); } to { opacity: 1; transform: translateY(0); } }
.msg.user { align-self: flex-end; }
.msg.assistant { align-self: flex-start; }
.msg-bubble {
padding: 10px 14px;
border-radius: 14px;
font-size: 0.9rem;
line-height: 1.55;
word-break: break-word;
}
.msg.user .msg-bubble {
background: var(--indigo-500);
color: var(--white);
border-bottom-right-radius: 4px;
}
.msg.assistant .msg-bubble {
background: var(--gray-100);
color: var(--gray-800);
border-bottom-left-radius: 4px;
}
/* User message attachment thumbnails */
.msg-attachments {
display: flex;
flex-wrap: wrap;
gap: 6px;
margin-bottom: 6px;
}
.msg-attachments img {
width: 80px;
height: 80px;
object-fit: cover;
border-radius: 8px;
border: 2px solid rgba(255,255,255,0.3);
}
.msg-attachments .audio-tag {
display: flex;
align-items: center;
gap: 4px;
background: rgba(255,255,255,0.15);
border-radius: 8px;
padding: 6px 10px;
font-size: 0.75rem;
color: rgba(255,255,255,0.9);
}
/* Markdown in assistant messages */
.msg.assistant .msg-bubble code {
background: rgba(0,0,0,0.06);
padding: 1px 5px;
border-radius: 4px;
font-size: 0.82rem;
font-family: 'SF Mono', SFMono-Regular, Consolas, 'Liberation Mono', Menlo, monospace;
}
.msg.assistant .msg-bubble pre {
background: var(--gray-900);
color: var(--gray-200);
padding: 12px 14px;
border-radius: 8px;
overflow-x: auto;
margin: 8px 0;
font-size: 0.8rem;
line-height: 1.5;
}
.msg.assistant .msg-bubble pre code {
background: none;
padding: 0;
color: inherit;
font-size: inherit;
}
.msg.assistant .msg-bubble pre {
position: relative;
}
.code-download-btn {
position: absolute;
top: 6px;
right: 6px;
background: rgba(255,255,255,0.8);
border: 1px solid var(--gray-200);
border-radius: 4px;
width: 24px;
height: 24px;
font-size: 0.75rem;
cursor: pointer;
display: flex;
align-items: center;
justify-content: center;
opacity: 0;
transition: opacity 0.15s;
}
.msg.assistant .msg-bubble pre:hover .code-download-btn { opacity: 1; }
.save-md-btn {
display: inline-block;
margin-top: 6px;
padding: 2px 8px;
font-size: 0.68rem;
color: var(--gray-400);
border: 1px solid var(--gray-200);
border-radius: 4px;
background: none;
cursor: pointer;
transition: color 0.15s, border-color 0.15s;
}
.save-md-btn:hover {
color: var(--gray-600);
border-color: var(--gray-400);
}
/* Collapsible tool-trace block (step labels + tool-call blocks) that
sits above the bubble when expanded. Default collapsed; toggle
button sits next to Save as MD. */
.tool-trace-container { margin: 0 0 8px 0; }
.tool-trace-container.collapsed { display: none; }
.msg.assistant .msg-bubble ol,
.msg.assistant .msg-bubble ul {
padding-left: 20px;
margin: 6px 0;
}
.msg.assistant .msg-bubble li { margin-bottom: 2px; }
.msg.assistant .msg-bubble strong { font-weight: 600; }
.msg.assistant .msg-bubble em { font-style: italic; }
.msg.assistant .msg-bubble p { margin-bottom: 8px; }
.msg.assistant .msg-bubble p:last-child { margin-bottom: 0; }
/* Thinking block */
.thinking-block {
background: rgba(102, 126, 234, 0.06);
border-left: 3px solid var(--indigo-500);
padding: 8px 12px;
border-radius: 0 8px 8px 0;
margin-bottom: 8px;
font-size: 0.82rem;
color: var(--gray-600);
font-style: italic;
}
.thinking-toggle {
cursor: pointer;
font-size: 0.75rem;
color: var(--indigo-500);
font-weight: 500;
display: flex;
align-items: center;
gap: 4px;
margin-bottom: 4px;
font-style: normal;
}
.thinking-content {
display: none;
white-space: pre-wrap;
}
.thinking-content.open { display: block; }
/* Tool call block */
.tool-call-block {
background: rgba(245, 158, 11, 0.06);
border-left: 3px solid var(--loading-border);
padding: 8px 12px;
border-radius: 0 8px 8px 0;
margin: 8px 0;
font-size: 0.82rem;
color: var(--gray-600);
}
.tool-call-toggle {
cursor: pointer;
font-size: 0.75rem;
color: var(--loading-text);
font-weight: 500;
display: flex;
align-items: center;
gap: 4px;
margin-bottom: 4px;
}
.tool-call-toggle strong {
font-family: 'SF Mono', 'Cascadia Code', 'Consolas', monospace;
}
.tool-call-result {
display: none;
white-space: pre-wrap;
font-size: 0.78rem;
margin-top: 4px;
}
.tool-call-result.open { display: block; }
/* Planner blocks (multi-step planning) β€” mirror tool-call-block but indigo */
.planner-block {
background: rgba(99, 102, 241, 0.06);
border-left: 3px solid var(--indigo-500);
padding: 8px 12px;
border-radius: 0 8px 8px 0;
margin: 8px 0;
font-size: 0.82rem;
color: var(--gray-600);
}
.planner-toggle {
cursor: pointer;
font-size: 0.75rem;
color: var(--indigo-500);
font-weight: 500;
display: flex;
align-items: center;
gap: 4px;
}
.planner-body {
display: none;
margin-top: 6px;
white-space: pre-wrap;
font-size: 0.78rem;
color: var(--gray-600);
}
.planner-body.open { display: block; }
.planner-status {
font-size: 0.78rem;
color: var(--gray-500);
font-style: italic;
padding: 6px 0;
}
/* Mermaid + KaTeX */
.mermaid {
background: var(--white);
border: 1px solid var(--gray-200);
border-radius: 6px;
padding: 12px;
margin: 8px 0;
overflow-x: auto;
text-align: center;
}
.mermaid-error, .katex-error {
color: var(--error-text);
background: rgba(185, 28, 28, 0.08);
padding: 2px 6px;
border-radius: 4px;
font-family: monospace;
font-size: 0.78rem;
}
/* Artifact preview (sandboxed iframe for ```html, ```svg, ```artifact) */
.artifact-preview {
margin: 8px 0;
border: 1px solid var(--gray-200);
border-radius: 6px;
overflow: hidden;
background: var(--white);
}
.artifact-header {
display: flex;
align-items: center;
justify-content: space-between;
padding: 6px 10px;
background: var(--gray-50);
border-bottom: 1px solid var(--gray-200);
font-size: 0.72rem;
color: var(--gray-600);
}
.artifact-header strong {
font-family: 'SF Mono', 'Cascadia Code', 'Consolas', monospace;
color: var(--indigo-500);
}
.artifact-frame {
width: 100%;
min-height: 200px;
border: 0;
display: block;
background: var(--white);
}
/* Message context menu (right-click / long-press) */
.msg-context-menu {
position: fixed;
display: none;
background: var(--white);
border: 1px solid var(--gray-200);
border-radius: 6px;
box-shadow: 0 4px 12px rgba(0, 0, 0, 0.1);
padding: 4px 0;
font-size: 0.82rem;
z-index: 1000;
min-width: 160px;
}
.msg-context-menu.visible { display: block; }
.msg-context-menu button {
display: block;
width: 100%;
text-align: left;
background: none;
border: none;
padding: 8px 12px;
cursor: pointer;
color: var(--gray-700);
font-size: inherit;
}
.msg-context-menu button:hover { background: var(--gray-50); }
.tool-call-result pre {
margin: 0;
padding: 6px 8px;
background: rgba(245, 158, 11, 0.04);
border-radius: 4px;
font-size: 0.78rem;
overflow-x: auto;
}
/* Segmentation overlay */
.segmentation-result {
position: relative;
display: inline-block;
margin: 8px 0;
max-width: 100%;
}
.segmentation-result img {
max-width: 100%;
border-radius: 8px;
display: block;
}
.segmentation-result canvas {
position: absolute;
top: 0;
left: 0;
width: 100%;
height: 100%;
border-radius: 8px;
pointer-events: none;
}
.segmentation-download {
display: inline-block;
margin-top: 4px;
font-size: 0.72rem;
color: var(--indigo-500);
cursor: pointer;
border: none;
background: none;
padding: 2px 0;
}
.segmentation-download:hover {
text-decoration: underline;
}
/* Typing indicator */
.typing-dot {
display: inline-block;
width: 6px;
height: 6px;
border-radius: 50%;
background: var(--gray-400);
animation: blink 1.4s infinite;
margin-right: 3px;
}
.typing-dot:nth-child(2) { animation-delay: 0.2s; }
.typing-dot:nth-child(3) { animation-delay: 0.4s; margin-right: 0; }
@keyframes blink {
0%, 60%, 100% { opacity: 0.3; }
30% { opacity: 1; }
}
/* Attachment bar */
.attachment-bar {
display: none;
padding: 8px 16px 4px;
border-top: 1px solid var(--gray-200);
background: var(--gray-50);
flex-shrink: 0;
overflow-x: auto;
gap: 8px;
}
.attachment-bar.visible {
display: flex;
}
.attachment-bar::-webkit-scrollbar { height: 4px; }
.attachment-bar::-webkit-scrollbar-thumb { background: var(--gray-200); border-radius: 2px; }
.attachment-chip {
position: relative;
flex-shrink: 0;
width: 56px;
height: 56px;
border-radius: 10px;
overflow: hidden;
border: 1.5px solid var(--gray-200);
background: var(--gray-100);
display: flex;
align-items: center;
justify-content: center;
}
.attachment-chip.video-chip {
width: 100px;
height: 64px;
}
.attachment-chip.audio-chip {
width: 180px;
height: 48px;
padding: 4px 6px;
gap: 4px;
}
.attachment-chip img {
width: 100%;
height: 100%;
object-fit: cover;
}
.attachment-chip video {
width: 100%;
height: 100%;
object-fit: cover;
pointer-events: none;
}
.attachment-chip .video-badge {
position: absolute;
bottom: 3px;
left: 3px;
background: rgba(0,0,0,0.6);
color: var(--white);
font-size: 0.5rem;
font-weight: 600;
padding: 1px 4px;
border-radius: 3px;
line-height: 1.4;
}
.attachment-chip audio {
width: 100%;
height: 28px;
}
.attachment-chip .audio-label {
font-size: 0.55rem;
color: var(--gray-600);
overflow: hidden;
text-overflow: ellipsis;
white-space: nowrap;
width: 100%;
text-align: center;
line-height: 1.2;
}
.attachment-chip .audio-icon {
font-size: 1.2rem;
color: var(--gray-600);
}
.attachment-chip .audio-name {
position: absolute;
bottom: 2px;
left: 2px;
right: 2px;
font-size: 0.55rem;
color: var(--gray-600);
text-align: center;
overflow: hidden;
text-overflow: ellipsis;
white-space: nowrap;
background: rgba(255,255,255,0.85);
border-radius: 3px;
padding: 0 2px;
}
.attachment-chip .remove-btn {
position: absolute;
top: 2px;
right: 2px;
width: 18px;
height: 18px;
border-radius: 50%;
background: rgba(0,0,0,0.55);
color: var(--white);
border: none;
font-size: 0.65rem;
cursor: pointer;
display: flex;
align-items: center;
justify-content: center;
line-height: 1;
opacity: 0;
transition: opacity 0.15s;
}
.attachment-chip:hover .remove-btn { opacity: 1; }
.attachment-chip .experimental-badge {
position: absolute;
top: 2px;
left: 2px;
background: var(--loading-border);
color: var(--white);
font-size: 0.5rem;
font-weight: 600;
padding: 0 3px;
border-radius: 3px;
line-height: 1.4;
}
/* Input bar */
.input-bar {
display: flex;
align-items: flex-end;
gap: 8px;
padding: 12px 16px;
border-top: 1px solid var(--gray-200);
background: var(--gray-50);
flex-shrink: 0;
}
.input-bar textarea {
flex: 1;
border: 1.5px solid var(--gray-200);
border-radius: 12px;
padding: 10px 14px;
font-size: 0.9rem;
font-family: inherit;
line-height: 1.4;
resize: none;
outline: none;
min-height: 42px;
max-height: 140px;
transition: border-color 0.15s;
background: var(--white);
}
.input-bar textarea:focus {
border-color: var(--indigo-500);
box-shadow: 0 0 0 3px var(--indigo-focus-ring);
}
.input-bar textarea:disabled {
background: var(--gray-100);
color: var(--gray-400);
cursor: not-allowed;
}
.input-bar textarea::placeholder { color: var(--gray-400); }
/* Input affordance buttons */
.input-affordances {
display: none;
align-items: flex-end;
gap: 4px;
flex-shrink: 0;
}
.input-affordances.visible { display: flex; }
.afford-btn {
width: 36px;
height: 36px;
border-radius: 10px;
border: 1.5px solid var(--gray-200);
background: var(--white);
color: var(--gray-600);
font-size: 1rem;
cursor: pointer;
display: flex;
align-items: center;
justify-content: center;
transition: border-color 0.15s, color 0.15s, background 0.15s;
flex-shrink: 0;
}
.afford-btn:hover:not(:disabled) {
border-color: var(--indigo-500);
color: var(--indigo-500);
}
.afford-btn:disabled { opacity: 0.3; cursor: not-allowed; }
.afford-btn.recording {
background: var(--error-bg);
border-color: var(--error-text);
color: var(--error-text);
animation: pulse-rec 1.2s ease-in-out infinite;
}
@keyframes pulse-rec {
0%, 100% { box-shadow: 0 0 0 0 rgba(229, 62, 62, 0.3); }
50% { box-shadow: 0 0 0 6px rgba(229, 62, 62, 0); }
}
.send-btn {
width: 42px;
height: 42px;
border-radius: 12px;
border: none;
background: var(--indigo-500);
color: var(--white);
font-size: 1.1rem;
cursor: pointer;
display: flex;
align-items: center;
justify-content: center;
transition: background 0.15s, opacity 0.15s;
flex-shrink: 0;
}
.send-btn:hover:not(:disabled) { background: var(--indigo-600); }
.send-btn:disabled { opacity: 0.4; cursor: not-allowed; }
.send-btn.stop { background: var(--error-text); }
.send-btn.stop:hover:not(:disabled) { background: var(--error-text); }
/* Action buttons bar */
.actions-bar {
display: flex;
align-items: center;
gap: 6px;
padding: 6px 16px;
border-top: 1px solid var(--gray-200);
background: var(--gray-50);
flex-shrink: 0;
}
.btn-icon {
background: none;
border: none;
border-radius: 6px;
font-size: 0.75rem;
color: var(--gray-600);
padding: 4px 8px;
cursor: pointer;
transition: background 0.15s, color 0.15s;
}
.btn-icon:hover:not(:disabled) {
background: var(--gray-200);
color: var(--gray-800);
}
.btn-icon:disabled { opacity: 0.4; cursor: not-allowed; }
.btn-icon.folder-open {
background: var(--indigo-100);
color: var(--indigo-600);
}
/* Model selector */
.model-select {
appearance: none;
-webkit-appearance: none;
background: var(--white);
border: 1.5px solid var(--gray-200);
border-radius: 8px;
padding: 3px 24px 3px 8px;
font-size: 0.72rem;
font-family: inherit;
color: var(--gray-700);
cursor: pointer;
outline: none;
transition: border-color 0.15s;
background-image: url("data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' width='10' height='6'%3E%3Cpath d='M0 0l5 6 5-6z' fill='%23718096'/%3E%3C/svg%3E");
background-repeat: no-repeat;
background-position: right 8px center;
}
.model-select:hover { border-color: var(--gray-400); }
.model-select:focus { border-color: var(--indigo-500); box-shadow: 0 0 0 3px var(--indigo-focus-ring); }
.model-select:disabled { opacity: 0.4; cursor: not-allowed; }
/* Progress section */
.progress-section {
padding: 12px 16px;
border-top: 1px solid var(--gray-200);
background: var(--gray-50);
flex-shrink: 0;
display: none;
}
.progress-section.visible { display: block; }
.progress-bar {
height: 6px;
background: var(--gray-200);
border-radius: 8px;
overflow: hidden;
}
.progress-fill {
height: 100%;
background: linear-gradient(90deg, var(--indigo-500), #b81848);
border-radius: 8px;
transition: width 0.3s ease;
width: 0%;
}
.progress-text-row {
display: flex;
align-items: center;
justify-content: space-between;
gap: 8px;
margin-top: 6px;
}
.progress-text {
font-size: 0.72rem;
color: var(--gray-600);
flex: 1;
}
.progress-cancel-btn {
padding: 2px 10px;
font-size: 0.7rem;
background: transparent;
border: 1px solid var(--gray-200);
color: var(--gray-600);
border-radius: 4px;
cursor: pointer;
flex-shrink: 0;
transition: border-color 0.15s, color 0.15s;
}
.progress-cancel-btn:hover {
border-color: var(--gray-400);
color: var(--gray-800);
}
/* Settings panel */
.settings-panel {
display: none;
padding: 12px 16px;
border-top: 1px solid var(--gray-200);
background: var(--gray-50);
flex-shrink: 1;
max-height: 50vh;
overflow-y: auto;
}
.settings-panel.open { display: block; }
/* API chip next to the model selector β€” only visible when the
window.localmind API is enabled in Settings. Flashes on each call. */
.api-chip {
display: none;
align-items: center;
gap: 4px;
padding: 2px 8px;
margin-right: 6px;
font-size: 0.7rem;
font-weight: 600;
letter-spacing: 0.04em;
color: var(--indigo-600);
background: var(--indigo-100);
border: 1px solid var(--indigo-500);
border-radius: 10px;
cursor: pointer;
transition: background-color 0.2s, color 0.2s;
}
.api-chip.on { display: inline-flex; }
.api-chip.flash {
background: var(--indigo-500);
color: var(--white);
}
.api-chip::before {
content: '';
width: 6px;
height: 6px;
border-radius: 50%;
background: var(--indigo-500);
}
.api-chip.flash::before { background: var(--white); }
/* Activity log modal β€” reuses the share-modal backdrop pattern */
.api-log-modal {
display: none;
position: fixed;
inset: 0;
z-index: 200;
background: rgba(26, 32, 44, 0.55);
align-items: center;
justify-content: center;
padding: 24px;
}
.api-log-modal.open { display: flex; }
.api-log-modal-content {
background: var(--white);
border-radius: 12px;
max-width: 720px;
width: 100%;
max-height: 80vh;
display: flex;
flex-direction: column;
overflow: hidden;
box-shadow: 0 20px 40px rgba(0,0,0,0.2);
}
.api-log-header {
padding: 14px 18px;
border-bottom: 1px solid var(--gray-200);
display: flex;
align-items: center;
justify-content: space-between;
}
.api-log-header strong { font-size: 0.95rem; }
.api-log-list {
overflow-y: auto;
padding: 8px 0;
font-size: 0.75rem;
font-family: ui-monospace, SFMono-Regular, Menlo, monospace;
}
.api-log-empty {
padding: 30px 18px;
text-align: center;
color: var(--gray-500);
}
.api-log-row {
display: grid;
grid-template-columns: 90px 1fr 80px 70px 70px;
gap: 12px;
padding: 6px 18px;
border-bottom: 1px solid var(--gray-100);
}
.api-log-row:hover { background: var(--gray-50); }
.api-log-row .t { color: var(--gray-500); }
.api-log-row .m { color: var(--gray-800); overflow: hidden; text-overflow: ellipsis; white-space: nowrap; }
.api-log-row .dur { color: var(--gray-600); text-align: right; }
.api-log-row .ok { color: var(--success-border); }
.api-log-row .err { color: var(--error-text); }
.api-log-row .busy { color: var(--loading-text); }
.settings-panel label {
display: block;
font-size: 0.75rem;
font-weight: 500;
color: var(--gray-600);
margin-bottom: 4px;
}
.settings-panel textarea {
width: 100%;
border: 1.5px solid var(--gray-200);
border-radius: 8px;
padding: 8px 10px;
font-size: 0.8rem;
font-family: inherit;
line-height: 1.4;
resize: vertical;
outline: none;
min-height: 60px;
max-height: 120px;
background: var(--white);
}
.settings-panel textarea:focus {
border-color: var(--indigo-500);
box-shadow: 0 0 0 3px var(--indigo-focus-ring);
}
.toggle-row {
display: flex;
align-items: center;
gap: 8px;
margin-top: 10px;
font-size: 0.8rem;
color: var(--gray-700);
cursor: pointer;
}
.toggle-row input[type="checkbox"] {
width: 16px;
height: 16px;
accent-color: var(--indigo-500);
cursor: pointer;
}
.toggle-row.hidden { display: none; }
.settings-divider {
border-top: 1px solid var(--gray-200);
margin-top: 12px;
padding-top: 12px;
}
.settings-select, .settings-input {
width: 100%;
padding: 8px 10px;
border: 1px solid var(--gray-200);
border-radius: 8px;
font-size: 0.82rem;
background: var(--white);
margin-top: 4px;
margin-bottom: 8px;
outline: none;
}
.settings-select:focus, .settings-input:focus {
border-color: var(--indigo-500);
box-shadow: 0 0 0 3px var(--indigo-focus-ring);
}
/* Dual send buttons */
.send-group {
display: flex;
gap: 4px;
}
.search-send-btn {
width: 40px;
height: 40px;
border-radius: 50%;
border: 1.5px solid var(--indigo-400);
background: var(--white);
color: var(--indigo-600);
font-size: 1rem;
cursor: pointer;
display: flex;
align-items: center;
justify-content: center;
transition: background 0.15s, transform 0.1s;
flex-shrink: 0;
}
.search-send-btn:hover:not(:disabled) {
background: rgba(102, 126, 234, 0.08);
}
.search-send-btn:disabled {
opacity: 0.4;
cursor: default;
}
/* Source badges on messages */
.msg-source-badge {
display: inline-block;
font-size: 0.65rem;
padding: 2px 6px;
border-radius: 4px;
margin-bottom: 4px;
font-weight: 500;
}
.msg-source-badge.on-device {
background: var(--gray-100);
color: var(--gray-500);
}
.msg-source-badge.web-enriched {
background: rgba(16, 185, 129, 0.1);
color: var(--success-border);
}
.source-links {
margin-top: 8px;
padding-top: 6px;
border-top: 1px solid var(--gray-200);
font-size: 0.72rem;
}
.source-links a {
color: var(--indigo-600);
text-decoration: none;
display: block;
margin-bottom: 2px;
overflow: hidden;
text-overflow: ellipsis;
white-space: nowrap;
}
.source-links a:hover { text-decoration: underline; }
/* Memory inspector panel */
.memory-panel {
display: none;
padding: 12px 16px;
border-top: 1px solid var(--gray-200);
max-height: 400px;
overflow-y: auto;
}
.memory-panel.open { display: block; }
.memory-header {
display: flex;
justify-content: space-between;
align-items: center;
margin-bottom: 8px;
font-size: 0.82rem;
color: var(--gray-900);
}
.memory-count {
font-size: 0.72rem;
color: var(--gray-400);
}
.memory-search-row {
display: flex;
gap: 6px;
margin-bottom: 6px;
}
.memory-search-input {
flex: 1;
padding: 6px 10px;
border: 1px solid var(--gray-200);
border-radius: 8px;
font-size: 0.78rem;
background: var(--white);
outline: none;
}
.memory-search-input:focus {
border-color: var(--indigo-500);
}
.memory-clear-btn {
font-size: 0.7rem;
color: var(--gray-500);
}
/* Category filter pills */
.memory-cat-pills {
display: flex;
flex-wrap: wrap;
gap: 4px;
margin-bottom: 8px;
}
.memory-cat-pill {
padding: 2px 8px;
border-radius: 20px;
border: 1px solid var(--gray-200);
background: var(--white);
font-size: 0.7rem;
color: var(--gray-600);
cursor: pointer;
white-space: nowrap;
transition: background 0.12s, border-color 0.12s, color 0.12s;
}
.memory-cat-pill:hover { border-color: var(--gray-400); }
.memory-cat-pill.active {
background: var(--indigo-100);
border-color: var(--indigo-500);
color: var(--indigo-600);
font-weight: 600;
}
/* Source group (for document categories) */
.memory-source-group {
border: 1px solid var(--gray-200);
border-radius: 10px;
overflow: hidden;
margin-bottom: 6px;
}
.memory-source-header {
display: flex;
align-items: center;
gap: 6px;
padding: 6px 10px;
background: var(--gray-100);
font-size: 0.75rem;
}
.memory-source-name {
flex: 1;
font-weight: 600;
color: var(--gray-700);
overflow: hidden;
text-overflow: ellipsis;
white-space: nowrap;
}
.memory-source-count {
color: var(--gray-400);
flex-shrink: 0;
}
.memory-source-del {
font-size: 0.68rem;
color: var(--gray-400);
background: none;
border: 1px solid var(--gray-200);
border-radius: 4px;
padding: 1px 6px;
cursor: pointer;
flex-shrink: 0;
}
.memory-source-del:hover { color: var(--error-border); border-color: var(--error-border); }
.memory-source-group .memory-item {
border-radius: 0;
border-top: 1px solid var(--gray-200);
}
.memory-list {
display: flex;
flex-direction: column;
gap: 6px;
}
.memory-item {
display: flex;
align-items: flex-start;
gap: 8px;
padding: 8px 10px;
background: var(--gray-50);
border-radius: 8px;
font-size: 0.78rem;
color: var(--gray-700);
line-height: 1.45;
}
.memory-item-text { flex: 1; min-width: 0; }
.memory-item-meta {
display: flex;
align-items: center;
flex-wrap: wrap;
gap: 4px;
font-size: 0.68rem;
color: var(--gray-400);
margin-top: 2px;
}
.memory-item-del {
cursor: pointer;
color: var(--gray-400);
font-size: 0.8rem;
padding: 0 4px;
border: none;
background: none;
line-height: 1;
flex-shrink: 0;
}
.memory-item-del:hover { color: var(--error-border); }
/* Category badges */
.mem-cat {
display: inline-block;
padding: 0 5px;
border-radius: 4px;
font-size: 0.65rem;
font-weight: 600;
text-transform: uppercase;
letter-spacing: 0.02em;
}
.mem-cat-fact { background: #dbeafe; color: #1d4ed8; }
.mem-cat-preference { background: #ede9fe; color: #6d28d9; }
.mem-cat-finding { background: #d1fae5; color: #065f46; }
.mem-cat-document { background: #ffedd5; color: #c2410c; }
.mem-cat-document_summary { background: #fef3c7; color: #92400e; }
.mem-cat-conversation { background: var(--gray-200); color: var(--gray-700); }
.memory-empty {
text-align: center;
color: var(--gray-400);
font-size: 0.78rem;
padding: 16px 0;
}
/* Audit view */
.audit-section { margin-bottom: 10px; }
.audit-section-header {
display: flex;
align-items: center;
gap: 6px;
padding: 5px 0;
font-size: 0.75rem;
font-weight: 600;
color: var(--gray-700);
border-bottom: 1px solid var(--gray-200);
margin-bottom: 6px;
}
.audit-section-header span { flex: 1; }
.audit-clean {
text-align: center;
background: #d1fae5;
color: #065f46;
border-radius: 8px;
padding: 14px;
font-size: 0.82rem;
font-weight: 500;
}
.audit-warn {
font-size: 0.7rem;
color: var(--gray-500);
background: var(--loading-bg);
border: 1px solid var(--loading-border);
border-radius: 6px;
padding: 6px 10px;
margin-bottom: 8px;
}
.history-item {
display: flex;
align-items: center;
gap: 8px;
padding: 8px 10px;
background: var(--gray-50);
border-radius: 8px;
font-size: 0.78rem;
color: var(--gray-700);
cursor: pointer;
transition: background 0.15s;
}
.history-item:hover { background: rgba(102, 126, 234, 0.06); }
.history-item-text { flex: 1; overflow: hidden; }
.history-item-title {
white-space: nowrap;
overflow: hidden;
text-overflow: ellipsis;
font-weight: 500;
}
.history-item-meta {
font-size: 0.68rem;
color: var(--gray-400);
margin-top: 1px;
}
.history-item-del {
cursor: pointer;
color: var(--gray-400);
font-size: 0.8rem;
padding: 0 4px;
border: none;
background: none;
line-height: 1;
flex-shrink: 0;
}
.history-item-del:hover { color: var(--error-border); }
/* History sidebar */
.history-backdrop {
display: none;
position: absolute;
inset: 0;
background: rgba(0,0,0,0.2);
z-index: 60;
border-radius: 16px;
}
.history-backdrop.open { display: block; }
.history-sidebar {
position: absolute;
top: 0;
left: 0;
bottom: 0;
width: 280px;
background: var(--white);
border-right: 1px solid var(--gray-200);
border-radius: 16px 0 0 16px;
z-index: 70;
display: flex;
flex-direction: column;
transform: translateX(-100%);
transition: transform 0.25s ease;
overflow: hidden;
}
.history-sidebar.open { transform: translateX(0); }
.history-sidebar-header {
display: flex;
justify-content: space-between;
align-items: center;
padding: 14px 16px 10px;
border-bottom: 1px solid var(--gray-200);
font-size: 0.85rem;
color: var(--gray-900);
flex-shrink: 0;
}
.history-sidebar-list {
flex: 1;
overflow-y: auto;
padding: 8px;
display: flex;
flex-direction: column;
gap: 4px;
}
.btn-danger-text { color: var(--gray-400); }
.btn-danger-text:hover { color: var(--error-border); }
/* Camera overlay */
.camera-overlay {
display: none;
position: fixed;
inset: 0;
z-index: 200;
background: rgba(0,0,0,0.85);
flex-direction: column;
align-items: center;
justify-content: center;
gap: 16px;
}
.camera-overlay.open { display: flex; }
.camera-overlay video {
max-width: 90vw;
max-height: 60vh;
border-radius: 12px;
background: #000;
}
.camera-overlay .cam-controls {
display: flex;
gap: 16px;
}
.camera-overlay button {
padding: 10px 24px;
border-radius: 24px;
border: none;
font-size: 0.9rem;
font-weight: 500;
cursor: pointer;
transition: opacity 0.15s;
}
.camera-overlay button:hover { opacity: 0.85; }
.cam-capture-btn {
background: var(--indigo-500);
color: var(--white);
}
.cam-cancel-btn {
background: var(--gray-200);
color: var(--gray-800);
}
/* Batch panel */
.batch-panel {
display: none;
padding: 12px 16px;
border-top: 1px solid var(--gray-200);
background: var(--gray-50);
flex-shrink: 0;
}
.batch-panel.open { display: block; }
.batch-textarea {
width: 100%;
min-height: 96px;
max-height: 200px;
border: 1.5px solid var(--gray-200);
border-radius: 8px;
padding: 8px 10px;
font-size: 0.78rem;
font-family: inherit;
color: var(--gray-800);
background: var(--white);
resize: vertical;
outline: none;
line-height: 1.5;
margin: 8px 0;
}
.batch-textarea:focus { border-color: var(--indigo-500); }
.batch-controls {
display: flex;
align-items: center;
gap: 8px;
flex-wrap: wrap;
}
.batch-chain-label {
display: flex;
align-items: center;
gap: 5px;
font-size: 0.75rem;
color: var(--gray-600);
cursor: pointer;
flex: 1;
}
.batch-progress {
font-size: 0.72rem;
color: var(--gray-500);
min-width: 80px;
text-align: right;
}
.btn-run {
padding: 4px 14px;
border-radius: 7px;
border: none;
background: var(--indigo-500);
color: white;
font-size: 0.78rem;
font-weight: 600;
cursor: pointer;
transition: background 0.15s;
}
.btn-run:hover:not(:disabled) { background: var(--indigo-600); }
.btn-run:disabled { opacity: 0.45; cursor: not-allowed; }
/* Share modal */
.share-backdrop {
position: fixed; inset: 0;
background: rgba(0,0,0,0.45);
z-index: 600;
display: none;
align-items: center;
justify-content: center;
}
.share-backdrop.open { display: flex; }
.share-modal {
background: var(--white);
border-radius: 14px;
padding: 24px;
width: min(480px, 90vw);
box-shadow: 0 12px 40px rgba(0,0,0,0.18);
display: flex;
flex-direction: column;
gap: 14px;
}
.share-modal h3 {
font-size: 1rem;
font-weight: 700;
color: var(--gray-900);
margin: 0;
}
.share-modal label {
font-size: 0.82rem;
color: var(--gray-600);
display: flex;
align-items: center;
gap: 8px;
cursor: pointer;
}
.share-modal input[type="text"],
.share-modal input[type="password"] {
width: 100%;
border: 1.5px solid var(--gray-200);
border-radius: 8px;
padding: 8px 10px;
font-size: 0.82rem;
color: var(--gray-800);
background: var(--gray-50);
outline: none;
transition: border-color 0.15s;
font-family: inherit;
}
.share-modal input:focus { border-color: var(--indigo-500); }
.share-url-row {
display: flex;
gap: 6px;
}
.share-url-row input { flex: 1; font-family: monospace; font-size: 0.75rem; }
.share-modal-footer {
display: flex;
justify-content: flex-end;
gap: 8px;
}
.share-modal-footer button {
padding: 7px 16px;
border-radius: 8px;
font-size: 0.82rem;
font-weight: 600;
cursor: pointer;
border: 1.5px solid var(--gray-200);
background: var(--white);
color: var(--gray-700);
transition: background 0.15s, border-color 0.15s;
}
.share-modal-footer button:hover { background: var(--gray-100); border-color: var(--gray-400); }
.share-modal-footer .btn-primary {
background: var(--indigo-500);
color: white;
border-color: var(--indigo-500);
}
.share-modal-footer .btn-primary:hover { background: var(--indigo-600); border-color: var(--indigo-600); }
.share-modal-footer .btn-primary:disabled { opacity: 0.5; cursor: not-allowed; }
.share-meta { font-size: 0.78rem; color: var(--gray-600); }
/* Import banner */
.import-banner {
position: fixed;
top: 0; left: 0; right: 0;
background: var(--indigo-500);
color: white;
padding: 12px 20px;
display: none;
align-items: center;
gap: 12px;
z-index: 700;
font-size: 0.85rem;
flex-wrap: wrap;
}
.import-banner.open { display: flex; }
.import-banner strong { flex: 1; min-width: 180px; }
.import-banner input[type="password"] {
border: none;
border-radius: 6px;
padding: 5px 10px;
font-size: 0.82rem;
width: 160px;
}
.import-banner button {
background: var(--white);
color: var(--indigo-600);
border: none;
border-radius: 6px;
padding: 5px 14px;
font-size: 0.82rem;
font-weight: 600;
cursor: pointer;
}
.import-banner .dismiss-btn {
background: transparent;
color: rgba(255,255,255,0.8);
font-weight: 400;
padding: 5px 8px;
}
/* Drag-drop overlay */
.drag-overlay {
display: none;
position: absolute;
inset: 0;
z-index: 50;
background: rgba(102, 126, 234, 0.08);
border: 2px dashed var(--indigo-500);
border-radius: 16px;
align-items: center;
justify-content: center;
font-size: 0.9rem;
color: var(--indigo-500);
font-weight: 500;
pointer-events: none;
}
.drag-overlay.visible { display: flex; }
/* WebGPU error */
.webgpu-error {
text-align: center;
padding: 40px 20px;
color: var(--error-text);
}
.webgpu-error h2 {
font-size: 1.1rem;
margin-bottom: 8px;
}
.webgpu-error p {
font-size: 0.88rem;
color: var(--gray-600);
max-width: 400px;
margin: 0 auto;
}
/* Footer */
.footer-nav {
position: fixed;
bottom: 12px;
left: 50%;
transform: translateX(-50%);
display: flex;
gap: 24px;
background: rgba(255,255,255,0.92);
padding: 6px 18px;
border-radius: 20px;
box-shadow: 0 1px 6px rgba(0,0,0,0.06);
font-size: 0.72rem;
white-space: nowrap;
}
.footer-nav a {
color: #0366d6;
text-decoration: none;
transition: opacity 0.2s;
}
.footer-nav a:hover { opacity: 0.7; }
/* Mobile */
@media (max-width: 600px) {
body { padding: 12px 8px 56px; }
.app { height: calc(100vh - 80px); }
.header h1 { font-size: 1.2rem; }
.msg { max-width: 92%; }
.chat-area { padding: 14px 12px; }
.help-popover { width: calc(100vw - 32px); right: -8px; max-width: 340px; }
.afford-btn { width: 32px; height: 32px; font-size: 0.9rem; }
.attachment-chip { width: 48px; height: 48px; }
.attachment-chip.video-chip { width: 80px; height: 52px; }
.attachment-chip.audio-chip { width: 150px; height: 44px; }
}
</style>
</head>
<body>
<div class="app">
<!-- Header -->
<div class="header">
<h1>LocalMind</h1>
<div class="header-right">
<div class="help-wrap">
<button class="help-btn" id="helpBtn" aria-label="Help">?</button>
<div class="help-popover" id="helpPopover">
<div class="help-tabs">
<button class="help-tab active" data-tab="about">About</button>
<button class="help-tab" data-tab="models">Models</button>
<button class="help-tab" data-tab="features">Features</button>
<button class="help-tab" data-tab="api">JS API</button>
<button class="help-tab" data-tab="try">Things to Try</button>
</div>
<div class="help-tab-content active" data-tab="about">
<p>A private AI research agent running <strong>Gemma</strong> entirely in your browser via WebGPU. Tool calling, persistent memory, and web search &mdash; all on-device.</p>
<p>Only your search queries touch the network (and only when you choose to). All reasoning stays on your device.</p>
<p>Models are cached after first download &mdash; future visits load instantly.</p>
<p><strong>Requirements:</strong> Chrome 113+, Edge 113+, or Firefox 130+ with WebGPU.</p>
<div class="help-note">v2.0.0 &middot; Powered by Transformers.js + Google Gemma (Apache 2.0).</div>
</div>
<div class="help-tab-content" data-tab="models">
<h4>Available Models</h4>
<ul>
<li><strong>Ternary Bonsai 1.7B</strong> (~470 MB, default) &mdash; text + agent (tool calling). Smallest download with tool calling. 1.58-bit ternary weights, Qwen3 backbone, Apache-2.0.</li>
<li><strong>Ternary Bonsai 4B</strong> (~1.1 GB) &mdash; same capabilities, better quality.</li>
<li><strong>Ternary Bonsai 8B</strong> (~2.2 GB) &mdash; best Bonsai quality, 65K context.</li>
<li><strong>Gemma 3 1B</strong> (~760 MB) &mdash; text-only, no tool calling. Fallback option.</li>
<li><strong>Gemma 4 E2B</strong> (~1.5 GB) &mdash; multimodal (image + audio) + agent.</li>
<li><strong>Gemma 4 E4B</strong> (~4.9 GB) &mdash; multimodal + agent, best quality.</li>
</ul>
<h4>Agent Tools (Ternary Bonsai + Gemma 4)</h4>
<ul>
<li><strong>calculate</strong> &mdash; math, percentages, conversions</li>
<li><strong>get_current_time</strong> &mdash; date/time with timezone</li>
<li><strong>store_memory</strong> &mdash; save facts to persistent memory</li>
<li><strong>search_memory</strong> &mdash; recall from stored memories</li>
<li><strong>web_search</strong> &mdash; search the web (requires API key)</li>
<li><strong>fetch_page</strong> &mdash; read a web page&rsquo;s content</li>
<li><strong>set_reminder</strong> &mdash; browser notification after N minutes</li>
<li><strong>list_memories</strong> &mdash; show what&rsquo;s stored in memory</li>
<li><strong>delete_memory</strong> &mdash; forget specific memories</li>
<li><strong>segment_image</strong> &mdash; segment objects in attached images (SAM)</li>
</ul>
<h4>Image Segmentation (SAM)</h4>
<p>Gemma 4 can call SAM (Segment Anything Model) to segment objects in attached images. Choose your SAM model in Settings &mdash; loaded on first use.</p>
<ul>
<li><strong>SlimSAM 50</strong> (~10 MB) &mdash; fastest, good enough for most tasks</li>
<li><strong>SlimSAM 77</strong> (~14 MB) &mdash; default, better accuracy</li>
<li><strong>SAM ViT-Base</strong> (~350 MB) &mdash; full quality, slower download</li>
<li><strong>SAM 3</strong> (latest) &mdash; newest architecture</li>
</ul>
<h4>Things to try</h4>
<ul>
<li>&ldquo;Segment the main object in this image&rdquo;</li>
<li>&ldquo;Outline the person on the left&rdquo;</li>
<li>&ldquo;Isolate the background&rdquo;</li>
<li>&ldquo;How many distinct objects are in this image?&rdquo;</li>
</ul>
<p>Translation works directly &mdash; Gemma 4 speaks 140+ languages natively, no tool needed.</p>
<p>Gemma 3 1B works as a simple chatbot &mdash; no agent tools. Prefer Ternary Bonsai 1.7B if you want tool calling at a similar size.</p>
</div>
<div class="help-tab-content" data-tab="features">
<h4>Document Upload</h4>
<ul>
<li><strong>Text</strong> &mdash; .txt, .md, .json, .csv</li>
<li><strong>PDF</strong> &mdash; extracted via PDF.js, auto-summarized</li>
<li><strong>DOCX</strong> &mdash; extracted via mammoth.js, auto-summarized</li>
</ul>
<p>Documents are chunked, embedded, and stored as searchable knowledge. A summary is generated on upload.</p>
<h4>Multimodal (Gemma 4)</h4>
<ul>
<li><strong>&#128206; Attach</strong> &mdash; images, audio, MP4 video, or documents</li>
<li><strong>&#128247; Camera</strong> / <strong>&#127908; Mic</strong> / <strong>Paste</strong> / <strong>Drag &amp; drop</strong></li>
</ul>
<h4>Document Upload</h4>
<ul>
<li><strong>Text</strong> &mdash; .txt, .md, .json, .csv</li>
<li><strong>PDF</strong> &mdash; extracted via PDF.js, auto-summarized</li>
<li><strong>DOCX</strong> &mdash; extracted via mammoth.js, auto-summarized</li>
<li><strong>Folder</strong> &mdash; open a local folder to ingest all .md/.txt/.pdf files at once; re-open to sync only changed files (incremental)</li>
</ul>
<h4>Conversations</h4>
<ul>
<li><strong>New Chat</strong> &mdash; archives to History + starts fresh</li>
<li><strong>Clear</strong> &mdash; deletes without saving</li>
<li><strong>History</strong> &mdash; sidebar, click to resume any past chat</li>
<li><strong>Share</strong> &mdash; generate an encrypted or plain link to share any conversation; recipient opens the URL to load it</li>
</ul>
<h4>Memory browser</h4>
<ul>
<li><strong>Category pills</strong> &mdash; filter by fact / preference / finding / document / conversation with live counts</li>
<li><strong>Source grouping</strong> &mdash; document chunks grouped by file; bulk &ldquo;Delete all&rdquo; per source</li>
<li><strong>Audit</strong> &mdash; flags stale (&gt;60 days), near-duplicates (cosine sim &ge;0.92), and outliers (low avg similarity to category); bulk or per-item delete; auto-reruns after each deletion</li>
</ul>
<h4>Output &amp; Export</h4>
<ul>
<li><strong>Save as MD</strong> &mdash; download any response as Markdown (or write directly to open folder if one is active)</li>
<li><strong>Code download</strong> &mdash; hover code blocks for download button</li>
<li><strong>Export / Import</strong> &mdash; in Memory panel, full data as JSON</li>
<li><strong>Auto-backup</strong> &mdash; toggle in Settings to download on New Chat</li>
</ul>
<h4>Batch Prompts</h4>
<ul>
<li>Enter one prompt per line in the Batch panel &mdash; they run sequentially through the full agent loop</li>
<li><strong>{{previous}}</strong> &mdash; explicit placeholder substituted with the previous response text</li>
<li><strong>Auto-inject</strong> &mdash; checkbox (on by default) appends the previous response as context even without a placeholder; disabled for any prompt that already contains {{previous}}</li>
<li><strong>Stop</strong> &mdash; halts after the current generation finishes; progress shown live</li>
</ul>
<h4>Other</h4>
<ul>
<li><strong>Web Search</strong> &mdash; Settings &rarr; provider + API key &rarr; &#127760; button</li>
<li><strong>Thinking Mode</strong> &mdash; see chain-of-thought (collapses when done)</li>
<li><strong>Multi-step planning</strong> (experimental, Gemma 4 only) &mdash; Settings &rarr; tick the toggle. Each message is planned into 2&ndash;5 steps, each step executed (with tools), then synthesised. Plan + per-step outputs render as collapsible blocks below the answer. Slower (3&times;+ model calls) but handles research-style queries better.</li>
<li><strong>Branch from here</strong> &mdash; right-click (or long-press) a user message &rarr; "Branch from here". Archives the current conversation, then forks a new one containing messages up to that point. Continue the new branch from that question.</li>
<li><strong>Custom tools</strong> (agent-capable models) &mdash; Settings &rarr; Custom tools. Paste a tool definition as JSON (<code>name</code>, <code>description</code>, <code>parameters</code>, <code>endpoint</code>). On a tool call the model's args are <code>POST</code>ed to your endpoint as a JSON body; the response is fed back to the model. CORS must allow this origin.</li>
<li><strong>MCP servers</strong> (agent-capable models) &mdash; Settings &rarr; MCP servers. Paste a Streamable HTTP MCP endpoint URL (plus optional bearer). LocalMind opens a JSON-RPC 2.0 session, discovers tools via <code>tools/list</code>, and registers each with an <code>mcp_</code> prefix so the agent loop can use them alongside built-ins.</li>
<li><strong>Math &amp; diagrams</strong> &mdash; inline <code>$\int x^2 dx$</code> and display <code>$$\\sum_{i=1}^n i$$</code> math render via KaTeX; <code>```mermaid</code> blocks render as SVG via lazy-loaded Mermaid.</li>
<li><strong>Artifact preview</strong> &mdash; <code>```html</code> / <code>```svg</code> / <code>```artifact</code> code blocks get a live sandboxed iframe below the code (<code>sandbox=&quot;allow-scripts&quot;</code>, no same-origin). Safe to run model-generated UI inline.</li>
<li><strong>Voice to text</strong> &mdash; πŸ—£ button left of the input records mic audio, decodes to 16 kHz mono PCM on-device, and runs Whisper-base via WebGPU to transcribe into the input. ~80 MB first-use download; nothing leaves the device.</li>
<li><strong>Python code tool</strong> (agent-capable models) &mdash; model can call <code>run_python</code> to execute Python in a sandboxed Pyodide worker. numpy / pandas / matplotlib auto-install on import. ~10 MB first-use download.</li>
<li><strong>Cache management</strong> &mdash; view/clear cached models in Settings</li>
<li><strong>Custom models</strong> &mdash; Settings &rarr; paste a Hugging Face ONNX repo id (causal LMs only). The validator probes the HF API, picks the best available quantisation, estimates real load size, and hard-blocks anything that exceeds the device&rsquo;s WebGPU buffer limit or the 6 GB ceiling.</li>
<li><strong>Response badges</strong> &mdash; On-device / Agent / Web-enriched</li>
</ul>
</div>
<div class="help-tab-content" data-tab="api">
<h4>JavaScript API <span style="font-weight:400;color:var(--gray-500)">(experimental)</span></h4>
<p>Settings &rarr; tick <strong>Expose <code>window.localmind</code></strong>. Same-tab only &mdash; cross-origin scripts cannot reach it. The object is frozen and non-writable; disable the toggle to detach.</p>
<h4>Surface (v1.0)</h4>
<ul>
<li><strong>version</strong> &middot; <strong>ready</strong> &middot; <strong>model</strong> &mdash; live state getters</li>
<li><strong>listModels()</strong> &mdash; full registry incl. custom models with <code>loaded</code> flag</li>
<li><strong>load(idOrKey)</strong> &mdash; loads a model (short key or HF id); resolves when ready</li>
<li><strong>chat.completions.create({ messages, max_tokens, temperature, top_p, model })</strong> &mdash; non-streaming, returns OpenAI-shaped <code>chat.completion</code></li>
<li><strong>chat.completions.create({ &hellip;, stream: true })</strong> &mdash; async iterator yielding <code>chat.completion.chunk</code> objects</li>
</ul>
<h4>Not exposed</h4>
<ul>
<li>Tools / tool calling</li>
<li>Memory read/write</li>
<li>File system, web search, search API keys, user profile</li>
<li>Multimodal input</li>
</ul>
<h4>Activity log</h4>
<p>Every API call is logged in-memory (last 50). Click the <code>&bull; API</code> chip in the toolbar or <strong>Settings &rarr; View activity log</strong>. Each call shows method, prompt length, tokens generated, duration, and outcome (ok / err / busy).</p>
<h4>Demo</h4>
<p>Open <code>demo.html</code> in the same folder. It iframes LocalMind, auto-flips the toggle, waits for the model, and runs both a non-streaming and a streaming completion against <code>iframe.contentWindow.localmind</code>.</p>
<div class="help-note">Experimental &mdash; the shape may change before a stable v1.1.</div>
</div>
<div class="help-tab-content" data-tab="try">
<h4>Math &amp; Conversions</h4>
<span class="try-prompt" data-prompt="What is 15% of 2450?">What is 15% of 2450?</span>
<span class="try-prompt" data-prompt="Convert 72 Fahrenheit to Celsius">Convert 72 Fahrenheit to Celsius</span>
<span class="try-prompt" data-prompt="If I invest $10,000 at 7% annual return, how much after 5 years with compound interest?">Compound interest: $10K at 7% for 5 years?</span>
<h4>Time &amp; Reminders</h4>
<span class="try-prompt" data-prompt="What time is it in Tokyo?">What time is it in Tokyo?</span>
<span class="try-prompt" data-prompt="Remind me in 5 minutes to check the oven">Remind me in 5 minutes to check the oven</span>
<h4>Memory</h4>
<span class="try-prompt" data-prompt="Remember that I'm a software engineer working on a React project called Dashboard Pro">Remember: I'm a software engineer on Dashboard Pro</span>
<span class="try-prompt" data-prompt="What do you know about me and my projects?">What do you know about me?</span>
<span class="try-prompt" data-prompt="Forget everything about my preferences">Forget my preferences</span>
<h4>Translation</h4>
<span class="try-prompt" data-prompt="Translate 'Good morning, how are you?' to Japanese, French, and Hindi">Translate "Good morning" to Japanese, French, Hindi</span>
<span class="try-prompt" data-prompt="How do you say 'Where is the nearest train station?' in Spanish and German?">Train station directions in Spanish &amp; German</span>
<h4>Writing &amp; Analysis</h4>
<span class="try-prompt" data-prompt="Write a professional email declining a meeting invitation politely">Write a polite meeting decline email</span>
<span class="try-prompt" data-prompt="Summarize the pros and cons of microservices vs monolithic architecture">Microservices vs monolith: pros and cons</span>
<span class="try-prompt" data-prompt="Explain the concept of WebGPU to a non-technical person in 3 sentences">Explain WebGPU in 3 simple sentences</span>
<h4>Documents (attach a PDF, DOCX, or text file)</h4>
<span class="try-prompt" data-prompt="Summarize the key points from the document I just uploaded">Summarize the uploaded document</span>
<span class="try-prompt" data-prompt="What are the main conclusions or recommendations in my document?">Main conclusions from my document</span>
<h4>Multimodal (attach an image first)</h4>
<span class="try-prompt" data-prompt="Describe this image in detail">Describe this image in detail</span>
<span class="try-prompt" data-prompt="What text can you see in this image? Transcribe it.">Transcribe text from this image</span>
<h4>Web Research (requires API key)</h4>
<span class="try-prompt" data-prompt="What are the top tech news stories today?">Top tech news today</span>
<span class="try-prompt" data-prompt="Search for the latest WebGPU browser support status and summarize">Latest WebGPU browser support status</span>
<span class="try-prompt" data-prompt="Find recent articles about AI running locally in the browser and give me a summary with sources">AI in the browser: recent articles with sources</span>
<h4>Coding</h4>
<span class="try-prompt" data-prompt="Write a Python function that finds all prime numbers up to n using the Sieve of Eratosthenes">Sieve of Eratosthenes in Python</span>
<span class="try-prompt" data-prompt="Explain the difference between async/await and Promises in JavaScript with examples">async/await vs Promises explained</span>
<h4>Math &amp; Diagrams</h4>
<span class="try-prompt" data-prompt="Derive the quadratic formula step by step, using inline and display LaTeX math.">Derive the quadratic formula with LaTeX</span>
<span class="try-prompt" data-prompt="Show the Pythagorean identity sin^2 x + cos^2 x = 1 and sketch a short proof using LaTeX display math.">Pythagorean identity with display math</span>
<span class="try-prompt" data-prompt="Draw a Mermaid flowchart for a login flow: user submits credentials, server validates, issues a JWT, client stores it, requests authorized resources.">Mermaid: login + JWT flow</span>
<span class="try-prompt" data-prompt="Draw a Mermaid sequence diagram for the classic producer-consumer pattern with a bounded buffer.">Mermaid: producer-consumer sequence</span>
<h4>Live HTML / SVG Artifacts</h4>
<span class="try-prompt" data-prompt="Output only a full standalone HTML document in a ```html block: a click counter with +, -, and reset buttons. Style it nicely.">Interactive counter (HTML artifact)</span>
<span class="try-prompt" data-prompt="Output only a complete HTML document in a ```html block: a Pomodoro timer with 25-minute countdown, start, pause, reset.">Pomodoro timer (HTML artifact)</span>
<span class="try-prompt" data-prompt="Output only an SVG in a ```svg block: a sunrise scene with gradient sky, sun, and three mountain silhouettes.">Sunrise scene (SVG artifact)</span>
<h4>Python (Gemma 4, run_python tool)</h4>
<span class="try-prompt" data-prompt="Use run_python to compute the first 20 Fibonacci numbers and print them as a comma-separated list.">First 20 Fibonacci via Python</span>
<span class="try-prompt" data-prompt="Use run_python with pandas to generate a synthetic dataframe of 1000 rows with columns age (18-80), income (30k-200k, roughly log-normal), and compute the correlation.">Pandas synthetic data + correlation</span>
<span class="try-prompt" data-prompt="Use run_python to solve this system of equations with numpy: 2x + 3y - z = 5, x - y + 2z = 3, 3x + y + z = 10">Solve a 3x3 linear system</span>
<h4>Multi-step planning (Gemma 4, Settings toggle)</h4>
<span class="try-prompt" data-prompt="Compare cold brew, pour-over, and French press coffee: typical brew time, caffeine content, flavor profile, and equipment cost. Give a recommendation for someone new to specialty coffee.">Compare 3 coffee brewing methods</span>
<span class="try-prompt" data-prompt="Research the history of the Silk Road in 3 phases (ancient origins, medieval peak, modern revival), then summarize the key goods traded at each phase.">Silk Road in 3 phases</span>
<h4>MCP tools (after adding a server in Settings)</h4>
<span class="try-prompt" data-prompt="List every MCP tool you have available and what each one does.">List all MCP tools I have</span>
<span class="try-prompt" data-prompt="Use mcp_fetch_url (or whatever fetch-like MCP tool is available) to grab https://example.com and summarise the response.">Use an MCP fetch tool</span>
<h4>Voice to text (no prompt needed)</h4>
<p style="font-size:0.78rem;color:var(--gray-500);margin:4px 0 8px">Click the πŸ—£ button to the left of the input, speak a sentence, click again to stop. Whisper runs on-device. Try saying: <em>&ldquo;Write a short summary of the Great Pyramid of Giza in three sentences.&rdquo;</em></p>
<div class="help-note">
<strong>Tips.</strong>
<ul style="margin:4px 0 0 16px;padding:0;font-size:0.72rem;line-height:1.5">
<li>For artifact and Mermaid prompts, start with <em>&ldquo;Output only this&hellip;&rdquo;</em> or <em>&ldquo;Copy this exactly&hellip;&rdquo;</em> &mdash; smaller models tend to wrap code blocks in prose otherwise.</li>
<li>If a <code>```mermaid</code> block renders as plain code, the model stripped the language label. Re-send with <em>&ldquo;Preserve the language tag after the backticks&rdquo;</em>.</li>
<li>Click any prompt above to paste it. Web search needs a provider in Settings. Multimodal needs a Gemma 4 model + an attached image. Artifact, math, and Mermaid rendering work on any model; <code>run_python</code>, MCP tools, and multi-step planning require an agent-capable (Gemma 4) model.</li>
</ul>
</div>
</div>
</div>
</div>
<div class="status-badge loading" id="statusBadge">
<div class="spinner" id="statusSpinner"></div>
<span id="statusText">Initializing...</span>
</div>
</div>
</div>
<!-- Main card -->
<div class="card" style="position:relative">
<!-- Drag-drop overlay -->
<div class="drag-overlay" id="dragOverlay">Drop image, audio, or video here</div>
<!-- Chat messages -->
<div class="chat-area" id="chatArea">
<div class="welcome" id="welcomeMsg">
<div class="welcome-icon">&#129504;</div>
<h2>Private AI, right in your browser</h2>
<p>Gemma runs entirely on your device. Nothing is sent to any server.</p>
<p style="font-size:0.78rem;color:var(--gray-400);margin-top:4px">Switch to a Gemma 4 model for tool calling, memory, and multimodal input.<br>Web search requires an API key (Tavily, Brave) or a self-hosted SearXNG instance.</p>
</div>
</div>
<!-- Progress bar (model loading) -->
<div class="progress-section" id="progressSection">
<div class="progress-bar">
<div class="progress-fill" id="progressFill"></div>
</div>
<div class="progress-text-row">
<div class="progress-text" id="progressText">Preparing to download model...</div>
<button class="progress-cancel-btn" id="progressCancelBtn" title="Cancel download (partial chunks are preserved for resume)">Cancel</button>
</div>
</div>
<!-- Progress bar (SAM loading) -->
<div class="progress-section" id="samProgressSection">
<div class="progress-bar">
<div class="progress-fill" id="samProgressFill" style="background:linear-gradient(90deg,#059669,#10b981)"></div>
</div>
<div class="progress-text" id="samProgressText">Loading SAM model...</div>
</div>
<!-- Settings (system prompt + thinking toggle) -->
<div class="settings-panel" id="settingsPanel">
<label for="systemPrompt">System prompt (optional)</label>
<textarea id="systemPrompt" placeholder="e.g. You are a helpful coding assistant."></textarea>
<label class="toggle-row hidden" id="thinkingRow">
<input type="checkbox" id="thinkingToggle">
Show reasoning (thinking mode)
</label>
<label class="toggle-row hidden" id="plannerRow">
<input type="checkbox" id="plannerToggle">
Multi-step planning <span style="font-weight:400;color:var(--gray-500)">(experimental &mdash; slower but handles complex research tasks better)</span>
</label>
<div class="settings-divider" id="searchSettingsSection" style="display:none">
<label for="searchProvider">Web search provider</label>
<select id="searchProvider" class="settings-select">
<option value="none">None (offline only)</option>
<option value="tavily">Tavily (free tier, no card)</option>
<option value="brave">Brave Search (privacy-first)</option>
<option value="searxng">SearXNG (self-hosted)</option>
</select>
<div id="apiKeyRow" style="display:none">
<label for="searchApiKey">API key</label>
<input type="password" id="searchApiKey" class="settings-input" placeholder="Paste your API key">
</div>
<div id="searxngUrlRow" style="display:none">
<label for="searxngUrl">SearXNG instance URL</label>
<input type="text" id="searxngUrl" class="settings-input" placeholder="https://searx.example.com">
</div>
</div>
<div class="settings-divider" id="samSettingsSection" style="display:none">
<label for="samModelSelect">Segmentation model (SAM)</label>
<select id="samModelSelect" class="settings-select">
<option value="Xenova/slimsam-50-uniform">SlimSAM 50 &middot; ~10 MB (fastest)</option>
<option value="Xenova/slimsam-77-uniform" selected>SlimSAM 77 &middot; ~14 MB (default)</option>
<option value="Xenova/sam-vit-base">SAM ViT-Base &middot; ~350 MB</option>
<option value="onnx-community/sam3-tracker-ONNX">SAM 3 &middot; Latest</option>
</select>
<div style="font-size:0.7rem;color:var(--gray-500);margin-top:4px;line-height:1.5">
Loaded on first use. Gemma calls this when you ask it to segment, outline, or identify objects in attached images.
</div>
</div>
<label class="toggle-row" style="border-top:1px solid var(--gray-200);padding-top:10px;margin-top:10px">
<input type="checkbox" id="autoBackupToggle">
Auto-download backup on New Chat
</label>
<div class="settings-divider">
<label>Model cache</label>
<div id="cacheInfo" style="font-size:0.75rem;color:var(--gray-500);margin-top:4px">Loading...</div>
<label class="toggle-row" style="padding-top:8px">
<input type="checkbox" id="resumableDownloadsToggle">
Resumable downloads <span style="font-weight:400;color:var(--gray-500)">(5 MB chunks cached in IndexedDB; interrupted downloads resume from the last chunk)</span>
</label>
</div>
<div class="settings-divider">
<label for="voiceLanguageSelect">Voice to text language <span style="font-weight:400;color:var(--gray-500)">(Whisper &mdash; leaving as English improves accuracy on short clips)</span></label>
<select id="voiceLanguageSelect" class="settings-select">
<option value="en">English</option>
<option value="es">Spanish</option>
<option value="fr">French</option>
<option value="de">German</option>
<option value="it">Italian</option>
<option value="pt">Portuguese</option>
<option value="ru">Russian</option>
<option value="ja">Japanese</option>
<option value="zh">Chinese</option>
<option value="hi">Hindi</option>
<option value="ar">Arabic</option>
<option value="ko">Korean</option>
</select>
</div>
<div class="settings-divider">
<label>Custom models <span style="font-weight:400;color:var(--gray-500)">(causal LMs with ONNX exports on Hugging Face)</span></label>
<div style="display:flex;gap:6px;margin-top:4px">
<input type="text" id="customModelInput" class="settings-input" placeholder="owner/repo e.g. onnx-community/Llama-3.2-1B-Instruct" style="flex:1">
<button class="btn-icon" id="customModelAddBtn">Add</button>
</div>
<div id="customModelStatus" style="font-size:0.7rem;color:var(--gray-500);margin-top:4px;line-height:1.5;min-height:1em"></div>
<div id="customModelList" style="margin-top:6px"></div>
<div style="font-size:0.7rem;color:var(--gray-500);margin-top:4px;line-height:1.5">
Must be a Hugging Face repo with ONNX files under <code>onnx/</code>. Multimodal
custom models are not yet supported. Added models appear in the model selector.
</div>
</div>
<div class="settings-divider">
<label>Custom tools <span style="font-weight:400;color:var(--gray-500)">(agent-capable models; HTTP POST to your endpoint)</span></label>
<textarea id="customToolInput" class="settings-input" rows="6" placeholder='{
"name": "my_tool",
"description": "What this tool does",
"parameters": { "type": "object", "properties": { "query": { "type": "string" } }, "required": ["query"] },
"endpoint": "https://example.com/tool"
}' style="margin-top:4px;font-family:monospace;font-size:0.72rem"></textarea>
<div style="display:flex;gap:6px;margin-top:4px">
<button class="btn-icon" id="customToolAddBtn">Add tool</button>
</div>
<div id="customToolStatus" style="font-size:0.7rem;color:var(--gray-500);margin-top:4px;line-height:1.5;min-height:1em"></div>
<div id="customToolList" style="margin-top:6px"></div>
<div style="font-size:0.7rem;color:var(--gray-500);margin-top:4px;line-height:1.5">
On a tool call, LocalMind does <code>POST &lt;endpoint&gt;</code> with the model-generated
args as JSON body; the response JSON is fed back to the model. The endpoint must send
CORS headers for this origin. Name must be <code>[a-zA-Z_][a-zA-Z0-9_]*</code> and not
collide with a built-in.
</div>
</div>
<div class="settings-divider">
<label>MCP servers <span style="font-weight:400;color:var(--gray-500)">(Streamable HTTP transport; tools discovered via JSON-RPC <code>tools/list</code>)</span></label>
<div style="display:flex;gap:6px;margin-top:4px">
<input type="text" id="mcpUrlInput" class="settings-input" placeholder="https://mcp.example.com" style="flex:1">
<input type="password" id="mcpAuthInput" class="settings-input" placeholder="Bearer (optional)" style="flex:0 0 40%">
<button class="btn-icon" id="mcpAddBtn">Add server</button>
</div>
<div id="mcpStatus" style="font-size:0.7rem;color:var(--gray-500);margin-top:4px;line-height:1.5;min-height:1em"></div>
<div id="mcpList" style="margin-top:6px"></div>
<div style="font-size:0.7rem;color:var(--gray-500);margin-top:4px;line-height:1.5">
Tools discovered from an MCP server are registered with the prefix <code>mcp_</code>
to avoid collisions with built-ins. The server must allow CORS for this origin and
accept JSON-RPC 2.0 requests at the given URL. Connections re-established on page load.
</div>
</div>
<div class="settings-divider">
<label>JavaScript API <span style="font-weight:400;color:var(--gray-500)">(experimental)</span></label>
<label class="toggle-row" style="padding-top:4px">
<input type="checkbox" id="apiEnabledToggle">
Expose <code>window.localmind</code> to scripts in this page
</label>
<div style="font-size:0.7rem;color:var(--gray-500);margin-top:4px;line-height:1.5">
Lets other JavaScript on this page call the loaded model via an OpenAI-shaped
method. Same-tab only β€” cross-origin scripts cannot reach it. No tool calling,
memory access, or web search is exposed.
</div>
<button class="btn-icon" id="apiLogBtn" style="margin-top:8px">View activity log</button>
</div>
</div>
<!-- Attachment bar -->
<div class="attachment-bar" id="attachmentBar"></div>
<!-- Memory inspector panel -->
<div class="memory-panel" id="memoryPanel">
<div class="memory-header">
<strong>Memory</strong>
<span class="memory-count" id="memoryCount">0 memories</span>
</div>
<div class="memory-search-row">
<input type="text" id="memorySearch" placeholder="Search memories..." class="memory-search-input">
<button class="btn-icon memory-clear-btn" id="memoryClearAll" title="Clear all memories">Clear All</button>
</div>
<div class="memory-cat-pills" id="memoryCatPills"></div>
<div class="memory-list" id="memoryList"></div>
<div class="memory-search-row" style="margin-top:8px;border-top:1px solid var(--gray-200);padding-top:8px">
<button class="btn-icon" id="memoryExport" title="Export all data">Export</button>
<button class="btn-icon" id="memoryImport" title="Import data">Import</button>
<button class="btn-icon" id="memoryAuditBtn" title="Audit memory for stale, duplicate, and outlier chunks">Audit</button>
<input type="file" id="importFileInput" accept=".json" style="display:none">
</div>
</div>
<!-- Batch panel -->
<div class="batch-panel" id="batchPanel">
<div class="memory-header">
<strong>Batch Prompts</strong>
<span class="memory-count" id="batchCount">0 prompts</span>
</div>
<textarea class="batch-textarea" id="batchTextarea" placeholder="One prompt per line.&#10;Use {{previous}} to insert the previous response into the next prompt.&#10;&#10;Example:&#10;Summarise the history of WebGPU&#10;Now translate this to French: {{previous}}"></textarea>
<div class="batch-controls">
<label class="batch-chain-label">
<input type="checkbox" id="batchChainToggle" checked>
Auto-inject {{previous}}
</label>
<span class="batch-progress" id="batchProgress"></span>
<button class="btn-icon" id="batchStopBtn" disabled>Stop</button>
<button class="btn-run" id="batchRunBtn" disabled>Run</button>
</div>
</div>
<!-- Actions bar -->
<div class="actions-bar">
<button class="btn-icon" id="newChatBtn" disabled title="Archive and start new chat">New Chat</button>
<button class="btn-icon btn-danger-text" id="clearBtn" disabled title="Delete current chat">Clear</button>
<button class="btn-icon" id="settingsBtn" title="System prompt settings">Settings</button>
<button class="btn-icon" id="memoryBtn" title="View stored memories">Memory</button>
<button class="btn-icon" id="folderBtn" title="Open folder β€” ingest all .md/.txt/.pdf files into memory">Folder</button>
<button class="btn-icon" id="historyBtn" title="Past conversations">History</button>
<button class="btn-icon" id="shareBtn" title="Share conversation (encrypted link)">Share</button>
<button class="btn-icon" id="batchBtn" title="Batch prompts β€” run a list of questions sequentially">Batch</button>
<span style="flex:1"></span>
<span class="api-chip" id="apiChip" title="window.localmind API is enabled β€” click to view activity log">API</span>
<select class="model-select" id="modelSelect" title="Choose model">
<option value="bonsai-ternary-1.7b" selected>Ternary Bonsai 1.7B &middot; Agent (~470 MB)</option>
<option value="bonsai-ternary-4b">Ternary Bonsai 4B &middot; Agent (~1.1 GB)</option>
<option value="bonsai-ternary-8b">Ternary Bonsai 8B &middot; Agent (~2.2 GB)</option>
<option value="gemma3-1b">Gemma 3 1B &middot; Fast (~760 MB)</option>
<option value="gemma4-e2b">Gemma 4 E2B &middot; Multimodal (~1.5 GB)</option>
<option value="gemma4-e4b">Gemma 4 E4B &middot; Multimodal (~4.9 GB)</option>
</select>
</div>
<!-- Input -->
<div class="input-bar">
<div class="input-affordances" id="inputAffordances">
<button class="afford-btn" id="attachBtn" title="Attach file" aria-label="Attach file">&#128206;</button>
<button class="afford-btn" id="cameraBtn" title="Camera" aria-label="Take photo">&#128247;</button>
<button class="afford-btn" id="micBtn" title="Record audio" aria-label="Record audio">&#127908;</button>
</div>
<button class="afford-btn" id="voiceBtn" title="Voice to text (Whisper, on-device)" aria-label="Voice to text">&#x1F5E3;</button>
<textarea id="chatInput" rows="1" placeholder="Type a message..." disabled></textarea>
<div class="send-group">
<button class="send-btn" id="sendBtn" disabled aria-label="Send" title="Send (offline)">&#9654;</button>
<button class="search-send-btn" id="searchSendBtn" disabled aria-label="Search and Send" title="Search web + Send (makes an external API call)" style="display:none">&#127760;</button>
</div>
</div>
</div>
<!-- History sidebar -->
<div class="history-backdrop" id="historyBackdrop"></div>
<div class="history-sidebar" id="historySidebar">
<div class="history-sidebar-header">
<strong>History</strong>
<span class="memory-count" id="historyCount">0 conversations</span>
</div>
<div class="history-sidebar-list" id="historyList"></div>
</div>
</div>
<!-- Import banner (shown when page loads with a share link) -->
<div class="import-banner" id="importBanner">
<strong id="importBannerMsg">Shared conversation detected</strong>
<input type="password" id="importPassphrase" placeholder="Passphrase" style="display:none">
<button id="importConfirmBtn">Load conversation</button>
<button class="dismiss-btn" id="importDismissBtn">Dismiss</button>
</div>
<!-- Message context menu (right-click / long-press on a message) -->
<div class="msg-context-menu" id="msgContextMenu">
<button id="msgBranchBtn">Branch from here</button>
</div>
<!-- API activity log modal -->
<div class="api-log-modal" id="apiLogModal">
<div class="api-log-modal-content">
<div class="api-log-header">
<strong>API activity log</strong>
<div>
<button class="btn-icon" id="apiLogClearBtn">Clear</button>
<button class="btn-icon" id="apiLogCloseBtn">Close</button>
</div>
</div>
<div class="api-log-list" id="apiLogList">
<div class="api-log-empty">No API calls yet.</div>
</div>
</div>
</div>
<!-- Share modal -->
<div class="share-backdrop" id="shareBackdrop">
<div class="share-modal">
<h3>Share Conversation</h3>
<p class="share-meta" id="shareMetaText"></p>
<label>
<input type="checkbox" id="shareUsePassphrase">
Encrypt with passphrase (AES-256-GCM)
</label>
<input type="password" id="sharePassphrase" placeholder="Passphrase" style="display:none" autocomplete="new-password">
<div class="share-url-row">
<input type="text" id="shareUrlInput" readonly placeholder="Click Generate to create link…">
</div>
<div class="share-modal-footer">
<button id="shareCopyBtn" disabled>Copy Link</button>
<button class="btn-primary" id="shareGenerateBtn">Generate</button>
<button id="shareCloseBtn">Close</button>
</div>
</div>
</div>
<!-- Camera overlay -->
<div class="camera-overlay" id="cameraOverlay">
<video id="cameraPreview" autoplay playsinline></video>
<div class="cam-controls">
<button class="cam-capture-btn" id="camCaptureBtn">Capture</button>
<button class="cam-cancel-btn" id="camCancelBtn">Cancel</button>
</div>
</div>
<!-- Hidden file input -->
<input type="file" id="fileInput" style="display:none" multiple accept="image/*,audio/*,video/mp4,.txt,.md,.json,.csv,.pdf,.docx">
<!-- Footer -->
<div class="footer-nav">
<a href="https://naklitechie.github.io/" target="_blank" rel="noopener">&#11013; More Projects</a>
<a href="https://github.com/NakliTechie/LocalMind" target="_blank" rel="noopener">&#9000; Source</a>
</div>
<script type="module">
// ── WebGPU check ────────────────────────────────────────────
const hasWebGPU = !!navigator.gpu;
if (!hasWebGPU) {
document.getElementById('chatArea').innerHTML = `
<div class="webgpu-error">
<div style="font-size:2rem;margin-bottom:12px">&#9888;</div>
<h2>WebGPU Not Available</h2>
<p>LocalMind requires WebGPU to run the AI model in your browser.
Please use Chrome 113+, Edge 113+, or Firefox 130+ on a device with a compatible GPU.</p>
</div>`;
document.getElementById('statusBadge').className = 'status-badge error';
document.getElementById('statusSpinner').style.display = 'none';
document.getElementById('statusText').textContent = 'Not supported';
throw new Error('WebGPU not available');
}
// ── Model registry ──────────────────────────────────────────
const MODELS = {
'bonsai-ternary-1.7b': {
id: 'onnx-community/Ternary-Bonsai-1.7B-ONNX',
label: 'Ternary Bonsai 1.7B',
dtype: 'q2f16',
size: '~470 MB',
type: 'causal',
multimodal: false,
agentCapable: true,
contextSize: 32768,
// Qwen3 recommended sampling for chat/agent. min_p not exposed by
// Transformers.js generate(), top_p covers it adequately.
genConfig: { temperature: 0.7, top_k: 20, top_p: 0.8, max_new_tokens: 2048, repetition_penalty: 1.05 },
},
'bonsai-ternary-4b': {
id: 'onnx-community/Ternary-Bonsai-4B-ONNX',
label: 'Ternary Bonsai 4B',
dtype: 'q2f16',
size: '~1.1 GB',
type: 'causal',
multimodal: false,
agentCapable: true,
contextSize: 32768,
genConfig: { temperature: 0.7, top_k: 20, top_p: 0.8, max_new_tokens: 2048, repetition_penalty: 1.05 },
},
'bonsai-ternary-8b': {
id: 'onnx-community/Ternary-Bonsai-8B-ONNX',
label: 'Ternary Bonsai 8B',
dtype: 'q2f16',
size: '~2.2 GB',
type: 'causal',
multimodal: false,
agentCapable: true,
contextSize: 65536,
genConfig: { temperature: 0.7, top_k: 20, top_p: 0.8, max_new_tokens: 2048, repetition_penalty: 1.05 },
},
'gemma3-1b': {
id: 'onnx-community/gemma-3-1b-it-ONNX-GQA',
label: 'Gemma 3 1B',
dtype: 'q4f16',
size: '~760 MB',
type: 'causal', // runtime loads this as a text-only causal model
multimodal: false,
agentCapable: false,
contextSize: 4096,
genConfig: { temperature: 0.7, top_k: 50, top_p: 0.95, max_new_tokens: 2048 },
},
'gemma4-e2b': {
id: 'onnx-community/gemma-4-E2B-it-ONNX',
label: 'Gemma 4 E2B',
dtype: 'q4f16',
size: '~1.5 GB',
type: 'multimodal', // runtime loads this via the multimodal processor path
multimodal: true,
agentCapable: true,
contextSize: 8192,
genConfig: { temperature: 0.7, top_k: 40, top_p: 0.95, max_new_tokens: 2048, repetition_penalty: 1.1 },
},
'gemma4-e4b': {
id: 'onnx-community/gemma-4-E4B-it-ONNX',
label: 'Gemma 4 E4B',
dtype: 'q4f16',
size: '~4.9 GB',
type: 'multimodal', // runtime loads this via the multimodal processor path
multimodal: true,
agentCapable: true,
contextSize: 12288,
genConfig: { temperature: 0.7, top_k: 40, top_p: 0.95, max_new_tokens: 2048, repetition_penalty: 1.1 },
},
};
// ── State ───────────────────────────────────────────────────
let messages = [];
let generating = false;
let msgIdCounter = 0;
let currentAssistantEl = null;
let currentAssistantText = '';
let activeModelKey = null;
let worker = null;
let modelReady = false;
let attachments = []; // [{type:'image'|'audio', blob:Blob, thumb:string, name:string, experimental?:bool}]
let mediaRecorder = null;
let cameraStream = null;
let generateResolve = null; // Promise resolve for agentic loop
let dirHandle = null; // FileSystemDirectoryHandle for open folder
let loadResolvers = null; // { resolve, reject } for window.localmind.load() awaiters
let currentInferenceContext = null; // { onToken } when a streaming API caller wants per-token notifications
// Most recent file the worker reported activity on β€” used so the
// 'progress_total' event (aggregate across all files, no `file` field)
// can still show *which* file is currently streaming bytes.
let currentDownloadFile = '';
// ── JavaScript API state ────────────────────────────────────
// Feature-flag persisted in localStorage. The real window.localmind
// object is attached in a later step; this step only builds the
// toggle, chip, and activity log infrastructure.
let apiEnabled = false;
try { apiEnabled = localStorage.getItem('lm_api_enabled') === '1'; } catch {}
// Multi-step planning: when on + agent-capable model, sendMessage routes
// through runPlannerAgent (plan β†’ per-step execute β†’ synthesize) instead
// of the single-pass agent loop. Default off; Gemma 4 at ~4.5B produces
// inconsistent plans, so this is labelled experimental.
let plannerEnabled = false;
try { plannerEnabled = localStorage.getItem('lm_planner_enabled') === '1'; } catch {}
const API_LOG_MAX = 50;
const apiLog = []; // newest first: [{ ts, method, promptLen, tokens, durationMs, outcome }]
// Restore from sessionStorage
try {
const saved = sessionStorage.getItem('localmind_chat');
if (saved) messages = JSON.parse(saved);
} catch {}
// ── Agent: Tool Registry ──────────────────────────────────
const TOOL_REGISTRY = {
calculate: {
description: 'Evaluate a mathematical expression. Use for arithmetic, unit conversions, percentages.',
parameters: {
type: 'object',
properties: { expression: { type: 'string', description: 'Math expression to evaluate, e.g. "2 * 3 + 1"' } },
required: ['expression'],
},
execute(args) {
try {
const expr = String(args.expression).replace(/[^0-9+\-*/.()% ]/g, '');
const result = Function('"use strict"; return (' + expr + ')')();
return { result: Number(result) };
} catch (e) { return { error: e.message }; }
},
},
get_current_time: {
description: 'Get the current date and time.',
parameters: {
type: 'object',
properties: { timezone: { type: 'string', description: 'IANA timezone, e.g. "America/New_York". Defaults to local.' } },
required: [],
},
execute(args) {
try {
const opts = { dateStyle: 'full', timeStyle: 'long' };
if (args.timezone) opts.timeZone = args.timezone;
const formatted = new Intl.DateTimeFormat('en-US', opts).format(new Date());
return { datetime: formatted, iso: new Date().toISOString() };
} catch (e) { return { error: e.message }; }
},
},
store_memory: {
description: 'Store an important fact, preference, or finding for later recall. Memories persist across sessions.',
parameters: {
type: 'object',
properties: {
content: { type: 'string', description: 'The information to remember' },
category: { type: 'string', enum: ['fact', 'preference', 'finding', 'task'], description: 'Category of memory' },
},
required: ['content'],
},
async execute(args) {
try {
const count = await embedAndStore(args.content, args.category || 'fact', 'assistant');
return { stored: true, chunks: count };
} catch (e) { return { error: e.message }; }
},
},
search_memory: {
description: 'Search your stored memories, past conversations, and documents. Use when the user asks about something you may have been told before.',
parameters: {
type: 'object',
properties: { query: { type: 'string', description: 'What to search for in memory' } },
required: ['query'],
},
async execute(args) {
try {
const results = await searchMemory(args.query, 5);
if (results.length === 0) return { found: false, message: 'No relevant memories found.' };
return {
found: true,
count: results.length,
memories: results.map(r => ({
content: r.text,
category: r.category,
score: Math.round(r.score * 100) / 100,
source: r.source,
})),
};
} catch (e) { return { error: e.message }; }
},
},
set_reminder: {
description: 'Set a reminder that will notify the user after a specified number of minutes. Uses browser notifications.',
parameters: {
type: 'object',
properties: {
message: { type: 'string', description: 'The reminder message to show' },
minutes: { type: 'number', description: 'Minutes from now (1-120)' },
},
required: ['message', 'minutes'],
},
execute(args) {
const mins = Math.max(1, Math.min(120, Number(args.minutes) || 5));
const msg = args.message || 'Reminder';
// Request notification permission if needed
if ('Notification' in window && Notification.permission === 'default') {
Notification.requestPermission();
}
setTimeout(() => {
// Browser notification
if ('Notification' in window && Notification.permission === 'granted') {
new Notification('LocalMind Reminder', { body: msg, icon: 'data:image/svg+xml,<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 100 100"><text y=".9em" font-size="90">🧠</text></svg>' });
}
// Also show a toast in case notifications are blocked
showToast('Reminder: ' + msg);
}, mins * 60 * 1000);
const time = new Date(Date.now() + mins * 60 * 1000);
return { set: true, message: msg, fires_at: time.toLocaleTimeString(), minutes: mins };
},
},
list_memories: {
description: 'List all stored memories grouped by category with counts. Use to see what you remember about the user.',
parameters: { type: 'object', properties: {}, required: [] },
async execute() {
try {
const chunks = await getAllChunks();
if (chunks.length === 0) return { total: 0, message: 'No memories stored.' };
const byCategory = {};
for (const c of chunks) {
byCategory[c.category] = (byCategory[c.category] || 0) + 1;
}
const recent = chunks.sort((a, b) => b.timestamp - a.timestamp).slice(0, 5);
return {
total: chunks.length,
categories: byCategory,
recent: recent.map(r => ({ text: r.text.slice(0, 80), category: r.category, source: r.source })),
};
} catch (e) { return { error: e.message }; }
},
},
delete_memory: {
description: 'Delete memories matching a search query. Use when the user asks you to forget something.',
parameters: {
type: 'object',
properties: { query: { type: 'string', description: 'What to forget β€” will delete matching memories' } },
required: ['query'],
},
async execute(args) {
try {
const results = await searchMemory(args.query, 10);
if (results.length === 0) return { deleted: 0, message: 'No matching memories found.' };
// Delete matches above 0.5 similarity (confident matches only)
const toDelete = results.filter(r => r.score >= 0.5);
for (const r of toDelete) await deleteChunk(r.id);
return { deleted: toDelete.length, message: `Deleted ${toDelete.length} matching memor${toDelete.length === 1 ? 'y' : 'ies'}.` };
} catch (e) { return { error: e.message }; }
},
},
segment_image: {
description: 'Segment objects in the attached image using SAM. YOU MUST estimate the point coordinates yourself by looking at the image β€” do NOT ask the user for coordinates. Coordinates are [x, y] in normalized [0,1] range: (0,0)=top-left, (1,1)=bottom-right, (0.5,0.5)=center. Example: to segment a dog in the center-left of the image, use points [[0.3, 0.5]] with labels [1]. Always call this tool immediately when the user asks to segment, outline, isolate, or identify objects β€” pick your best estimate for the object center.',
parameters: {
type: 'object',
properties: {
points: {
type: 'array',
description: 'Array of [x, y] pairs in [0,1] range. YOU choose these by estimating where the target object is in the image. Place at the center of each object to segment.',
items: { type: 'array', items: { type: 'number' }, minItems: 2, maxItems: 2 },
},
labels: {
type: 'array',
description: 'Same length as points. 1 = foreground (the object to segment), 0 = background (area to exclude).',
items: { type: 'integer', enum: [0, 1] },
},
},
required: ['points', 'labels'],
},
requiresImage: true,
async execute(args, context) {
if (!context || !context.imageBlob) {
return { error: 'No image attached. Ask the user to attach an image first.' };
}
const { points, labels } = args;
if (!Array.isArray(points) || !Array.isArray(labels) || points.length === 0) {
return { error: 'points and labels must be non-empty arrays of equal length.' };
}
if (points.length !== labels.length) {
return { error: `points (${points.length}) and labels (${labels.length}) must have the same length.` };
}
try {
// Get image dimensions to scale normalized coords to pixels
const bmp = await createImageBitmap(context.imageBlob);
const imgW = bmp.width;
const imgH = bmp.height;
bmp.close();
showToast('Loading SAM model...');
// Run SAM once per foreground point to get separate masks per object.
// SAM returns 3 alternative masks per prompt β€” we pick the best one each time.
const objectMasks = [];
const objectScores = [];
for (let i = 0; i < points.length; i++) {
if (labels[i] !== 1) continue; // skip background points
const px = Math.round(Math.max(0, Math.min(1, points[i][0])) * imgW);
const py = Math.round(Math.max(0, Math.min(1, points[i][1])) * imgH);
const { masks, scores: sc } = await LocalMind.runtime.segmentImage(context.imageBlob, [[px, py]], [1]);
// Pick best-scoring mask from the 3 alternatives
let bestIdx = 0, bestScore = -1;
for (let j = 0; j < masks.length; j++) {
if ((sc[j] || 0) > bestScore) { bestScore = sc[j] || 0; bestIdx = j; }
}
objectMasks.push(masks[bestIdx]);
objectScores.push(bestScore);
}
if (objectMasks.length === 0) {
return { error: 'No foreground points provided (all labels were 0).' };
}
// Build JSON-safe result for the model
const maskSummaries = objectMasks.map((m, i) => {
const totalPixels = m.width * m.height;
const fgPixels = m.data.reduce((sum, v) => sum + (v > 128 ? 1 : 0), 0);
return {
object: i + 1,
score: Math.round(objectScores[i] * 1000) / 1000,
area_percent: Math.round((fgPixels / totalPixels) * 10000) / 100,
};
});
const result = {
success: true,
image_width: imgW,
image_height: imgH,
objects_found: objectMasks.length,
objects: maskSummaries,
_rawMasks: objectMasks,
_imageBlob: context.imageBlob,
_scores: objectScores,
};
return result;
} catch (e) {
return { error: 'Segmentation failed: ' + e.message };
}
},
},
};
// ── Web: Search Provider Adapters ─────────────────────────
async function parseApiError(res, provider) {
const status = res.status;
let detail = '';
try {
const body = await res.text();
try {
const json = JSON.parse(body);
detail = json.message || json.error?.message || json.detail || json.error || '';
} catch { detail = body.slice(0, 200); }
} catch {}
// Human-readable messages for common HTTP errors
if (status === 401 || status === 403) return `${provider}: Invalid or expired API key. Check your key in Settings.`;
if (status === 429) return `${provider}: Rate limit or quota exceeded. ${detail || 'Try again later or check your plan.'}`;
if (status === 402) return `${provider}: Payment required. Your free tier may be exhausted.`;
if (status === 503 || status === 502) return `${provider}: Service temporarily unavailable. Try again in a moment.`;
return `${provider} error ${status}: ${detail || res.statusText}`;
}
const SearchProviders = {
async brave(query, apiKey) {
const res = await fetch(`https://api.search.brave.com/res/v1/web/search?q=${encodeURIComponent(query)}&count=5`, {
headers: { 'Accept': 'application/json', 'Accept-Encoding': 'gzip', 'X-Subscription-Token': apiKey },
});
if (!res.ok) throw new Error(await parseApiError(res, 'Brave Search'));
const data = await res.json();
return (data.web?.results || []).slice(0, 5).map(r => ({
title: r.title, url: r.url, snippet: r.description || '', content: null, age: r.age || null,
}));
},
async tavily(query, apiKey) {
const res = await fetch('https://api.tavily.com/search', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ api_key: apiKey, query, search_depth: 'basic', include_raw_content: false, max_results: 5 }),
});
if (!res.ok) throw new Error(await parseApiError(res, 'Tavily'));
const data = await res.json();
return (data.results || []).map(r => ({
title: r.title, url: r.url, snippet: r.content || '', content: r.raw_content || null, age: null,
}));
},
async searxng(query, instanceUrl) {
const base = instanceUrl.replace(/\/+$/, '');
const res = await fetch(`${base}/search?q=${encodeURIComponent(query)}&format=json&categories=general`);
if (!res.ok) throw new Error(await parseApiError(res, 'SearXNG'));
const data = await res.json();
return (data.results || []).slice(0, 5).map(r => ({
title: r.title, url: r.url, snippet: r.content || '', content: null, age: null,
}));
},
};
function getSearchConfig() {
const provider = localStorage.getItem('lm_search_provider') || 'none';
const apiKey = localStorage.getItem('lm_search_key') || '';
const searxngUrl = localStorage.getItem('lm_searxng_url') || '';
return { provider, apiKey, searxngUrl };
}
function isSearchConfigured() {
const { provider, apiKey, searxngUrl } = getSearchConfig();
if (provider === 'none') return false;
if (provider === 'searxng') return !!searxngUrl;
return !!apiKey;
}
async function executeWebSearch(query) {
const { provider, apiKey, searxngUrl } = getSearchConfig();
if (provider === 'brave') return SearchProviders.brave(query, apiKey);
if (provider === 'tavily') return SearchProviders.tavily(query, apiKey);
if (provider === 'searxng') return SearchProviders.searxng(query, searxngUrl);
throw new Error('No search provider configured');
}
// ── Web: Page Fetcher ───────────────────────────────────────
let readabilityLoaded = false;
async function ensureReadability() {
if (readabilityLoaded) return;
return new Promise((resolve, reject) => {
const script = document.createElement('script');
script.src = 'https://cdn.jsdelivr.net/npm/@mozilla/readability@0.5.0/Readability.min.js';
// SRI: if the CDN is compromised or the file is tampered with, the
// browser will refuse to execute the script and fire onerror.
script.integrity = 'sha384-DHTbAPJmhf9shXtIK080V86CoLVziMLp/Gdn1EVZCKVyvIMiENtuUVSfckMLbkYO';
script.crossOrigin = 'anonymous';
script.onload = () => { readabilityLoaded = true; resolve(); };
script.onerror = () => reject(new Error('Failed to load or verify Readability.js'));
document.head.appendChild(script);
});
}
async function fetchAndExtract(url, userQuery) {
// Try direct fetch, fallback to CORS proxy
let html;
try {
const res = await fetch(url, { signal: AbortSignal.timeout(8000) });
html = await res.text();
} catch {
try {
const res = await fetch(`https://api.allorigins.win/raw?url=${encodeURIComponent(url)}`, { signal: AbortSignal.timeout(10000) });
html = await res.text();
} catch (e) {
return { error: `Could not fetch page: ${e.message}`, url };
}
}
// Extract with Readability
try {
await ensureReadability();
const doc = new DOMParser().parseFromString(html, 'text/html');
const reader = new Readability(doc);
const article = reader.parse();
let text = article ? article.textContent : doc.body?.textContent || '';
text = text.replace(/\s+/g, ' ').trim();
// Semantic pre-filter if embedding is ready
if (LocalMind.runtime.embeddingsReady && userQuery && text.length > 1500) {
try {
const paragraphs = text.match(/.{200,1000}[.!?\n]|.{200,1000}$/g) || [text.slice(0, 3000)];
const paraVecs = await LocalMind.runtime.embed(paragraphs);
const queryVec = await LocalMind.runtime.embed(userQuery);
const scored = paragraphs.map((p, i) => ({ text: p, score: cosineSimilarity(queryVec, paraVecs[i]) }));
scored.sort((a, b) => b.score - a.score);
text = scored.slice(0, 5).map(s => s.text).join('\n\n');
} catch {
text = text.slice(0, 3500);
}
} else {
text = text.slice(0, 3500);
}
return { title: article?.title || '', content: text, url };
} catch (e) {
return { content: html.replace(/<[^>]+>/g, ' ').replace(/\s+/g, ' ').slice(0, 3500), url };
}
}
// ── Web: Register web_search + fetch_page tools ─────────────
// Added dynamically so they only appear in the tool schema when configured
TOOL_REGISTRY.web_search = {
description: 'Search the web for current information. Use when you need facts, news, or data you don\'t know.',
parameters: {
type: 'object',
properties: { query: { type: 'string', description: 'Search query, 3-8 words' } },
required: ['query'],
},
requiresWeb: true,
async execute(args) {
try {
const results = await executeWebSearch(args.query);
// Cache results in RAG for future offline use
if (LocalMind.runtime.embeddingsReady && results.length > 0) {
const snippetText = results.map(r => `${r.title}: ${r.snippet}`).join('\n');
embedAndStore(snippetText, 'finding', 'web-search').catch(() => {});
}
return { provider: getSearchConfig().provider, query: args.query, results };
} catch (e) { return { error: e.message }; }
},
};
TOOL_REGISTRY.fetch_page = {
description: 'Fetch and read content from a URL. Use after web_search to get details from a promising result.',
parameters: {
type: 'object',
properties: { url: { type: 'string', description: 'Full URL to fetch' } },
required: ['url'],
},
requiresWeb: true,
async execute(args, context) {
try {
const result = await fetchAndExtract(args.url, context?.userQuery || '');
// Cache fetched content in RAG
if (LocalMind.runtime.embeddingsReady && result.content) {
embedAndStore(result.content, 'finding', args.url).catch(() => {});
}
return result;
} catch (e) { return { error: e.message }; }
},
};
// ── Agent: tool-call helpers ─────────────────────────────────
// The system-prompt scaffolding, output parser, and tool-result
// formatting now live inside the runtime adapter (search for
// _adapterBuildToolPrompt / _adapterParseToolCalls /
// _adapterFormatToolResultMessage). The agent loop in
// sendMessage() consumes tool_call events from runtime.chat()
// instead of parsing model output itself.
// Singleton Pyodide worker β€” instantiated lazily the first time the
// run_python tool is called. The blob worker imports Pyodide ESM and
// holds the interpreter across calls so subsequent runs reuse the
// already-loaded runtime and installed packages.
let __pyodideWorker = null;
let __pyodideLoadingToastShown = false;
function runPython(userCode) {
if (!__pyodideWorker) {
__pyodideWorker = createPyodideWorker();
__pyodideWorker.addEventListener('message', (e) => {
if (e.data.type === 'loading' && !__pyodideLoadingToastShown) {
showToast('Loading Python runtime (first use)\u2026');
__pyodideLoadingToastShown = true;
} else if (e.data.type === 'loaded') {
showToast('Python runtime ready');
}
});
}
return new Promise((resolve) => {
const id = Math.random().toString(36).slice(2);
const onMsg = (e) => {
if (e.data.id !== id) return;
if (e.data.type === 'result') {
__pyodideWorker.removeEventListener('message', onMsg);
resolve(e.data);
}
};
__pyodideWorker.addEventListener('message', onMsg);
__pyodideWorker.postMessage({ type: 'run', code: userCode, id });
});
}
TOOL_REGISTRY.run_python = {
description: 'Execute Python code in a sandboxed in-browser runtime (Pyodide). Imported packages (numpy, pandas, matplotlib, etc.) auto-install on first use. Returns stdout, stderr, and the last-expression result as strings. Good for arithmetic, data manipulation, CSV parsing, quick simulations. First call downloads ~10 MB of Python runtime.',
parameters: {
type: 'object',
properties: {
code: { type: 'string', description: 'Python source to execute. The last expression (if any) is returned in `result`.' },
},
required: ['code'],
},
async execute(args) {
const code = typeof args.code === 'string' ? args.code : '';
if (!code.trim()) return { error: 'code parameter is required' };
try {
const out = await runPython(code);
if (out.error) return { error: out.error };
const compact = { stdout: out.stdout || '', stderr: out.stderr || '' };
if (out.result) compact.result = out.result;
return compact;
} catch (e) {
return { error: 'Python execution failed: ' + (e.message || e) };
}
},
};
// Helper used only by the agent loop to build the OpenAI-shaped
// tools array from TOOL_REGISTRY. Stays outside the adapter β€” the
// adapter receives an explicit tools[] array per call and never
// sees TOOL_REGISTRY directly.
function buildToolsForRuntime() {
const webConfigured = isSearchConfigured();
const caps = LocalMind.runtime.capabilities();
const hasImage = !!(caps && caps.image);
return Object.entries(TOOL_REGISTRY)
.filter(([_, t]) => !t.requiresWeb || webConfigured)
.filter(([_, t]) => !t.requiresImage || hasImage)
.map(([name, t]) => ({
name,
description: t.description,
parameters: t.parameters,
}));
}
// ── Agent: Context Budget Helper ────────────────────────────
function approxTokens(text) {
if (!text) return 0;
return Math.ceil(String(text).length / 3.5);
}
// ── Agent: Sliding Window Conversation Builder ────────────
let conversationSummary = '';
function buildContextMessages(allMessages, systemPrompt, modelKey) {
const m = MODELS[modelKey];
const maxCtx = m ? m.contextSize : 4096;
// Reserve tokens: system ~500, response ~2048, headroom ~300
const budgetForHistory = maxCtx - 2048 - 300;
const out = [];
if (systemPrompt) out.push({ role: 'system', content: systemPrompt });
// Calculate system prompt cost
let used = approxTokens(systemPrompt);
// Always include all messages if they fit
let historyTokens = 0;
for (const msg of allMessages) {
historyTokens += approxTokens(typeof msg.content === 'string' ? msg.content : JSON.stringify(msg.content));
}
if (used + historyTokens <= budgetForHistory) {
// Everything fits β€” send it all
for (const msg of allMessages) {
out.push({ role: msg.role === 'assistant' ? 'assistant' : 'user', content: msg.content });
}
return out;
}
// Doesn't fit β€” sliding window: keep last 3 turn-pairs verbatim, summarize the rest
// A "turn pair" = one user + one assistant message
const pairs = [];
let i = 0;
while (i < allMessages.length) {
if (allMessages[i].role === 'user') {
const user = allMessages[i];
const asst = (i + 1 < allMessages.length && allMessages[i + 1].role === 'assistant') ? allMessages[i + 1] : null;
pairs.push({ user, assistant: asst });
i += asst ? 2 : 1;
} else {
// Orphan assistant message
pairs.push({ user: null, assistant: allMessages[i] });
i++;
}
}
const keepPairs = Math.min(3, pairs.length);
const recentPairs = pairs.slice(-keepPairs);
// If we have a conversation summary from older turns, prepend it
if (conversationSummary && pairs.length > keepPairs) {
out.push({ role: 'user', content: '[Previous conversation summary]\n' + conversationSummary });
out.push({ role: 'assistant', content: 'Understood, I have context from our earlier conversation.' });
}
// Add recent turns
for (const pair of recentPairs) {
if (pair.user) out.push({ role: 'user', content: pair.user.content });
if (pair.assistant) out.push({ role: 'assistant', content: pair.assistant.content });
}
return out;
}
// ── RAG: Embedding Worker (MiniLM, WASM/CPU) ─────────────
// Lives entirely inside LocalMind.runtime now. Use:
// await LocalMind.runtime.embed(text) -> Float32Array
// await LocalMind.runtime.embed(textArray) -> Float32Array[]
// LocalMind.runtime.embeddingsReady -> boolean (gate)
// The worker is loaded lazily on the first embed() call.
// ── RAG: IndexedDB Vector Store ─────────────────────────────
const DB_NAME = 'localmind_rag';
const DB_VERSION = 2;
function openRAGDB() {
return new Promise((resolve, reject) => {
const req = indexedDB.open(DB_NAME, DB_VERSION);
req.onupgradeneeded = (e) => {
const db = e.target.result;
if (!db.objectStoreNames.contains('chunks')) {
const store = db.createObjectStore('chunks', { keyPath: 'id' });
store.createIndex('category', 'category', { unique: false });
store.createIndex('source', 'source', { unique: false });
}
if (!db.objectStoreNames.contains('profile')) {
db.createObjectStore('profile', { keyPath: 'key' });
}
if (!db.objectStoreNames.contains('conversations')) {
const convStore = db.createObjectStore('conversations', { keyPath: 'id' });
convStore.createIndex('updated', 'updated', { unique: false });
}
};
req.onsuccess = () => resolve(req.result);
req.onerror = () => reject(req.error);
});
}
async function storeChunks(chunks) {
const db = await openRAGDB();
const tx = db.transaction('chunks', 'readwrite');
const store = tx.objectStore('chunks');
for (const chunk of chunks) store.put(chunk);
return new Promise((resolve, reject) => {
tx.oncomplete = resolve;
tx.onerror = () => reject(tx.error);
});
}
async function getAllChunks() {
const db = await openRAGDB();
return new Promise((resolve, reject) => {
const tx = db.transaction('chunks', 'readonly');
const req = tx.objectStore('chunks').getAll();
req.onsuccess = () => resolve(req.result);
req.onerror = () => reject(req.error);
});
}
async function deleteChunk(id) {
const db = await openRAGDB();
const tx = db.transaction('chunks', 'readwrite');
tx.objectStore('chunks').delete(id);
return new Promise((resolve, reject) => {
tx.oncomplete = resolve;
tx.onerror = () => reject(tx.error);
});
}
async function clearAllChunks() {
const db = await openRAGDB();
const tx = db.transaction('chunks', 'readwrite');
tx.objectStore('chunks').clear();
return new Promise((resolve, reject) => {
tx.oncomplete = resolve;
tx.onerror = () => reject(tx.error);
});
}
// ── Conversation History (IndexedDB) ──────────────────────
let activeConversationId = null;
function newConversationId() { return Date.now().toString(36) + '-' + Math.random().toString(36).slice(2, 6); }
async function saveConversation(msgs, id) {
if (!msgs || msgs.length === 0) return;
const db = await openRAGDB();
const tx = db.transaction('conversations', 'readwrite');
const now = Date.now();
// Generate title from first user message
const firstUser = msgs.find(m => m.role === 'user');
let title = 'Untitled';
if (firstUser) {
const text = typeof firstUser.content === 'string' ? firstUser.content : firstUser.content.filter(c => c.type === 'text').map(c => c.text).join(' ');
title = text.slice(0, 60) + (text.length > 60 ? '...' : '');
}
tx.objectStore('conversations').put({
id: id || activeConversationId || newConversationId(),
title,
messages: msgs.map(m => {
if (m.role === 'assistant') return m;
if (typeof m.content === 'string') return m;
const text = m.content.filter(c => c.type === 'text').map(c => c.text).join(' ');
return { role: m.role, content: text };
}),
modelKey: activeModelKey,
messageCount: msgs.length,
created: now,
updated: now,
});
return new Promise((resolve, reject) => { tx.oncomplete = resolve; tx.onerror = () => reject(tx.error); });
}
async function getAllConversations() {
const db = await openRAGDB();
return new Promise((resolve, reject) => {
const tx = db.transaction('conversations', 'readonly');
const req = tx.objectStore('conversations').getAll();
req.onsuccess = () => resolve(req.result.sort((a, b) => b.updated - a.updated));
req.onerror = () => reject(req.error);
});
}
async function getConversation(id) {
const db = await openRAGDB();
return new Promise((resolve, reject) => {
const tx = db.transaction('conversations', 'readonly');
const req = tx.objectStore('conversations').get(id);
req.onsuccess = () => resolve(req.result);
req.onerror = () => reject(req.error);
});
}
async function deleteConversation(id) {
const db = await openRAGDB();
const tx = db.transaction('conversations', 'readwrite');
tx.objectStore('conversations').delete(id);
return new Promise((resolve, reject) => { tx.oncomplete = resolve; tx.onerror = () => reject(tx.error); });
}
async function exportAllData() {
const [chunks, convs, profile] = await Promise.all([getAllChunks(), getAllConversations(), getProfile()]);
return { version: 1, exported: new Date().toISOString(), memories: chunks, conversations: convs, profile };
}
async function importData(data) {
if (!data || data.version !== 1) throw new Error('Invalid export format');
if (data.memories?.length) await storeChunks(data.memories);
if (data.profile) await saveProfile(data.profile);
if (data.conversations?.length) {
const db = await openRAGDB();
const tx = db.transaction('conversations', 'readwrite');
const store = tx.objectStore('conversations');
for (const c of data.conversations) store.put(c);
await new Promise((resolve, reject) => { tx.oncomplete = resolve; tx.onerror = () => reject(tx.error); });
}
}
function cosineSimilarity(a, b) {
let dot = 0, normA = 0, normB = 0;
for (let i = 0; i < a.length; i++) {
dot += a[i] * b[i];
normA += a[i] * a[i];
normB += b[i] * b[i];
}
return dot / (Math.sqrt(normA) * Math.sqrt(normB) + 1e-8);
}
async function searchByVector(queryVec, topK = 5, threshold = 0.3) {
const chunks = await getAllChunks();
const scored = chunks.map(c => ({
...c,
score: cosineSimilarity(queryVec, c.embedding),
})).filter(c => c.score >= threshold);
scored.sort((a, b) => b.score - a.score);
return scored.slice(0, topK);
}
// ── RAG: User Profile (Working Memory) ──────────────────────
async function getProfile() {
const db = await openRAGDB();
return new Promise((resolve, reject) => {
const tx = db.transaction('profile', 'readonly');
const req = tx.objectStore('profile').get('user');
req.onsuccess = () => resolve(req.result || { key: 'user', name: '', preferences: [], facts: [] });
req.onerror = () => reject(req.error);
});
}
async function saveProfile(profile) {
const db = await openRAGDB();
const tx = db.transaction('profile', 'readwrite');
tx.objectStore('profile').put({ ...profile, key: 'user' });
return new Promise((resolve, reject) => {
tx.oncomplete = resolve;
tx.onerror = () => reject(tx.error);
});
}
// ── RAG: Chunk Pipeline ─────────────────────────────────────
function chunkText(text, maxChars = 900, overlapChars = 200) {
// Split at sentence boundaries, then group into chunks
const sentences = text.match(/[^.!?\n]+[.!?\n]+|[^.!?\n]+$/g) || [text];
const chunks = [];
let current = '';
for (const sentence of sentences) {
if (current.length + sentence.length > maxChars && current.length > 0) {
chunks.push(current.trim());
// Keep overlap from end of current chunk
const words = current.split(/\s+/);
const overlapWords = [];
let len = 0;
for (let i = words.length - 1; i >= 0 && len < overlapChars; i--) {
overlapWords.unshift(words[i]);
len += words[i].length + 1;
}
current = overlapWords.join(' ') + ' ' + sentence;
} else {
current += (current ? ' ' : '') + sentence;
}
}
if (current.trim()) chunks.push(current.trim());
return chunks;
}
async function embedAndStore(text, category, source) {
const textChunks = chunkText(text);
const vectors = await LocalMind.runtime.embed(textChunks);
const now = Date.now();
const chunks = textChunks.map((t, i) => ({
id: `${now}-${i}-${Math.random().toString(36).slice(2, 6)}`,
text: t,
embedding: vectors[i],
category: category || 'fact',
source: source || 'user',
timestamp: now,
}));
await storeChunks(chunks);
return chunks.length;
}
async function searchMemory(query, topK = 5) {
const vec = await LocalMind.runtime.embed(query);
return searchByVector(vec, topK);
}
// ── DOM refs ────────────────────────────────────────────────
const chatArea = document.getElementById('chatArea');
const chatInput = document.getElementById('chatInput');
const sendBtn = document.getElementById('sendBtn');
const clearBtn = document.getElementById('clearBtn');
const settingsBtn = document.getElementById('settingsBtn');
const settingsPanel = document.getElementById('settingsPanel');
const systemPrompt = document.getElementById('systemPrompt');
const statusBadge = document.getElementById('statusBadge');
const statusSpinner = document.getElementById('statusSpinner');
const statusText = document.getElementById('statusText');
const progressSection = document.getElementById('progressSection');
const progressFill = document.getElementById('progressFill');
const progressText = document.getElementById('progressText');
const welcomeMsg = document.getElementById('welcomeMsg');
const helpBtn = document.getElementById('helpBtn');
const helpPopover = document.getElementById('helpPopover');
const modelSelect = document.getElementById('modelSelect');
const attachmentBar = document.getElementById('attachmentBar');
const inputAffordances = document.getElementById('inputAffordances');
const attachBtn = document.getElementById('attachBtn');
const cameraBtn = document.getElementById('cameraBtn');
const micBtn = document.getElementById('micBtn');
const fileInput = document.getElementById('fileInput');
const thinkingToggle = document.getElementById('thinkingToggle');
const thinkingRow = document.getElementById('thinkingRow');
const plannerRow = document.getElementById('plannerRow');
const plannerToggle = document.getElementById('plannerToggle');
plannerToggle.checked = plannerEnabled;
plannerToggle.addEventListener('change', () => {
plannerEnabled = plannerToggle.checked;
try { localStorage.setItem('lm_planner_enabled', plannerEnabled ? '1' : '0'); } catch {}
});
const cameraOverlay = document.getElementById('cameraOverlay');
const cameraPreview = document.getElementById('cameraPreview');
const camCaptureBtn = document.getElementById('camCaptureBtn');
const camCancelBtn = document.getElementById('camCancelBtn');
const dragOverlay = document.getElementById('dragOverlay');
// ── Help popover ────────────────────────────────────────────
helpBtn.addEventListener('click', e => {
e.stopPropagation();
helpPopover.classList.toggle('open');
});
document.addEventListener('click', () => helpPopover.classList.remove('open'));
// Help popover: tab switching
helpPopover.addEventListener('click', e => {
e.stopPropagation();
const tab = e.target.closest('.help-tab');
if (tab) {
helpPopover.querySelectorAll('.help-tab').forEach(t => t.classList.remove('active'));
helpPopover.querySelectorAll('.help-tab-content').forEach(c => c.classList.remove('active'));
tab.classList.add('active');
helpPopover.querySelector(`.help-tab-content[data-tab="${tab.dataset.tab}"]`).classList.add('active');
return;
}
// Click-to-paste prompt
const prompt = e.target.closest('.try-prompt');
if (prompt && prompt.dataset.prompt) {
chatInput.value = prompt.dataset.prompt;
chatInput.dispatchEvent(new Event('input'));
helpPopover.classList.remove('open');
chatInput.focus();
}
});
// ── Settings panel ──────────────────────────────────────────
settingsBtn.addEventListener('click', () => {
settingsPanel.classList.toggle('open');
memoryPanel.classList.remove('open');
closeHistorySidebar();
});
// ── Model cache management ────────────────────────────────
const cacheInfo = document.getElementById('cacheInfo');
async function refreshCacheInfo() {
try {
const keys = await caches.keys();
const modelCaches = keys.filter(k => k.includes('transformers'));
if (modelCaches.length === 0) {
cacheInfo.innerHTML = 'No cached models';
return;
}
let totalSize = 0;
const entries = [];
for (const key of modelCaches) {
const cache = await caches.open(key);
const requests = await cache.keys();
let size = 0;
for (const req of requests) {
try {
const res = await cache.match(req);
if (res) {
const blob = await res.blob();
size += blob.size;
}
} catch {}
}
totalSize += size;
entries.push({ key, size, count: requests.length });
}
const sizeMB = (totalSize / 1024 / 1024).toFixed(0);
let html = `<strong>${sizeMB} MB</strong> cached (${entries.reduce((a, e) => a + e.count, 0)} files)<br>`;
html += `<button class="btn-icon btn-danger-text" style="font-size:0.7rem;margin-top:4px" id="clearCacheBtn">Clear model cache</button>`;
cacheInfo.innerHTML = html;
document.getElementById('clearCacheBtn')?.addEventListener('click', async () => {
if (!confirm('Delete all cached models? They will re-download on next use.')) return;
for (const key of modelCaches) await caches.delete(key);
refreshCacheInfo();
showToast('Model cache cleared');
});
} catch (e) {
cacheInfo.innerHTML = 'Cache info unavailable';
}
}
// Refresh cache info when settings panel opens
settingsBtn.addEventListener('click', () => { if (settingsPanel.classList.contains('open')) refreshCacheInfo(); });
refreshCacheInfo();
// Resumable downloads toggle (default on; set to '0' to disable)
const resumableToggle = document.getElementById('resumableDownloadsToggle');
resumableToggle.checked = localStorage.getItem('lm_resumable_downloads') !== '0';
resumableToggle.addEventListener('change', () => {
try { localStorage.setItem('lm_resumable_downloads', resumableToggle.checked ? '1' : '0'); } catch {}
showToast(resumableToggle.checked ? 'Resumable downloads on' : 'Resumable downloads off β€” takes effect on next model load');
});
// Voice-to-text language. Whisper's auto-detect is unreliable on short
// clips and often degenerates to filler tokens ("and", "the"); forcing
// an explicit language avoids that. Default English.
const voiceLanguageSelect = document.getElementById('voiceLanguageSelect');
voiceLanguageSelect.value = localStorage.getItem('lm_voice_language') || 'en';
voiceLanguageSelect.addEventListener('change', () => {
try { localStorage.setItem('lm_voice_language', voiceLanguageSelect.value); } catch {}
});
// ── JavaScript API: chip, toggle, activity log ──────────────
const apiChip = document.getElementById('apiChip');
const apiEnabledToggle = document.getElementById('apiEnabledToggle');
const apiLogBtn = document.getElementById('apiLogBtn');
const apiLogModal = document.getElementById('apiLogModal');
const apiLogList = document.getElementById('apiLogList');
const apiLogCloseBtn = document.getElementById('apiLogCloseBtn');
const apiLogClearBtn = document.getElementById('apiLogClearBtn');
function setApiEnabled(on) {
apiEnabled = !!on;
try { localStorage.setItem('lm_api_enabled', apiEnabled ? '1' : '0'); } catch {}
apiChip.classList.toggle('on', apiEnabled);
apiEnabledToggle.checked = apiEnabled;
if (apiEnabled) {
attachLocalmindAPI();
// One-time console notice the first time it's enabled in a session
if (!window.__lm_api_notice_shown) {
window.__lm_api_notice_shown = true;
console.info('%cLocalMind API enabled', 'color:#5a67d8;font-weight:600', 'β€” window.localmind is experimental and unstable. Activity is logged and visible via Settings β†’ View activity log.');
}
} else {
detachLocalmindAPI();
}
}
function logApiCall(entry) {
// Entry shape: { method, promptLen, tokens, durationMs, outcome, error }
apiLog.unshift({ ts: Date.now(), ...entry });
if (apiLog.length > API_LOG_MAX) apiLog.length = API_LOG_MAX;
flashApiChip();
// If the modal is open, refresh it live
if (apiLogModal.classList.contains('open')) renderApiLog();
}
function flashApiChip() {
if (!apiEnabled) return;
apiChip.classList.add('flash');
setTimeout(() => apiChip.classList.remove('flash'), 400);
}
function renderApiLog() {
if (apiLog.length === 0) {
apiLogList.innerHTML = '<div class="api-log-empty">No API calls yet.</div>';
return;
}
const frag = document.createDocumentFragment();
for (const e of apiLog) {
const row = document.createElement('div');
row.className = 'api-log-row';
const time = new Date(e.ts).toLocaleTimeString();
const outcomeClass = e.outcome === 'ok' ? 'ok' : (e.outcome === 'busy' ? 'busy' : 'err');
// All user-facing fields are escaped before being written as text.
row.innerHTML =
'<span class="t"></span>' +
'<span class="m"></span>' +
'<span class="dur"></span>' +
'<span class="dur"></span>' +
'<span class="' + outcomeClass + '"></span>';
row.children[0].textContent = time;
row.children[1].textContent = e.method + (e.promptLen != null ? ' (' + e.promptLen + ' ch)' : '');
row.children[2].textContent = e.tokens != null ? e.tokens + ' tok' : '';
row.children[3].textContent = e.durationMs != null ? e.durationMs + ' ms' : '';
row.children[4].textContent = e.outcome + (e.error ? ': ' + e.error.slice(0, 40) : '');
frag.appendChild(row);
}
apiLogList.innerHTML = '';
apiLogList.appendChild(frag);
}
// Resolve a user-supplied model identifier to a short MODELS key.
// Accepts either the short key ("gemma3-1b") or the full HF id
// ("onnx-community/gemma-3-1b-it-ONNX-GQA"). Returns null if unknown.
function resolveModelKey(idOrKey) {
if (MODELS[idOrKey]) return idOrKey;
for (const [k, m] of Object.entries(MODELS)) {
if (m.id === idOrKey) return k;
}
return null;
}
// Promise wrapper around loadModel(). loadModel itself is fire-and-forget
// (success/failure is signalled later via worker messages); we install
// resolvers right after calling it. The order matters: loadModel() is
// synchronous, so no worker message can arrive between the two lines.
function loadModelViaApi(idOrKey) {
const key = resolveModelKey(idOrKey);
if (!key) return Promise.reject(new Error('Unknown model: ' + idOrKey));
// Already loaded β€” fast path, no worker churn
if (key === activeModelKey && modelReady) return Promise.resolve();
return new Promise((resolve, reject) => {
loadModel(key);
loadResolvers = { resolve, reject };
});
}
// Tiny push/pull async iterator for streaming responses. Producers call
// push() with each chunk and finish() when done (optionally with an
// error). Consumers iterate via `for await`. Pulls outpace pushes by
// queueing pending promises; pushes outpace pulls by buffering.
// onReturn fires exactly once if the consumer exits the loop early
// (break, return, throw); producers use it to cancel upstream work.
function makeAsyncIterator(onReturn) {
const buffer = [];
const pending = [];
let done = false;
let error = null;
const it = {
push(item) {
if (done) return;
if (pending.length) pending.shift().resolve({ value: item, done: false });
else buffer.push(item);
},
finish(err) {
if (done) return;
done = true;
error = err || null;
while (pending.length) {
const p = pending.shift();
if (error) p.reject(error);
else p.resolve({ value: undefined, done: true });
}
},
next() {
if (buffer.length) return Promise.resolve({ value: buffer.shift(), done: false });
if (done) return error ? Promise.reject(error) : Promise.resolve({ value: undefined, done: true });
return new Promise((resolve, reject) => pending.push({ resolve, reject }));
},
return() {
const wasDone = done;
this.finish();
if (!wasDone && typeof onReturn === 'function') {
try { onReturn(); } catch {}
}
return Promise.resolve({ value: undefined, done: true });
},
[Symbol.asyncIterator]() { return this; },
};
return it;
}
// Validate and normalise the messages array passed to chat.completions.create.
// Accepts the OpenAI shape: [{ role, content }]. Roles limited to
// system/user/assistant. Content must be a string in v1 (no multimodal).
function validateChatMessages(messages) {
if (!Array.isArray(messages) || messages.length === 0) {
throw new Error('messages must be a non-empty array');
}
const VALID_ROLES = new Set(['system', 'user', 'assistant']);
const out = [];
for (let i = 0; i < messages.length; i++) {
const m = messages[i];
if (!m || typeof m !== 'object') throw new Error('messages[' + i + '] must be an object');
if (!VALID_ROLES.has(m.role)) throw new Error('messages[' + i + '].role must be one of system|user|assistant');
if (typeof m.content !== 'string') throw new Error('messages[' + i + '].content must be a string (multimodal not supported in v1)');
out.push({ role: m.role, content: m.content });
}
return out;
}
// Build an OpenAI-shaped chat.completion.chunk
function makeStreamChunk(id, created, modelId, delta, finishReason) {
return {
id,
object: 'chat.completion.chunk',
created,
model: modelId,
choices: [
{ index: 0, delta, finish_reason: finishReason },
],
};
}
// Streaming variant of chat.completions.create. Returns an async iterator
// immediately (well, the create() function awaits this and returns it),
// and pushes chunks as the worker emits tokens. The runInference call is
// fired without await β€” completion is signalled by its .then. Errors are
// propagated to the iterator's next() call.
function makeStreamingCompletion(chatMessages, genConfig, modelEntry, startTs) {
const id = 'lmcc-' + Math.random().toString(36).slice(2, 12);
const created = Math.floor(Date.now() / 1000);
const modelId = modelEntry.id;
const promptChars = chatMessages.reduce((s, m) => s + m.content.length, 0);
// If the consumer breaks out of `for await` before generation ends,
// tell the worker to stop so the next API call isn't queued behind
// an abandoned generation that keeps running to max_tokens.
const iter = makeAsyncIterator(() => {
try {
const w = LocalMind.runtime._worker;
if (w) w.postMessage({ type: 'stop' });
} catch {}
});
let tokenCount = 0;
runInference({
chatMessages,
attData: null,
enableThinking: false,
genConfig,
setup: () => {
// Detach the chat-UI bindings β€” same as the non-streaming path
currentAssistantEl = null;
currentAssistantText = '';
// First chunk is role-only, matching OpenAI's convention
iter.push(makeStreamChunk(id, created, modelId, { role: 'assistant' }, null));
// Wire each subsequent token to a content delta
currentInferenceContext = {
onToken: (token) => {
tokenCount++;
iter.push(makeStreamChunk(id, created, modelId, { content: token }, null));
},
};
},
}).then(
() => {
// Final chunk: empty delta + stop
iter.push(makeStreamChunk(id, created, modelId, {}, 'stop'));
iter.finish();
logApiCall({
method: 'chat.completions.create (stream)',
promptLen: promptChars,
tokens: tokenCount,
outcome: 'ok',
durationMs: Date.now() - startTs,
});
},
(err) => {
iter.finish(err);
logApiCall({
method: 'chat.completions.create (stream)',
promptLen: promptChars,
outcome: 'err',
error: err.message,
durationMs: Date.now() - startTs,
});
}
);
return iter;
}
// chat.completions.create β€” non-streaming or streaming depending on
// params.stream. Mirrors the OpenAI SDK shape closely enough that
// simple clients can drop us in by swapping the base.
// Tools, multimodal, and response_format are intentionally rejected
// in v1 β€” see ROADMAP.md and the SECURITY notes in the README.
async function chatCompletionsCreate(params) {
const start = Date.now();
params = params || {};
// Hard-rejects (logged as err)
const reject = (msg) => {
const e = new Error(msg);
logApiCall({ method: 'chat.completions.create', outcome: 'err', error: msg, durationMs: Date.now() - start });
throw e;
};
if (!modelReady || !activeModelKey) reject('no model loaded β€” call window.localmind.load() first');
if (params.tools) reject('tools not supported in v1');
if (params.tool_choice) reject('tool_choice not supported in v1');
if (params.response_format) reject('response_format not supported in v1');
if (params.n != null && params.n !== 1) reject('only n=1 is supported');
// Optional model param: if present, must match loaded model (by id or key)
if (params.model) {
const requested = resolveModelKey(params.model);
if (!requested) reject('unknown model: ' + params.model);
if (requested !== activeModelKey) reject('model mismatch: requested ' + params.model + ' but ' + MODELS[activeModelKey].id + ' is loaded β€” call load() first');
}
let chatMessages;
try { chatMessages = validateChatMessages(params.messages); }
catch (e) { reject(e.message); }
const m = MODELS[activeModelKey];
const baseGen = m.genConfig;
// Clamp/sanitise generation parameters
const temperature = (typeof params.temperature === 'number') ? Math.max(0, Math.min(2, params.temperature)) : baseGen.temperature;
const top_p = (typeof params.top_p === 'number') ? Math.max(0, Math.min(1, params.top_p)) : baseGen.top_p;
const reqMaxTokens = (typeof params.max_tokens === 'number' && params.max_tokens > 0) ? Math.floor(params.max_tokens) : baseGen.max_new_tokens;
// Hard cap: never let a caller exceed the model's default ceiling
const max_new_tokens = Math.min(reqMaxTokens, baseGen.max_new_tokens);
const genConfig = {
...baseGen,
temperature,
top_p,
max_new_tokens,
};
// Streaming branch: return an async iterator that yields OpenAI-shaped
// chat.completion.chunk objects as tokens arrive.
if (params.stream) {
return makeStreamingCompletion(chatMessages, genConfig, m, start);
}
// Run through the queue with a setup callback that detaches the chat
// UI bindings inside the critical section, so concurrent chat sends
// can't trample our generation.
let responseText;
try {
responseText = await runInference({
chatMessages,
attData: null,
enableThinking: false,
genConfig,
setup: () => {
currentAssistantEl = null;
currentAssistantText = '';
},
});
} catch (e) {
reject(e.message);
}
// Approximate token counts: prompt by char/4 heuristic, completion by
// tokenizer streaming events isn't surfaced to main thread, so use the
// same heuristic. Marked as approximate in the README.
const promptChars = chatMessages.reduce((s, m) => s + m.content.length, 0);
const promptTokensApprox = Math.max(1, Math.round(promptChars / 4));
const completionTokensApprox = Math.max(1, Math.round((responseText || '').length / 4));
const id = 'lmcc-' + Math.random().toString(36).slice(2, 12);
const created = Math.floor(Date.now() / 1000);
const out = {
id,
object: 'chat.completion',
created,
model: m.id,
choices: [
{
index: 0,
message: { role: 'assistant', content: responseText || '' },
finish_reason: 'stop',
},
],
usage: {
prompt_tokens: promptTokensApprox,
completion_tokens: completionTokensApprox,
total_tokens: promptTokensApprox + completionTokensApprox,
// Marker so callers can tell these aren't exact
_approximate: true,
},
};
logApiCall({
method: 'chat.completions.create',
promptLen: promptChars,
tokens: completionTokensApprox,
outcome: 'ok',
durationMs: Date.now() - start,
});
return out;
}
// window.localmind v1 β€” read-only inference surface.
// Live properties (`ready`, `model`) use getters so they reflect current
// state without needing a re-attach on every model change.
const localmindAPI = Object.freeze({
version: '1.0',
get ready() { return modelReady; },
get model() { return (activeModelKey && modelReady) ? MODELS[activeModelKey].id : null; },
listModels() {
return Object.entries(MODELS).map(([key, m]) => ({
id: m.id,
key,
label: m.label,
size: m.size,
multimodal: !!m.multimodal,
contextSize: m.contextSize,
loaded: key === activeModelKey && modelReady,
}));
},
load(idOrKey) {
const start = Date.now();
return loadModelViaApi(idOrKey).then(
() => {
logApiCall({ method: 'load', outcome: 'ok', durationMs: Date.now() - start });
},
(err) => {
logApiCall({ method: 'load', outcome: 'err', error: err.message, durationMs: Date.now() - start });
throw err;
}
);
},
chat: Object.freeze({
completions: Object.freeze({
create: chatCompletionsCreate,
}),
}),
});
function attachLocalmindAPI() {
try {
Object.defineProperty(window, 'localmind', {
value: localmindAPI,
writable: false,
configurable: true,
enumerable: false,
});
} catch (e) {
console.error('Could not attach window.localmind:', e);
}
}
function detachLocalmindAPI() {
try { delete window.localmind; } catch {}
}
apiEnabledToggle.addEventListener('change', (e) => setApiEnabled(e.target.checked));
apiLogBtn.addEventListener('click', () => {
renderApiLog();
apiLogModal.classList.add('open');
});
apiChip.addEventListener('click', () => {
renderApiLog();
apiLogModal.classList.add('open');
});
apiLogCloseBtn.addEventListener('click', () => apiLogModal.classList.remove('open'));
apiLogModal.addEventListener('click', (e) => { if (e.target === apiLogModal) apiLogModal.classList.remove('open'); });
apiLogClearBtn.addEventListener('click', () => { apiLog.length = 0; renderApiLog(); });
// Apply the persisted flag on load
setApiEnabled(apiEnabled);
// ── Custom models (paste a HF ONNX model id) ────────────────
const customModelInput = document.getElementById('customModelInput');
const customModelAddBtn = document.getElementById('customModelAddBtn');
const customModelStatus = document.getElementById('customModelStatus');
const customModelList = document.getElementById('customModelList');
function setCustomStatus(msg, kind) {
customModelStatus.textContent = msg || '';
customModelStatus.style.color = kind === 'err' ? 'var(--error-text)' : (kind === 'ok' ? '#2f855a' : 'var(--gray-500)');
}
// Query the WebGPU adapter once for its hard buffer limits. Cached so
// we don't pay the cost on every validation.
let __webgpuLimits = null;
async function getWebGPULimits() {
if (__webgpuLimits) return __webgpuLimits;
try {
if (!navigator.gpu) return (__webgpuLimits = { maxBufferSize: null, ok: false });
const adapter = await navigator.gpu.requestAdapter();
if (!adapter) return (__webgpuLimits = { maxBufferSize: null, ok: false });
__webgpuLimits = {
maxBufferSize: adapter.limits.maxBufferSize || null,
maxStorageBufferBindingSize: adapter.limits.maxStorageBufferBindingSize || null,
ok: true,
};
} catch {
__webgpuLimits = { maxBufferSize: null, ok: false };
}
return __webgpuLimits;
}
function formatBytes(n) {
if (!n || n <= 0) return 'unknown';
if (n >= 1024 * 1024 * 1024) return (n / 1024 / 1024 / 1024).toFixed(1) + ' GB';
if (n >= 1024 * 1024) return Math.round(n / 1024 / 1024) + ' MB';
return Math.round(n / 1024) + ' KB';
}
// Inspect a list of HF siblings and group ONNX files by quantization.
// Returns { quants: { q4f16: {files, bytes}, q4: {...}, ... }, all: [...] }.
function classifyOnnxFiles(siblings) {
const out = { quants: {}, all: [] };
// Detect dtype suffix from filename. Map several common conventions to
// the dtype string Transformers.js expects.
const detectDtype = (name) => {
const f = name.toLowerCase();
if (/_q4f16(\.onnx|_data\b)/.test(f)) return 'q4f16';
if (/_q4(\.onnx|_data\b)/.test(f) && !/q4f16/.test(f)) return 'q4';
if (/_int8(\.onnx|_data\b)/.test(f)) return 'int8';
if (/_uint8(\.onnx|_data\b)/.test(f)) return 'uint8';
if (/_fp16(\.onnx|_data\b)/.test(f)) return 'fp16';
if (/_quantized(\.onnx|_data\b)/.test(f)) return 'quantized';
if (/_bnb4(\.onnx|_data\b)/.test(f)) return 'bnb4';
if (/_q8(\.onnx|_data\b)/.test(f)) return 'q8';
if (/_q4f8(\.onnx|_data\b)/.test(f)) return 'q4f8';
// Bare model.onnx (no suffix) β†’ fp32
if (/(^|\/)model\.onnx(_data)?$/.test(f)) return 'fp32';
return null;
};
for (const s of siblings) {
if (!s || typeof s.rfilename !== 'string') continue;
if (!/\.onnx(_data)?$/.test(s.rfilename)) continue;
out.all.push(s);
const dtype = detectDtype(s.rfilename);
if (!dtype) continue;
if (!out.quants[dtype]) out.quants[dtype] = { files: [], bytes: 0 };
out.quants[dtype].files.push(s);
if (typeof s.size === 'number') out.quants[dtype].bytes += s.size;
}
return out;
}
// Validate a user-supplied HF repo id and probe the HF API to confirm
// it exists, has usable ONNX files, picks a workable quantization, and
// checks the chosen size against the device's WebGPU buffer limits.
// Returns { entry, warnings: [] } on success, throws on hard failure.
async function probeHuggingFaceModel(id) {
if (!/^[\w.-]+\/[\w.-]+$/.test(id)) {
throw new Error('invalid id β€” use owner/repo');
}
// 1. HF model info. ?blobs=true populates each sibling's .size.
let info;
try {
const res = await fetch('https://huggingface.co/api/models/' + id + '?blobs=true', { signal: AbortSignal.timeout(10000) });
if (!res.ok) throw new Error('HTTP ' + res.status);
info = await res.json();
} catch (e) {
throw new Error('model not found or network error: ' + e.message);
}
// 2. Find ONNX files anywhere in the repo (not just under onnx/).
const siblings = Array.isArray(info.siblings) ? info.siblings : [];
const classified = classifyOnnxFiles(siblings);
if (classified.all.length === 0) {
throw new Error('no ONNX files found in this repo. Hugging Face has many ONNX exports under the onnx-community/ org and from Xenova β€” try one of those.');
}
// 3. Pick a quantization. Order = preference for in-browser inference:
// smallest-and-fastest first, largest last.
const PREFERENCE = ['q4f16', 'q4', 'int8', 'q8', 'uint8', 'bnb4', 'q4f8', 'fp16', 'quantized', 'fp32'];
let chosenDtype = null;
for (const dt of PREFERENCE) {
if (classified.quants[dt] && classified.quants[dt].files.length > 0) { chosenDtype = dt; break; }
}
if (!chosenDtype) {
// ONNX files exist but none match a known dtype convention. Let the
// user know exactly what we found so they can investigate.
const sample = classified.all.slice(0, 4).map(s => s.rfilename).join(', ');
throw new Error('ONNX files found but none match a known quantization (looked for q4f16/q4/int8/fp16/quantized/fp32). Files: ' + sample);
}
// Estimate size as MAX(.onnx) + MAX(.onnx_data) across files in the
// chosen dtype group. Many repos publish multiple variants of the
// same model (model_, decoder_model_, decoder_model_merged_, ...) and
// Transformers.js loads exactly ONE β€” summing them all overcounts by
// 3-4Γ—. The largest single .onnx is the actual loaded weights file;
// .onnx_data (if any) is its companion external-weights file.
let chosenBytes = 0;
{
let maxOnnx = 0, maxData = 0;
for (const f of classified.quants[chosenDtype].files) {
if (typeof f.size !== 'number') continue;
if (/\.onnx_data$/.test(f.rfilename)) {
if (f.size > maxData) maxData = f.size;
} else if (/\.onnx$/.test(f.rfilename)) {
if (f.size > maxOnnx) maxOnnx = f.size;
}
}
chosenBytes = maxOnnx + maxData;
}
const warnings = [];
// 4. Size check vs WebGPU limits.
const limits = await getWebGPULimits();
if (limits.ok && limits.maxBufferSize && chosenBytes > 0) {
// The single biggest weight tensor must fit in maxBufferSize. We
// approximate by assuming the largest file is one buffer's worth.
// If even the smallest quant exceeds the limit, hard-block.
const largestFile = Math.max(0, ...classified.quants[chosenDtype].files.map(f => f.size || 0));
if (largestFile > limits.maxBufferSize) {
throw new Error('largest weight file (' + formatBytes(largestFile) + ') exceeds this device\'s WebGPU buffer limit (' + formatBytes(limits.maxBufferSize) + '). Try a smaller model or a device with more GPU memory.');
}
// If the total exceeds 4Γ— the buffer limit, it almost certainly
// won't fit overall β€” block it. (Models are usually split across
// many buffers, so 4Γ— headroom is generous.)
if (chosenBytes > limits.maxBufferSize * 4) {
throw new Error('model is ' + formatBytes(chosenBytes) + ' but this device\'s WebGPU buffer limit is ' + formatBytes(limits.maxBufferSize) + ' β€” too large to load.');
}
// Warn if chosen size is over half the buffer limit β€” likely tight
if (chosenBytes > limits.maxBufferSize) {
warnings.push('size (' + formatBytes(chosenBytes) + ') exceeds the per-buffer WebGPU limit (' + formatBytes(limits.maxBufferSize) + '); load may fail');
}
} else if (!limits.ok) {
warnings.push('could not query WebGPU limits β€” load may fail if model exceeds device memory');
}
// 5. Absolute soft cap: anything bigger than 6 GB is impractical in
// a browser tab regardless of GPU.
if (chosenBytes > 6 * 1024 * 1024 * 1024) {
throw new Error('model is ' + formatBytes(chosenBytes) + ' β€” too large for in-browser inference. The hard ceiling for LocalMind is 6 GB.');
}
if (chosenBytes > 2 * 1024 * 1024 * 1024) {
warnings.push('large model (' + formatBytes(chosenBytes) + ') β€” first download will take a while and may not fit on lower-end GPUs');
}
// 6. config.json β€” architecture hint + context size
let config = null;
try {
const res = await fetch('https://huggingface.co/' + id + '/resolve/main/config.json', { signal: AbortSignal.timeout(10000) });
if (res.ok) config = await res.json();
} catch {}
// 7. Reject multimodal in v1 (the runtime adapter currently
// only knows how to load the multimodal class for Gemma 4)
const modelType = config && config.model_type ? String(config.model_type) : '';
if (config && (config.vision_config || config.audio_config)) {
throw new Error('multimodal custom models are not supported in v1 (model_type=' + modelType + ')');
}
// 8. Build the MODELS entry
const repoShort = id.split('/').pop();
const sizeLabel = chosenBytes ? '~' + formatBytes(chosenBytes) : 'size unknown';
const entry = {
id,
label: repoShort + ' (custom)',
dtype: chosenDtype,
size: sizeLabel,
type: 'causal',
multimodal: false,
agentCapable: false,
contextSize: (config && (config.max_position_embeddings || config.max_seq_len)) || 4096,
genConfig: { temperature: 0.7, top_k: 50, top_p: 0.95, max_new_tokens: 1024 },
custom: true,
};
return { entry, warnings };
}
function loadCustomModelsFromStorage() {
try {
const raw = localStorage.getItem('lm_custom_models');
if (!raw) return [];
const arr = JSON.parse(raw);
if (!Array.isArray(arr)) return [];
return arr.filter(m => m && typeof m.id === 'string');
} catch { return []; }
}
function saveCustomModelsToStorage() {
const custom = Object.values(MODELS).filter(m => m.custom);
try { localStorage.setItem('lm_custom_models', JSON.stringify(custom)); } catch {}
}
function addCustomModelToRegistry(entry) {
// The HF id is the registry key for custom models β€” it's unique, stable,
// and matches what the worker will pass to from_pretrained.
MODELS[entry.id] = entry;
// Append to the model selector if not already there
if (!Array.from(modelSelect.options).some(o => o.value === entry.id)) {
const opt = document.createElement('option');
opt.value = entry.id;
opt.textContent = entry.label + ' Β· ' + entry.size;
modelSelect.appendChild(opt);
}
}
function removeCustomModelFromRegistry(id) {
if (!MODELS[id] || !MODELS[id].custom) return;
// If this is the active model, we can't remove it β€” the worker is tied to it
if (id === activeModelKey) {
setCustomStatus('cannot remove β€” this model is currently loaded. Switch to another first.', 'err');
return;
}
delete MODELS[id];
const opt = Array.from(modelSelect.options).find(o => o.value === id);
if (opt) opt.remove();
saveCustomModelsToStorage();
renderCustomModelList();
}
function renderCustomModelList() {
customModelList.innerHTML = '';
const custom = Object.values(MODELS).filter(m => m.custom);
if (custom.length === 0) {
customModelList.textContent = '';
return;
}
for (const m of custom) {
const row = document.createElement('div');
row.style.cssText = 'display:flex;align-items:center;justify-content:space-between;gap:8px;padding:4px 6px;margin-top:4px;background:var(--gray-50);border:1px solid var(--gray-200);border-radius:4px;font-size:0.72rem';
const left = document.createElement('div');
left.style.cssText = 'flex:1;overflow:hidden;text-overflow:ellipsis;white-space:nowrap';
left.textContent = m.id + ' Β· ' + m.size;
const del = document.createElement('button');
del.className = 'btn-icon';
del.textContent = 'Remove';
del.style.cssText = 'flex-shrink:0;padding:2px 8px;font-size:0.7rem';
del.addEventListener('click', () => removeCustomModelFromRegistry(m.id));
row.appendChild(left);
row.appendChild(del);
customModelList.appendChild(row);
}
}
async function handleAddCustomModel() {
const id = customModelInput.value.trim();
if (!id) { setCustomStatus('enter a model id', 'err'); return; }
if (MODELS[id]) { setCustomStatus('already added', 'err'); return; }
customModelAddBtn.disabled = true;
setCustomStatus('Probing Hugging Face…');
try {
const { entry, warnings } = await probeHuggingFaceModel(id);
addCustomModelToRegistry(entry);
saveCustomModelsToStorage();
renderCustomModelList();
customModelInput.value = '';
let msg = 'Added (' + entry.dtype + ', ' + entry.size + '). Select it from the model dropdown to load.';
if (warnings && warnings.length) {
msg += '\nWarning: ' + warnings.join('; ');
// Use a softer colour so the user notices but knows it's not a fatal error
customModelStatus.textContent = msg;
customModelStatus.style.color = '#975a16';
customModelStatus.style.whiteSpace = 'pre-wrap';
} else {
setCustomStatus(msg, 'ok');
}
} catch (e) {
setCustomStatus('Failed: ' + e.message, 'err');
customModelStatus.style.whiteSpace = 'pre-wrap';
} finally {
customModelAddBtn.disabled = false;
}
}
customModelAddBtn.addEventListener('click', handleAddCustomModel);
customModelInput.addEventListener('keydown', (e) => {
if (e.key === 'Enter') { e.preventDefault(); handleAddCustomModel(); }
});
// Restore any previously added custom models BEFORE the initial loadModel
// runs β€” otherwise a saved active selection pointing at a custom model
// would silently fall back to the default.
for (const m of loadCustomModelsFromStorage()) {
try { addCustomModelToRegistry(m); } catch {}
}
renderCustomModelList();
// ── Custom tools (user-defined, HTTP POST) ────────────────
const customToolInput = document.getElementById('customToolInput');
const customToolAddBtn = document.getElementById('customToolAddBtn');
const customToolStatus = document.getElementById('customToolStatus');
const customToolList = document.getElementById('customToolList');
function setCustomToolStatus(msg, kind) {
customToolStatus.textContent = msg || '';
customToolStatus.style.color = kind === 'err' ? 'var(--red-600, #dc2626)' : 'var(--gray-500)';
}
function validateCustomToolDef(def) {
if (!def || typeof def !== 'object') return 'must be a JSON object';
if (typeof def.name !== 'string' || !/^[a-zA-Z_][a-zA-Z0-9_]*$/.test(def.name)) {
return 'name must match /^[a-zA-Z_][a-zA-Z0-9_]*$/';
}
if (TOOL_REGISTRY[def.name] && !TOOL_REGISTRY[def.name]._custom) {
return 'name collides with a built-in tool';
}
if (typeof def.description !== 'string' || !def.description.trim()) return 'description required';
if (typeof def.endpoint !== 'string' || !/^https?:\/\//i.test(def.endpoint)) return 'endpoint must be http(s) URL';
if (!def.parameters || typeof def.parameters !== 'object') return 'parameters required (JSON schema object)';
return null;
}
function makeCustomToolExecute(endpoint, toolName) {
return async function execute(args) {
try {
const res = await fetch(endpoint, {
method: 'POST',
headers: { 'content-type': 'application/json' },
body: JSON.stringify(args || {}),
});
const ct = res.headers.get('content-type') || '';
const body = ct.includes('application/json') ? await res.json() : await res.text();
if (!res.ok) return { error: 'Tool "' + toolName + '" returned ' + res.status, body };
return body;
} catch (e) {
return { error: 'Tool "' + toolName + '" fetch failed: ' + (e.message || e) };
}
};
}
function registerCustomTool(def) {
TOOL_REGISTRY[def.name] = {
description: def.description,
parameters: def.parameters,
execute: makeCustomToolExecute(def.endpoint, def.name),
_custom: true,
_endpoint: def.endpoint,
};
}
function loadCustomToolsFromStorage() {
try {
const raw = localStorage.getItem('lm_custom_tools');
if (!raw) return [];
const arr = JSON.parse(raw);
return Array.isArray(arr) ? arr : [];
} catch { return []; }
}
function saveCustomToolsToStorage() {
const arr = Object.entries(TOOL_REGISTRY)
.filter(([, t]) => t._custom)
.map(([name, t]) => ({ name, description: t.description, parameters: t.parameters, endpoint: t._endpoint }));
try { localStorage.setItem('lm_custom_tools', JSON.stringify(arr)); } catch {}
}
function renderCustomToolList() {
customToolList.innerHTML = '';
const entries = Object.entries(TOOL_REGISTRY).filter(([, t]) => t._custom);
if (entries.length === 0) return;
for (const [name, t] of entries) {
const row = document.createElement('div');
row.style.cssText = 'display:flex;align-items:center;justify-content:space-between;gap:8px;padding:4px 6px;margin-top:4px;background:var(--gray-50);border:1px solid var(--gray-200);border-radius:4px;font-size:0.72rem';
const left = document.createElement('div');
left.style.cssText = 'flex:1;overflow:hidden;text-overflow:ellipsis;white-space:nowrap';
left.textContent = name + ' \u2192 ' + t._endpoint;
const del = document.createElement('button');
del.className = 'btn-icon';
del.textContent = 'Remove';
del.style.cssText = 'flex-shrink:0;padding:2px 8px;font-size:0.7rem';
del.addEventListener('click', () => {
delete TOOL_REGISTRY[name];
saveCustomToolsToStorage();
renderCustomToolList();
});
row.appendChild(left);
row.appendChild(del);
customToolList.appendChild(row);
}
}
customToolAddBtn.addEventListener('click', () => {
let def;
try { def = JSON.parse(customToolInput.value); }
catch (e) { setCustomToolStatus('Invalid JSON: ' + e.message, 'err'); return; }
const err = validateCustomToolDef(def);
if (err) { setCustomToolStatus(err, 'err'); return; }
registerCustomTool(def);
saveCustomToolsToStorage();
renderCustomToolList();
customToolInput.value = '';
setCustomToolStatus('Added "' + def.name + '". Agent-capable models will see it on next message.');
});
for (const def of loadCustomToolsFromStorage()) {
const err = validateCustomToolDef(def);
if (!err) registerCustomTool(def);
}
renderCustomToolList();
// ── MCP servers (Streamable HTTP JSON-RPC 2.0) ─────────────
// Discover tools on page load from each registered server and add them
// to TOOL_REGISTRY with an mcp_ prefix. Tool execution POSTs a
// tools/call JSON-RPC request back to the same URL.
const mcpUrlInput = document.getElementById('mcpUrlInput');
const mcpAuthInput = document.getElementById('mcpAuthInput');
const mcpAddBtn = document.getElementById('mcpAddBtn');
const mcpStatus = document.getElementById('mcpStatus');
const mcpListDiv = document.getElementById('mcpList');
// Keep server connection state here so removeMcpServer can wipe only
// the tools that belong to that server.
const __mcpServers = []; // [{ url, auth, toolNames: [] }]
function mcpSetStatus(msg, kind) {
mcpStatus.textContent = msg || '';
mcpStatus.style.color = kind === 'err' ? '#dc2626' : 'var(--gray-500)';
}
async function mcpRpc(url, auth, method, params) {
const body = { jsonrpc: '2.0', id: Math.floor(Math.random() * 1e9), method };
if (params !== undefined) body.params = params;
const headers = { 'content-type': 'application/json', 'accept': 'application/json, text/event-stream' };
if (auth) headers['authorization'] = 'Bearer ' + auth;
const res = await fetch(url, { method: 'POST', headers, body: JSON.stringify(body) });
if (!res.ok) throw new Error('HTTP ' + res.status + ' ' + res.statusText);
const ct = res.headers.get('content-type') || '';
// MVP: only parse JSON responses. SSE streams aren't supported yet β€”
// callers that need them should pick a stateless MCP server.
const text = await res.text();
if (ct.includes('text/event-stream')) {
// Pull the first `data: …` line's JSON β€” sufficient for discrete replies
const line = text.split(/\r?\n/).find((l) => l.startsWith('data:'));
if (!line) throw new Error('MCP SSE response had no data frame');
const j = JSON.parse(line.slice(5).trim());
if (j.error) throw new Error('MCP error: ' + (j.error.message || JSON.stringify(j.error)));
return j.result;
}
const j = JSON.parse(text);
if (j.error) throw new Error('MCP error: ' + (j.error.message || JSON.stringify(j.error)));
return j.result;
}
function mcpMakeExecute(url, auth, remoteName) {
return async function execute(args) {
try {
const result = await mcpRpc(url, auth, 'tools/call', { name: remoteName, arguments: args || {} });
// MCP tool result shape: { content: [{ type: 'text', text: '…' }], isError? }
if (result && Array.isArray(result.content)) {
const textParts = result.content.filter((c) => c && c.type === 'text').map((c) => c.text).join('\n');
return result.isError ? { error: textParts || 'tool reported error' } : (textParts || result.content);
}
return result;
} catch (e) {
return { error: 'MCP call failed: ' + (e.message || e) };
}
};
}
async function mcpConnectServer(url, auth) {
// Initialize handshake + list tools, then register each.
await mcpRpc(url, auth, 'initialize', {
protocolVersion: '2025-03-26',
capabilities: {},
clientInfo: { name: 'LocalMind', version: '2.0' },
});
const list = await mcpRpc(url, auth, 'tools/list');
const tools = (list && list.tools) || [];
const registered = [];
for (const t of tools) {
if (!t || !t.name) continue;
// Sanitise the MCP tool name into a valid identifier, prefix with mcp_.
const clean = String(t.name).replace(/[^a-zA-Z0-9_]/g, '_');
const local = 'mcp_' + clean;
if (TOOL_REGISTRY[local] && !TOOL_REGISTRY[local]._mcp) continue; // skip collisions
TOOL_REGISTRY[local] = {
description: t.description || ('MCP tool from ' + url),
parameters: t.inputSchema || { type: 'object', properties: {}, required: [] },
execute: mcpMakeExecute(url, auth, t.name),
_mcp: true,
_mcpUrl: url,
};
registered.push(local);
}
return registered;
}
function mcpLoadFromStorage() {
try {
const raw = localStorage.getItem('lm_mcp_servers');
if (!raw) return [];
const arr = JSON.parse(raw);
return Array.isArray(arr) ? arr : [];
} catch { return []; }
}
function mcpSaveToStorage() {
const list = __mcpServers.map((s) => ({ url: s.url, auth: s.auth || '' }));
try { localStorage.setItem('lm_mcp_servers', JSON.stringify(list)); } catch {}
}
function renderMcpList() {
mcpListDiv.innerHTML = '';
if (__mcpServers.length === 0) return;
for (const server of __mcpServers) {
const row = document.createElement('div');
row.style.cssText = 'display:flex;align-items:center;justify-content:space-between;gap:8px;padding:4px 6px;margin-top:4px;background:var(--gray-50);border:1px solid var(--gray-200);border-radius:4px;font-size:0.72rem';
const left = document.createElement('div');
left.style.cssText = 'flex:1;overflow:hidden;text-overflow:ellipsis;white-space:nowrap';
left.textContent = server.url + ' \u2014 ' + server.toolNames.length + ' tool(s)';
const del = document.createElement('button');
del.className = 'btn-icon';
del.textContent = 'Remove';
del.style.cssText = 'flex-shrink:0;padding:2px 8px;font-size:0.7rem';
del.addEventListener('click', () => removeMcpServer(server.url));
row.appendChild(left);
row.appendChild(del);
mcpListDiv.appendChild(row);
}
}
async function addMcpServer(url, auth) {
if (__mcpServers.some((s) => s.url === url)) { mcpSetStatus('already added', 'err'); return; }
mcpSetStatus('Connecting to ' + url + '\u2026');
mcpAddBtn.disabled = true;
try {
const names = await mcpConnectServer(url, auth);
__mcpServers.push({ url, auth, toolNames: names });
mcpSaveToStorage();
renderMcpList();
mcpSetStatus('Connected. Discovered ' + names.length + ' tool(s).');
mcpUrlInput.value = '';
mcpAuthInput.value = '';
} catch (e) {
mcpSetStatus('Failed: ' + (e.message || e), 'err');
} finally {
mcpAddBtn.disabled = false;
}
}
function removeMcpServer(url) {
const idx = __mcpServers.findIndex((s) => s.url === url);
if (idx < 0) return;
for (const name of __mcpServers[idx].toolNames) {
if (TOOL_REGISTRY[name] && TOOL_REGISTRY[name]._mcp) delete TOOL_REGISTRY[name];
}
__mcpServers.splice(idx, 1);
mcpSaveToStorage();
renderMcpList();
}
mcpAddBtn.addEventListener('click', () => {
const url = mcpUrlInput.value.trim();
if (!url || !/^https?:\/\//i.test(url)) { mcpSetStatus('URL must be http(s)://', 'err'); return; }
addMcpServer(url, mcpAuthInput.value.trim());
});
// Auto-reconnect saved servers on page load. Silent on failures β€”
// users will see the stale count in the list and can retry manually.
(async () => {
for (const s of mcpLoadFromStorage()) {
try {
const names = await mcpConnectServer(s.url, s.auth);
__mcpServers.push({ url: s.url, auth: s.auth, toolNames: names });
} catch {
__mcpServers.push({ url: s.url, auth: s.auth, toolNames: [] });
}
}
renderMcpList();
})();
// ── Web search settings ───────────────────────────────────
const searchProvider = document.getElementById('searchProvider');
const searchApiKey = document.getElementById('searchApiKey');
const searxngUrl = document.getElementById('searxngUrl');
const apiKeyRow = document.getElementById('apiKeyRow');
const searxngUrlRow = document.getElementById('searxngUrlRow');
const searchSettingsSection = document.getElementById('searchSettingsSection');
const searchSendBtn = document.getElementById('searchSendBtn');
let webEnrichedMode = false;
// Restore saved settings
searchProvider.value = localStorage.getItem('lm_search_provider') || 'none';
searchApiKey.value = localStorage.getItem('lm_search_key') || '';
searxngUrl.value = localStorage.getItem('lm_searxng_url') || '';
function updateSearchUI() {
const p = searchProvider.value;
apiKeyRow.style.display = (p === 'brave' || p === 'tavily') ? '' : 'none';
searxngUrlRow.style.display = p === 'searxng' ? '' : 'none';
const m = MODELS[activeModelKey];
const isAgent = m && m.agentCapable;
const configured = isSearchConfigured();
searchSendBtn.style.display = (configured && isAgent) ? '' : 'none';
}
searchProvider.addEventListener('change', () => {
localStorage.setItem('lm_search_provider', searchProvider.value);
updateSearchUI();
});
searchApiKey.addEventListener('input', () => localStorage.setItem('lm_search_key', searchApiKey.value));
searxngUrl.addEventListener('input', () => localStorage.setItem('lm_searxng_url', searxngUrl.value));
// SAM model selector
const samModelSelect = document.getElementById('samModelSelect');
const samSettingsSection = document.getElementById('samSettingsSection');
const savedSamModel = localStorage.getItem('lm_sam_model');
if (savedSamModel) samModelSelect.value = savedSamModel;
samModelSelect.addEventListener('change', () => {
localStorage.setItem('lm_sam_model', samModelSelect.value);
});
// Search+Send button
searchSendBtn.addEventListener('click', () => {
if (generating) return;
webEnrichedMode = true;
sendMessage();
});
updateSearchUI();
// ── Memory inspector ────────────────────────────────────────
const memoryBtn = document.getElementById('memoryBtn');
const memoryPanel = document.getElementById('memoryPanel');
const memoryList = document.getElementById('memoryList');
const memoryCount = document.getElementById('memoryCount');
const memorySearch = document.getElementById('memorySearch');
const memoryClearAll = document.getElementById('memoryClearAll');
const memoryCatPills = document.getElementById('memoryCatPills');
let memoryCatFilter = 'all';
memoryBtn.addEventListener('click', () => {
memoryPanel.classList.toggle('open');
settingsPanel.classList.remove('open');
closeHistorySidebar();
if (memoryPanel.classList.contains('open')) {
refreshMemoryPanel();
} else if (auditMode) {
// Reset audit state when panel is closed
auditMode = false;
memoryAuditBtn.textContent = 'Audit';
memoryCatPills.style.display = '';
memorySearch.closest('.memory-search-row').style.display = '';
}
});
function relTime(ts) {
const m = Math.floor((Date.now() - ts) / 60000);
if (m < 1) return 'just now';
if (m < 60) return `${m}m ago`;
const h = Math.floor(m / 60);
if (h < 24) return `${h}h ago`;
const d = Math.floor(h / 24);
if (d < 30) return `${d}d ago`;
return new Date(ts).toLocaleDateString();
}
function srcBasename(src) {
if (!src) return 'unknown';
return src.includes('/') ? src.split('/').pop() : src;
}
function catBadgeHtml(cat) {
return `<span class="mem-cat mem-cat-${escapeHtml(cat)}">${escapeHtml(cat)}</span>`;
}
function makeChunkItem(chunk, showCat) {
const item = document.createElement('div');
item.className = 'memory-item';
const preview = chunk.text.length > 150 ? chunk.text.slice(0, 150) + '…' : chunk.text;
const src = srcBasename(chunk.source);
item.innerHTML = `
<div class="memory-item-text">
${escapeHtml(preview)}
<div class="memory-item-meta">
${showCat ? catBadgeHtml(chunk.category) : ''}
<span title="${escapeHtml(chunk.source || '')}">${escapeHtml(src)}</span>
&middot; ${relTime(chunk.timestamp)}
</div>
</div>
<button class="memory-item-del" title="Delete">&times;</button>
`;
item.querySelector('.memory-item-del').addEventListener('click', async () => {
await deleteChunk(chunk.id);
refreshMemoryPanel();
});
return item;
}
async function refreshMemoryPanel() {
const textQ = memorySearch.value.trim().toLowerCase();
const catF = memoryCatFilter;
try {
const all = await getAllChunks();
memoryCount.textContent = `${all.length} chunk${all.length === 1 ? '' : 's'}`;
// Build category counts for pills
const catCounts = {};
for (const c of all) catCounts[c.category] = (catCounts[c.category] || 0) + 1;
// Render pills
const CAT_ORDER = ['all', 'fact', 'preference', 'finding', 'document', 'document_summary', 'conversation'];
const cats = ['all', ...CAT_ORDER.slice(1).filter(c => catCounts[c])];
memoryCatPills.innerHTML = '';
for (const cat of cats) {
const pill = document.createElement('button');
pill.className = 'memory-cat-pill' + (cat === catF ? ' active' : '');
const label = cat === 'document_summary' ? 'doc summary' : cat;
const count = cat === 'all' ? all.length : (catCounts[cat] || 0);
pill.textContent = `${label} (${count})`;
pill.addEventListener('click', () => {
memoryCatFilter = cat;
refreshMemoryPanel();
});
memoryCatPills.appendChild(pill);
}
// Filter
let display = catF === 'all' ? all : all.filter(c => c.category === catF);
if (textQ) {
display = display.filter(c =>
c.text.toLowerCase().includes(textQ) ||
c.category.includes(textQ) ||
(c.source && c.source.toLowerCase().includes(textQ))
);
}
memoryList.innerHTML = '';
if (display.length === 0) {
memoryList.innerHTML = '<div class="memory-empty">No memories found</div>';
return;
}
const isDocCat = catF === 'document' || catF === 'document_summary';
if (isDocCat) {
// Group by source
const groups = {};
for (const c of display) {
const src = c.source || 'unknown';
if (!groups[src]) groups[src] = [];
groups[src].push(c);
}
for (const [src, items] of Object.entries(groups).sort()) {
const basename = srcBasename(src);
const group = document.createElement('div');
group.className = 'memory-source-group';
const header = document.createElement('div');
header.className = 'memory-source-header';
header.innerHTML = `
<span class="memory-source-name" title="${escapeHtml(src)}">${escapeHtml(basename)}</span>
<span class="memory-source-count">${items.length} chunk${items.length === 1 ? '' : 's'}</span>
<button class="memory-source-del">Delete all</button>
`;
header.querySelector('.memory-source-del').addEventListener('click', async () => {
if (!confirm(`Delete all ${items.length} chunk(s) from "${basename}"?`)) return;
for (const c of items) await deleteChunk(c.id);
refreshMemoryPanel();
});
group.appendChild(header);
for (const chunk of items.sort((a, b) => b.timestamp - a.timestamp)) {
group.appendChild(makeChunkItem(chunk, false));
}
memoryList.appendChild(group);
}
} else {
// Flat list sorted newest-first
const sorted = display.sort((a, b) => b.timestamp - a.timestamp);
const showCat = catF === 'all';
const LIMIT = 200;
for (const chunk of sorted.slice(0, LIMIT)) {
memoryList.appendChild(makeChunkItem(chunk, showCat));
}
if (sorted.length > LIMIT) {
memoryList.insertAdjacentHTML('beforeend',
`<div class="memory-empty">${sorted.length - LIMIT} more β€” use a category filter or search to narrow</div>`);
}
}
} catch (e) {
memoryList.innerHTML = `<div class="memory-empty">Error loading memories</div>`;
console.error('Memory panel error:', e);
}
}
memorySearch.addEventListener('input', () => refreshMemoryPanel());
memoryClearAll.addEventListener('click', async () => {
if (!confirm('Delete all stored memories? This cannot be undone.')) return;
await clearAllChunks();
refreshMemoryPanel();
});
// ── Memory audit ────────────────────────────────────────────
const memoryAuditBtn = document.getElementById('memoryAuditBtn');
let auditMode = false;
memoryAuditBtn.addEventListener('click', async () => {
if (auditMode) {
auditMode = false;
memoryAuditBtn.textContent = 'Audit';
memoryCatPills.style.display = '';
memorySearch.closest('.memory-search-row').style.display = '';
refreshMemoryPanel();
return;
}
auditMode = true;
memoryAuditBtn.textContent = '← Back';
memoryCatPills.style.display = 'none';
memorySearch.closest('.memory-search-row').style.display = 'none';
memoryList.innerHTML = '<div class="memory-empty">Running audit…</div>';
try {
const results = await runMemoryAudit();
renderAuditResults(results);
} catch (e) {
memoryList.innerHTML = `<div class="memory-empty">Audit failed: ${escapeHtml(e.message)}</div>`;
console.error('Audit error:', e);
}
});
async function runMemoryAudit() {
const STALE_DAYS = 60;
const DUPE_THRESHOLD = 0.92;
const OUTLIER_MAX_SIM = 0.20;
const MIN_CAT_SIZE = 5;
const CAP = 600;
const all = await getAllChunks();
const withEmb = all.filter(c => c.embedding && c.embedding.length > 0);
const capped = withEmb.length > CAP;
const work = capped ? withEmb.slice(0, CAP) : withEmb;
// Stale
const stale = work.filter(c => Date.now() - c.timestamp > STALE_DAYS * 86400000);
// Group by category for pairwise ops
const byCat = {};
for (const c of work) {
(byCat[c.category] = byCat[c.category] || []).push(c);
}
const dupeIds = new Set();
const dupePairs = []; // [[kept, flagged], ...]
const outliers = [];
for (const members of Object.values(byCat)) {
const n = members.length;
const avgSims = new Float32Array(n);
for (let i = 0; i < n; i++) {
let sum = 0;
for (let j = 0; j < n; j++) {
if (i === j) continue;
const sim = cosineSimilarity(members[i].embedding, members[j].embedding);
sum += sim;
if (sim >= DUPE_THRESHOLD && i < j
&& !dupeIds.has(members[i].id)
&& !dupeIds.has(members[j].id)) {
dupePairs.push([members[i], members[j]]);
dupeIds.add(members[j].id);
}
}
avgSims[i] = n > 1 ? sum / (n - 1) : 1;
}
if (n >= MIN_CAT_SIZE) {
for (let i = 0; i < n; i++) {
if (avgSims[i] < OUTLIER_MAX_SIM && !dupeIds.has(members[i].id)) {
outliers.push(members[i]);
}
}
}
}
return { total: work.length, capped, stale, dupePairs, dupeIds, outliers };
}
function renderAuditResults({ total, capped, stale, dupePairs, outliers }) {
memoryList.innerHTML = '';
if (capped) {
memoryList.insertAdjacentHTML('beforeend',
`<div class="audit-warn">Only the first 600 chunks were analysed. Use category filters to audit subsets of larger stores.</div>`);
}
const flagCount = stale.length + dupePairs.length + outliers.length;
if (flagCount === 0) {
memoryList.insertAdjacentHTML('beforeend',
`<div class="audit-clean">βœ“ Memory looks clean β€” no stale, duplicate, or outlier chunks found across ${total} chunk(s).</div>`);
return;
}
if (stale.length > 0) {
memoryList.appendChild(makeAuditSection(
`${stale.length} stale β€” older than 60 days`, stale,
async () => { for (const c of stale) await deleteChunk(c.id); rerunAudit(); }
));
}
if (dupePairs.length > 0) {
const flagged = dupePairs.map(([, b]) => b);
memoryList.appendChild(makeAuditSection(
`${dupePairs.length} near-duplicate(s) β€” keeping one of each pair`, flagged,
async () => { for (const c of flagged) await deleteChunk(c.id); rerunAudit(); }
));
}
if (outliers.length > 0) {
memoryList.appendChild(makeAuditSection(
`${outliers.length} outlier(s) β€” low similarity to their category`, outliers,
async () => { for (const c of outliers) await deleteChunk(c.id); rerunAudit(); }
));
}
}
async function rerunAudit() {
memoryList.innerHTML = '<div class="memory-empty">Re-running…</div>';
try {
renderAuditResults(await runMemoryAudit());
} catch (e) {
memoryList.innerHTML = `<div class="memory-empty">Audit failed: ${escapeHtml(e.message)}</div>`;
}
}
function makeAuditSection(label, chunks, onDeleteAll) {
const section = document.createElement('div');
section.className = 'audit-section';
const header = document.createElement('div');
header.className = 'audit-section-header';
const delBtn = document.createElement('button');
delBtn.className = 'memory-source-del';
delBtn.textContent = 'Delete all';
delBtn.addEventListener('click', async () => {
if (!confirm(`Delete ${chunks.length} item(s)?`)) return;
await onDeleteAll();
});
header.innerHTML = `<span>${escapeHtml(label)}</span>`;
header.appendChild(delBtn);
section.appendChild(header);
const PREVIEW = 20;
for (const chunk of chunks.slice(0, PREVIEW)) {
const item = makeChunkItem(chunk, true);
// Override the delete handler so it reruns audit afterward
item.querySelector('.memory-item-del').replaceWith((() => {
const btn = document.createElement('button');
btn.className = 'memory-item-del';
btn.title = 'Delete';
btn.textContent = 'Γ—';
btn.addEventListener('click', async () => {
await deleteChunk(chunk.id);
rerunAudit();
});
return btn;
})());
section.appendChild(item);
}
if (chunks.length > PREVIEW) {
section.insertAdjacentHTML('beforeend',
`<div class="memory-empty">…and ${chunks.length - PREVIEW} more</div>`);
}
return section;
}
// ── Export / Import ─────────────────────────────────────────
const memoryExport = document.getElementById('memoryExport');
const memoryImport = document.getElementById('memoryImport');
const importFileInput = document.getElementById('importFileInput');
memoryExport.addEventListener('click', async () => {
try {
const data = await exportAllData();
const blob = new Blob([JSON.stringify(data, null, 2)], { type: 'application/json' });
const url = URL.createObjectURL(blob);
const a = document.createElement('a');
a.href = url; a.download = `localmind-export-${new Date().toISOString().slice(0, 10)}.json`;
a.click();
URL.revokeObjectURL(url);
showToast('Data exported');
} catch (e) { showToast('Export failed: ' + e.message); }
});
memoryImport.addEventListener('click', () => importFileInput.click());
importFileInput.addEventListener('change', async () => {
const file = importFileInput.files[0];
if (!file) return;
try {
const text = await file.text();
const data = JSON.parse(text);
await importData(data);
showToast(`Imported ${data.memories?.length || 0} memories, ${data.conversations?.length || 0} conversations`);
refreshMemoryPanel();
} catch (e) { showToast('Import failed: ' + e.message); }
importFileInput.value = '';
});
// ── History sidebar ────────────────────────────────────────
const historyBtn = document.getElementById('historyBtn');
const historySidebar = document.getElementById('historySidebar');
const historyBackdrop = document.getElementById('historyBackdrop');
const historyList = document.getElementById('historyList');
const historyCount = document.getElementById('historyCount');
const newChatBtn = document.getElementById('newChatBtn');
function openHistorySidebar() {
historySidebar.classList.add('open');
historyBackdrop.classList.add('open');
memoryPanel.classList.remove('open');
settingsPanel.classList.remove('open');
refreshHistoryPanel();
}
function closeHistorySidebar() {
historySidebar.classList.remove('open');
historyBackdrop.classList.remove('open');
}
historyBtn.addEventListener('click', () => {
if (historySidebar.classList.contains('open')) closeHistorySidebar();
else openHistorySidebar();
});
historyBackdrop.addEventListener('click', closeHistorySidebar);
async function refreshHistoryPanel() {
try {
const convs = await getAllConversations();
historyCount.textContent = `${convs.length} conversation${convs.length === 1 ? '' : 's'}`;
historyList.innerHTML = '';
if (convs.length === 0) {
historyList.innerHTML = '<div class="memory-empty">No saved conversations</div>';
return;
}
for (const conv of convs.slice(0, 50)) {
const item = document.createElement('div');
item.className = 'history-item';
const date = new Date(conv.updated).toLocaleDateString();
const model = MODELS[conv.modelKey]?.label || conv.modelKey || '';
item.innerHTML = `
<div class="history-item-text">
<div class="history-item-title">${escapeHtml(conv.title)}</div>
<div class="history-item-meta">${conv.messageCount} messages &middot; ${model} &middot; ${date}</div>
</div>
<button class="history-item-del" title="Delete">&times;</button>
`;
item.addEventListener('click', (e) => {
if (e.target.closest('.history-item-del')) return;
resumeConversation(conv);
});
item.querySelector('.history-item-del').addEventListener('click', async (e) => {
e.stopPropagation();
await deleteConversation(conv.id);
refreshHistoryPanel();
});
historyList.appendChild(item);
}
} catch (e) {
historyList.innerHTML = '<div class="memory-empty">Error loading history</div>';
}
}
function resetChatUI() {
messages = [];
conversationSummary = '';
activeConversationId = newConversationId();
saveChat();
clearAttachments();
chatArea.innerHTML = '';
chatArea.appendChild(welcomeMsg);
welcomeMsg.style.display = '';
}
function resumeConversation(conv) {
messages = conv.messages || [];
activeConversationId = conv.id;
conversationSummary = '';
saveChat();
chatArea.innerHTML = '';
chatArea.appendChild(welcomeMsg);
welcomeMsg.style.display = 'none';
renderRestoredMessages();
closeHistorySidebar();
}
// ── Branch from message ────────────────────────────────────
// Right-click / long-press on a user message β†’ archive the current
// conversation, then slice messages[0..index] into a new conversation
// and switch to it. The original conversation keeps its full history.
async function branchFromMessage(index) {
if (generating) { showToast('Finish the current message first'); return; }
if (!Number.isFinite(index) || index < 0 || index >= messages.length) return;
// Archive the current conversation first so the branch doesn't
// lose messages after the branch point. If no activeConversationId
// exists yet (fresh session, user hasn't hit New Chat), mint one
// now so the original isn't orphaned.
if (messages.length >= 2) {
if (!activeConversationId) activeConversationId = newConversationId();
try { await saveConversation(messages, activeConversationId); } catch {}
}
const branched = messages.slice(0, index + 1);
const newId = newConversationId();
messages = branched;
activeConversationId = newId;
conversationSummary = '';
saveChat();
try { await saveConversation(messages, newId); } catch {}
chatArea.innerHTML = '';
chatArea.appendChild(welcomeMsg);
welcomeMsg.style.display = messages.length ? 'none' : '';
renderRestoredMessages();
showToast('Branched to new conversation');
}
const msgContextMenu = document.getElementById('msgContextMenu');
const msgBranchBtn = document.getElementById('msgBranchBtn');
let msgContextTargetIndex = null;
function showMsgContextMenu(x, y, index) {
msgContextTargetIndex = index;
// Position near the cursor, clamping to viewport
const menuW = 170, menuH = 40;
const left = Math.min(x, window.innerWidth - menuW - 8);
const top = Math.min(y, window.innerHeight - menuH - 8);
msgContextMenu.style.left = left + 'px';
msgContextMenu.style.top = top + 'px';
msgContextMenu.classList.add('visible');
}
function hideMsgContextMenu() {
msgContextMenu.classList.remove('visible');
msgContextTargetIndex = null;
}
chatArea.addEventListener('contextmenu', (e) => {
const msgEl = e.target.closest('.msg.user');
if (!msgEl) return;
const idxStr = msgEl.dataset.msgIndex;
if (idxStr == null) return;
e.preventDefault();
showMsgContextMenu(e.clientX, e.clientY, parseInt(idxStr, 10));
});
msgBranchBtn.addEventListener('click', () => {
const idx = msgContextTargetIndex;
hideMsgContextMenu();
if (idx != null) branchFromMessage(idx);
});
document.addEventListener('click', (e) => {
if (!msgContextMenu.contains(e.target)) hideMsgContextMenu();
});
document.addEventListener('scroll', hideMsgContextMenu, true);
window.addEventListener('resize', hideMsgContextMenu);
// New Chat: archive current β†’ start fresh
// Auto-backup toggle
const autoBackupToggle = document.getElementById('autoBackupToggle');
autoBackupToggle.checked = localStorage.getItem('lm_auto_backup') === 'true';
autoBackupToggle.addEventListener('change', () => localStorage.setItem('lm_auto_backup', autoBackupToggle.checked));
newChatBtn.addEventListener('click', async () => {
if (generating) return;
if (messages.length >= 2) {
try { await saveConversation(messages, activeConversationId); } catch (e) { console.warn('Failed to archive:', e); }
// Post-session summarization
if (LocalMind.runtime.embeddingsReady) {
try {
const textParts = messages.map(m => {
const role = m.role === 'user' ? 'User' : 'Assistant';
const content = typeof m.content === 'string' ? m.content : m.content.filter(c => c.type === 'text').map(c => c.text).join(' ');
return `${role}: ${content}`;
}).filter(s => s.length > 10);
const summary = textParts.slice(-10).join('\n').slice(0, 2000);
await embedAndStore(summary, 'conversation', 'session-' + new Date().toISOString().slice(0, 10));
} catch {}
}
// Auto-backup if enabled
if (autoBackupToggle.checked) {
try {
const data = await exportAllData();
const blob = new Blob([JSON.stringify(data)], { type: 'application/json' });
const url = URL.createObjectURL(blob);
const a = document.createElement('a');
a.href = url; a.download = `localmind-backup-${new Date().toISOString().slice(0, 10)}.json`;
a.click();
URL.revokeObjectURL(url);
} catch {}
}
}
resetChatUI();
});
// ── Auto-resize textarea ────────────────────────────────────
chatInput.addEventListener('input', () => {
chatInput.style.height = 'auto';
chatInput.style.height = Math.min(chatInput.scrollHeight, 140) + 'px';
});
// ── Model UI visibility ───────────────────────────────────
// Capability gates read from LocalMind.runtime.capabilities() so
// that any future runtime adapter (WebLLM/wllama) can advertise
// its own caps without the UI needing to know which backend is
// active. The runtime returns a snapshot β€” re-read on every call.
function updateModelUI() {
const caps = LocalMind.runtime.capabilities();
const isMultimodal = !!(caps && (caps.image || caps.audio || caps.video));
const isAgent = !!(caps && caps.toolCalling);
inputAffordances.classList.toggle('visible', isMultimodal && modelReady);
thinkingRow.classList.toggle('hidden', !isMultimodal);
plannerRow.classList.toggle('hidden', !isAgent);
searchSettingsSection.style.display = isAgent ? '' : 'none';
samSettingsSection.style.display = isAgent ? '' : 'none';
updateSearchUI();
}
// ── Attachment management ───────────────────────────────────
function addAttachment(att) {
attachments.push(att);
renderAttachmentBar();
}
function removeAttachment(index) {
const att = attachments[index];
if (att.thumb) URL.revokeObjectURL(att.thumb);
attachments.splice(index, 1);
renderAttachmentBar();
}
function clearAttachments() {
for (const att of attachments) {
if (att.thumb) URL.revokeObjectURL(att.thumb);
}
attachments = [];
renderAttachmentBar();
}
function renderAttachmentBar() {
attachmentBar.innerHTML = '';
if (attachments.length === 0) {
attachmentBar.classList.remove('visible');
return;
}
attachmentBar.classList.add('visible');
attachments.forEach((att, i) => {
const chip = document.createElement('div');
chip.className = 'attachment-chip';
if (att.type === 'image') {
const img = document.createElement('img');
img.src = att.thumb;
img.alt = att.name;
chip.appendChild(img);
} else if (att.type === 'video') {
chip.classList.add('video-chip');
const vid = document.createElement('video');
vid.src = att.thumb;
vid.muted = true;
vid.preload = 'metadata';
// Show first frame as poster
vid.addEventListener('loadeddata', () => { vid.currentTime = 0.1; });
chip.appendChild(vid);
const badge = document.createElement('span');
badge.className = 'video-badge';
badge.textContent = '\u25B6 Video';
chip.appendChild(badge);
} else if (att.type === 'audio') {
chip.classList.add('audio-chip');
chip.style.flexDirection = 'column';
const label = document.createElement('span');
label.className = 'audio-label';
label.textContent = att.name;
chip.appendChild(label);
const audio = document.createElement('audio');
audio.src = att.thumb || URL.createObjectURL(att.blob);
audio.controls = true;
audio.preload = 'metadata';
chip.appendChild(audio);
}
const removeBtn = document.createElement('button');
removeBtn.className = 'remove-btn';
removeBtn.textContent = '\u00D7';
removeBtn.addEventListener('click', () => removeAttachment(i));
chip.appendChild(removeBtn);
attachmentBar.appendChild(chip);
});
}
// ── File input handler ──────────────────────────────────────
attachBtn.addEventListener('click', () => fileInput.click());
fileInput.addEventListener('change', () => {
handleFiles(fileInput.files);
fileInput.value = '';
});
function handleFiles(files) {
for (const file of files) {
if (file.type.startsWith('image/')) {
addAttachment({
type: 'image',
blob: file,
thumb: URL.createObjectURL(file),
name: file.name,
});
} else if (file.type.startsWith('audio/')) {
addAttachment({
type: 'audio',
blob: file,
thumb: URL.createObjectURL(file),
name: file.name,
});
} else if (file.type === 'video/mp4' || file.name.endsWith('.mp4')) {
addAttachment({
type: 'video',
blob: file,
thumb: URL.createObjectURL(file),
name: file.name,
});
} else if (/\.(txt|md|json|csv)$/i.test(file.name) || file.type.startsWith('text/')) {
// Text document β†’ embed into RAG memory
ingestDocument(file.name, file.text());
} else if (/\.pdf$/i.test(file.name) || file.type === 'application/pdf') {
// PDF β†’ extract text β†’ embed
(async () => {
try {
showToast(`Extracting text from ${file.name}...`);
const { text, pageCount } = await extractPDFText(file);
if (!text.trim()) { showToast('PDF appears to be empty or image-only'); return; }
const count = await embedAndStore(text, 'document', file.name);
const summary = extractiveSummary(text);
await embedAndStore(summary, 'document_summary', file.name);
showToast(`Stored "${file.name}" (${pageCount} pages, ${count} chunks) β€” ${summary.slice(0, 80)}...`);
} catch (e) {
console.error('PDF error:', e);
showToast('Failed to process PDF: ' + e.message);
}
})();
} else if (/\.docx$/i.test(file.name) || file.type === 'application/vnd.openxmlformats-officedocument.wordprocessingml.document') {
// DOCX β†’ extract text β†’ embed
(async () => {
try {
showToast(`Extracting text from ${file.name}...`);
const text = await extractDOCXText(file);
if (!text.trim()) { showToast('Document appears to be empty'); return; }
const count = await embedAndStore(text, 'document', file.name);
const summary = extractiveSummary(text);
await embedAndStore(summary, 'document_summary', file.name);
showToast(`Stored "${file.name}" (${count} chunks) β€” ${summary.slice(0, 80)}...`);
} catch (e) {
console.error('DOCX error:', e);
showToast('Failed to process DOCX: ' + e.message);
}
})();
}
}
}
// ── Folder ingestion (FS API) ──────────────────────────────
const folderBtn = document.getElementById('folderBtn');
folderBtn.addEventListener('click', async () => {
if (dirHandle) {
dirHandle = null;
folderBtn.classList.remove('folder-open');
folderBtn.title = 'Open folder β€” ingest all .md/.txt/.pdf files into memory';
showToast('Folder closed');
return;
}
if (!window.showDirectoryPicker) {
showToast('Folder picker not supported in this browser');
return;
}
try {
dirHandle = await window.showDirectoryPicker({ mode: 'readwrite' });
folderBtn.classList.add('folder-open');
folderBtn.title = `Folder: ${dirHandle.name} (click to close)`;
await ingestFolder(dirHandle);
} catch (e) {
if (e.name !== 'AbortError') showToast('Could not open folder: ' + e.message);
dirHandle = null;
folderBtn.classList.remove('folder-open');
}
});
async function ingestFolder(handle) {
const SUPPORTED = /\.(md|txt|pdf|docx)$/i;
const fpKey = 'lm_folder_fp';
const fingerprints = JSON.parse(localStorage.getItem(fpKey) || '{}');
let ingested = 0, skipped = 0, failed = 0;
async function walk(dir, prefix) {
for await (const [name, entry] of dir.entries()) {
if (name.startsWith('.')) continue;
if (entry.kind === 'directory') {
await walk(entry, prefix ? `${prefix}/${name}` : name);
} else if (SUPPORTED.test(name)) {
const fullPath = prefix ? `${prefix}/${name}` : name;
const file = await entry.getFile();
const fp = `${file.lastModified}-${file.size}`;
if (fingerprints[fullPath] === fp) { skipped++; continue; }
try {
if (/\.pdf$/i.test(name)) {
const { text, pageCount } = await extractPDFText(file);
if (text.trim()) {
const count = await embedAndStore(text, 'document', fullPath);
const summary = extractiveSummary(text);
if (summary) await embedAndStore(summary, 'document_summary', fullPath);
}
} else if (/\.docx$/i.test(name)) {
const text = await extractDOCXText(file);
if (text.trim()) {
const count = await embedAndStore(text, 'document', fullPath);
const summary = extractiveSummary(text);
if (summary) await embedAndStore(summary, 'document_summary', fullPath);
}
} else {
const text = await file.text();
if (text.trim()) {
const count = await embedAndStore(text, 'document', fullPath);
const summary = extractiveSummary(text);
if (summary) await embedAndStore(summary, 'document_summary', fullPath);
}
}
fingerprints[fullPath] = fp;
ingested++;
} catch (e) {
console.error(`Failed to ingest ${fullPath}:`, e);
failed++;
}
}
}
}
showToast(`Scanning "${handle.name}"…`);
await walk(handle, '');
localStorage.setItem(fpKey, JSON.stringify(fingerprints));
const parts = [`"${handle.name}": ${ingested} file(s) ingested`];
if (skipped) parts.push(`${skipped} unchanged`);
if (failed) parts.push(`${failed} failed`);
showToast(parts.join(', '));
}
// ── Document extractors (lazy-loaded) ─────────────────────
let pdfjsLoaded = false;
async function ensurePDFJS() {
if (pdfjsLoaded) return;
return new Promise((resolve, reject) => {
const script = document.createElement('script');
script.src = 'https://cdn.jsdelivr.net/npm/pdfjs-dist@4.4.168/build/pdf.min.mjs';
script.type = 'module';
// SRI on the main module. Chrome 89+ / Firefox 113+ enforce this for
// module scripts the same as classic scripts.
script.integrity = 'sha384-fzqD3KLclV6zyTSuToNuSG370CipiqN572vfT2zti7ORKnQPaJrvsz8deY+/Tgwe';
script.crossOrigin = 'anonymous';
script.onload = async () => {
try {
// The PDF.js worker is loaded by pdf.js via `new Worker(src)`, which
// does not accept an SRI attribute. Fetch it with Fetch-API SRI
// instead, turn it into a blob URL, and hand that to workerSrc.
// Fetch will throw if the hash doesn't match.
const workerUrl = 'https://cdn.jsdelivr.net/npm/pdfjs-dist@4.4.168/build/pdf.worker.min.mjs';
const res = await fetch(workerUrl, {
integrity: 'sha384-RTiLhGDoA0SEd0K0E9qrg2HwSEEnAo91MwUh79PteR4v01eEByhHOdEkFKxeaeqw',
});
if (!res.ok) throw new Error('HTTP ' + res.status);
const blob = await res.blob();
window.pdfjsLib.GlobalWorkerOptions.workerSrc = URL.createObjectURL(blob);
pdfjsLoaded = true;
resolve();
} catch (e) {
reject(new Error('Failed to load or verify PDF.js worker: ' + e.message));
}
};
script.onerror = () => reject(new Error('Failed to load or verify PDF.js'));
document.head.appendChild(script);
});
}
let mammothLoaded = false;
async function ensureMammoth() {
if (mammothLoaded) return;
return new Promise((resolve, reject) => {
const script = document.createElement('script');
script.src = 'https://cdn.jsdelivr.net/npm/mammoth@1.8.0/mammoth.browser.min.js';
script.integrity = 'sha384-/cXAMbzovUIKbBERjPmR3SnPTh8siWr5lsvFYj1Uq4XP0yaJUZJmsh0YXyGv5P0y';
script.crossOrigin = 'anonymous';
script.onload = () => { mammothLoaded = true; resolve(); };
script.onerror = () => reject(new Error('Failed to load or verify mammoth.js'));
document.head.appendChild(script);
});
}
async function extractPDFText(blob) {
await ensurePDFJS();
const arrayBuffer = await blob.arrayBuffer();
const pdf = await window.pdfjsLib.getDocument({ data: arrayBuffer }).promise;
const pages = [];
for (let i = 1; i <= pdf.numPages; i++) {
const page = await pdf.getPage(i);
const content = await page.getTextContent();
pages.push(content.items.map(item => item.str).join(' '));
}
return { text: pages.join('\n\n'), pageCount: pdf.numPages };
}
async function extractDOCXText(blob) {
await ensureMammoth();
const arrayBuffer = await blob.arrayBuffer();
const result = await mammoth.extractRawText({ arrayBuffer });
return result.value;
}
function extractiveSummary(text, maxSentences = 3) {
const sentences = text.match(/[^.!?\n]+[.!?]+/g) || [];
if (sentences.length <= maxSentences) return text.slice(0, 300);
// Score by word count (longer = more informative) and position (earlier = more important)
const scored = sentences.map((s, i) => ({
text: s.trim(),
score: s.split(/\s+/).length * (1 - i / sentences.length * 0.3),
}));
scored.sort((a, b) => b.score - a.score);
return scored.slice(0, maxSentences).map(s => s.text).join(' ');
}
async function ingestDocument(fileName, textPromise) {
try {
const text = await textPromise;
if (!text || !text.trim()) { showToast(`${fileName} appears to be empty`); return; }
const count = await embedAndStore(text, 'document', fileName);
const summary = extractiveSummary(text);
if (summary) await embedAndStore(summary, 'document_summary', fileName);
showToast(`Stored "${fileName}" (${count} chunks) β€” ${summary.slice(0, 80)}...`);
} catch (e) {
console.error('Document ingest error:', e);
showToast('Failed to store document: ' + e.message);
}
}
function showToast(msg) {
const toast = document.createElement('div');
toast.textContent = msg;
toast.style.cssText = 'position:fixed;bottom:80px;left:50%;transform:translateX(-50%);background:var(--gray-800);color:white;padding:10px 20px;border-radius:8px;font-size:0.82rem;z-index:1000;opacity:0;transition:opacity 0.3s';
document.body.appendChild(toast);
requestAnimationFrame(() => toast.style.opacity = '1');
setTimeout(() => { toast.style.opacity = '0'; setTimeout(() => toast.remove(), 300); }, 3000);
}
// ── Clipboard paste (images) ────────────────────────────────
chatInput.addEventListener('paste', (e) => {
const m = MODELS[activeModelKey];
if (!m || !m.multimodal) return;
const items = e.clipboardData?.items;
if (!items) return;
for (const item of items) {
if (item.type.startsWith('image/')) {
e.preventDefault();
const file = item.getAsFile();
if (file) {
addAttachment({
type: 'image',
blob: file,
thumb: URL.createObjectURL(file),
name: 'pasted-image.png',
});
}
}
}
});
// ── Drag and drop ───────────────────────────────────────────
let dragCounter = 0;
const card = document.querySelector('.card');
card.addEventListener('dragenter', (e) => {
e.preventDefault();
const m = MODELS[activeModelKey];
if (!m || !m.multimodal) return;
dragCounter++;
dragOverlay.classList.add('visible');
});
card.addEventListener('dragleave', (e) => {
e.preventDefault();
dragCounter--;
if (dragCounter <= 0) {
dragCounter = 0;
dragOverlay.classList.remove('visible');
}
});
card.addEventListener('dragover', (e) => e.preventDefault());
card.addEventListener('drop', (e) => {
e.preventDefault();
dragCounter = 0;
dragOverlay.classList.remove('visible');
const m = MODELS[activeModelKey];
if (!m || !m.multimodal) return;
if (e.dataTransfer?.files?.length) {
handleFiles(e.dataTransfer.files);
}
});
// ── Camera capture ──────────────────────────────────────────
cameraBtn.addEventListener('click', async () => {
try {
cameraStream = await navigator.mediaDevices.getUserMedia({ video: { facingMode: 'environment' } });
cameraPreview.srcObject = cameraStream;
cameraOverlay.classList.add('open');
} catch (err) {
console.error('Camera error:', err);
alert('Could not access camera: ' + err.message);
}
});
camCaptureBtn.addEventListener('click', () => {
const video = cameraPreview;
const canvas = document.createElement('canvas');
canvas.width = video.videoWidth;
canvas.height = video.videoHeight;
canvas.getContext('2d').drawImage(video, 0, 0);
canvas.toBlob((blob) => {
if (blob) {
addAttachment({
type: 'image',
blob,
thumb: URL.createObjectURL(blob),
name: 'camera-photo.jpg',
});
}
closeCamera();
}, 'image/jpeg', 0.9);
});
camCancelBtn.addEventListener('click', closeCamera);
function closeCamera() {
cameraOverlay.classList.remove('open');
if (cameraStream) {
cameraStream.getTracks().forEach(t => t.stop());
cameraStream = null;
}
cameraPreview.srcObject = null;
}
// ── Voice to text (Whisper WebGPU, on-device) ──────────────
// Separate from the mic-for-audio-attachment flow above: this records
// audio, decodes it to 16 kHz mono PCM, and runs local Whisper to
// transcribe text back into the input field. Works on any model.
const voiceBtn = document.getElementById('voiceBtn');
let voiceRecorder = null;
let voiceStream = null;
let voiceWorker = null;
let voiceReady = false;
let voiceReadyPromise = null;
function ensureVoiceWorker() {
if (voiceReadyPromise) return voiceReadyPromise;
voiceWorker = createTranscriptionWorker();
voiceReadyPromise = new Promise((resolve, reject) => {
const onMsg = (e) => {
if (e.data.type === 'progress') {
const p = e.data.data;
if (p.status === 'downloading' && p.total) {
const mb = (p.loaded / 1024 / 1024).toFixed(0);
const total = (p.total / 1024 / 1024).toFixed(0);
showToast('Loading Whisper ' + mb + ' / ' + total + ' MB');
}
} else if (e.data.type === 'ready') {
voiceReady = true;
voiceWorker.removeEventListener('message', onMsg);
resolve();
} else if (e.data.type === 'error') {
voiceWorker.removeEventListener('message', onMsg);
reject(new Error(e.data.message));
}
};
voiceWorker.addEventListener('message', onMsg);
voiceWorker.postMessage({ type: 'load' });
});
return voiceReadyPromise;
}
async function decodeToWhisperPCM(blob) {
const AC = window.AudioContext || window.webkitAudioContext;
if (!AC) throw new Error('AudioContext unavailable');
const audioCtx = new AC();
try {
const arrayBuf = await blob.arrayBuffer();
const audioBuf = await audioCtx.decodeAudioData(arrayBuf);
const targetRate = 16000;
if (audioBuf.sampleRate === targetRate && audioBuf.numberOfChannels === 1) {
return new Float32Array(audioBuf.getChannelData(0));
}
// Resample + downmix via OfflineAudioContext
const length = Math.ceil(audioBuf.duration * targetRate);
const offline = new OfflineAudioContext(1, length, targetRate);
const src = offline.createBufferSource();
src.buffer = audioBuf;
src.connect(offline.destination);
src.start();
const rendered = await offline.startRendering();
return new Float32Array(rendered.getChannelData(0));
} finally {
try { audioCtx.close(); } catch {}
}
}
async function transcribeAndInsert(blob) {
try {
voiceBtn.classList.remove('recording');
voiceBtn.innerHTML = '&#x23F3;'; // hourglass
showToast(voiceReady ? 'Transcribing\u2026' : 'Loading Whisper \u2014 first use may take a minute');
await ensureVoiceWorker();
const pcm = await decodeToWhisperPCM(blob);
const text = await new Promise((resolve, reject) => {
const id = Math.random().toString(36).slice(2);
const onMsg = (e) => {
if (e.data.id && e.data.id !== id) return;
if (e.data.type === 'transcription') {
voiceWorker.removeEventListener('message', onMsg);
resolve(e.data.text || '');
} else if (e.data.type === 'error') {
voiceWorker.removeEventListener('message', onMsg);
reject(new Error(e.data.message));
}
};
voiceWorker.addEventListener('message', onMsg);
const lang = localStorage.getItem('lm_voice_language') || 'en';
voiceWorker.postMessage({ type: 'transcribe', pcm, lang, id }, [pcm.buffer]);
});
if (text) {
const existing = chatInput.value;
chatInput.value = existing ? existing + (existing.endsWith(' ') ? '' : ' ') + text : text;
chatInput.dispatchEvent(new Event('input'));
chatInput.focus();
showToast('Transcribed');
} else {
showToast('No speech detected');
}
} catch (e) {
console.error('Whisper error:', e);
showToast('Transcription failed: ' + (e.message || e));
} finally {
voiceBtn.innerHTML = '&#x1F5E3;'; // speaking head
}
}
voiceBtn.addEventListener('click', async () => {
if (voiceRecorder && voiceRecorder.state === 'recording') {
voiceRecorder.stop();
return;
}
try {
voiceStream = await navigator.mediaDevices.getUserMedia({ audio: true });
const chunks = [];
voiceRecorder = new MediaRecorder(voiceStream);
voiceRecorder.addEventListener('dataavailable', (e) => { if (e.data.size > 0) chunks.push(e.data); });
voiceRecorder.addEventListener('stop', () => {
voiceStream.getTracks().forEach(t => t.stop());
voiceStream = null;
const blob = new Blob(chunks, { type: voiceRecorder.mimeType || 'audio/webm' });
voiceRecorder = null;
if (blob.size === 0) { showToast('No audio captured'); voiceBtn.classList.remove('recording'); return; }
transcribeAndInsert(blob);
});
voiceRecorder.start();
voiceBtn.classList.add('recording');
} catch (err) {
showToast('Mic error: ' + err.message);
}
});
// ── Microphone recording (audio attachment for multimodal) ───
micBtn.addEventListener('click', () => {
if (mediaRecorder && mediaRecorder.state === 'recording') {
mediaRecorder.stop();
return;
}
startMicRecording();
});
async function startMicRecording() {
try {
const stream = await navigator.mediaDevices.getUserMedia({ audio: true });
const chunks = [];
mediaRecorder = new MediaRecorder(stream);
mediaRecorder.addEventListener('dataavailable', (e) => {
if (e.data.size > 0) chunks.push(e.data);
});
mediaRecorder.addEventListener('stop', () => {
stream.getTracks().forEach(t => t.stop());
micBtn.classList.remove('recording');
micBtn.textContent = '\u{1F3A4}';
if (chunks.length > 0) {
const blob = new Blob(chunks, { type: mediaRecorder.mimeType || 'audio/webm' });
addAttachment({
type: 'audio',
blob,
thumb: URL.createObjectURL(blob),
name: 'recording.webm',
});
}
mediaRecorder = null;
});
mediaRecorder.start();
micBtn.classList.add('recording');
micBtn.textContent = '\u{23F9}';
} catch (err) {
console.error('Mic error:', err);
alert('Could not access microphone: ' + err.message);
}
}
// ── Markdown renderer (lightweight) ─────────────────────────
// ── Math & diagram rendering ────────────────────────────────
// KaTeX loads via a <script defer> in <head>. If a render happens
// before it finishes, math stays literal; once it loads we re-render
// any already-painted assistant bubbles. Mermaid is heavier (~2 MB),
// so it's lazy-loaded on the first message that contains a ```mermaid
// block.
let __mermaidPromise = null;
async function ensureMermaid() {
if (!__mermaidPromise) {
__mermaidPromise = import('https://cdn.jsdelivr.net/npm/mermaid@11/dist/mermaid.esm.min.mjs')
.then((m) => {
m.default.initialize({ startOnLoad: false, theme: 'default', securityLevel: 'strict' });
return m.default;
});
}
return __mermaidPromise;
}
// Re-render all assistant bubbles that already contain literal math
// markers ($…$ or $$…$$) once KaTeX finishes loading. Called once from
// the KaTeX onload hook installed below.
function rerenderAssistantBubblesWithMath() {
document.querySelectorAll('.msg.assistant').forEach((el) => {
const idxStr = el.dataset.msgIndex;
if (idxStr == null) return;
const msg = messages[parseInt(idxStr, 10)];
if (!msg || msg.role !== 'assistant' || typeof msg.content !== 'string') return;
if (!/\$[^$\n]/.test(msg.content)) return;
const bubble = el.querySelector('.msg-bubble');
if (bubble) { bubble.innerHTML = renderMarkdown(msg.content); postProcessBubble(bubble); }
});
}
// Transform `<pre data-lang="mermaid">` blocks into rendered Mermaid
// diagrams, and attach sandboxed preview iframes beneath `<pre>` blocks
// tagged html / svg / artifact. Called after each bubble innerHTML update;
// guarded with data attributes so repeated calls are cheap.
async function postProcessBubble(bubble) {
if (!bubble) return;
// Artifact previews β€” cheap, pure DOM.
const ARTIFACT_LANGS = new Set(['html', 'svg', 'artifact']);
bubble.querySelectorAll('pre[data-lang]:not([data-af-processed])').forEach((pre) => {
const lang = pre.dataset.lang;
if (!ARTIFACT_LANGS.has(lang)) return;
pre.dataset.afProcessed = '1';
const code = pre.querySelector('code');
const src = code ? code.textContent : pre.textContent;
const doc = lang === 'svg'
? '<!DOCTYPE html><html><body style="margin:0;padding:12px">' + src + '</body></html>'
: src;
const wrap = document.createElement('div');
wrap.className = 'artifact-preview';
const header = document.createElement('div');
header.className = 'artifact-header';
const badge = document.createElement('span');
badge.innerHTML = 'Preview \u2014 <strong>' + escapeHtml(lang) + '</strong> <span style="color:var(--gray-400)">(sandboxed)</span>';
header.appendChild(badge);
const frame = document.createElement('iframe');
frame.className = 'artifact-frame';
// allow-scripts only β€” no allow-same-origin so the frame runs in an
// opaque origin and can't touch localStorage, IDB, cookies, or parent.
frame.setAttribute('sandbox', 'allow-scripts');
frame.srcdoc = doc;
wrap.appendChild(header);
wrap.appendChild(frame);
pre.after(wrap);
});
// Mermaid β€” async, lazy-loaded.
const pendingMermaid = bubble.querySelectorAll('pre[data-lang="mermaid"]:not([data-mm-processed])');
if (pendingMermaid.length === 0) return;
const nodes = [];
pendingMermaid.forEach((pre) => {
pre.dataset.mmProcessed = '1';
const code = pre.querySelector('code');
const src = code ? code.textContent : pre.textContent;
const div = document.createElement('div');
div.className = 'mermaid';
div.textContent = src;
pre.replaceWith(div);
nodes.push(div);
});
try {
const m = await ensureMermaid();
await m.run({ nodes });
} catch (e) {
nodes.forEach((n) => {
n.innerHTML = '<span class="mermaid-error">Mermaid: ' + escapeHtml(e.message || String(e)) + '</span>';
});
}
}
// Once KaTeX finishes loading, re-paint any already-rendered bubbles so
// math shows up. No-op if katex never loads (offline, CDN blocked).
if (typeof katex === 'undefined') {
const poll = setInterval(() => {
if (typeof katex !== 'undefined') { clearInterval(poll); rerenderAssistantBubblesWithMath(); }
}, 200);
// Stop polling after 30 s so we don't burn CPU forever when offline
setTimeout(() => clearInterval(poll), 30000);
}
function renderMarkdown(text) {
let thinkContent = '';
let mainContent = text;
// Handle <|think|>...<|/think|> format
const thinkMatch = text.match(/<\|think\|>([\s\S]*?)(<\|\/think\|>|$)/);
if (thinkMatch) {
thinkContent = thinkMatch[1].trim();
mainContent = text.replace(/<\|think\|>[\s\S]*?(<\|\/think\|>|$)/, '').trim();
}
// Also handle Gemma 4 <|channel>thought ... <channel|> format
if (!thinkContent) {
const channelMatch = text.match(/<\|channel>thought\n?([\s\S]*?)(<channel\|>|$)/);
if (channelMatch) {
thinkContent = channelMatch[1].trim();
mainContent = text.replace(/<\|channel>thought\n?[\s\S]*?(<channel\|>|$)/, '').trim();
}
}
let html = '';
if (thinkContent) {
const thinkId = 'think-' + Math.random().toString(36).slice(2, 8);
// During streaming: mainContent is empty β†’ thinking is in progress β†’ show open
// After streaming: mainContent has text β†’ thinking is done β†’ show collapsed
const isStillThinking = !mainContent;
const openClass = isStillThinking ? ' open' : '';
const arrow = isStillThinking ? '&#9660;' : '&#9654;';
const label = isStillThinking ? 'Thinking...' : 'Thought process';
html += `<div class="thinking-block">
<div class="thinking-toggle" onclick="document.getElementById('${thinkId}').classList.toggle('open'); this.querySelector('span').textContent = document.getElementById('${thinkId}').classList.contains('open') ? '&#9660;' : '&#9654;'">
<span>${arrow}</span> ${label}
</div>
<div class="thinking-content${openClass}" id="${thinkId}">${escapeHtml(thinkContent)}</div>
</div>`;
}
if (!mainContent) return html || '<span style="color:var(--gray-400)">...</span>';
const cbNonce = Math.random().toString(36).slice(2, 12);
mainContent = mainContent.replace(/```(\w*)\n?([\s\S]*?)```/g, (_, lang, code) => {
// SAFETY: sanitise `lang` to [a-zA-Z0-9] so it can't break out of
// the onclick handler's single-quoted string argument.
const ext = (lang || 'txt').replace(/[^a-zA-Z0-9]/g, '') || 'txt';
const codeId = 'code-' + Math.random().toString(36).slice(2, 8);
// data-lang carries the original language for post-processors
// (mermaid, artifact preview). Separate from `ext` which feeds
// the download filename.
const langAttr = ext === 'txt' && lang ? '' : ' data-lang="' + ext + '"';
return `<pre class="cb-${cbNonce}"${langAttr}><code id="${codeId}">${escapeHtml(code.trim())}</code><button class="code-download-btn" onclick="downloadCodeBlock('${codeId}','${ext}')" title="Download">&#8595;</button></pre>`;
});
// Split on our nonce-stamped code-block wrapper. The nonce ensures the
// split matches only code blocks WE emitted in this call, not any
// <pre>-looking HTML the model may have produced verbatim via prompt
// injection. The [^>]* allows for optional attributes (e.g. data-lang).
const splitRe = new RegExp('(<pre class="cb-' + cbNonce + '"[^>]*>[\\s\\S]*?<\\/pre>)', 'g');
const parts = mainContent.split(splitRe);
const preTag = '<pre class="cb-' + cbNonce + '"';
html += parts.map(part => {
if (part.startsWith(preTag)) return part;
// Extract math expressions BEFORE escapeHtml so LaTeX survives. The
// `Β§Β§KM…§§` marker is vanishingly unlikely in model output and
// contains no HTML-meaningful characters, so it survives escape.
// Display math ($$…$$) is checked first to avoid `$$` being eaten
// by the inline rule. Inline math must contain a LaTeX-ish token
// (`\`, `^`, `_`, `{`, `}`, or a Greek name) so plain prose like
// "cost $5 and $10" doesn't get treated as math.
const maths = [];
part = part.replace(/\$\$([\s\S]+?)\$\$/g, (_, expr) => {
maths.push({ expr, display: true });
return '\u00A7\u00A7KM' + (maths.length - 1) + '\u00A7\u00A7';
});
part = part.replace(/\$([^\n$]+?)\$/g, (full, expr) => {
if (!/[\\^_{}]|\\(alpha|beta|gamma|delta|theta|lambda|mu|pi|sigma|sum|int|frac|sqrt)/i.test(expr)) {
return full;
}
maths.push({ expr, display: false });
return '\u00A7\u00A7KM' + (maths.length - 1) + '\u00A7\u00A7';
});
// SAFETY: escape untrusted text before running markdown regexes.
// Model output can contain raw HTML via prompt injection from fetched
// pages, search results, or ingested documents. Inline markdown
// regexes match literal * ` - digits \n which all survive escape.
part = escapeHtml(part);
part = part.replace(/`([^`]+)`/g, '<code>$1</code>');
part = part.replace(/\*\*(.+?)\*\*/g, '<strong>$1</strong>');
part = part.replace(/\*(.+?)\*/g, '<em>$1</em>');
part = part.replace(/(?:^|\n)((?:[-*] .+\n?)+)/g, (_, list) => {
const items = list.trim().split('\n').map(l => `<li>${l.replace(/^[-*] /, '')}</li>`).join('');
return `<ul>${items}</ul>`;
});
part = part.replace(/(?:^|\n)((?:\d+\. .+\n?)+)/g, (_, list) => {
const items = list.trim().split('\n').map(l => `<li>${l.replace(/^\d+\. /, '')}</li>`).join('');
return `<ol>${items}</ol>`;
});
part = part.replace(/\n\n+/g, '</p><p>');
part = part.replace(/\n/g, '<br>');
if (part.trim() && !part.startsWith('<')) part = `<p>${part}</p>`;
// Substitute math placeholders with KaTeX HTML (or literal $… if
// KaTeX isn't loaded yet; a poll re-renders once it's ready).
if (maths.length) {
part = part.replace(/\u00A7\u00A7KM(\d+)\u00A7\u00A7/g, (_, i) => {
const m = maths[parseInt(i, 10)];
if (!m) return '';
if (typeof katex === 'undefined') {
const d = m.display ? '$$' : '$';
return d + escapeHtml(m.expr) + d;
}
try {
return katex.renderToString(m.expr, { displayMode: m.display, throwOnError: false, output: 'html' });
} catch (e) {
return '<span class="katex-error">' + escapeHtml((m.display ? '$$' : '$') + m.expr) + '</span>';
}
});
}
return part;
}).join('');
return html;
}
// Global: download a code block by ID
window.downloadCodeBlock = function(codeId, ext) {
const el = document.getElementById(codeId);
if (!el) return;
const text = el.textContent;
const blob = new Blob([text], { type: 'text/plain' });
const url = URL.createObjectURL(blob);
const a = document.createElement('a');
a.href = url; a.download = `code.${ext}`;
a.click();
URL.revokeObjectURL(url);
};
function escapeHtml(s) {
return s.replace(/&/g, '&amp;').replace(/</g, '&lt;').replace(/>/g, '&gt;').replace(/"/g, '&quot;');
}
// ── Chat rendering ──────────────────────────────────────────
function renderRestoredMessages() {
if (messages.length === 0) return;
welcomeMsg.style.display = 'none';
for (let i = 0; i < messages.length; i++) {
const msg = messages[i];
const role = msg.role === 'user' ? 'user' : 'assistant';
const el = document.createElement('div');
el.className = `msg ${role}`;
el.dataset.msgIndex = String(i);
const bubble = document.createElement('div');
bubble.className = 'msg-bubble';
if (role === 'user') {
// For restored messages, content is always text (blobs not persisted)
const textContent = typeof msg.content === 'string' ? msg.content :
msg.content.filter(c => c.type === 'text').map(c => c.text).join(' ');
bubble.textContent = textContent;
} else {
bubble.innerHTML = renderMarkdown(msg.content);
postProcessBubble(bubble);
}
el.appendChild(bubble);
chatArea.appendChild(el);
}
chatArea.scrollTop = chatArea.scrollHeight;
}
function addUserMessage(text, hasAttachments) {
welcomeMsg.style.display = 'none';
const el = document.createElement('div');
el.className = 'msg user';
// Index stored so right-click/long-press can find the corresponding
// entry in the messages array for "Branch from here".
el.dataset.msgIndex = String(messages.length - 1);
const bubble = document.createElement('div');
bubble.className = 'msg-bubble';
// Show attachment thumbnails in the message
if (hasAttachments && attachments.length > 0) {
const attDiv = document.createElement('div');
attDiv.className = 'msg-attachments';
for (const att of attachments) {
if (att.type === 'image' && att.thumb) {
const img = document.createElement('img');
img.src = att.thumb;
img.alt = att.name;
attDiv.appendChild(img);
} else if (att.type === 'video' && att.thumb) {
const tag = document.createElement('span');
tag.className = 'audio-tag';
tag.textContent = '\u{1F3AC} ' + att.name;
attDiv.appendChild(tag);
} else if (att.type === 'audio') {
const tag = document.createElement('span');
tag.className = 'audio-tag';
tag.textContent = '\u{1F3B5} ' + att.name;
attDiv.appendChild(tag);
}
}
bubble.appendChild(attDiv);
}
const textNode = document.createElement('span');
textNode.textContent = text;
bubble.appendChild(textNode);
el.appendChild(bubble);
chatArea.appendChild(el);
chatArea.scrollTop = chatArea.scrollHeight;
}
function addAssistantPlaceholder() {
const el = document.createElement('div');
el.className = 'msg assistant';
const bubble = document.createElement('div');
bubble.className = 'msg-bubble';
bubble.innerHTML = '<span class="typing-dot"></span><span class="typing-dot"></span><span class="typing-dot"></span>';
el.appendChild(bubble);
chatArea.appendChild(el);
chatArea.scrollTop = chatArea.scrollHeight;
return { el, bubble };
}
function updateAssistantBubble(bubble, text) {
bubble.innerHTML = renderMarkdown(text);
// Mermaid blocks render async once the bubble is in the DOM. Safe to
// call every token β€” postProcessBubble guards with data-mm-processed.
postProcessBubble(bubble);
chatArea.scrollTop = chatArea.scrollHeight;
}
// Strip the model's raw <tool_call>...</tool_call> blocks from text so
// the tags don't leak into the visible bubble or saved message history.
// (They're still present in loopMessages when feeding the model its own
// turn back, so the model continues to see its own output format.)
function stripToolCallTags(text) {
if (!text) return text;
return text.replace(/<tool_call>[\s\S]*?<\/tool_call>/g, '').trim();
}
function saveChat() {
try {
// Strip blobs from messages for sessionStorage (only persist text)
const serializable = messages.map(m => {
if (m.role === 'assistant') return m;
if (typeof m.content === 'string') return m;
// Multimodal user message: extract text only
const textParts = m.content.filter(c => c.type === 'text');
const text = textParts.map(c => c.text).join(' ');
return { role: m.role, content: text };
});
sessionStorage.setItem('localmind_chat', JSON.stringify(serializable));
} catch {}
}
// ===========================================================
// === RUNTIME ADAPTER: Transformers.js ======================
// ===========================================================
// This section is the ONLY place in the file that may reference
// Transformers.js. Everything else β€” agent loop, chat UI, JS API,
// RAG ingestion, document upload β€” must go through
// `LocalMind.runtime.*`. The rule exists so a second backend
// (WebLLM, wllama, …) can be slotted in by adding another adapter
// section without touching anything above. If you need to add new
// model-side functionality, the API surface below is the contract;
// bend the adapter to fit it, not the other way around.
//
// Public surface:
// runtime.loadModel(modelId, options) -> Promise<{ modelId, capabilities }>
// runtime.unload()
// runtime.chat(request) -> async iterator of events
// events: { type: 'token', text }
// | { type: 'tool_call', id, name, arguments }
// | { type: 'done', reason, usage }
// | { type: 'error', error }
// runtime.embed(string) -> Promise<Float32Array>
// runtime.embed(string[]) -> Promise<Float32Array[]>
// runtime.embeddingsReady -> boolean (gate; embed is lazy-loaded)
// runtime.capabilities() -> { text, image, audio, video, toolCalling, maxContextTokens }
//
// Tool calling is faked via prompt injection + parse-after-completion
// (Transformers.js has no native tool API). The scaffolding and the
// parser both live in this file as `_adapter*` helpers β€” the agent
// loop above sees only the {type:'tool_call'} events.
//
// Multimodal preprocessing (audio decode, video frame extraction)
// also lives here. The chat UI passes content blocks containing raw
// Blobs; the adapter translates them to whatever the worker expects.
// -----------------------------------------------------------
const LocalMind = (typeof window !== 'undefined' && window.LocalMind) || {};
if (typeof window !== 'undefined') window.LocalMind = LocalMind;
// ── Tool-calling helpers (adapter-internal) ─────────────────
// Transformers.js has no native tool-calling API. The strategy is
// prompt-injection: we describe the available tools and the
// expected <tool_call> XML format in the system prompt, generate,
// then scan the model's output for tool-call blocks. Both the
// scaffolding and the parser live here so the agent loop above
// never has to know how a particular runtime fakes tool calling.
// A WebLLM/wllama adapter would replace these with whatever
// native shape that runtime exposes.
function _adapterRepairJSON(str) {
try { return JSON.parse(str); } catch {}
const fixed = String(str)
.replace(/,\s*([}\]])/g, '$1') // trailing commas
.replace(/'/g, '"') // single β†’ double quotes
.replace(/(\w+)\s*:/g, '"$1":'); // unquoted keys
try { return JSON.parse(fixed); } catch {}
return null;
}
function _adapterBuildToolPrompt(tools, userPrompt) {
// tools: [{ name, description, parameters }]
const toolDefs = (tools || []).map(t => ({
type: 'function',
function: { name: t.name, description: t.description, parameters: t.parameters },
}));
let prompt = `You are a helpful AI assistant with access to tools. Use them when they would help answer the user's question. If tools aren't needed, respond directly.
<tools>
${JSON.stringify(toolDefs, null, 2)}
</tools>
To call a function, output EXACTLY this format (not inside thinking):
<tool_call>
{"name": "function_name", "arguments": {"arg1": "value1"}}
</tool_call>
CRITICAL: The <tool_call> block must appear in your main response, NEVER inside your thinking/reasoning. Think first, then output the <tool_call> block as your response.
Rules:
- Call one tool at a time. Wait for the result before calling another.
- When citing sources, mention where the information came from.
- If you don't need tools, just answer directly.
- Never fabricate tool results.
- Always use the exact <tool_call> XML format shown above. Do not invent other formats like "tool.call:" or function-call syntax.
- You can translate text between 140+ languages directly in your response β€” no tool needed for translation.`;
if (userPrompt) prompt += '\n\n' + userPrompt;
return prompt;
}
function _adapterParseToolCalls(text, knownToolNames) {
const toolCalls = [];
let cleanText = text;
// 1. Match <tool_call>…</tool_call> blocks
const tagRegex = /<tool_call>\s*([\s\S]*?)\s*<\/tool_call>/g;
let match;
while ((match = tagRegex.exec(text)) !== null) {
const parsed = _adapterRepairJSON(match[1].trim());
const tcName = parsed && (parsed.name || parsed.function);
if (tcName) {
toolCalls.push({ name: tcName, arguments: parsed.arguments || {} });
}
cleanText = cleanText.replace(match[0], '');
}
// 2. Bare JSON without tags
if (toolCalls.length === 0) {
const bareRegex = /\{\s*"(?:name|function)"\s*:\s*"(\w+)"\s*,\s*"arguments"\s*:\s*(\{[\s\S]*?\})\s*\}/g;
while ((match = bareRegex.exec(text)) !== null) {
const args = _adapterRepairJSON(match[2]);
if (args) {
toolCalls.push({ name: match[1], arguments: args });
cleanText = cleanText.replace(match[0], '');
}
}
}
// 3. Inline tool.call:name{args} buried in thinking
if (toolCalls.length === 0 && knownToolNames && knownToolNames.length) {
const mainContent = text
.replace(/<\|think\|>[\s\S]*?(<\|\/think\|>|$)/, '')
.replace(/<\|channel>thought\n?[\s\S]*?(<channel\|>|$)/, '')
.trim();
if (!mainContent) {
const namesAlt = knownToolNames.map(n => n.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')).join('|');
const intentRegex = new RegExp(`tool[._]?call\\s*[:.]\\s*(${namesAlt})\\s*[{(]([^})]*)`, 'i');
const intentMatch = text.match(intentRegex);
if (intentMatch) {
const args = {};
intentMatch[2].split(/[,;]/).forEach(pair => {
const [k, ...rest] = pair.split(':');
if (k && rest.length) args[k.trim()] = rest.join(':').trim().replace(/["']/g, '');
});
toolCalls.push({ name: intentMatch[1], arguments: args });
}
}
}
return { toolCalls, cleanText: cleanText.trim() };
}
function _adapterFormatToolResultMessage(name, result) {
// The string the model sees as the tool result. Wrapped as a
// user-role message in apply_chat_template β€” Gemma's convention.
return `<tool_response>
${JSON.stringify({ name, result })}
</tool_response>`;
}
// ── Multimodal helpers (adapter-internal) ───────────────────
// The Transformers.js multimodal processor expects:
// - images: ArrayBuffer of an image file (jpg/png), reconstructed
// in the worker via load_image() on a blob URL
// - audio: Float32Array of mono PCM at 16 kHz
// - video: decomposed into N frames (images) + 1 audio track
// The translation lives here so the agent loop never has to know
// about decode pipelines or sample rates.
async function _adapterDecodeAudioTo16k(blob) {
const arrayBuffer = await blob.arrayBuffer();
const audioCtx = new AudioContext({ sampleRate: 16000 });
const decoded = await audioCtx.decodeAudioData(arrayBuffer);
const mono = decoded.getChannelData(0);
const pcm = new Float32Array(mono);
audioCtx.close();
return pcm;
}
async function _adapterDecomposeVideo(blob) {
const url = URL.createObjectURL(blob);
const video = document.createElement('video');
video.muted = true;
video.preload = 'auto';
video.src = url;
await new Promise((resolve, reject) => {
video.addEventListener('loadedmetadata', resolve);
video.addEventListener('error', reject);
});
const results = { images: [], audio: null };
const duration = video.duration;
const numFrames = Math.min(4, Math.max(1, Math.floor(duration / 2)));
const canvas = document.createElement('canvas');
const ctx = canvas.getContext('2d');
for (let i = 0; i < numFrames; i++) {
const t = (duration / (numFrames + 1)) * (i + 1);
video.currentTime = t;
await new Promise(r => video.addEventListener('seeked', r, { once: true }));
canvas.width = video.videoWidth;
canvas.height = video.videoHeight;
ctx.drawImage(video, 0, 0);
const frameBlob = await new Promise(r => canvas.toBlob(r, 'image/jpeg', 0.85));
if (frameBlob) {
results.images.push(await frameBlob.arrayBuffer());
}
}
try {
const arrayBuffer = await blob.arrayBuffer();
const audioCtx = new AudioContext({ sampleRate: 16000 });
const audioBuffer = await audioCtx.decodeAudioData(arrayBuffer);
results.audio = new Float32Array(audioBuffer.getChannelData(0));
audioCtx.close();
} catch {}
URL.revokeObjectURL(url);
return results;
}
// Walk a messages array. For any content array containing media
// blocks (image/audio/video) with `data` set, process the data and
// build a parallel attData list. The returned chatMessages have
// the same structure but with media blocks stripped of their data
// field β€” that's what the chat template needs to emit positional
// tokens. Stale placeholder media blocks (no data) are preserved
// in chatMessages but contribute nothing to attData; this matches
// the agent-loop behaviour where iteration > 0 carries position
// tokens forward but skips re-processing.
async function _adapterProcessContentBlocks(messages) {
const attData = [];
let hasMedia = false;
const out = [];
for (const m of messages) {
if (Array.isArray(m.content)) {
const cleanBlocks = [];
for (const block of m.content) {
if (!block || typeof block !== 'object') continue;
if (block.type === 'text') {
cleanBlocks.push({ type: 'text', text: block.text || '' });
continue;
}
if (block.type !== 'image' && block.type !== 'audio' && block.type !== 'video') {
continue;
}
hasMedia = true;
if (!block.data) {
// Stale placeholder from a prior agent-loop iteration β€”
// keep the chat-template position, no attData entry.
if (block.type === 'video') {
// Videos were already expanded on first pass, so they
// shouldn't appear as stale; treat defensively as one
// image position.
cleanBlocks.push({ type: 'image' });
} else {
cleanBlocks.push({ type: block.type });
}
continue;
}
const blob = block.data instanceof Blob
? block.data
: new Blob([block.data], block.mimeType ? { type: block.mimeType } : undefined);
if (block.type === 'image') {
const buf = await blob.arrayBuffer();
attData.push({ type: 'image', data: buf, mimeType: blob.type || 'image/jpeg' });
cleanBlocks.push({ type: 'image' });
} else if (block.type === 'audio') {
const pcm = await _adapterDecodeAudioTo16k(blob);
attData.push({ type: 'audio', pcmData: pcm });
cleanBlocks.push({ type: 'audio' });
} else if (block.type === 'video') {
const decomp = await _adapterDecomposeVideo(blob);
for (const imgBuf of decomp.images) {
attData.push({ type: 'image', data: imgBuf, mimeType: 'image/jpeg' });
cleanBlocks.push({ type: 'image' });
}
if (decomp.audio) {
attData.push({ type: 'audio', pcmData: decomp.audio });
cleanBlocks.push({ type: 'audio' });
}
}
}
out.push({ role: m.role, content: cleanBlocks });
} else {
out.push(m);
}
}
return { chatMessages: out, attData, hasMedia };
}
LocalMind.runtime = {
// --- internal state -----------------------------------------
_worker: null,
_activeModelId: null,
_activeCaps: {
text: true, image: false, audio: false, video: false,
toolCalling: false, maxContextTokens: 0,
},
// --- loadModel ----------------------------------------------
// Synchronous up to the first await: terminates any previous
// worker, creates a fresh one, posts the load message, and stores
// the worker on `_worker` BEFORE returning the promise. This lets
// legacy callers grab the worker via `_getWorker()` synchronously
// after invoking loadModel and attach their own message handlers
// before any worker message is dispatched (postMessage and
// addEventListener inside the executor both run in the current
// microtask, so handlers attached by the caller in the same tick
// are guaranteed to receive every event).
loadModel(modelId, options) {
options = options || {};
const dtype = options.dtype || 'q4f16';
const modelType = options.modelType || 'causal';
const onProgress = options.onProgress;
const caps = options.capabilities || {
text: true, image: false, audio: false, video: false,
toolCalling: false, maxContextTokens: 0,
};
// Terminate previous worker if any
if (this._worker) {
try { this._worker.terminate(); } catch {}
this._worker = null;
}
const w = createWorker();
this._worker = w;
this._activeModelId = modelId;
this._activeCaps = caps;
const promise = new Promise((resolve, reject) => {
let settled = false;
const cleanup = () => {
settled = true;
w.removeEventListener('message', onMsg);
};
const onMsg = (e) => {
const d = e.data;
if (settled) return;
if (d.type === 'progress' && typeof onProgress === 'function') {
try { onProgress(d.data); } catch {}
} else if (d.type === 'ready') {
cleanup();
resolve({ modelId, capabilities: caps });
} else if (d.type === 'error') {
cleanup();
reject(new Error(d.message || 'Model load failed'));
}
};
w.addEventListener('message', onMsg);
w.addEventListener('error', (ev) => {
if (settled) return;
cleanup();
reject(new Error(ev.message || 'Worker startup failed'));
});
const resumable = localStorage.getItem('lm_resumable_downloads') !== '0';
w.postMessage({ type: 'load', modelId, dtype, modelType, resumable });
});
return promise;
},
// --- unload -------------------------------------------------
// Frees the chat worker. Embedding worker is independent and
// unaffected. Idempotent.
unload() {
if (this._worker) {
try { this._worker.terminate(); } catch {}
this._worker = null;
}
this._activeModelId = null;
this._activeCaps = {
text: true, image: false, audio: false, video: false,
toolCalling: false, maxContextTokens: 0,
};
},
// --- chat ---------------------------------------------------
// Returns an async iterator that yields:
// { type: 'token', text }
// { type: 'tool_call', id, name, arguments }
// { type: 'done', reason: 'stop'|'length'|'tool_calls'|'abort',
// usage: { promptTokens, completionTokens } }
// { type: 'error', error }
//
// Tool calling: parse-after-completion. We accumulate the text
// stream, then run _adapterParseToolCalls on the full text once
// generation finishes. If any calls are detected we emit them
// as tool_call events BEFORE the done event, and set the done
// reason to 'tool_calls'. Tokens still stream live to consumers
// β€” the chat-UI bubble keeps painting tool-call XML in real
// time, exactly as it does today.
//
// Multimodal: messages may carry content arrays with image/audio/
// video blocks (each with `data` set to a Blob or ArrayBuffer).
// The adapter walks them via _adapterProcessContentBlocks before
// posting to the worker. Stale placeholder blocks (no data field)
// are kept in the chat-template message but contribute nothing
// to attData β€” that's how follow-up agent iterations carry the
// image position tokens forward without re-decoding.
chat(request) {
request = request || {};
const messages = Array.isArray(request.messages) ? request.messages : [];
const enableThinking = !!request.enableThinking;
const signal = request.signal;
const tools = Array.isArray(request.tools) ? request.tools : null;
const toolsEnabled = tools && tools.length > 0;
const knownToolNames = toolsEnabled ? tools.map(t => t.name) : [];
// Detect whether ANY message has a media block. If yes, the
// multimodal processing path runs (asynchronously) before we
// post to the worker; otherwise we take the fast text-only
// path. The multimodal pre-processing (audio decode, video
// frame extraction) requires await, so we wrap the rest of
// setup in an async IIFE β€” the iterator is still returned
// synchronously and will start emitting events as soon as the
// worker begins streaming.
let hasMediaInput = false;
for (const m of messages) {
if (Array.isArray(m.content)) {
for (const b of m.content) {
if (b && (b.type === 'image' || b.type === 'audio' || b.type === 'video')) {
hasMediaInput = true;
break;
}
}
}
if (hasMediaInput) break;
}
// Build genConfig: start from the active model's defaults so
// model-specific tuning (top_k, repetition_penalty, etc) is
// preserved, then layer request-level overrides.
const activeModel = (typeof MODELS !== 'undefined' && typeof activeModelKey === 'string') ? MODELS[activeModelKey] : null;
const baseGen = activeModel && activeModel.genConfig ? activeModel.genConfig : {};
const genConfig = Object.assign({}, baseGen);
if (typeof request.maxTokens === 'number' && request.maxTokens > 0) {
genConfig.max_new_tokens = Math.floor(request.maxTokens);
}
if (typeof request.temperature === 'number') {
genConfig.temperature = request.temperature;
}
const iter = makeAsyncIterator();
let completionTokens = 0;
// Approximate prompt tokens from the raw input β€” exact count
// would require tokenising on the worker, which we don't
// surface to the main thread.
const promptTokens = messages.reduce((s, x) => {
let txt = '';
if (typeof x.content === 'string') txt = x.content;
else if (Array.isArray(x.content)) txt = x.content.filter(c => c && c.type === 'text').map(c => c.text || '').join(' ');
return s + Math.max(1, Math.round(txt.length / 4));
}, 0);
// Wire up cancellation: an aborted signal posts {type:'stop'}
// to the worker. The legacy 'complete' handler still fires
// afterwards (with whatever was generated so far), which lets
// runInference resolve and the iterator finish cleanly.
let aborted = false;
const onAbort = () => {
aborted = true;
try {
const w = LocalMind.runtime._worker;
if (w) w.postMessage({ type: 'stop' });
} catch {}
};
if (signal) {
if (signal.aborted) onAbort();
else signal.addEventListener('abort', onAbort, { once: true });
}
// Async IIFE: do (optional) multimodal preprocessing, then
// build the final chatMessages, inject tools if needed, fire
// runInference. The iterator was created above and is returned
// synchronously below β€” events stream into it as the worker
// produces tokens.
(async () => {
let chatMessages;
let attDataForWorker = null;
try {
if (hasMediaInput) {
// Multimodal path: keep content arrays as positional
// placeholder blocks, extract binary data into attData.
const processed = await _adapterProcessContentBlocks(messages);
chatMessages = processed.chatMessages.map(m => {
if (m.role === 'tool') {
// Tool-role messages don't carry media; collapse via
// the same translation as the text path.
const content = typeof m.content === 'string' ? m.content : '';
const toolPayload = (() => {
try { return JSON.parse(content); } catch { return content; }
})();
return {
role: 'user',
content: _adapterFormatToolResultMessage(m.name || 'tool', toolPayload),
};
}
return m;
});
attDataForWorker = processed.attData;
} else {
// Text-only path: collapse content arrays to strings,
// translate tool-role messages.
chatMessages = messages.map(m => {
let content = m.content;
if (Array.isArray(content)) {
content = content
.filter(c => c && c.type === 'text')
.map(c => c.text || '')
.join(' ');
} else if (typeof content !== 'string') {
content = String(content == null ? '' : content);
}
if (m.role === 'tool') {
const toolPayload = (() => {
try { return JSON.parse(content); } catch { return content; }
})();
return {
role: 'user',
content: _adapterFormatToolResultMessage(m.name || 'tool', toolPayload),
};
}
return { role: m.role, content };
});
}
// Tool prompt injection. Replaces (or inserts) the system
// message at index 0 with the tool-scaffolded version. The
// original system content (if any) becomes the suffix.
if (toolsEnabled) {
let userSysContent = '';
if (chatMessages.length && chatMessages[0].role === 'system') {
const c = chatMessages[0].content;
userSysContent = typeof c === 'string' ? c : '';
chatMessages = chatMessages.slice(1);
}
const toolSys = _adapterBuildToolPrompt(tools, userSysContent);
chatMessages = [{ role: 'system', content: toolSys }].concat(chatMessages);
}
} catch (e) {
const err = e instanceof Error ? e : new Error(String(e));
iter.push({ type: 'error', error: err });
iter.finish(err);
return;
}
const inferencePromise = runInference({
chatMessages,
attData: attDataForWorker,
enableThinking,
genConfig,
setup: () => {
// Reset the legacy accumulator so `currentAssistantText`
// (which the 'complete' handler resolves runInference
// with) starts this call from empty. Without this,
// direct programmatic callers of runtime.chat would see
// the text of the previous call prepended to theirs and
// the tool-call parser could double-match. The chat UI
// and JS API paths also reset this themselves β€” doing
// it again here is harmless and makes the adapter
// self-contained.
currentAssistantText = '';
// Install the per-token sink. We deliberately do NOT
// touch currentAssistantEl here β€” the legacy chat-UI
// bubble keeps updating for callers that placed a bubble
// before invoking us. Callers that don't want a bubble
// (the JS API, sub-agents) null currentAssistantEl
// themselves before the call.
currentInferenceContext = {
onToken: (token) => {
completionTokens++;
iter.push({ type: 'token', text: token });
},
};
},
});
inferencePromise.then(
(fullText) => {
if (signal) signal.removeEventListener && signal.removeEventListener('abort', onAbort);
// Tool-call detection (parse-after-completion). Only emit
// tool_call events when the caller actually asked for
// tools β€” otherwise the same model output is just plain
// text and we don't want phantom calls from regex hits.
let reason = aborted ? 'abort' : 'stop';
if (toolsEnabled && !aborted && typeof fullText === 'string' && fullText.length) {
const { toolCalls } = _adapterParseToolCalls(fullText, knownToolNames);
if (toolCalls.length > 0) {
reason = 'tool_calls';
for (const tc of toolCalls) {
iter.push({
type: 'tool_call',
id: 'tc-' + Math.random().toString(36).slice(2, 10),
name: tc.name,
arguments: tc.arguments || {},
});
}
}
}
iter.push({
type: 'done',
reason,
usage: { promptTokens, completionTokens },
});
iter.finish();
},
(err) => {
if (signal) signal.removeEventListener && signal.removeEventListener('abort', onAbort);
const e = err instanceof Error ? err : new Error(String(err));
iter.push({ type: 'error', error: e });
iter.finish(e);
}
);
})();
return iter;
},
// --- embed --------------------------------------------------
// Lazy: the embedding worker is created on first call, kept
// alive for the lifetime of the page, and serialised internally
// so concurrent callers can't race the underlying postMessage
// request/response pairing. The chat worker is independent β€”
// unload() does NOT touch this one.
//
// Spec contract:
// embed(string) -> Promise<Float32Array>
// embed(string[]) -> Promise<Float32Array[]>
_embWorker: null,
_embReady: false,
_embQueue: Promise.resolve(),
get embeddingsReady() { return this._embReady; },
_ensureEmbWorker() {
if (this._embReady) return Promise.resolve();
const self = this;
if (this._embWorker) {
return new Promise((resolve, reject) => {
const handler = (e) => {
if (e.data.type === 'ready') {
self._embReady = true;
self._embWorker.removeEventListener('message', handler);
resolve();
} else if (e.data.type === 'error') {
self._embWorker.removeEventListener('message', handler);
reject(new Error(e.data.message));
}
};
self._embWorker.addEventListener('message', handler);
});
}
this._embWorker = createEmbeddingWorker();
return new Promise((resolve, reject) => {
const handler = (e) => {
if (e.data.type === 'ready') {
self._embReady = true;
self._embWorker.removeEventListener('message', handler);
resolve();
} else if (e.data.type === 'error') {
self._embWorker.removeEventListener('message', handler);
reject(new Error(e.data.message));
}
};
self._embWorker.addEventListener('message', handler);
self._embWorker.postMessage({ type: 'load' });
});
},
embed(texts /*, options */) {
const self = this;
const wasString = typeof texts === 'string';
const arr = wasString ? [texts] : texts;
if (!Array.isArray(arr) || arr.length === 0) {
return Promise.resolve(wasString ? new Float32Array(0) : []);
}
// Serialise: each call chains onto the previous so worker
// request IDs cannot collide across concurrent callers.
this._embQueue = this._embQueue.then(() => self._ensureEmbWorker().then(() => {
return new Promise((resolve, reject) => {
const id = Math.random().toString(36).slice(2);
const handler = (e) => {
if (e.data.id === id) {
self._embWorker.removeEventListener('message', handler);
if (e.data.type === 'embeddings') {
// Worker returns Array<Array<number>>; cast each to
// Float32Array per the runtime contract. cosineSimilarity
// and IndexedDB both handle typed arrays transparently.
resolve(e.data.vectors.map(v => new Float32Array(v)));
} else {
reject(new Error(e.data.message || 'Embedding failed'));
}
}
};
self._embWorker.addEventListener('message', handler);
self._embWorker.postMessage({ type: 'embed', texts: arr, id });
});
}));
return this._embQueue.then(vectors => wasString ? vectors[0] : vectors);
},
// --- SAM segmentation ----------------------------------------
// Lazy: the SAM worker is created on first segmentImage() call.
// Independent of the chat and embedding workers. Auto-switches
// if the user picks a different SAM model in settings.
_samWorker: null,
_samReady: false,
_samQueue: Promise.resolve(),
_samModelId: null,
_ensureSAMWorker(modelId) {
const self = this;
const samSection = document.getElementById('samProgressSection');
const samFill = document.getElementById('samProgressFill');
const samText = document.getElementById('samProgressText');
function showBar(text, pct) {
if (samSection) samSection.classList.add('visible');
if (samText) samText.textContent = text;
if (samFill && pct != null) samFill.style.width = pct + '%';
}
function hideBar() {
if (samSection) setTimeout(() => samSection.classList.remove('visible'), 1200);
}
// Model changed β€” tear down old worker
if (this._samWorker && this._samModelId !== modelId) {
try { this._samWorker.terminate(); } catch {}
this._samWorker = null;
this._samReady = false;
this._samModelId = null;
}
if (this._samReady && this._samModelId === modelId) return Promise.resolve();
function handleProgress(e, resolve, reject) {
if (e.data.type === 'progress' && e.data.data) {
const p = e.data.data;
// v4 emits 'progress_total' as the aggregate across files; the
// older 'downloading' branch is kept as a belt-and-braces fallback.
if ((p.status === 'progress_total' || p.status === 'downloading') && p.total) {
const pct = Math.min(100, Math.round((p.loaded / p.total) * 100));
const loadedMB = (p.loaded / 1e6).toFixed(1);
const totalMB = (p.total / 1e6).toFixed(1);
showBar('Downloading SAM \u2014 ' + loadedMB + ' / ' + totalMB + ' MB (' + pct + '%)', pct);
} else if (p.status === 'loading') {
showBar('Loading SAM model\u2026', 100);
} else if (p.status === 'initiate') {
showBar('Fetching SAM ' + (p.file || '') + '\u2026', null);
}
} else if (e.data.type === 'ready') {
self._samReady = true;
self._samWorker.removeEventListener('message', handleProgress._bound);
showBar('SAM model ready', 100);
hideBar();
resolve();
} else if (e.data.type === 'error') {
self._samWorker.removeEventListener('message', handleProgress._bound);
self._samReady = false;
showBar('SAM load failed: ' + (e.data.message || 'unknown error'), 0);
reject(new Error(e.data.message));
}
}
if (this._samWorker) {
showBar('Loading SAM model\u2026', null);
return new Promise((resolve, reject) => {
const bound = (e) => handleProgress(e, resolve, reject);
handleProgress._bound = bound;
self._samWorker.addEventListener('message', bound);
});
}
showBar('Preparing to download SAM model\u2026', 0);
this._samWorker = createSAMWorker();
this._samModelId = modelId;
return new Promise((resolve, reject) => {
const bound = (e) => handleProgress(e, resolve, reject);
handleProgress._bound = bound;
self._samWorker.addEventListener('message', bound);
self._samWorker.postMessage({ type: 'load', modelId });
});
},
segmentImage(imageBlob, points, labels) {
const self = this;
const modelId = localStorage.getItem('lm_sam_model') || 'Xenova/slimsam-77-uniform';
const samSection = document.getElementById('samProgressSection');
const samText = document.getElementById('samProgressText');
this._samQueue = this._samQueue.then(() => self._ensureSAMWorker(modelId).then(async () => {
if (samSection) samSection.classList.add('visible');
if (samText) samText.textContent = 'Segmenting image\u2026';
const arrayBuf = await imageBlob.arrayBuffer();
return new Promise((resolve, reject) => {
const id = Math.random().toString(36).slice(2);
const handler = (e) => {
if (e.data.id === id) {
self._samWorker.removeEventListener('message', handler);
if (e.data.type === 'masks') {
if (samText) samText.textContent = 'Segmentation complete';
if (samSection) setTimeout(() => samSection.classList.remove('visible'), 1200);
resolve({ masks: e.data.masks, scores: e.data.scores });
} else {
if (samSection) samSection.classList.remove('visible');
reject(new Error(e.data.message || 'Segmentation failed'));
}
}
};
self._samWorker.addEventListener('message', handler);
self._samWorker.postMessage(
{ type: 'segment', imageBuffer: arrayBuf, mimeType: imageBlob.type, points, labels, id },
[arrayBuf]
);
});
}));
return this._samQueue;
},
// --- capabilities -------------------------------------------
capabilities() {
return Object.assign({}, this._activeCaps);
},
// --- transitional helpers (removed in later steps) ----------
// These exist only so the legacy code that still pokes the
// worker directly (attachWorkerHandlers, generateOnce, the stop
// button) can keep working until Step 5 routes everything
// through chat(). They are NOT part of the public surface.
_getWorker() { return this._worker; },
};
// ── Embedding worker factory (MiniLM via @huggingface/transformers) ──
// Called once, lazily, by runtime.embed() on the first embed request.
// Lives inside the adapter because its blob source code imports from
// the Transformers.js CDN.
function createEmbeddingWorker() {
const code = `
import {
env,
pipeline,
} from 'https://cdn.jsdelivr.net/npm/@huggingface/transformers@4/+esm';
env.allowLocalModels = true;
env.localModelPath = '/models/';
env.allowRemoteModels = true;
let embedder = null;
self.addEventListener('message', async (e) => {
const { type, texts, id } = e.data;
if (type === 'load') {
try {
embedder = await pipeline('feature-extraction', 'Xenova/all-MiniLM-L6-v2', {
device: 'wasm',
progress_callback: (p) => self.postMessage({ type: 'progress', data: p }),
});
self.postMessage({ type: 'ready' });
} catch (err) {
self.postMessage({ type: 'error', message: err.message || String(err) });
}
} else if (type === 'embed') {
try {
const results = [];
for (const text of texts) {
const output = await embedder(text, { pooling: 'mean', normalize: true });
results.push(Array.from(output.data));
}
self.postMessage({ type: 'embeddings', vectors: results, id });
} catch (err) {
self.postMessage({ type: 'error', message: err.message || String(err), id });
}
}
});
`;
const blob = new Blob([code], { type: 'application/javascript' });
const url = URL.createObjectURL(blob);
const w = new Worker(url, { type: 'module' });
w.addEventListener('error', (e) => {
console.error('Embedding worker error:', e.message);
});
return w;
}
// ── SAM segmentation worker factory ────────────────────────
// Lazy-loaded on first segment_image tool call. Runs on WASM
// (WebGPU is occupied by Gemma). Handles both SAM 1 (SamModel)
// and SAM 3 (Sam3TrackerModel) via a modelId check.
function createSAMWorker() {
const code = `
import {
env,
SamModel,
Sam3TrackerModel,
AutoProcessor,
RawImage,
} from 'https://cdn.jsdelivr.net/npm/@huggingface/transformers@4/+esm';
env.allowLocalModels = true;
env.localModelPath = '/models/';
env.allowRemoteModels = true;
let model = null;
let processor = null;
let loadedModelId = null;
self.addEventListener('message', async (e) => {
const { type } = e.data;
if (type === 'load') {
const { modelId } = e.data;
try {
const isSam3 = modelId.toLowerCase().includes('sam3');
const ModelClass = isSam3 ? Sam3TrackerModel : SamModel;
const opts = {
device: 'wasm',
progress_callback: (p) => self.postMessage({ type: 'progress', data: p }),
};
processor = await AutoProcessor.from_pretrained(modelId);
model = await ModelClass.from_pretrained(modelId, opts);
loadedModelId = modelId;
self.postMessage({ type: 'ready' });
} catch (err) {
self.postMessage({ type: 'error', message: 'SAM load failed: ' + (err.message || String(err)) });
}
}
else if (type === 'segment') {
const { imageBuffer, mimeType, points, labels, id } = e.data;
if (!model || !processor) {
self.postMessage({ type: 'error', message: 'SAM model not loaded', id });
return;
}
try {
const blob = new Blob([imageBuffer], { type: mimeType || 'image/jpeg' });
const raw_image = await RawImage.fromBlob(blob);
// Point format differs by model:
// SAM 1 (SamModel): input_points [[[x,y]]], input_labels [[1]]
// SAM 3 (Sam3TrackerModel): input_points [[[[x,y]]]], input_labels [[[1]]]
const isSam3 = loadedModelId && loadedModelId.toLowerCase().includes('sam3');
const ptsList = points.map(p => [p[0], p[1]]);
const lblList = labels.map(l => l);
const input_points = isSam3 ? [[ptsList]] : [ptsList];
const input_labels = isSam3 ? [[lblList]] : [lblList];
const inputs = await processor(raw_image, { input_points, input_labels });
const outputs = await model(inputs);
const masks = await processor.post_process_masks(
outputs.pred_masks,
inputs.original_sizes,
inputs.reshaped_input_sizes,
);
// masks from post_process_masks: masks[0] is a Tensor.
// Shape varies: could be [num_masks, H, W] or [1, num_masks, H, W].
// Normalize to always work with [num_masks, H, W].
let maskTensor = masks[0];
const dims = maskTensor.dims;
// If 4D [1, num_masks, H, W], squeeze the batch dim
let numMasks, maskH, maskW, dataOffset;
if (dims.length === 4) {
numMasks = dims[1];
maskH = dims[2];
maskW = dims[3];
dataOffset = dims[0] > 1 ? 0 : 0; // always first batch
} else {
numMasks = dims[0];
maskH = dims[1];
maskW = dims[2];
}
const iou_scores = outputs.iou_scores?.data
? Array.from(outputs.iou_scores.data)
: [];
const result = [];
const transferables = [];
const stridePerMask = maskH * maskW;
// For 4D tensor, skip the batch dimension in the flat data array
const batchStart = dims.length === 4 ? 0 : 0; // batch 0
for (let i = 0; i < numMasks; i++) {
const maskData = new Uint8Array(stridePerMask);
const offset = batchStart + i * stridePerMask;
for (let j = 0; j < stridePerMask; j++) {
// Data can be booleans (true/false) or logits (floats).
// Booleans: true β†’ 255. Logits: > 0 β†’ foreground.
const val = maskTensor.data[offset + j];
maskData[j] = (val === true || val > 0) ? 255 : 0;
}
result.push({ width: maskW, height: maskH, data: maskData });
transferables.push(maskData.buffer);
}
self.postMessage(
{ type: 'masks', masks: result, scores: iou_scores, id },
transferables
);
} catch (err) {
self.postMessage({ type: 'error', message: 'Segmentation failed: ' + (err.message || String(err)), id });
}
}
});
`;
const blob = new Blob([code], { type: 'application/javascript' });
const url = URL.createObjectURL(blob);
const w = new Worker(url, { type: 'module' });
w.addEventListener('error', (e) => {
console.error('SAM worker error:', e.message);
});
return w;
}
// ── Pyodide worker factory (run_python agent tool) ──────
// Lazy-loaded on first tool call. Loads Pyodide + auto-loads imports
// (numpy, pandas, matplotlib via loadPackagesFromImports). stdout /
// stderr buffered and returned; no plots yet.
function createPyodideWorker() {
const code = `
import { loadPyodide } from 'https://cdn.jsdelivr.net/pyodide/v0.26.4/full/pyodide.mjs';
let pyodide = null;
let pyodideReadyPromise = null;
function ensurePyodide() {
if (!pyodideReadyPromise) {
self.postMessage({ type: 'loading' });
pyodideReadyPromise = loadPyodide({ indexURL: 'https://cdn.jsdelivr.net/pyodide/v0.26.4/full/' }).then((p) => {
pyodide = p;
self.postMessage({ type: 'loaded' });
return p;
});
}
return pyodideReadyPromise;
}
self.addEventListener('message', async (e) => {
const { type, code, id } = e.data;
if (type === 'run') {
try {
await ensurePyodide();
let out = '';
let err = '';
pyodide.setStdout({ batched: (s) => { out += s + '\\n'; } });
pyodide.setStderr({ batched: (s) => { err += s + '\\n'; } });
try { await pyodide.loadPackagesFromImports(code); } catch (loadErr) {
err += 'Package load warning: ' + (loadErr.message || loadErr) + '\\n';
}
let result = undefined;
try {
result = await pyodide.runPythonAsync(code);
} catch (pyErr) {
err += (pyErr.message || String(pyErr)) + '\\n';
}
const resultStr = (result !== undefined && result !== null) ? String(result) : '';
self.postMessage({ type: 'result', stdout: out, stderr: err, result: resultStr, id });
} catch (outerErr) {
self.postMessage({ type: 'result', error: outerErr.message || String(outerErr), id });
}
}
});
`;
const blob = new Blob([code], { type: 'application/javascript' });
const url = URL.createObjectURL(blob);
const w = new Worker(url, { type: 'module' });
w.addEventListener('error', (e) => {
console.error('Pyodide worker error:', e.message);
});
return w;
}
// ── Transcription worker factory (Whisper via WebGPU) ───
// Lazy-loaded on first voice-to-text click. Expects Float32Array PCM
// at 16 kHz mono (decoded+resampled on the main thread via OfflineAudioContext).
// Follows the HF realtime-whisper-webgpu pattern: bypasses the pipeline()
// wrapper (which drops language/task kwargs silently on v4) and calls
// model.generate() directly with language/task/max_new_tokens.
function createTranscriptionWorker() {
const code = `
import {
env,
AutoProcessor,
AutoTokenizer,
WhisperForConditionalGeneration,
full,
} from 'https://cdn.jsdelivr.net/npm/@huggingface/transformers@4/+esm';
env.allowLocalModels = true;
env.localModelPath = '/models/';
env.allowRemoteModels = true;
const MODEL_ID = 'onnx-community/whisper-base';
const MAX_NEW_TOKENS = 64;
let processor = null;
let tokenizer = null;
let model = null;
async function load() {
const onProgress = (p) => self.postMessage({ type: 'progress', data: p });
processor = await AutoProcessor.from_pretrained(MODEL_ID, { progress_callback: onProgress });
tokenizer = await AutoTokenizer.from_pretrained(MODEL_ID, { progress_callback: onProgress });
model = await WhisperForConditionalGeneration.from_pretrained(MODEL_ID, {
// fp32 encoder + q4 decoder is the known-good combo (HF realtime demo).
// fp16 encoder produced subtle feature drift that biased the decoder to
// filler tokens ("and", "the") on short clips.
dtype: { encoder_model: 'fp32', decoder_model_merged: 'q4' },
device: 'webgpu',
progress_callback: onProgress,
});
// Warmup: compile shaders against a real mel-shape tensor so the first
// real transcription doesn't take the compile hit.
await model.generate({
input_features: full([1, 80, 3000], 0.0),
max_new_tokens: 1,
});
self.postMessage({ type: 'ready' });
}
async function transcribe(pcm, lang) {
if (!model || !processor || !tokenizer) throw new Error('Whisper not loaded');
// processor builds log-mel features from the Float32Array PCM.
const inputs = await processor(pcm);
// Pass language/task/max_new_tokens directly to generate() β€” the pipeline
// wrapper used to silently drop these kwargs in v4.
const outputs = await model.generate({
...inputs,
max_new_tokens: MAX_NEW_TOKENS,
language: lang || 'en',
task: 'transcribe',
});
const text = tokenizer.batch_decode(outputs, { skip_special_tokens: true })[0] || '';
return text.trim();
}
self.addEventListener('message', async (e) => {
const { type, pcm, lang, id } = e.data;
try {
if (type === 'load') {
await load();
} else if (type === 'transcribe') {
const text = await transcribe(pcm, lang);
self.postMessage({ type: 'transcription', text, id });
}
} catch (err) {
self.postMessage({ type: 'error', message: err.message || String(err), id });
}
});
`;
const blob = new Blob([code], { type: 'application/javascript' });
const url = URL.createObjectURL(blob);
const w = new Worker(url, { type: 'module' });
w.addEventListener('error', (e) => {
console.error('Transcription worker error:', e.message);
});
return w;
}
// ── Generation worker factory ───────────────────────────────
// Pattern from huggingface/transformers.js-examples (Llama 3.2 WebGPU).
// Uses AutoModelForCausalLM + AutoTokenizer (not pipeline) for causal models.
// Uses Gemma4ForConditionalGeneration + AutoProcessor for multimodal models.
// GQA variant required for Gemma 3 1B (issue #1469: regular variant crashes).
function createWorker() {
const code = `
// ── Resumable download cache ────────────────────────────────
// Monkey-patches self.fetch so HF CDN downloads checkpoint to IndexedDB
// every 5 MB. If the tab closes mid-download, the next load resumes from
// the last saved chunk via HTTP Range instead of restarting from byte 0.
// Any failure in this path falls back to the original fetch, so the model
// still loads β€” it just won't resume if re-interrupted. Toggle via the
// 'resumable' flag on the load message; default on.
const __RDB_NAME = 'localmind-downloads';
const __RDB_CHUNK = 5 * 1024 * 1024;
let __resumableEnabled = true;
function __openRDB() {
return new Promise((resolve, reject) => {
const r = indexedDB.open(__RDB_NAME, 1);
r.onupgradeneeded = () => {
const db = r.result;
if (!db.objectStoreNames.contains('meta')) db.createObjectStore('meta', { keyPath: 'url' });
if (!db.objectStoreNames.contains('chunks')) db.createObjectStore('chunks', { keyPath: 'key' });
};
r.onsuccess = () => resolve(r.result);
r.onerror = () => reject(r.error);
});
}
function __rdbGet(db, store, key) {
return new Promise((resolve, reject) => {
const req = db.transaction(store, 'readonly').objectStore(store).get(key);
req.onsuccess = () => resolve(req.result || null);
req.onerror = () => reject(req.error);
});
}
function __rdbPut(db, store, value) {
return new Promise((resolve, reject) => {
const tx = db.transaction(store, 'readwrite');
tx.objectStore(store).put(value);
tx.oncomplete = () => resolve();
tx.onerror = () => reject(tx.error);
});
}
function __rdbClearFile(db, url) {
return new Promise((resolve, reject) => {
const tx = db.transaction(['chunks', 'meta'], 'readwrite');
const req = tx.objectStore('chunks').openCursor();
req.onsuccess = (e) => {
const c = e.target.result;
if (!c) return;
if (c.value.key.indexOf(url + '|') === 0) c.delete();
c.continue();
};
tx.objectStore('meta').delete(url);
tx.oncomplete = () => resolve();
tx.onerror = () => reject(tx.error);
});
}
// On worker start: drop IDB chunks for files already fully cached in Cache
// Storage. Recovers disk after transformers.js has cache.put()'d the whole
// response independently of our chunk store.
async function __rdbCleanup() {
let db;
try { db = await __openRDB(); } catch { return; }
try {
const urls = await new Promise((resolve, reject) => {
const r = db.transaction('meta', 'readonly').objectStore('meta').getAllKeys();
r.onsuccess = () => resolve(r.result || []);
r.onerror = () => reject(r.error);
});
for (const url of urls) {
try {
const hit = await caches.match(url);
if (hit) await __rdbClearFile(db, url);
} catch {}
}
} finally {
try { db.close(); } catch {}
}
}
function __shouldIntercept(url) {
return typeof url === 'string' && url.indexOf('https://huggingface.co/') === 0;
}
const __origFetch = self.fetch.bind(self);
self.fetch = async function(input, init) {
const urlStr = typeof input === 'string' ? input : (input && input.url);
const method = (init && init.method) || (input && input.method) || 'GET';
if (!__resumableEnabled || method !== 'GET' || !__shouldIntercept(urlStr)) {
return __origFetch(input, init);
}
let db;
try { db = await __openRDB(); }
catch { return __origFetch(input, init); }
const closeDb = () => { try { db.close(); } catch {} };
try {
const headResp = await __origFetch(urlStr, { method: 'HEAD' });
if (!headResp.ok) { closeDb(); return __origFetch(input, init); }
const totalSize = parseInt(headResp.headers.get('content-length'), 10);
const etag = headResp.headers.get('etag') || headResp.headers.get('last-modified') || '';
const contentType = headResp.headers.get('content-type') || 'application/octet-stream';
if (!totalSize || totalSize < __RDB_CHUNK) {
closeDb();
return __origFetch(input, init);
}
let meta = await __rdbGet(db, 'meta', urlStr);
if (meta && meta.etag !== etag) {
await __rdbClearFile(db, urlStr);
meta = null;
}
const chunkCount = Math.ceil(totalSize / __RDB_CHUNK);
const existing = new Array(chunkCount).fill(null);
if (meta) {
for (let i = 0; i < chunkCount; i++) {
const rec = await __rdbGet(db, 'chunks', urlStr + '|' + i);
if (rec) existing[i] = rec.data;
}
}
const firstMissing = existing.findIndex(c => !c);
if (firstMissing === -1) {
closeDb();
return new Response(new Blob(existing, { type: contentType }), {
status: 200,
headers: new Headers({
'content-length': String(totalSize),
'content-type': contentType,
}),
});
}
if (!meta) {
await __rdbPut(db, 'meta', { url: urlStr, etag, totalSize, chunkSize: __RDB_CHUNK, chunkCount });
}
const startByte = firstMissing * __RDB_CHUNK;
const rangeResp = await __origFetch(urlStr, { headers: { Range: 'bytes=' + startByte + '-' } });
if (!rangeResp.ok) { closeDb(); return __origFetch(input, init); }
const stream = new ReadableStream({
async start(controller) {
try {
for (let i = 0; i < firstMissing; i++) {
const buf = await existing[i].arrayBuffer();
controller.enqueue(new Uint8Array(buf));
}
const reader = rangeResp.body.getReader();
let accum = new Uint8Array(__RDB_CHUNK);
let accumLen = 0;
let chunkIdx = firstMissing;
while (true) {
const { done, value } = await reader.read();
if (done) break;
let pos = 0;
while (pos < value.length) {
const take = Math.min(__RDB_CHUNK - accumLen, value.length - pos);
accum.set(value.subarray(pos, pos + take), accumLen);
accumLen += take;
pos += take;
if (accumLen === __RDB_CHUNK) {
const slice = accum.slice(0, accumLen);
await __rdbPut(db, 'chunks', { key: urlStr + '|' + chunkIdx, data: new Blob([slice]) });
controller.enqueue(slice);
chunkIdx++;
accumLen = 0;
accum = new Uint8Array(__RDB_CHUNK);
}
}
}
if (accumLen > 0) {
const slice = accum.slice(0, accumLen);
await __rdbPut(db, 'chunks', { key: urlStr + '|' + chunkIdx, data: new Blob([slice]) });
controller.enqueue(slice);
}
controller.close();
} catch (e) {
controller.error(e);
} finally {
closeDb();
}
},
cancel() { closeDb(); },
});
return new Response(stream, {
status: 200,
headers: new Headers({
'content-length': String(totalSize),
'content-type': contentType,
}),
});
} catch {
closeDb();
return __origFetch(input, init);
}
};
__rdbCleanup();
// Dynamic import *after* the fetch patch is installed. Using top-level await
// would race the 'load' postMessage from the main thread β€” it can arrive
// before the module-body handler is registered and get dropped. Kick off
// the import eagerly and have the message handler await it.
let env, AutoTokenizer, AutoModelForCausalLM, AutoProcessor, Gemma4ForConditionalGeneration, load_image, TextStreamer, InterruptableStoppingCriteria;
let stopping_criteria;
const __modReady = import('https://cdn.jsdelivr.net/npm/@huggingface/transformers@4/+esm').then((m) => {
env = m.env;
AutoTokenizer = m.AutoTokenizer;
AutoModelForCausalLM = m.AutoModelForCausalLM;
AutoProcessor = m.AutoProcessor;
Gemma4ForConditionalGeneration = m.Gemma4ForConditionalGeneration;
load_image = m.load_image;
TextStreamer = m.TextStreamer;
InterruptableStoppingCriteria = m.InterruptableStoppingCriteria;
env.allowLocalModels = true;
env.localModelPath = '/models/';
env.allowRemoteModels = true;
// Transformers.js captures globalThis.fetch at module init via .bind(), so
// overriding self.fetch alone doesn't reach it. Replace env.fetch so its
// internal getFile() routes through our resumable path.
env.fetch = self.fetch;
stopping_criteria = new InterruptableStoppingCriteria();
});
let processor = null;
let tokenizer = null;
let model = null;
let loadedType = null;
function progressCallback(p) {
self.postMessage({ type: 'progress', data: p });
}
async function loadCausal(modelId, dtype) {
processor = null; tokenizer = null; model = null;
tokenizer = await AutoTokenizer.from_pretrained(modelId, {
progress_callback: progressCallback,
});
model = await AutoModelForCausalLM.from_pretrained(modelId, {
dtype: dtype,
device: 'webgpu',
progress_callback: progressCallback,
});
// Warmup: compile WebGPU shaders with a single token
self.postMessage({ type: 'warmup' });
const warmupInputs = tokenizer('a');
await model.generate({ ...warmupInputs, max_new_tokens: 1 });
loadedType = 'causal';
self.postMessage({ type: 'ready' });
}
async function loadMultimodal(modelId, dtype) {
processor = null; tokenizer = null; model = null;
processor = await AutoProcessor.from_pretrained(modelId, {
progress_callback: progressCallback,
});
tokenizer = processor.tokenizer;
model = await Gemma4ForConditionalGeneration.from_pretrained(modelId, {
dtype: dtype,
device: 'webgpu',
progress_callback: progressCallback,
});
// Warmup
self.postMessage({ type: 'warmup' });
const warmupInputs = tokenizer('a');
await model.generate({ ...warmupInputs, max_new_tokens: 1 });
loadedType = 'multimodal';
self.postMessage({ type: 'ready' });
}
async function generateCausal(chatMessages, id, genConfig) {
stopping_criteria.reset();
try {
const inputs = tokenizer.apply_chat_template(chatMessages, {
add_generation_prompt: true,
return_dict: true,
});
const streamer = new TextStreamer(tokenizer, {
skip_prompt: true,
skip_special_tokens: true,
callback_function: (text) => {
self.postMessage({ type: 'token', token: text, id });
},
});
const gc = genConfig || {};
await model.generate({
...inputs,
max_new_tokens: gc.max_new_tokens || 2048,
do_sample: true,
temperature: gc.temperature ?? 0.7,
top_k: gc.top_k ?? 50,
top_p: gc.top_p ?? 0.95,
repetition_penalty: gc.repetition_penalty ?? 1.0,
streamer,
stopping_criteria,
});
self.postMessage({ type: 'complete', id });
} catch (err) {
self.postMessage({ type: 'error', message: err.message || String(err) });
self.postMessage({ type: 'complete', id });
}
}
async function generateMultimodal(chatMessages, id, attachmentData, enableThinking, genConfig) {
stopping_criteria.reset();
try {
// Build multimodal content blocks for the last user message
// chatMessages is already in the right format with content arrays
// Apply chat template via processor (returns a prompt string, not tokens)
const prompt = processor.apply_chat_template(chatMessages, {
add_generation_prompt: true,
enable_thinking: enableThinking || false,
});
// Reconstruct images and audio from transferred data
const images = [];
const audios = [];
if (attachmentData) {
for (const att of attachmentData) {
if (att.type === 'image') {
// Create blob URL and use load_image() which properly constructs
// a RawImage with .rgb() and other methods the processor needs
const blob = new Blob([att.data], { type: att.mimeType || 'image/jpeg' });
const blobUrl = URL.createObjectURL(blob);
const img = await load_image(blobUrl);
URL.revokeObjectURL(blobUrl);
images.push(img);
} else if (att.type === 'audio') {
// Reconstruct Float32Array from transferred data
// (postMessage may transfer as ArrayBuffer, losing typed array identity)
const pcm = att.pcmData instanceof Float32Array
? att.pcmData
: new Float32Array(att.pcmData);
audios.push(pcm);
}
}
}
// Process multimodal inputs: processor(prompt, image|null, audio|null, options)
// Must pass null for missing modalities β€” positional args, not named.
// The processor dispatches arg[1] to vision and arg[2] to audio extractor.
const imageArg = images.length > 0 ? (images.length === 1 ? images[0] : images) : null;
const audioArg = audios.length > 0 ? (audios.length === 1 ? audios[0] : audios) : null;
const inputs = await processor(prompt, imageArg, audioArg, { add_special_tokens: false });
const streamer = new TextStreamer(processor.tokenizer, {
skip_prompt: true,
skip_special_tokens: true,
callback_function: (text) => {
self.postMessage({ type: 'token', token: text, id });
},
});
const gc = genConfig || {};
await model.generate({
...inputs,
max_new_tokens: gc.max_new_tokens || 2048,
do_sample: true,
temperature: gc.temperature ?? 1.0,
top_k: gc.top_k ?? 64,
top_p: gc.top_p ?? 0.95,
repetition_penalty: gc.repetition_penalty ?? 1.0,
streamer,
stopping_criteria,
});
self.postMessage({ type: 'complete', id });
} catch (err) {
self.postMessage({ type: 'error', message: err.message || String(err) });
self.postMessage({ type: 'complete', id });
}
}
self.addEventListener('message', async (e) => {
await __modReady; // Transformers.js may still be loading on first message
const { type, messages, id, modelId, dtype, modelType, attachments: attData, enableThinking, generationConfig, resumable } = e.data;
if (type === 'load') {
if (resumable === false) __resumableEnabled = false;
try {
if (modelType === 'multimodal') {
await loadMultimodal(modelId, dtype);
} else {
await loadCausal(modelId, dtype);
}
} catch (err) {
self.postMessage({ type: 'error', message: 'Failed to load model: ' + (err.message || String(err)) });
}
} else if (type === 'generate') {
if (loadedType === 'multimodal') {
await generateMultimodal(messages, id, attData, enableThinking, generationConfig);
} else {
await generateCausal(messages, id, generationConfig);
}
} else if (type === 'stop') {
stopping_criteria.interrupt();
}
});
`;
const blob = new Blob([code], { type: 'application/javascript' });
const url = URL.createObjectURL(blob);
return new Worker(url, { type: 'module' });
}
// ===========================================================
// === END RUNTIME ADAPTER ===================================
// ===========================================================
// ── Worker message handling ─────────────────────────────────
function attachWorkerHandlers(w) {
// Catch silent worker failures (e.g. bad import, missing export)
w.addEventListener('error', (e) => {
console.error('Worker error event:', e.message, e);
statusBadge.className = 'status-badge error';
statusSpinner.style.display = 'none';
statusText.textContent = 'Worker error';
progressSection.classList.remove('visible');
progressText.textContent = e.message || 'Worker failed to start';
modelSelect.disabled = false;
});
w.addEventListener('message', (e) => {
const data = e.data;
if (data.type === 'progress') {
const p = data.data;
progressSection.classList.add('visible');
const fileName = p.file ? p.file.split('/').pop() : '';
if (fileName) currentDownloadFile = fileName;
// Transformers.js v4 emits: 'initiate' (fetch starting), 'progress'
// (per-file bytes), 'progress_total' (aggregate bytes across all
// files β€” no `file` field), and 'loading' (file parsing into
// memory after download). We drive the bar off progress_total so
// it moves monotonically across the whole model instead of
// resetting per file.
if (p.status === 'initiate') {
progressText.textContent = `Fetching ${fileName}...`;
} else if (p.status === 'progress_total' && p.loaded && p.total) {
const pct = Math.min(100, Math.round((p.loaded / p.total) * 100));
progressFill.style.width = pct + '%';
const loadedMB = (p.loaded / 1024 / 1024).toFixed(0);
const totalMB = (p.total / 1024 / 1024).toFixed(0);
const label = currentDownloadFile ? ` ${currentDownloadFile}` : '';
progressText.textContent = `Downloading${label} β€” ${loadedMB} / ${totalMB} MB (${pct}%)`;
statusText.textContent = 'Downloading...';
} else if (p.status === 'loading') {
progressText.textContent = `Loading ${fileName}...`;
statusText.textContent = 'Loading into memory...';
}
}
else if (data.type === 'warmup') {
progressText.textContent = 'Compiling shaders and warming up...';
statusText.textContent = 'Warming up...';
}
else if (data.type === 'ready') {
modelReady = true;
progressSection.classList.remove('visible');
statusBadge.className = 'status-badge ready';
statusSpinner.style.display = 'none';
const m = MODELS[activeModelKey];
statusText.textContent = m.label;
chatInput.disabled = false;
const readyCaps = LocalMind.runtime.capabilities();
const supportsMedia = !!(readyCaps && (readyCaps.image || readyCaps.audio || readyCaps.video));
chatInput.placeholder = supportsMedia ? 'Type a message, or attach images/audio...' : 'Type a message...';
sendBtn.disabled = false; searchSendBtn.disabled = false;
clearBtn.disabled = false;
newChatBtn.disabled = false;
modelSelect.disabled = false;
updateModelUI();
updateBatchCount();
chatInput.focus();
// Resolve any pending window.localmind.load() promise
if (loadResolvers) {
const r = loadResolvers; loadResolvers = null;
r.resolve();
}
}
else if (data.type === 'token') {
// Always accumulate so non-streaming callers (chat UI, non-streaming
// API) still receive the full text via the 'complete' handler.
currentAssistantText += data.token;
if (currentAssistantEl) {
updateAssistantBubble(currentAssistantEl.bubble, currentAssistantText);
}
// Streaming consumer (window.localmind.chat.completions.create with
// stream: true) β€” route this token to its async iterator.
if (currentInferenceContext && typeof currentInferenceContext.onToken === 'function') {
try { currentInferenceContext.onToken(data.token); } catch (e) { console.error('streaming onToken threw:', e); }
}
}
else if (data.type === 'complete') {
// If agentic loop is awaiting, resolve promise (loop manages state)
if (generateResolve) {
const resolve = generateResolve;
generateResolve = null;
resolve(currentAssistantText);
} else {
// Fallback: no agentic loop (shouldn't happen, but safe)
generating = false;
if (currentAssistantText) {
messages.push({ role: 'assistant', content: currentAssistantText });
saveChat();
}
currentAssistantEl = null;
currentAssistantText = '';
sendBtn.innerHTML = '&#9654;';
sendBtn.classList.remove('stop');
sendBtn.disabled = false; searchSendBtn.disabled = false;
chatInput.disabled = false;
modelSelect.disabled = false;
chatInput.focus();
}
}
else if (data.type === 'error') {
if (!modelReady) {
statusBadge.className = 'status-badge error';
statusSpinner.style.display = 'none';
statusText.textContent = 'Error';
progressSection.classList.remove('visible');
modelSelect.disabled = false;
}
console.error('Worker error:', data.message);
if (generating && currentAssistantEl) {
currentAssistantText += `\n\n*Error: ${data.message}*`;
updateAssistantBubble(currentAssistantEl.bubble, currentAssistantText);
}
// Reject any pending window.localmind.load() promise
if (loadResolvers) {
const r = loadResolvers; loadResolvers = null;
r.reject(new Error(data.message || 'Model load failed'));
}
}
});
}
// ── Load model ──────────────────────────────────────────────
// Thin shell over LocalMind.runtime.loadModel that drives the
// existing UI state machine. The runtime owns worker lifecycle;
// we still attach the legacy message handlers to the freshly
// created worker so streaming tokens, progress, and 'ready'
// events update the UI exactly as before. Step 5 collapses the
// legacy handlers in favour of the chat() async iterator.
function loadModel(key) {
const m = MODELS[key];
if (!m) return;
// Any in-flight API load promise is now stale β€” reject it before the
// new load starts so the caller is notified and the new caller (if any)
// can attach fresh resolvers right after this returns.
if (loadResolvers) {
const r = loadResolvers; loadResolvers = null;
try { r.reject(new Error('Load superseded by another load')); } catch {}
}
activeModelKey = key;
modelReady = false;
// Clear attachments when switching away from multimodal
if (!m.multimodal) clearAttachments();
// Reset aggregate-progress state for this load
currentDownloadFile = '';
// Reset UI
statusBadge.className = 'status-badge loading';
statusSpinner.style.display = '';
statusText.textContent = 'Loading...';
progressFill.style.width = '0%';
progressText.textContent = `Preparing to download ${m.label} (${m.size})...`;
progressSection.classList.add('visible');
chatInput.disabled = true;
chatInput.placeholder = `Loading ${m.label}...`;
sendBtn.disabled = true; searchSendBtn.disabled = true;
// modelSelect stays enabled so users can hot-swap to a different model
// mid-download (runtime.loadModel terminates the existing worker first).
// Partial chunks already written to IDB stay available for resume.
updateModelUI();
// Delegate worker creation + load message to the runtime adapter.
// We deliberately do NOT await the returned promise here β€” the
// legacy attachWorkerHandlers() below already drives the UI from
// worker events (progress / ready / error). The .catch is purely
// defensive against unhandled rejections; UI error state is
// already handled by the legacy 'error' message handler.
LocalMind.runtime.loadModel(m.id, {
dtype: m.dtype,
modelType: m.type,
capabilities: {
text: true,
image: !!m.multimodal,
audio: !!m.multimodal,
video: !!m.multimodal,
toolCalling: !!m.agentCapable,
maxContextTokens: m.contextSize || 0,
},
}).catch((e) => {
console.warn('runtime.loadModel rejected:', e && e.message);
});
// Pull the freshly created worker from the runtime and attach
// the legacy UI handlers. Synchronous after loadModel() returns,
// so handlers are wired before any worker message dispatches.
worker = LocalMind.runtime._getWorker();
attachWorkerHandlers(worker);
}
// ── Model selector ──────────────────────────────────────────
modelSelect.addEventListener('change', () => {
if (generating) return;
const key = modelSelect.value;
if (key === activeModelKey && modelReady) return;
loadModel(key);
});
// ── Cancel button on the download progress panel ────────────
// Terminates the worker (which aborts in-flight fetches) and returns
// the app to an idle state. Partial chunks already saved to IDB stay
// there, so reselecting the same model picks up roughly where we left
// off via the resumable-fetch path.
const progressCancelBtn = document.getElementById('progressCancelBtn');
progressCancelBtn.addEventListener('click', () => {
try { LocalMind.runtime.unload(); } catch {}
worker = null;
modelReady = false;
activeModelKey = null;
progressSection.classList.remove('visible');
// Base `.status-badge` class with no state modifier β€” neutral grey
// instead of red (this is user-initiated, not an error).
statusBadge.className = 'status-badge';
statusSpinner.style.display = 'none';
statusText.textContent = 'Cancelled β€” pick a model';
chatInput.disabled = true;
chatInput.placeholder = 'Select a model to load';
sendBtn.disabled = true;
searchSendBtn.disabled = true;
showToast('Download cancelled. Partial chunks kept for resume.');
});
// Attachment preprocessing (audio decode, video frame extraction,
// image blob reads) has moved inside the runtime adapter β€” see
// _adapterDecodeAudioTo16k / _adapterDecomposeVideo /
// _adapterProcessContentBlocks. The chat UI now passes raw Blobs
// in content-block form and the adapter handles the translation.
// ── Send message ────────────────────────────────────────────
function generateOnce(chatMessages, attData, enableThinking, genConfig) {
return new Promise((resolve) => {
generateResolve = resolve;
const id = ++msgIdCounter;
worker.postMessage({
type: 'generate',
messages: chatMessages,
id,
attachments: attData || null,
enableThinking,
generationConfig: genConfig,
});
});
}
// FIFO wrapper around generateOnce. All inference callers β€” the chat UI,
// the agentic loop, and (soon) the window.localmind API β€” route through
// here so two callers can't race and clobber each other's generateResolve
// promise. Chat UI is already serialised by the `generating` flag; the
// queue adds correctness so that does not have to be the load-bearing
// guarantee. Bounded length keeps a misbehaving caller from piling up
// jobs. Items past MAX_INFER_QUEUE are rejected synchronously.
const inferQueue = [];
const MAX_INFER_QUEUE = 4;
let inferDraining = false;
// opts: { chatMessages, attData, enableThinking, genConfig, setup? }
// The optional `setup` callback runs INSIDE the queue critical section,
// right before the worker job posts. This is how the API path safely
// resets currentAssistantEl/Text without racing the chat path's own
// setup that lives in sendMessage().
function runInference(opts) {
return new Promise((resolve, reject) => {
if (inferQueue.length >= MAX_INFER_QUEUE) {
reject(new Error('inference queue full (max ' + MAX_INFER_QUEUE + ')'));
return;
}
inferQueue.push({ ...opts, resolve, reject });
drainInferQueue();
});
}
async function drainInferQueue() {
if (inferDraining) return;
inferDraining = true;
try {
while (inferQueue.length) {
const job = inferQueue.shift();
// Reset per-job context so an earlier streaming job's token sink
// can't leak into this one. The job's setup() may install a new
// currentInferenceContext on top.
currentInferenceContext = null;
try {
if (typeof job.setup === 'function') job.setup();
const result = await generateOnce(job.chatMessages, job.attData, job.enableThinking, job.genConfig);
job.resolve(result);
} catch (e) {
job.reject(e);
}
}
currentInferenceContext = null;
} finally {
inferDraining = false;
}
}
function renderToolCallBlock(bubble, toolName, args, result) {
const blockId = 'tc-' + Math.random().toString(36).slice(2, 8);
const argsStr = Object.entries(args).map(([k, v]) => `${k}: ${JSON.stringify(v)}`).join(', ');
const resultStr = typeof result === 'string' ? result : JSON.stringify(result, null, 2);
const block = document.createElement('div');
block.className = 'tool-call-block';
block.innerHTML = `
<div class="tool-call-toggle" onclick="document.getElementById('${blockId}').classList.toggle('open'); this.querySelector('span').textContent = document.getElementById('${blockId}').classList.contains('open') ? '&#9660;' : '&#9654;'">
<span>&#9654;</span> <strong>${escapeHtml(toolName)}</strong>(${escapeHtml(argsStr)})
</div>
<div class="tool-call-result" id="${blockId}"><pre>${escapeHtml(resultStr)}</pre></div>
`;
bubble.appendChild(block);
chatArea.scrollTop = chatArea.scrollHeight;
}
function renderSegmentationOverlay(bubble, imageBlob, masks, scores) {
if (!masks || masks.length === 0) return;
// Distinct colors for up to 6 masks
const COLORS = [
[234, 88, 12], // orange
[59, 130, 246], // blue
[16, 185, 129], // emerald
[168, 85, 247], // purple
[239, 68, 68], // red
[245, 158, 11], // amber
];
const container = document.createElement('div');
container.className = 'segmentation-result';
const img = document.createElement('img');
const imgUrl = URL.createObjectURL(imageBlob);
img.src = imgUrl;
img.alt = 'Segmented image';
container.appendChild(img);
img.onload = () => {
const displayW = img.naturalWidth;
const displayH = img.naturalHeight;
const canvas = document.createElement('canvas');
canvas.width = displayW;
canvas.height = displayH;
container.appendChild(canvas);
const ctx = canvas.getContext('2d');
// Each mask = one object (best-of-3 already selected in execute).
// Draw fill + bounding box per object in distinct colors.
const imgData = ctx.createImageData(displayW, displayH);
const boxes = []; // collect {minX, minY, maxX, maxY, colorIdx, score} per mask
for (let mi = 0; mi < masks.length; mi++) {
const mask = masks[mi];
const [r, g, b] = COLORS[mi % COLORS.length];
let bMinX = displayW, bMinY = displayH, bMaxX = 0, bMaxY = 0;
for (let y = 0; y < displayH; y++) {
for (let x = 0; x < displayW; x++) {
const mx = Math.floor((x / displayW) * mask.width);
const my = Math.floor((y / displayH) * mask.height);
if (mask.data[my * mask.width + mx] > 128) {
const idx = (y * displayW + x) * 4;
imgData.data[idx] = r;
imgData.data[idx + 1] = g;
imgData.data[idx + 2] = b;
imgData.data[idx + 3] = 80;
if (x < bMinX) bMinX = x;
if (x > bMaxX) bMaxX = x;
if (y < bMinY) bMinY = y;
if (y > bMaxY) bMaxY = y;
}
}
}
if (bMaxX > bMinX && bMaxY > bMinY) {
boxes.push({ minX: bMinX, minY: bMinY, maxX: bMaxX, maxY: bMaxY, ci: mi, score: scores[mi] || 0 });
}
}
ctx.putImageData(imgData, 0, 0);
// Draw bounding boxes and labels on top
const lw = Math.max(3, Math.round(displayW / 200));
const fontSize = Math.max(14, Math.round(displayW / 35));
ctx.font = `bold ${fontSize}px sans-serif`;
for (const box of boxes) {
const [r, g, b] = COLORS[box.ci % COLORS.length];
ctx.strokeStyle = `rgb(${r},${g},${b})`;
ctx.lineWidth = lw;
ctx.strokeRect(box.minX, box.minY, box.maxX - box.minX, box.maxY - box.minY);
const label = `#${box.ci + 1} ${Math.round(box.score * 100)}%`;
const tm = ctx.measureText(label);
const labelH = fontSize + 6;
const labelY = box.minY > labelH + 4 ? box.minY - labelH - 2 : box.minY + 2;
ctx.fillStyle = `rgb(${r},${g},${b})`;
ctx.fillRect(box.minX, labelY, tm.width + 12, labelH);
ctx.fillStyle = '#fff';
ctx.fillText(label, box.minX + 6, labelY + fontSize - 2);
}
// Download button
const dlBtn = document.createElement('button');
dlBtn.className = 'segmentation-download';
dlBtn.textContent = '\u2913 Download segmented image';
dlBtn.addEventListener('click', () => {
const exportCanvas = document.createElement('canvas');
exportCanvas.width = displayW;
exportCanvas.height = displayH;
const ectx = exportCanvas.getContext('2d');
ectx.drawImage(img, 0, 0);
ectx.drawImage(canvas, 0, 0);
exportCanvas.toBlob((blob) => {
const url = URL.createObjectURL(blob);
const a = document.createElement('a');
a.href = url;
a.download = 'segmented-' + new Date().toISOString().slice(0, 19).replace(/:/g, '') + '.png';
a.click();
URL.revokeObjectURL(url);
}, 'image/png');
});
container.appendChild(dlBtn);
chatArea.scrollTop = chatArea.scrollHeight;
};
bubble.appendChild(container);
}
// ── Multi-step planner ──────────────────────────────────────
// parsePlan is tolerant: picks up any line that starts with "N." or "N)"
// or "- N." and ignores everything else (prefaces, trailing commentary).
function parsePlan(text) {
const steps = [];
const lines = String(text || '').split(/\r?\n/);
for (const line of lines) {
const m = line.match(/^[\s\-*]*(\d+)[.):\]]\s+(.+?)\s*$/);
if (m && m[2]) steps.push(m[2].trim());
}
return steps;
}
// Append a collapsible block to the msg container (parent of bubble, so
// bubble.innerHTML updates from streaming tokens don't wipe it).
function renderPlannerBlock(container, title, body) {
const id = 'pb-' + Math.random().toString(36).slice(2, 8);
const block = document.createElement('div');
block.className = 'planner-block';
const safeBody = escapeHtml(body || '(no output)');
block.innerHTML =
'<div class="planner-toggle" onclick="document.getElementById(\'' + id +
'\').classList.toggle(\'open\'); this.querySelector(\'span\').textContent = ' +
'document.getElementById(\'' + id + '\').classList.contains(\'open\') ? \'\\u25bc\' : \'\\u25b6\'">' +
'<span>\u25b6</span> <strong>' + escapeHtml(title) + '</strong></div>' +
'<div class="planner-body" id="' + id + '">' + safeBody + '</div>';
container.appendChild(block);
chatArea.scrollTop = chatArea.scrollHeight;
}
// Drive one runtime.chat() call, collect text + any tool_calls. Pure β€”
// does no tool execution. Used by the planner for plan/synthesis passes
// and as the inner loop of runStepWithTools.
async function runPlannerChatTurn(messages, thinking, tools, genConfig) {
let fullText = '';
const toolCalls = [];
let error = null;
try {
for await (const ev of LocalMind.runtime.chat({
messages,
tools: tools || undefined,
enableThinking: thinking,
maxTokens: genConfig && genConfig.max_new_tokens,
temperature: genConfig && genConfig.temperature,
})) {
if (ev.type === 'token') fullText += ev.text;
else if (ev.type === 'tool_call') toolCalls.push(ev);
else if (ev.type === 'done') break;
else if (ev.type === 'error') { error = ev.error; break; }
}
} catch (e) { error = e; }
return { fullText, toolCalls, error };
}
// Run a planner step: up to 2 tool iterations, returns the text of the
// final (non-tool) assistant response along with collected sources and
// a flag for whether any tool was actually called.
async function runPlannerStepWithTools(stepMessages, enableThinking, tools, genConfig, lastImageBlob, userQuery) {
const STEP_MAX_ITER = 2;
let loopMessages = [...stepMessages];
let finalText = '';
let usedTools = false;
const sources = [];
for (let iter = 0; iter < STEP_MAX_ITER; iter++) {
if (!generating) return { fullText: finalText, usedTools, sources };
const turn = await runPlannerChatTurn(loopMessages, iter === 0 ? enableThinking : false, tools, genConfig);
if (turn.error) return { fullText: finalText || stripToolCallTags(turn.fullText) || '', usedTools, sources };
finalText = stripToolCallTags(turn.fullText);
if (turn.toolCalls.length === 0) return { fullText: finalText, usedTools, sources };
// Execute ALL tool calls emitted this turn, not just the first.
const stepResults = [];
for (const tc of turn.toolCalls) {
const tool = TOOL_REGISTRY[tc.name];
let toolResult;
if (!tool) {
toolResult = { error: 'Unknown tool: ' + tc.name };
} else {
try {
toolResult = await tool.execute(tc.arguments, { userQuery, imageBlob: lastImageBlob });
} catch (e) {
toolResult = { error: e.message };
}
}
usedTools = true;
if (tc.name === 'web_search' && toolResult && toolResult.results) {
sources.push(...toolResult.results);
}
stepResults.push({ tc, toolResult });
}
loopMessages.push({ role: 'assistant', content: turn.fullText });
for (const { tc, toolResult } of stepResults) {
loopMessages.push({ role: 'tool', name: tc.name, content: JSON.stringify(toolResult) });
}
if (iter === STEP_MAX_ITER - 1) {
loopMessages.push({ role: 'user', content: '[System: Tool limit reached β€” give final answer for this step now.]' });
const finalTurn = await runPlannerChatTurn(loopMessages, false, null, genConfig);
finalText = stripToolCallTags(finalTurn.fullText) || finalText;
}
}
return { fullText: finalText, usedTools, sources };
}
// Plan β†’ execute per step β†’ synthesize. Returns { handled, fullText,
// usedTools, sources }. handled=false means the caller should fall back
// to the single-pass agent loop (bad plan, plan phase errored, etc.).
async function runPlannerAgent(opts) {
const { text, chatMessages, tools, genConfig, enableThinking, msgEl, bubble, lastImageBlob, searchSources } = opts;
// Suppress token→bubble streaming during planner phases so the
// planner-block DOM we render to msgEl isn't wiped by each token
// arrival. Restore savedEl for the synthesis phase to stream
// normally into the bubble.
const savedEl = currentAssistantEl;
currentAssistantEl = null;
bubble.innerHTML = '<div class="planner-status">Planning\u2026</div>';
const plannerSystem =
'You are a task planner. Given the user task, break it into 2 to 5 sequential steps.\n' +
'Format: a numbered list, one step per line, each step written as a short imperative ' +
'("Find X", "Compare Y and Z", "Summarize results").\n' +
'Do NOT explain or add commentary. Do NOT perform the task. Only list the steps.';
const planTurn = await runPlannerChatTurn(
[{ role: 'system', content: plannerSystem }, { role: 'user', content: text }],
false, null, genConfig,
);
if (!generating) { currentAssistantEl = savedEl; return { handled: true, fullText: null }; }
if (planTurn.error) { currentAssistantEl = savedEl; return { handled: false }; }
const steps = parsePlan(planTurn.fullText);
if (steps.length < 2) {
// Bad / too-short plan β€” fall back to single-pass
currentAssistantEl = savedEl;
return { handled: false };
}
renderPlannerBlock(msgEl, 'Plan (' + steps.length + ' steps)',
steps.map((s, i) => (i + 1) + '. ' + s).join('\n'));
const stepOutputs = [];
const allSources = [...(searchSources || [])];
let anyToolsUsed = false;
for (let i = 0; i < steps.length; i++) {
if (!generating) { currentAssistantEl = savedEl; return { handled: true, fullText: null }; }
bubble.innerHTML = '<div class="planner-status">Step ' + (i + 1) + ' / ' + steps.length + ': ' + escapeHtml(steps[i]) + '</div>';
const stepMessages = [...chatMessages];
const priorBlock = stepOutputs.length
? '[Prior step outputs]\n' + stepOutputs.map((o, j) => 'Step ' + (j + 1) + ' result: ' + o).join('\n\n') + '\n\n'
: '';
stepMessages.push({ role: 'user', content: priorBlock + '[Current step]\n' + steps[i] });
const result = await runPlannerStepWithTools(stepMessages, enableThinking, tools, genConfig, lastImageBlob, text);
anyToolsUsed = anyToolsUsed || result.usedTools;
if (result.sources && result.sources.length) allSources.push(...result.sources);
stepOutputs.push(result.fullText || '');
renderPlannerBlock(msgEl, 'Step ' + (i + 1) + ': ' + steps[i], result.fullText || '(no output)');
}
if (!generating) { currentAssistantEl = savedEl; return { handled: true, fullText: null }; }
// Synthesis β€” restore bubble so tokens stream into it normally.
currentAssistantEl = savedEl;
currentAssistantText = '';
bubble.innerHTML = '';
const synthSystem =
'You just completed a multi-step task. Using the step outputs provided, write a direct final ' +
'answer for the user. Do not repeat the steps β€” deliver the answer. Cite sources by number if present.';
const synthMessages = [
{ role: 'system', content: synthSystem },
{ role: 'user', content: text },
{ role: 'assistant', content: 'Step outputs:\n\n' + stepOutputs.map((o, i) => 'Step ' + (i + 1) + ' (' + steps[i] + '): ' + (o || '(none)')).join('\n\n') },
{ role: 'user', content: 'Now give me the final answer.' },
];
const synthTurn = await runPlannerChatTurn(synthMessages, false, null, genConfig);
return {
handled: true,
fullText: synthTurn.fullText || '',
usedTools: anyToolsUsed,
sources: allSources,
};
}
async function sendMessage() {
const text = chatInput.value.trim();
if ((!text && attachments.length === 0) || !modelReady || generating) return;
const m = MODELS[activeModelKey];
const caps = LocalMind.runtime.capabilities();
const supportsMedia = !!(caps && (caps.image || caps.audio || caps.video));
const isMultimodal = supportsMedia && attachments.length > 0;
const hasAttachments = attachments.length > 0;
const isAgent = !!(caps && caps.toolCalling);
// Snapshot the attachment Blobs into a content-block array with
// real data β€” this is what runtime.chat() will translate into
// worker-ready inputs. The stored chat-history message stays as
// text-only ('(attachments only)') because Blobs can't be
// serialised to sessionStorage anyway, and the chat history
// display only renders attachments via the input bar at send
// time, not from the messages array.
let runtimeUserContent = null;
if (isMultimodal) {
const blocks = [];
for (const att of attachments) {
if (att.type === 'audio' && att.pcmData) {
// Pre-decoded mic recording β€” wrap as a Blob so the adapter
// re-decoder gets consistent input. The decoder will redo
// the work; we could pass pcmData directly via a custom
// block shape, but Blob keeps the contract simple.
blocks.push({ type: 'audio', data: att.blob });
} else if (att.type === 'image' || att.type === 'audio' || att.type === 'video') {
blocks.push({ type: att.type, data: att.blob });
}
}
if (text) blocks.push({ type: 'text', text });
runtimeUserContent = blocks;
}
// Build user message content for storage / chat history. Stored
// shape is text-only β€” the data-bearing version above is used
// only for the runtime.chat call.
const userContent = text || (hasAttachments ? '(attachments only)' : '');
messages.push({ role: 'user', content: userContent });
addUserMessage(text || '(attachments)', hasAttachments);
saveChat();
chatInput.value = '';
chatInput.style.height = 'auto';
// Build system prompt. The tool-calling scaffolding is added
// by runtime.chat() when we pass it a `tools` array β€” we just
// build the user-facing portion (custom prompt + optional RAG
// + optional web search results) here.
const userSysPrompt = systemPrompt.value.trim();
let sysPrompt = userSysPrompt;
// Auto-inject RAG context if embedding model is ready and agent-capable
if (isAgent && LocalMind.runtime.embeddingsReady) {
try {
const userText = typeof userContent === 'string' ? userContent : text;
if (userText) {
const ragResults = await searchMemory(userText, 3);
if (ragResults.length > 0) {
const ragBlock = ragResults.map(r => `[${r.category}] ${r.text}`).join('\n\n');
sysPrompt += `\n\n[Retrieved from memory β€” use if relevant]\n${ragBlock}`;
}
}
} catch (e) {
console.warn('RAG retrieval failed:', e);
}
}
// Web-enriched mode: pre-search and inject results into context
const isWebEnriched = webEnrichedMode && isAgent && isSearchConfigured();
let searchSources = [];
webEnrichedMode = false; // reset flag
if (isWebEnriched) {
try {
const userText = typeof userContent === 'string' ? userContent : text;
const results = await executeWebSearch(userText);
if (results.length > 0) {
searchSources = results;
const searchBlock = results.map((r, i) => `[${i + 1}] ${r.title}\n${r.snippet}\nURL: ${r.url}`).join('\n\n');
sysPrompt += `\n\n[Web search results for "${userText}" β€” cite sources by number]\n${searchBlock}`;
// Cache in RAG
if (LocalMind.runtime.embeddingsReady) {
const snippetText = results.map(r => `${r.title}: ${r.snippet}`).join('\n');
embedAndStore(snippetText, 'finding', 'web-search').catch(() => {});
}
}
} catch (e) {
console.warn('Web search failed:', e);
showToast(e.message || 'Web search failed');
}
}
// Build context-windowed chat history
const chatMessages = buildContextMessages(messages, sysPrompt, activeModelKey);
// For multimodal sends, replace the most-recent user message in
// chatMessages with the data-bearing content array we built
// above. The runtime adapter will detect the media blocks and
// process them. After the first agent-loop iteration, we strip
// the data field (see scrubMediaForFollowup below) so the chat
// template still emits position tokens but the adapter doesn't
// re-decode the same media on every pass.
if (runtimeUserContent) {
for (let i = chatMessages.length - 1; i >= 0; i--) {
if (chatMessages[i].role === 'user') {
chatMessages[i] = { role: 'user', content: runtimeUserContent };
break;
}
}
}
// UI: enter generating state
currentAssistantEl = addAssistantPlaceholder();
currentAssistantText = '';
generating = true;
sendBtn.innerHTML = '&#9632;';
sendBtn.classList.add('stop');
chatInput.disabled = true;
modelSelect.disabled = true;
const genConfig = m.genConfig;
const enableThinking = thinkingToggle.checked;
// Snapshot last image blob for segment_image tool before clearing
const lastImageBlob = runtimeUserContent
? runtimeUserContent.filter(b => b.type === 'image').map(b => b.data).pop()
: null;
// Clear attachments after capturing data
clearAttachments();
if (!isAgent) {
// Non-agent path: single generation, no tool calling.
// Routed through LocalMind.runtime.chat β€” the legacy bubble
// updater (attachWorkerHandlers token handler) still paints
// currentAssistantEl as tokens arrive; we consume the iterator
// purely to collect the final text and the done signal.
let fullText = '';
let chatErr = null;
try {
for await (const ev of LocalMind.runtime.chat({
messages: chatMessages,
enableThinking,
maxTokens: genConfig && genConfig.max_new_tokens,
temperature: genConfig && genConfig.temperature,
})) {
if (ev.type === 'token') {
fullText += ev.text;
} else if (ev.type === 'done') {
break;
} else if (ev.type === 'error') {
chatErr = ev.error;
break;
}
}
} catch (e) {
chatErr = e;
}
if (chatErr) {
fullText = (fullText || '') + `\n\n*Error: ${chatErr.message || chatErr}*`;
}
finishGeneration(fullText || null, searchSources);
return;
}
// ── Multi-step planner (opt-in, agent-capable only) ──────
// Returns handled=false on a malformed/short plan β€” in that case we
// fall through to the single-pass agent loop below.
if (plannerEnabled) {
const plannerTools = buildToolsForRuntime();
const msgElForPlanner = currentAssistantEl.bubble.parentElement;
const result = await runPlannerAgent({
text,
chatMessages,
tools: plannerTools,
genConfig,
enableThinking,
msgEl: msgElForPlanner,
bubble: currentAssistantEl.bubble,
lastImageBlob,
searchSources,
});
if (result.handled) {
finishGeneration(result.fullText || null, result.sources || searchSources, result.usedTools);
return;
}
// Fall through to single-pass agent loop on bad plan
}
// ── Agentic loop ──────────────────────────────────────────
// The agent loop is now runtime-agnostic: it calls
// LocalMind.runtime.chat() with the tool list, consumes
// {token, tool_call, done} events from the iterator, executes
// each tool through TOOL_REGISTRY, and feeds results back as
// tool-role messages on the next iteration.
const MAX_TOOL_ITERATIONS = 3;
let loopMessages = [...chatMessages];
const allSources = [...searchSources];
let toolsUsedInLoop = false;
let pendingSegOverlay = null;
const tools = buildToolsForRuntime();
// After iter 0, strip the `data` field from media blocks in any
// user message β€” the chat template still emits position tokens
// for the placeholder shape, but the adapter won't re-decode the
// same audio/video on every iteration (and the worker receives
// null images/audio, matching the pre-refactor iter>0 behaviour).
function scrubMediaForFollowup(msgs) {
return msgs.map(msg => {
if (!Array.isArray(msg.content)) return msg;
return {
...msg,
content: msg.content.map(b => {
if (b && (b.type === 'image' || b.type === 'audio' || b.type === 'video')) {
return { type: b.type };
}
return b;
}),
};
});
}
// Helper: drive one chat() call to completion, collecting text
// and any tool_call events from the iterator. Returns
// { fullText, toolCalls, doneReason, error }.
async function runOneAgentTurn(turnMessages, turnThinking) {
let fullText = '';
const toolCalls = [];
let doneReason = null;
let error = null;
try {
for await (const ev of LocalMind.runtime.chat({
messages: turnMessages,
tools,
enableThinking: turnThinking,
maxTokens: genConfig && genConfig.max_new_tokens,
temperature: genConfig && genConfig.temperature,
})) {
if (ev.type === 'token') {
fullText += ev.text;
} else if (ev.type === 'tool_call') {
toolCalls.push(ev);
} else if (ev.type === 'done') {
doneReason = ev.reason;
break;
} else if (ev.type === 'error') {
error = ev.error;
break;
}
}
} catch (e) {
error = e;
}
return { fullText, toolCalls, doneReason, error };
}
for (let iter = 0; iter < MAX_TOOL_ITERATIONS; iter++) {
// Disable thinking on follow-up iterations to avoid tool
// calls inside thinking blocks.
const iterThinking = iter === 0 ? enableThinking : false;
const turn = await runOneAgentTurn(loopMessages, iterThinking);
// After the first iteration, scrub data from media blocks so
// subsequent passes carry the chat-template position tokens
// forward without re-decoding the same audio/video.
if (iter === 0) loopMessages = scrubMediaForFollowup(loopMessages);
function finishWithOverlay(text, srcs, tools) {
// Capture the message container (parent of bubble) before finishGeneration nulls currentAssistantEl.next
// Append overlay to the msg container, NOT inside the bubble β€” innerHTML updates during
// streaming destroy bubble children, but the msg container is stable.
const msgEl = currentAssistantEl ? currentAssistantEl.bubble.parentElement : null;
finishGeneration(text, srcs, tools);
if (pendingSegOverlay && msgEl) {
// Short delay to ensure all pending DOM updates (streaming, badges) have completed
setTimeout(() => {
renderSegmentationOverlay(msgEl, pendingSegOverlay.imageBlob, pendingSegOverlay.masks, pendingSegOverlay.scores);
chatArea.scrollTop = chatArea.scrollHeight;
}, 50);
}
}
// User clicked stop while we were generating
if (!generating) {
finishWithOverlay(turn.fullText || null, allSources, toolsUsedInLoop);
return;
}
if (turn.error) {
finishWithOverlay(stripToolCallTags(turn.fullText || '') + `\n\n*Error: ${turn.error.message || turn.error}*`, allSources, toolsUsedInLoop);
return;
}
if (turn.toolCalls.length === 0) {
// No tool calls β€” we're done
finishWithOverlay(turn.fullText, allSources, toolsUsedInLoop);
return;
}
// Clean the raw <tool_call> tags out of the live bubble display so
// users see the model's prose (if any), not its raw call syntax.
const cleanedTurnText = stripToolCallTags(turn.fullText);
currentAssistantText = cleanedTurnText;
if (currentAssistantEl) updateAssistantBubble(currentAssistantEl.bubble, cleanedTurnText);
// Execute ALL tool calls the model emitted this turn (Qwen3-family
// models including Ternary Bonsai often batch multiple calls into
// one response). Blocks are appended to a tool-trace container
// inserted BEFORE the bubble β€” collapsed by default, expanded via
// a toggle added in finishGeneration. Container is a sibling of
// the bubble so streaming updates to bubble.innerHTML don't wipe it.
toolsUsedInLoop = true;
const msgContainer = currentAssistantEl ? currentAssistantEl.bubble.parentElement : null;
let traceContainer = msgContainer && msgContainer.querySelector('.tool-trace-container');
if (msgContainer && !traceContainer) {
traceContainer = document.createElement('div');
traceContainer.className = 'tool-trace-container collapsed';
msgContainer.insertBefore(traceContainer, currentAssistantEl.bubble);
}
const toolResults = [];
for (let tcIdx = 0; tcIdx < turn.toolCalls.length; tcIdx++) {
const tc = turn.toolCalls[tcIdx];
const tool = TOOL_REGISTRY[tc.name];
let toolResult;
if (!tool) {
toolResult = { error: `Unknown tool: ${tc.name}` };
} else {
try {
toolResult = await tool.execute(tc.arguments, { userQuery: text, imageBlob: lastImageBlob });
} catch (e) {
toolResult = { error: e.message };
}
}
if (tc.name === 'web_search' && toolResult && toolResult.results) {
allSources.push(...toolResult.results);
}
if (traceContainer) {
const stepLabel = document.createElement('div');
stepLabel.style.cssText = 'font-size:0.68rem;color:var(--gray-400);margin:6px 0 2px;font-weight:500';
const stepSuffix = turn.toolCalls.length > 1 ? ` \u00B7 Call ${tcIdx + 1}/${turn.toolCalls.length}` : '';
stepLabel.textContent = `Step ${iter + 1}/${MAX_TOOL_ITERATIONS}${stepSuffix}`;
traceContainer.appendChild(stepLabel);
renderToolCallBlock(traceContainer, tc.name, tc.arguments, toolResult);
}
if (tc.name === 'segment_image' && toolResult._rawMasks) {
pendingSegOverlay = { imageBlob: toolResult._imageBlob, masks: toolResult._rawMasks, scores: toolResult._scores };
delete toolResult._rawMasks;
delete toolResult._imageBlob;
delete toolResult._scores;
}
toolResults.push({ tc, toolResult });
}
// Append the assistant turn once (with raw tags so the model sees
// its own format), then all tool-role results in order.
loopMessages.push({ role: 'assistant', content: turn.fullText });
for (const { tc, toolResult } of toolResults) {
loopMessages.push({ role: 'tool', name: tc.name, content: JSON.stringify(toolResult) });
}
// Reset streaming state for next generation pass
currentAssistantText = '';
// If this is the last iteration, force a final response
if (iter === MAX_TOOL_ITERATIONS - 1) {
loopMessages.push({ role: 'user', content: '[System: Tool call limit reached. Please provide your final answer now.]' });
const finalTurn = await runOneAgentTurn(loopMessages, enableThinking);
finishWithOverlay(stripToolCallTags(finalTurn.fullText), allSources, toolsUsedInLoop);
return;
}
}
}
function finishGeneration(response, sources, usedTools) {
generating = false;
if (response) {
messages.push({ role: 'assistant', content: response });
saveChat();
}
// Force-collapse any open thinking blocks (they expand during streaming)
if (currentAssistantEl) {
const bubble = currentAssistantEl.bubble;
bubble.querySelectorAll('.thinking-content.open').forEach(el => {
el.classList.remove('open');
const toggle = el.previousElementSibling;
if (toggle) {
const arrow = toggle.querySelector('span');
if (arrow) arrow.innerHTML = '&#9654;';
toggle.childNodes.forEach(n => { if (n.nodeType === 3 && n.textContent.includes('Thinking')) n.textContent = ' Thought process'; });
}
});
}
// Transparency: add source badge and links to the assistant bubble
if (currentAssistantEl) {
const bubble = currentAssistantEl.bubble;
const badge = document.createElement('div');
if (sources && sources.length > 0) {
badge.innerHTML = `<span class="msg-source-badge web-enriched">Web-enriched &middot; ${sources.length} sources</span>`;
const linksDiv = document.createElement('div');
linksDiv.className = 'source-links';
linksDiv.innerHTML = sources.map(s => `<a href="${escapeHtml(s.url)}" target="_blank" rel="noopener">${escapeHtml(s.title || s.url)}</a>`).join('');
bubble.appendChild(linksDiv);
} else if (usedTools) {
badge.innerHTML = `<span class="msg-source-badge" style="background:rgba(102,126,234,0.1);color:var(--indigo-600)">Agent</span>`;
} else {
badge.innerHTML = `<span class="msg-source-badge on-device">On-device</span>`;
}
bubble.insertBefore(badge, bubble.firstChild);
// Save as Markdown button
if (response && response.length > 20) {
const saveBtn = document.createElement('button');
saveBtn.className = 'save-md-btn';
saveBtn.textContent = dirHandle ? `Save to ${dirHandle.name}` : 'Save as MD';
saveBtn.addEventListener('click', async () => {
const filename = `response-${new Date().toISOString().slice(0, 16).replace(/:/g, '')}.md`;
if (dirHandle) {
try {
const fh = await dirHandle.getFileHandle(filename, { create: true });
const writable = await fh.createWritable();
await writable.write(response);
await writable.close();
showToast(`Saved to ${dirHandle.name}/${filename}`);
} catch (e) {
showToast('Write failed: ' + e.message);
}
} else {
const blob = new Blob([response], { type: 'text/markdown' });
const url = URL.createObjectURL(blob);
const a = document.createElement('a');
a.href = url; a.download = filename;
a.click();
URL.revokeObjectURL(url);
}
});
bubble.appendChild(saveBtn);
}
// Tool-trace toggle: if this msg has a collapsed trace container
// above the bubble, expose a discreet button that expands it.
const msgEl = bubble.parentElement;
const traceContainer = msgEl && msgEl.querySelector('.tool-trace-container');
const callCount = traceContainer ? traceContainer.querySelectorAll('.tool-call-block').length : 0;
if (callCount > 0) {
const traceBtn = document.createElement('button');
traceBtn.className = 'save-md-btn';
traceBtn.style.marginLeft = '6px';
const label = () => (traceContainer.classList.contains('collapsed') ? '\u25B6' : '\u25BC') +
' ' + callCount + ' tool call' + (callCount > 1 ? 's' : '');
traceBtn.innerHTML = label();
traceBtn.addEventListener('click', () => {
traceContainer.classList.toggle('collapsed');
traceBtn.innerHTML = label();
});
bubble.appendChild(traceBtn);
}
}
currentAssistantEl = null;
currentAssistantText = '';
sendBtn.innerHTML = '&#9654;';
sendBtn.classList.remove('stop');
sendBtn.disabled = false; searchSendBtn.disabled = false;
chatInput.disabled = false;
modelSelect.disabled = false;
chatInput.focus();
}
sendBtn.addEventListener('click', () => {
if (generating) {
// Stop generation β€” interrupt worker, let complete handler resolve the promise.
// Set generating=false so the agentic loop knows to exit after awaiting.
worker.postMessage({ type: 'stop' });
generating = false;
} else {
sendMessage();
}
});
chatInput.addEventListener('keydown', (e) => {
if (e.key === 'Enter' && !e.shiftKey) {
e.preventDefault();
if (!generating) sendMessage();
}
});
// ── Clear chat (delete, no archive) ────────────────────────
clearBtn.addEventListener('click', () => {
if (generating) return;
if (messages.length > 0) showToast('Chat cleared (not saved to History)');
resetChatUI();
});
// ── Batch prompts ────────────────────────────────────────────
const batchBtn = document.getElementById('batchBtn');
const batchPanel = document.getElementById('batchPanel');
const batchTextarea = document.getElementById('batchTextarea');
const batchCount = document.getElementById('batchCount');
const batchProgress = document.getElementById('batchProgress');
const batchRunBtn = document.getElementById('batchRunBtn');
const batchStopBtn = document.getElementById('batchStopBtn');
const batchChainToggle = document.getElementById('batchChainToggle');
let batchRunning = false;
let batchShouldStop = false;
batchBtn.addEventListener('click', () => {
batchPanel.classList.toggle('open');
settingsPanel.classList.remove('open');
memoryPanel.classList.remove('open');
closeHistorySidebar();
});
function parseBatchPrompts() {
return batchTextarea.value.split('\n').map(l => l.trim()).filter(Boolean);
}
function updateBatchCount() {
const n = parseBatchPrompts().length;
batchCount.textContent = `${n} prompt${n === 1 ? '' : 's'}`;
batchRunBtn.disabled = n === 0 || !modelReady || batchRunning;
}
batchTextarea.addEventListener('input', updateBatchCount);
batchStopBtn.addEventListener('click', () => {
batchShouldStop = true;
batchStopBtn.disabled = true;
batchProgress.textContent = 'Stopping…';
});
batchRunBtn.addEventListener('click', async () => {
if (!modelReady) { showToast('Model not ready yet'); return; }
if (generating) { showToast('Already generating β€” wait for current response'); return; }
const prompts = parseBatchPrompts();
if (!prompts.length) return;
batchRunning = true;
batchShouldStop = false;
batchRunBtn.disabled = true;
batchStopBtn.disabled = false;
batchTextarea.disabled = true;
let lastResponse = '';
let ranCount = 0;
for (let i = 0; i < prompts.length; i++) {
if (batchShouldStop) break;
ranCount = i + 1;
let prompt = prompts[i];
// {{previous}} substitution (always β€” user put it there explicitly)
if (i > 0 && lastResponse) {
prompt = prompt.replace(/\{\{previous\}\}/g, lastResponse);
}
// Auto-inject: if chain toggle is on and no explicit {{previous}}, append previous as context
if (batchChainToggle.checked && i > 0 && lastResponse && !prompts[i].includes('{{previous}}')) {
prompt = `${prompt}\n\n[Previous response for context:\n${lastResponse}\n]`;
}
batchProgress.textContent = `${i + 1} / ${prompts.length}`;
chatInput.value = prompt;
await sendMessage();
// Grab the last assistant turn
const last = messages[messages.length - 1];
if (last?.role === 'assistant') {
lastResponse = typeof last.content === 'string' ? last.content : '';
}
}
const wasStopped = batchShouldStop;
batchRunning = false;
batchShouldStop = false;
batchRunBtn.disabled = false;
batchStopBtn.disabled = true;
batchTextarea.disabled = false;
const ran = ranCount;
batchProgress.textContent = wasStopped ? `Stopped at ${ran}/${prompts.length}.` : `Done (${prompts.length} prompts)`;
showToast(wasStopped ? 'Batch stopped' : 'Batch complete');
});
// ── Encrypted share links ───────────────────────────────────
// URL format:
// #lm:<base64> plain
// #lme:<b64salt>.<b64iv>.<b64ciphertext> AES-256-GCM + PBKDF2
function b64ToArr(b64) {
return Uint8Array.from(atob(b64), c => c.charCodeAt(0));
}
function arrToB64(arr) {
return btoa(String.fromCharCode(...new Uint8Array(arr)));
}
async function deriveKey(passphrase, salt) {
const enc = new TextEncoder();
const keyMat = await crypto.subtle.importKey(
'raw', enc.encode(passphrase), { name: 'PBKDF2' }, false, ['deriveKey']
);
return crypto.subtle.deriveKey(
{ name: 'PBKDF2', salt, hash: 'SHA-256', iterations: 200000 },
keyMat, { name: 'AES-GCM', length: 256 }, false, ['encrypt', 'decrypt']
);
}
async function encryptPayload(json, passphrase) {
const salt = crypto.getRandomValues(new Uint8Array(16));
const iv = crypto.getRandomValues(new Uint8Array(12));
const key = await deriveKey(passphrase, salt);
const data = new TextEncoder().encode(json);
const cipher = await crypto.subtle.encrypt({ name: 'AES-GCM', iv }, key, data);
return `${arrToB64(salt)}.${arrToB64(iv)}.${arrToB64(cipher)}`;
}
async function decryptPayload(encoded, passphrase) {
const parts = encoded.split('.');
if (parts.length !== 3) throw new Error('Invalid format');
const [saltB64, ivB64, cipherB64] = parts;
const salt = b64ToArr(saltB64), iv = b64ToArr(ivB64), cipher = b64ToArr(cipherB64);
const key = await deriveKey(passphrase, salt);
const plain = await crypto.subtle.decrypt({ name: 'AES-GCM', iv }, key, cipher);
return new TextDecoder().decode(plain);
}
function buildConversationPayload() {
// Strip blobs; keep text content only
const clean = messages.map(m => {
if (typeof m.content === 'string') return { role: m.role, content: m.content };
if (Array.isArray(m.content)) {
const text = m.content.filter(p => p.type === 'text').map(p => p.text).join('\n');
return { role: m.role, content: text };
}
return { role: m.role, content: String(m.content || '') };
}).filter(m => m.content);
return { v: 1, model: activeModelKey, created: new Date().toISOString(), messages: clean };
}
// Share modal
const shareBackdrop = document.getElementById('shareBackdrop');
const shareMetaText = document.getElementById('shareMetaText');
const sharePassphrase = document.getElementById('sharePassphrase');
const shareUsePassphrase = document.getElementById('shareUsePassphrase');
const shareUrlInput = document.getElementById('shareUrlInput');
const shareGenerateBtn = document.getElementById('shareGenerateBtn');
const shareCopyBtn = document.getElementById('shareCopyBtn');
const shareCloseBtn = document.getElementById('shareCloseBtn');
const shareBtn = document.getElementById('shareBtn');
shareBtn.addEventListener('click', () => {
if (!messages.length) { showToast('Nothing to share yet'); return; }
const userTurns = messages.filter(m => m.role === 'user').length;
shareMetaText.textContent = `${messages.length} message(s) Β· ${userTurns} turn(s)`;
shareUrlInput.value = '';
shareCopyBtn.disabled = true;
shareUsePassphrase.checked = false;
sharePassphrase.style.display = 'none';
sharePassphrase.value = '';
shareBackdrop.classList.add('open');
});
shareUsePassphrase.addEventListener('change', () => {
sharePassphrase.style.display = shareUsePassphrase.checked ? '' : 'none';
if (shareUsePassphrase.checked) sharePassphrase.focus();
shareUrlInput.value = '';
shareCopyBtn.disabled = true;
});
shareGenerateBtn.addEventListener('click', async () => {
const usePass = shareUsePassphrase.checked;
const pass = sharePassphrase.value.trim();
if (usePass && !pass) { showToast('Enter a passphrase first'); sharePassphrase.focus(); return; }
shareGenerateBtn.disabled = true;
shareGenerateBtn.textContent = 'Generating…';
try {
const payload = buildConversationPayload();
const json = JSON.stringify(payload);
let fragment;
if (usePass) {
const enc = await encryptPayload(json, pass);
fragment = '#lme:' + enc;
} else {
fragment = '#lm:' + btoa(unescape(encodeURIComponent(json)));
}
const url = location.origin + location.pathname + fragment;
shareUrlInput.value = url;
shareCopyBtn.disabled = false;
} catch (e) {
showToast('Failed to generate link: ' + e.message);
} finally {
shareGenerateBtn.disabled = false;
shareGenerateBtn.textContent = 'Generate';
}
});
shareCopyBtn.addEventListener('click', async () => {
try {
await navigator.clipboard.writeText(shareUrlInput.value);
shareCopyBtn.textContent = 'Copied!';
setTimeout(() => { shareCopyBtn.textContent = 'Copy Link'; }, 1800);
} catch {
shareUrlInput.select();
document.execCommand('copy');
}
});
shareCloseBtn.addEventListener('click', () => shareBackdrop.classList.remove('open'));
shareBackdrop.addEventListener('click', e => { if (e.target === shareBackdrop) shareBackdrop.classList.remove('open'); });
// Import banner (on load, detect share link in URL hash)
const importBanner = document.getElementById('importBanner');
const importBannerMsg = document.getElementById('importBannerMsg');
const importPassphrase = document.getElementById('importPassphrase');
const importConfirmBtn = document.getElementById('importConfirmBtn');
const importDismissBtn = document.getElementById('importDismissBtn');
let _pendingSharePayload = null; // decoded plain payload
let _pendingShareEncoded = null; // raw encoded string for encrypted links
async function checkShareLink() {
const hash = location.hash;
if (!hash) return;
if (hash.startsWith('#lm:')) {
try {
const json = decodeURIComponent(escape(atob(hash.slice(4))));
_pendingSharePayload = JSON.parse(json);
importBannerMsg.textContent = `Shared conversation (${_pendingSharePayload.messages?.length ?? '?'} messages) β€” load it?`;
importBanner.classList.add('open');
} catch (e) { console.warn('Bad share link', e); }
} else if (hash.startsWith('#lme:')) {
_pendingShareEncoded = hash.slice(5);
importBannerMsg.textContent = 'Encrypted shared conversation β€” enter passphrase to load:';
importPassphrase.style.display = '';
importBanner.classList.add('open');
}
}
importConfirmBtn.addEventListener('click', async () => {
try {
let payload = _pendingSharePayload;
if (!payload) {
const pass = importPassphrase.value;
if (!pass) { showToast('Enter the passphrase'); importPassphrase.focus(); return; }
const json = await decryptPayload(_pendingShareEncoded, pass);
payload = JSON.parse(json);
}
if (!payload?.messages?.length) { showToast('No messages found in link'); return; }
// Load into current session
resetChatUI();
for (const msg of payload.messages) {
if (msg.role === 'user') {
addUserMessage(msg.content, false);
} else if (msg.role === 'assistant') {
const { bubble } = addAssistantPlaceholder();
updateAssistantBubble(bubble, msg.content);
}
messages.push(msg);
}
saveChat();
importBanner.classList.remove('open');
history.replaceState(null, '', location.pathname);
showToast(`Loaded ${payload.messages.length} messages`);
} catch (e) {
showToast('Could not load: ' + e.message);
}
});
importDismissBtn.addEventListener('click', () => {
importBanner.classList.remove('open');
history.replaceState(null, '', location.pathname);
});
checkShareLink();
// ── Init ────────────────────────────────────────────────────
renderRestoredMessages();
loadModel(modelSelect.value);
</script>
</body>
</html>