init: oxagenttwo — voice chat with local qwen3 (Ollama) + OpenRouter, Whisper STT, Piper TTS (Thorsten), sessions, thinking mode

server/src/db.ts

@@ -0,0 +1,117 @@
+import { DatabaseSync } from "node:sqlite";
+import { randomUUID } from "node:crypto";
+import path from "node:path";
+
+const dbPath = path.join(import.meta.dirname, "..", "data.sqlite");
+export const db = new DatabaseSync(dbPath);
+
+db.exec(`
+  PRAGMA journal_mode = WAL;
+  CREATE TABLE IF NOT EXISTS sessions (
+    id TEXT PRIMARY KEY,
+    title TEXT NOT NULL DEFAULT 'Neuer Chat',
+    created_at INTEGER NOT NULL
+  );
+  CREATE TABLE IF NOT EXISTS messages (
+    id TEXT PRIMARY KEY,
+    session_id TEXT NOT NULL REFERENCES sessions(id) ON DELETE CASCADE,
+    role TEXT NOT NULL CHECK(role IN ('user','assistant','system')),
+    content TEXT NOT NULL DEFAULT '',
+    thinking TEXT,
+    created_at INTEGER NOT NULL
+  );
+  CREATE INDEX IF NOT EXISTS idx_messages_session ON messages(session_id, created_at);
+`);
+
+export interface SessionRow {
+  id: string;
+  title: string;
+  created_at: number;
+}
+
+export interface MessageRow {
+  id: string;
+  session_id: string;
+  role: string;
+  content: string;
+  thinking: string | null;
+  created_at: number;
+}
+
+export function listSessions(): SessionRow[] {
+  return db
+    .prepare("SELECT * FROM sessions ORDER BY created_at DESC")
+    .all() as unknown as SessionRow[];
+}
+
+export function createSession(title = "Neuer Chat"): SessionRow {
+  const row: SessionRow = { id: randomUUID(), title, created_at: Date.now() };
+  db.prepare("INSERT INTO sessions (id, title, created_at) VALUES (?, ?, ?)").run(
+    row.id,
+    row.title,
+    row.created_at,
+  );
+  return row;
+}
+
+export function getSession(id: string): SessionRow | undefined {
+  return db.prepare("SELECT * FROM sessions WHERE id = ?").get(id) as
+    | SessionRow
+    | undefined;
+}
+
+export function renameSessionIfDefault(id: string, firstMessage: string) {
+  const s = getSession(id);
+  if (!s || s.title !== "Neuer Chat") return;
+  const title =
+    firstMessage.trim().slice(0, 60) + (firstMessage.length > 60 ? "…" : "");
+  db.prepare("UPDATE sessions SET title = ? WHERE id = ?").run(
+    title || "Neuer Chat",
+    id,
+  );
+}
+
+export function deleteSession(id: string) {
+  db.prepare("DELETE FROM messages WHERE session_id = ?").run(id);
+  db.prepare("DELETE FROM sessions WHERE id = ?").run(id);
+}
+
+export function insertMessage(
+  sessionId: string,
+  role: string,
+  content: string,
+  thinking?: string | null,
+): MessageRow {
+  const row: MessageRow = {
+    id: randomUUID(),
+    session_id: sessionId,
+    role,
+    content,
+    thinking: thinking ?? null,
+    created_at: Date.now(),
+  };
+  db.prepare(
+    "INSERT INTO messages (id, session_id, role, content, thinking, created_at) VALUES (?, ?, ?, ?, ?, ?)",
+  ).run(row.id, row.session_id, row.role, row.content, row.thinking, row.created_at);
+  return row;
+}
+
+export function updateAssistantMessage(
+  id: string,
+  content: string,
+  thinking: string | null,
+) {
+  db.prepare("UPDATE messages SET content = ?, thinking = ? WHERE id = ?").run(
+    content,
+    thinking,
+    id,
+  );
+}
+
+export function listMessages(sessionId: string): MessageRow[] {
+  return db
+    .prepare(
+      "SELECT * FROM messages WHERE session_id = ? ORDER BY created_at ASC",
+    )
+    .all(sessionId) as unknown as MessageRow[];
+}

server/src/index.ts

@@ -0,0 +1,302 @@
+import Fastify from "fastify";
+import cors from "@fastify/cors";
+import { WebSocketServer, WebSocket } from "ws";
+import type { IncomingMessage } from "node:http";
+import { readFileSync } from "node:fs";
+import path from "node:path";
+import * as dbmod from "./db.js";
+import { streamChat, type OllamaOptions } from "./ollama.js";
+import { transcribeAudio, synthesizeSpeech } from "./voice.js";
+import {
+  streamOpenRouter,
+  listOpenRouterModels,
+  getOpenRouterKey,
+} from "./openrouter.js";
+
+// simple .env loader (project root)
+try {
+  for (const line of readFileSync(
+    path.join(import.meta.dirname, "..", "..", ".env"),
+    "utf8",
+  ).split("\n")) {
+    const m = line.match(/^\s*([A-Za-z_][A-Za-z0-9_]*)\s*=\s*(.*)\s*$/);
+    if (m && !(m[1] in process.env)) process.env[m[1]] = m[2].replace(/^["']|["']$/g, "");
+  }
+} catch {
+  /* keine .env vorhanden */
+}
+
+const PORT = Number(process.env.PORT ?? 8787);
+const OLLAMA_URL = process.env.OLLAMA_URL ?? "http://localhost:11434";
+
+const app = Fastify({ logger: false, bodyLimit: 32 * 1024 * 1024 });
+await app.register(cors, { origin: true });
+
+app.addContentTypeParser(
+  ["application/octet-stream", "audio/*", "video/*"],
+  { parseAs: "buffer" },
+  (_req, body, done) => done(null, body),
+);
+
+app.get("/api/health", async () => ({ ok: true }));
+
+app.get("/api/model", async () => {
+  try {
+    const res = await fetch(`${OLLAMA_URL}/api/show`, {
+      method: "POST",
+      headers: { "Content-Type": "application/json" },
+      body: JSON.stringify({ name: process.env.MODEL ?? "qwen3.5:latest" }),
+    });
+    if (!res.ok) return { ok: false, error: `Ollama HTTP ${res.status}` };
+    const data = (await res.json()) as {
+      details?: { parameter_size?: string; quantization_level?: string };
+      capabilities?: string[];
+    };
+    return {
+      ok: true,
+      model: process.env.MODEL ?? "qwen3.5:latest",
+      parameterSize: data.details?.parameter_size,
+      quantization: data.details?.quantization_level,
+      capabilities: data.capabilities ?? [],
+    };
+  } catch (err) {
+    return { ok: false, error: String(err) };
+  }
+});
+
+// --- Sessions REST ---
+app.get("/api/sessions", async () => dbmod.listSessions());
+app.post("/api/sessions", async () => dbmod.createSession());
+app.get("/api/sessions/:id/messages", async (req, reply) => {
+  const { id } = req.params as { id: string };
+  if (!dbmod.getSession(id)) return reply.code(404).send({ error: "not found" });
+  return dbmod.listMessages(id);
+});
+app.delete("/api/sessions/:id", async (req) => {
+  const { id } = req.params as { id: string };
+  dbmod.deleteSession(id);
+  broadcast({ type: "session-deleted", id });
+  return { ok: true };
+});
+
+// --- Voice ---
+app.post("/api/stt", async (req, reply) => {
+  const audio = req.body as Buffer;
+  if (!Buffer.isBuffer(audio) || audio.length === 0) {
+    return reply.code(400).send({ ok: false, error: "Kein Audio empfangen" });
+  }
+  try {
+    const text = await transcribeAudio(audio);
+    return { ok: true, text };
+  } catch (err) {
+    app.log.error(err);
+    return reply
+      .code(500)
+      .send({ ok: false, error: err instanceof Error ? err.message : String(err) });
+  }
+});
+
+app.post("/api/tts", async (req, reply) => {
+  const { text } = (req.body ?? {}) as { text?: string };
+  if (!text || !text.trim()) {
+    return reply.code(400).send({ error: "text fehlt" });
+  }
+  try {
+    const wav = await synthesizeSpeech(text);
+    reply.header("Content-Type", "audio/wav").send(wav);
+  } catch (err) {
+    app.log.error(err);
+    return reply
+      .code(500)
+      .send({ error: err instanceof Error ? err.message : String(err) });
+  }
+});
+
+// --- OpenRouter models ---
+app.get("/api/openrouter/models", async () => {
+  if (!getOpenRouterKey()) {
+    return { ok: false, error: "OPENROUTER_API_KEY nicht gesetzt (.env fehlt)" };
+  }
+  try {
+    return { ok: true, models: await listOpenRouterModels() };
+  } catch (err) {
+    return { ok: false, error: err instanceof Error ? err.message : String(err) };
+  }
+});
+
+// --- WebSocket ---
+interface ChatOptionsPayload {
+  think: boolean;
+  temperature: number;
+  numPredict: number;
+  provider?: string;
+  openrouterModel?: string;
+}
+
+function parseOptions(raw: unknown): OllamaOptions & {
+  provider: "ollama" | "openrouter";
+  openrouterModel: string;
+} {
+  const o = (raw ?? {}) as Partial<ChatOptionsPayload>;
+  return {
+    model: process.env.MODEL ?? "qwen3.5:latest",
+    think: o.think !== false,
+    temperature: clamp(Number(o.temperature ?? 0.7), 0, 2),
+    numPredict: Math.min(Math.max(Number(o.numPredict ?? 2048), 64), 16384),
+    provider: o.provider === "openrouter" ? "openrouter" : "ollama",
+    openrouterModel:
+      typeof o.openrouterModel === "string" &&
+      /^[\w./:-]{3,120}$/.test(o.openrouterModel)
+        ? o.openrouterModel
+        : "anthropic/claude-sonnet-4.5",
+  };
+}
+
+function clamp(v: number, min: number, max: number) {
+  if (!Number.isFinite(v)) return min;
+  return Math.min(Math.max(v, min), max);
+}
+
+const wss = new WebSocketServer({ noServer: true });
+
+function broadcast(data: unknown) {
+  for (const client of wss.clients) {
+    if (client.readyState === WebSocket.OPEN) client.send(JSON.stringify(data));
+  }
+}
+
+wss.on("connection", (socket: WebSocket, _req: IncomingMessage) => {
+  let activeAbort: AbortController | null = null;
+
+  socket.on("message", async (raw: Buffer) => {
+    let msg: Record<string, unknown>;
+    try {
+      msg = JSON.parse(raw.toString()) as Record<string, unknown>;
+    } catch {
+      socket.send(JSON.stringify({ type: "error", error: "invalid json" }));
+      return;
+    }
+
+    if (msg.type === "abort") {
+      activeAbort?.abort();
+      return;
+    }
+
+    if (msg.type !== "chat") return;
+
+    const sessionId = String(msg.sessionId ?? "");
+    const content = String(msg.content ?? "").trim();
+    if (!sessionId || !content) {
+      socket.send(JSON.stringify({ type: "error", error: "sessionId/content fehlt" }));
+      return;
+    }
+
+    let session = dbmod.getSession(sessionId);
+    if (!session) session = dbmod.createSession();
+    dbmod.renameSessionIfDefault(session.id, content);
+
+    const userMsg = dbmod.insertMessage(session.id, "user", content);
+    socket.send(JSON.stringify({ type: "user-message", message: userMsg }));
+    broadcast({ type: "sessions-changed" });
+
+    const opts = parseOptions(msg.options);
+    const history = dbmod
+      .listMessages(session.id)
+      .slice(-24)
+      .map((m) => ({ role: m.role, content: m.content }));
+
+    const assistantRow = dbmod.insertMessage(session.id, "assistant", "");
+    socket.send(JSON.stringify({ type: "assistant-start", message: assistantRow }));
+
+    activeAbort = new AbortController();
+    let full = "";
+    let thinking = "";
+    const callbacks = {
+      onThinking(text: string) {
+        thinking += text;
+        socket.send(
+          JSON.stringify({ type: "thinking", text, messageId: assistantRow.id }),
+        );
+      },
+      onToken(text: string) {
+        full += text;
+        socket.send(
+          JSON.stringify({ type: "token", text, messageId: assistantRow.id }),
+        );
+      },
+      onDone() {},
+    };
+
+    try {
+      if (opts.provider === "openrouter") {
+        const apiKey = getOpenRouterKey();
+        if (!apiKey) throw new Error("OPENROUTER_API_KEY nicht gesetzt (.env fehlt)");
+        await streamOpenRouter(
+          history,
+          typeof msg.systemPrompt === "string" && msg.systemPrompt.trim()
+            ? (msg.systemPrompt as string)
+            : undefined,
+          {
+            model: opts.openrouterModel,
+            temperature: opts.temperature,
+            numPredict: opts.numPredict,
+          },
+          apiKey,
+          callbacks,
+          activeAbort.signal,
+        );
+      } else {
+        await streamChat(
+          history,
+          typeof msg.systemPrompt === "string" && msg.systemPrompt.trim()
+            ? (msg.systemPrompt as string)
+            : undefined,
+          opts,
+          callbacks,
+          activeAbort.signal,
+        );
+      }
+      dbmod.updateAssistantMessage(assistantRow.id, full.trim(), thinking.trim() || null);
+      socket.send(
+        JSON.stringify({
+          type: "done",
+          messageId: assistantRow.id,
+          aborted: activeAbort.signal.aborted,
+        }),
+      );
+      broadcast({ type: "sessions-changed" });
+    } catch (err) {
+      const aborted =
+        activeAbort.signal.aborted ||
+        (err instanceof Error && err.name === "AbortError");
+      dbmod.updateAssistantMessage(assistantRow.id, full.trim(), thinking.trim() || null);
+      if (aborted) {
+        socket.send(JSON.stringify({ type: "done", messageId: assistantRow.id, aborted: true }));
+      } else {
+        socket.send(
+          JSON.stringify({
+            type: "error",
+            messageId: assistantRow.id,
+            error: err instanceof Error ? err.message : String(err),
+          }),
+        );
+      }
+    } finally {
+      activeAbort = null;
+    }
+  });
+});
+
+const server = app.server;
+server.on("upgrade", (req, socket, head) => {
+  const url = new URL(req.url ?? "", "http://localhost");
+  if (url.pathname === "/ws") {
+    wss.handleUpgrade(req, socket, head, (ws) => wss.emit("connection", ws, req));
+  } else {
+    socket.destroy();
+  }
+});
+
+app.listen({ port: PORT, host: "127.0.0.1" }, () => {
+  console.log(`[oxagenttwo] Server läuft auf http://127.0.0.1:${PORT}`);
+});

server/src/ollama.ts

@@ -0,0 +1,84 @@
+export interface OllamaOptions {
+  model: string;
+  think: boolean;
+  temperature: number;
+  numPredict: number;
+}
+
+export interface StreamCallbacks {
+  onThinking(text: string): void;
+  onToken(text: string): void;
+  onDone(): void;
+}
+
+interface ChatChunk {
+  message?: { content?: string; thinking?: string };
+  done?: boolean;
+  error?: string;
+}
+
+export async function streamChat(
+  history: { role: string; content: string }[],
+  systemPrompt: string | undefined,
+  opts: OllamaOptions,
+  cb: StreamCallbacks,
+  signal: AbortSignal,
+): Promise<void> {
+  const body: Record<string, unknown> = {
+    model: opts.model,
+    messages: [
+      ...(systemPrompt ? [{ role: "system", content: systemPrompt }] : []),
+      ...history,
+    ],
+    stream: true,
+    think: opts.think,
+    options: {
+      temperature: opts.temperature,
+      num_predict: opts.numPredict,
+    },
+  };
+
+  const res = await fetch("http://localhost:11434/api/chat", {
+    method: "POST",
+    headers: { "Content-Type": "application/json" },
+    body: JSON.stringify(body),
+    signal,
+  });
+
+  if (!res.ok || !res.body) {
+    const text = await res.text().catch(() => res.statusText);
+    throw new Error(`Ollama HTTP ${res.status}: ${text}`);
+  }
+
+  const reader = res.body.getReader();
+  const decoder = new TextDecoder();
+  let buffer = "";
+
+  while (true) {
+    const { value, done } = await reader.read();
+    if (done) break;
+    buffer += decoder.decode(value, { stream: true });
+
+    let nlIndex: number;
+    while ((nlIndex = buffer.indexOf("\n")) !== -1) {
+      const line = buffer.slice(0, nlIndex).trim();
+      buffer = buffer.slice(nlIndex + 1);
+      if (!line) continue;
+
+      let chunk: ChatChunk;
+      try {
+        chunk = JSON.parse(line) as ChatChunk;
+      } catch {
+        continue;
+      }
+      if (chunk.error) throw new Error(chunk.error);
+      if (chunk.message?.thinking) cb.onThinking(chunk.message.thinking);
+      if (chunk.message?.content) cb.onToken(chunk.message.content);
+      if (chunk.done) {
+        cb.onDone();
+        return;
+      }
+    }
+  }
+  cb.onDone();
+}

server/src/openrouter.ts

@@ -0,0 +1,116 @@
+export interface OpenRouterOptions {
+  model: string;
+  temperature: number;
+  numPredict: number;
+}
+
+export interface StreamCallbacks {
+  onThinking(text: string): void;
+  onToken(text: string): void;
+  onDone(): void;
+}
+
+export function getOpenRouterKey(): string | undefined {
+  return process.env.OPENROUTER_API_KEY;
+}
+
+export async function listOpenRouterModels(): Promise<
+  { id: string; name: string; contextLength: number; promptPrice: number }[]
+> {
+  const res = await fetch("https://openrouter.ai/api/v1/models");
+  if (!res.ok) throw new Error(`OpenRouter HTTP ${res.status}`);
+  const data = (await res.json()) as {
+    data: {
+      id: string;
+      name: string;
+      context_length: number;
+      pricing: { prompt: string };
+    }[];
+  };
+  return data.data
+    .map((m) => ({
+      id: m.id,
+      name: m.name,
+      contextLength: m.context_length,
+      promptPrice: Number(m.pricing?.prompt ?? 0) * 1_000_000,
+    }))
+    .sort((a, b) => a.name.localeCompare(b.name));
+}
+
+export async function streamOpenRouter(
+  history: { role: string; content: string }[],
+  systemPrompt: string | undefined,
+  opts: OpenRouterOptions,
+  apiKey: string,
+  cb: StreamCallbacks,
+  signal: AbortSignal,
+): Promise<void> {
+  const res = await fetch("https://openrouter.ai/api/v1/chat/completions", {
+    method: "POST",
+    headers: {
+      Authorization: `Bearer ${apiKey}`,
+      "Content-Type": "application/json",
+      "HTTP-Referer": "http://localhost:5173",
+      "X-Title": "oxagenttwo",
+    },
+    body: JSON.stringify({
+      model: opts.model,
+      stream: true,
+      messages: [
+        ...(systemPrompt ? [{ role: "system", content: systemPrompt }] : []),
+        ...history,
+      ],
+      temperature: opts.temperature,
+      max_tokens: opts.numPredict,
+    }),
+    signal,
+  });
+
+  if (!res.ok || !res.body) {
+    const text = await res.text().catch(() => res.statusText);
+    throw new Error(`OpenRouter HTTP ${res.status}: ${text.slice(0, 300)}`);
+  }
+
+  const reader = res.body.getReader();
+  const decoder = new TextDecoder();
+  let buffer = "";
+
+  while (true) {
+    const { value, done } = await reader.read();
+    if (done) break;
+    buffer += decoder.decode(value, { stream: true });
+
+    let nlIndex: number;
+    while ((nlIndex = buffer.indexOf("\n")) !== -1) {
+      const line = buffer.slice(0, nlIndex).trim();
+      buffer = buffer.slice(nlIndex + 1);
+      if (!line.startsWith("data:")) continue;
+      const payload = line.slice(5).trim();
+      if (payload === "[DONE]") {
+        cb.onDone();
+        return;
+      }
+      let chunk: {
+        choices?: {
+          delta?: { content?: string; reasoning?: string };
+          finish_reason?: string | null;
+        }[];
+        error?: { message?: string };
+      };
+      try {
+        chunk = JSON.parse(payload);
+      } catch {
+        continue;
+      }
+      if (chunk.error) throw new Error(chunk.error.message ?? "OpenRouter error");
+      const delta = chunk.choices?.[0]?.delta;
+      if (delta?.reasoning) cb.onThinking(delta.reasoning);
+      if (delta?.content) cb.onToken(delta.content);
+      if (chunk.choices?.[0]?.finish_reason && !delta?.content) {
+        cb.onDone();
+        return;
+      }
+    }
+  }
+  cb.onDone();
+}

server/src/voice.ts

@@ -0,0 +1,88 @@
+import { execFile, spawn } from "node:child_process";
+import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
+import { tmpdir, homedir } from "node:os";
+import path from "node:path";
+import { promisify } from "node:util";
+
+const pExecFile = promisify(execFile);
+
+const WHISPER_MODEL =
+  process.env.WHISPER_MODEL ??
+  path.join(homedir(), "whisper-models", "ggml-large-v3-turbo.bin");
+const PIPER_MODEL =
+  process.env.PIPER_MODEL ??
+  path.join(import.meta.dirname, "..", "voices", "de_DE-thorsten-high.onnx");
+const WHISPER_LANG = process.env.WHISPER_LANG ?? "de";
+
+export async function transcribeAudio(input: Buffer): Promise<string> {
+  const dir = await mkdtemp(path.join(tmpdir(), "oxa-voice-"));
+  const rawPath = path.join(dir, "in.raw");
+  const wavPath = path.join(dir, "in.wav");
+  try {
+    await writeFile(rawPath, input);
+    await pExecFile(
+      "ffmpeg",
+      ["-y", "-i", rawPath, "-ar", "16000", "-ac", "1", "-c:a", "pcm_s16le", wavPath],
+      { timeout: 30_000 },
+    );
+    const { stdout } = await pExecFile(
+      "whisper-cli",
+      ["-m", WHISPER_MODEL, "-f", wavPath, "-nt", "-np", "-l", WHISPER_LANG],
+      { timeout: 180_000, maxBuffer: 16 * 1024 * 1024 },
+    );
+    return stdout
+      .split("\n")
+      .map((l) => l.trim())
+      .filter(Boolean)
+      .join(" ")
+      .replace(/\s+/g, " ")
+      .trim();
+  } finally {
+    await rm(dir, { recursive: true, force: true });
+  }
+}
+
+export function stripMarkdownForSpeech(text: string): string {
+  return text
+    .replace(/```[\s\S]*?```/g, " ")
+    .replace(/`([^`]+)`/g, "$1")
+    .replace(/!\[[^\]]*\]\([^)]*\)/g, " ")
+    .replace(/\[([^\]]+)\]\([^)]*\)/g, "$1")
+    .replace(/^\s{0,3}#{1,6}\s+/gm, "")
+    .replace(/(\*\*|__|\*|_|~~)/g, "")
+    .replace(/^\s*[-*+]\s+/gm, "")
+    .replace(/^\s*\|.*\|\s*$/gm, "")
+    .replace(/https?:\/\/\S+/g, "Link")
+    .replace(/\s+/g, " ")
+    .trim();
+}
+
+export async function synthesizeSpeech(text: string): Promise<Buffer> {
+  const cleaned = stripMarkdownForSpeech(text);
+  if (!cleaned) throw new Error("Kein sprechbarer Text");
+
+  const dir = await mkdtemp(path.join(tmpdir(), "oxa-tts-"));
+  const wavPath = path.join(dir, "out.wav");
+  try {
+    await new Promise<void>((resolve, reject) => {
+      const proc = spawn(
+        "python3",
+        ["-m", "piper", "-m", PIPER_MODEL, "-f", wavPath, "--sentence-silence", "0.2"],
+        { stdio: ["pipe", "ignore", "pipe"] },
+      );
+      let err = "";
+      proc.stderr.on("data", (d) => (err += d.toString()));
+      proc.on("error", reject);
+      proc.on("close", (code) =>
+        code === 0
+          ? resolve()
+          : reject(new Error(`piper exit ${code}: ${err.slice(-400)}`)),
+      );
+      proc.stdin.write(cleaned);
+      proc.stdin.end();
+    });
+    return await readFile(wavPath);
+  } finally {
+    await rm(dir, { recursive: true, force: true });
+  }
+}