mirror of
https://github.com/Jeuners/agenttwo.git
synced 2026-09-09 15:02:31 +02:00
init: oxagenttwo — voice chat with local qwen3 (Ollama) + OpenRouter, Whisper STT, Piper TTS (Thorsten), sessions, thinking mode
This commit is contained in:
Jeuner
commit
752198c5fa
26 changed files with 7690 additions and 0 deletions
22
server/package.json
Normal file
22
server/package.json
Normal file
|
|
@ -0,0 +1,22 @@
|
|||
{
|
||||
"name": "server",
|
||||
"private": true,
|
||||
"version": "0.1.0",
|
||||
"type": "module",
|
||||
"scripts": {
|
||||
"dev": "tsx watch src/index.ts",
|
||||
"start": "tsx src/index.ts",
|
||||
"typecheck": "tsc --noEmit"
|
||||
},
|
||||
"dependencies": {
|
||||
"@fastify/cors": "^10.0.1",
|
||||
"fastify": "^5.2.1"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/node": "^22.13.0",
|
||||
"@types/ws": "^8.5.14",
|
||||
"tsx": "^4.19.3",
|
||||
"typescript": "^5.7.3",
|
||||
"ws": "^8.18.0"
|
||||
}
|
||||
}
|
||||
117
server/src/db.ts
Normal file
117
server/src/db.ts
Normal file
|
|
@ -0,0 +1,117 @@
|
|||
import { DatabaseSync } from "node:sqlite";
|
||||
import { randomUUID } from "node:crypto";
|
||||
import path from "node:path";
|
||||
|
||||
const dbPath = path.join(import.meta.dirname, "..", "data.sqlite");
|
||||
export const db = new DatabaseSync(dbPath);
|
||||
|
||||
db.exec(`
|
||||
PRAGMA journal_mode = WAL;
|
||||
CREATE TABLE IF NOT EXISTS sessions (
|
||||
id TEXT PRIMARY KEY,
|
||||
title TEXT NOT NULL DEFAULT 'Neuer Chat',
|
||||
created_at INTEGER NOT NULL
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS messages (
|
||||
id TEXT PRIMARY KEY,
|
||||
session_id TEXT NOT NULL REFERENCES sessions(id) ON DELETE CASCADE,
|
||||
role TEXT NOT NULL CHECK(role IN ('user','assistant','system')),
|
||||
content TEXT NOT NULL DEFAULT '',
|
||||
thinking TEXT,
|
||||
created_at INTEGER NOT NULL
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_messages_session ON messages(session_id, created_at);
|
||||
`);
|
||||
|
||||
export interface SessionRow {
|
||||
id: string;
|
||||
title: string;
|
||||
created_at: number;
|
||||
}
|
||||
|
||||
export interface MessageRow {
|
||||
id: string;
|
||||
session_id: string;
|
||||
role: string;
|
||||
content: string;
|
||||
thinking: string | null;
|
||||
created_at: number;
|
||||
}
|
||||
|
||||
export function listSessions(): SessionRow[] {
|
||||
return db
|
||||
.prepare("SELECT * FROM sessions ORDER BY created_at DESC")
|
||||
.all() as unknown as SessionRow[];
|
||||
}
|
||||
|
||||
export function createSession(title = "Neuer Chat"): SessionRow {
|
||||
const row: SessionRow = { id: randomUUID(), title, created_at: Date.now() };
|
||||
db.prepare("INSERT INTO sessions (id, title, created_at) VALUES (?, ?, ?)").run(
|
||||
row.id,
|
||||
row.title,
|
||||
row.created_at,
|
||||
);
|
||||
return row;
|
||||
}
|
||||
|
||||
export function getSession(id: string): SessionRow | undefined {
|
||||
return db.prepare("SELECT * FROM sessions WHERE id = ?").get(id) as
|
||||
| SessionRow
|
||||
| undefined;
|
||||
}
|
||||
|
||||
export function renameSessionIfDefault(id: string, firstMessage: string) {
|
||||
const s = getSession(id);
|
||||
if (!s || s.title !== "Neuer Chat") return;
|
||||
const title =
|
||||
firstMessage.trim().slice(0, 60) + (firstMessage.length > 60 ? "…" : "");
|
||||
db.prepare("UPDATE sessions SET title = ? WHERE id = ?").run(
|
||||
title || "Neuer Chat",
|
||||
id,
|
||||
);
|
||||
}
|
||||
|
||||
export function deleteSession(id: string) {
|
||||
db.prepare("DELETE FROM messages WHERE session_id = ?").run(id);
|
||||
db.prepare("DELETE FROM sessions WHERE id = ?").run(id);
|
||||
}
|
||||
|
||||
export function insertMessage(
|
||||
sessionId: string,
|
||||
role: string,
|
||||
content: string,
|
||||
thinking?: string | null,
|
||||
): MessageRow {
|
||||
const row: MessageRow = {
|
||||
id: randomUUID(),
|
||||
session_id: sessionId,
|
||||
role,
|
||||
content,
|
||||
thinking: thinking ?? null,
|
||||
created_at: Date.now(),
|
||||
};
|
||||
db.prepare(
|
||||
"INSERT INTO messages (id, session_id, role, content, thinking, created_at) VALUES (?, ?, ?, ?, ?, ?)",
|
||||
).run(row.id, row.session_id, row.role, row.content, row.thinking, row.created_at);
|
||||
return row;
|
||||
}
|
||||
|
||||
export function updateAssistantMessage(
|
||||
id: string,
|
||||
content: string,
|
||||
thinking: string | null,
|
||||
) {
|
||||
db.prepare("UPDATE messages SET content = ?, thinking = ? WHERE id = ?").run(
|
||||
content,
|
||||
thinking,
|
||||
id,
|
||||
);
|
||||
}
|
||||
|
||||
export function listMessages(sessionId: string): MessageRow[] {
|
||||
return db
|
||||
.prepare(
|
||||
"SELECT * FROM messages WHERE session_id = ? ORDER BY created_at ASC",
|
||||
)
|
||||
.all(sessionId) as unknown as MessageRow[];
|
||||
}
|
||||
302
server/src/index.ts
Normal file
302
server/src/index.ts
Normal file
|
|
@ -0,0 +1,302 @@
|
|||
import Fastify from "fastify";
|
||||
import cors from "@fastify/cors";
|
||||
import { WebSocketServer, WebSocket } from "ws";
|
||||
import type { IncomingMessage } from "node:http";
|
||||
import { readFileSync } from "node:fs";
|
||||
import path from "node:path";
|
||||
import * as dbmod from "./db.js";
|
||||
import { streamChat, type OllamaOptions } from "./ollama.js";
|
||||
import { transcribeAudio, synthesizeSpeech } from "./voice.js";
|
||||
import {
|
||||
streamOpenRouter,
|
||||
listOpenRouterModels,
|
||||
getOpenRouterKey,
|
||||
} from "./openrouter.js";
|
||||
|
||||
// simple .env loader (project root)
|
||||
try {
|
||||
for (const line of readFileSync(
|
||||
path.join(import.meta.dirname, "..", "..", ".env"),
|
||||
"utf8",
|
||||
).split("\n")) {
|
||||
const m = line.match(/^\s*([A-Za-z_][A-Za-z0-9_]*)\s*=\s*(.*)\s*$/);
|
||||
if (m && !(m[1] in process.env)) process.env[m[1]] = m[2].replace(/^["']|["']$/g, "");
|
||||
}
|
||||
} catch {
|
||||
/* keine .env vorhanden */
|
||||
}
|
||||
|
||||
const PORT = Number(process.env.PORT ?? 8787);
|
||||
const OLLAMA_URL = process.env.OLLAMA_URL ?? "http://localhost:11434";
|
||||
|
||||
const app = Fastify({ logger: false, bodyLimit: 32 * 1024 * 1024 });
|
||||
await app.register(cors, { origin: true });
|
||||
|
||||
app.addContentTypeParser(
|
||||
["application/octet-stream", "audio/*", "video/*"],
|
||||
{ parseAs: "buffer" },
|
||||
(_req, body, done) => done(null, body),
|
||||
);
|
||||
|
||||
app.get("/api/health", async () => ({ ok: true }));
|
||||
|
||||
app.get("/api/model", async () => {
|
||||
try {
|
||||
const res = await fetch(`${OLLAMA_URL}/api/show`, {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify({ name: process.env.MODEL ?? "qwen3.5:latest" }),
|
||||
});
|
||||
if (!res.ok) return { ok: false, error: `Ollama HTTP ${res.status}` };
|
||||
const data = (await res.json()) as {
|
||||
details?: { parameter_size?: string; quantization_level?: string };
|
||||
capabilities?: string[];
|
||||
};
|
||||
return {
|
||||
ok: true,
|
||||
model: process.env.MODEL ?? "qwen3.5:latest",
|
||||
parameterSize: data.details?.parameter_size,
|
||||
quantization: data.details?.quantization_level,
|
||||
capabilities: data.capabilities ?? [],
|
||||
};
|
||||
} catch (err) {
|
||||
return { ok: false, error: String(err) };
|
||||
}
|
||||
});
|
||||
|
||||
// --- Sessions REST ---
|
||||
app.get("/api/sessions", async () => dbmod.listSessions());
|
||||
app.post("/api/sessions", async () => dbmod.createSession());
|
||||
app.get("/api/sessions/:id/messages", async (req, reply) => {
|
||||
const { id } = req.params as { id: string };
|
||||
if (!dbmod.getSession(id)) return reply.code(404).send({ error: "not found" });
|
||||
return dbmod.listMessages(id);
|
||||
});
|
||||
app.delete("/api/sessions/:id", async (req) => {
|
||||
const { id } = req.params as { id: string };
|
||||
dbmod.deleteSession(id);
|
||||
broadcast({ type: "session-deleted", id });
|
||||
return { ok: true };
|
||||
});
|
||||
|
||||
// --- Voice ---
|
||||
app.post("/api/stt", async (req, reply) => {
|
||||
const audio = req.body as Buffer;
|
||||
if (!Buffer.isBuffer(audio) || audio.length === 0) {
|
||||
return reply.code(400).send({ ok: false, error: "Kein Audio empfangen" });
|
||||
}
|
||||
try {
|
||||
const text = await transcribeAudio(audio);
|
||||
return { ok: true, text };
|
||||
} catch (err) {
|
||||
app.log.error(err);
|
||||
return reply
|
||||
.code(500)
|
||||
.send({ ok: false, error: err instanceof Error ? err.message : String(err) });
|
||||
}
|
||||
});
|
||||
|
||||
app.post("/api/tts", async (req, reply) => {
|
||||
const { text } = (req.body ?? {}) as { text?: string };
|
||||
if (!text || !text.trim()) {
|
||||
return reply.code(400).send({ error: "text fehlt" });
|
||||
}
|
||||
try {
|
||||
const wav = await synthesizeSpeech(text);
|
||||
reply.header("Content-Type", "audio/wav").send(wav);
|
||||
} catch (err) {
|
||||
app.log.error(err);
|
||||
return reply
|
||||
.code(500)
|
||||
.send({ error: err instanceof Error ? err.message : String(err) });
|
||||
}
|
||||
});
|
||||
|
||||
// --- OpenRouter models ---
|
||||
app.get("/api/openrouter/models", async () => {
|
||||
if (!getOpenRouterKey()) {
|
||||
return { ok: false, error: "OPENROUTER_API_KEY nicht gesetzt (.env fehlt)" };
|
||||
}
|
||||
try {
|
||||
return { ok: true, models: await listOpenRouterModels() };
|
||||
} catch (err) {
|
||||
return { ok: false, error: err instanceof Error ? err.message : String(err) };
|
||||
}
|
||||
});
|
||||
|
||||
// --- WebSocket ---
|
||||
interface ChatOptionsPayload {
|
||||
think: boolean;
|
||||
temperature: number;
|
||||
numPredict: number;
|
||||
provider?: string;
|
||||
openrouterModel?: string;
|
||||
}
|
||||
|
||||
function parseOptions(raw: unknown): OllamaOptions & {
|
||||
provider: "ollama" | "openrouter";
|
||||
openrouterModel: string;
|
||||
} {
|
||||
const o = (raw ?? {}) as Partial<ChatOptionsPayload>;
|
||||
return {
|
||||
model: process.env.MODEL ?? "qwen3.5:latest",
|
||||
think: o.think !== false,
|
||||
temperature: clamp(Number(o.temperature ?? 0.7), 0, 2),
|
||||
numPredict: Math.min(Math.max(Number(o.numPredict ?? 2048), 64), 16384),
|
||||
provider: o.provider === "openrouter" ? "openrouter" : "ollama",
|
||||
openrouterModel:
|
||||
typeof o.openrouterModel === "string" &&
|
||||
/^[\w./:-]{3,120}$/.test(o.openrouterModel)
|
||||
? o.openrouterModel
|
||||
: "anthropic/claude-sonnet-4.5",
|
||||
};
|
||||
}
|
||||
|
||||
function clamp(v: number, min: number, max: number) {
|
||||
if (!Number.isFinite(v)) return min;
|
||||
return Math.min(Math.max(v, min), max);
|
||||
}
|
||||
|
||||
const wss = new WebSocketServer({ noServer: true });
|
||||
|
||||
function broadcast(data: unknown) {
|
||||
for (const client of wss.clients) {
|
||||
if (client.readyState === WebSocket.OPEN) client.send(JSON.stringify(data));
|
||||
}
|
||||
}
|
||||
|
||||
wss.on("connection", (socket: WebSocket, _req: IncomingMessage) => {
|
||||
let activeAbort: AbortController | null = null;
|
||||
|
||||
socket.on("message", async (raw: Buffer) => {
|
||||
let msg: Record<string, unknown>;
|
||||
try {
|
||||
msg = JSON.parse(raw.toString()) as Record<string, unknown>;
|
||||
} catch {
|
||||
socket.send(JSON.stringify({ type: "error", error: "invalid json" }));
|
||||
return;
|
||||
}
|
||||
|
||||
if (msg.type === "abort") {
|
||||
activeAbort?.abort();
|
||||
return;
|
||||
}
|
||||
|
||||
if (msg.type !== "chat") return;
|
||||
|
||||
const sessionId = String(msg.sessionId ?? "");
|
||||
const content = String(msg.content ?? "").trim();
|
||||
if (!sessionId || !content) {
|
||||
socket.send(JSON.stringify({ type: "error", error: "sessionId/content fehlt" }));
|
||||
return;
|
||||
}
|
||||
|
||||
let session = dbmod.getSession(sessionId);
|
||||
if (!session) session = dbmod.createSession();
|
||||
dbmod.renameSessionIfDefault(session.id, content);
|
||||
|
||||
const userMsg = dbmod.insertMessage(session.id, "user", content);
|
||||
socket.send(JSON.stringify({ type: "user-message", message: userMsg }));
|
||||
broadcast({ type: "sessions-changed" });
|
||||
|
||||
const opts = parseOptions(msg.options);
|
||||
const history = dbmod
|
||||
.listMessages(session.id)
|
||||
.slice(-24)
|
||||
.map((m) => ({ role: m.role, content: m.content }));
|
||||
|
||||
const assistantRow = dbmod.insertMessage(session.id, "assistant", "");
|
||||
socket.send(JSON.stringify({ type: "assistant-start", message: assistantRow }));
|
||||
|
||||
activeAbort = new AbortController();
|
||||
let full = "";
|
||||
let thinking = "";
|
||||
const callbacks = {
|
||||
onThinking(text: string) {
|
||||
thinking += text;
|
||||
socket.send(
|
||||
JSON.stringify({ type: "thinking", text, messageId: assistantRow.id }),
|
||||
);
|
||||
},
|
||||
onToken(text: string) {
|
||||
full += text;
|
||||
socket.send(
|
||||
JSON.stringify({ type: "token", text, messageId: assistantRow.id }),
|
||||
);
|
||||
},
|
||||
onDone() {},
|
||||
};
|
||||
|
||||
try {
|
||||
if (opts.provider === "openrouter") {
|
||||
const apiKey = getOpenRouterKey();
|
||||
if (!apiKey) throw new Error("OPENROUTER_API_KEY nicht gesetzt (.env fehlt)");
|
||||
await streamOpenRouter(
|
||||
history,
|
||||
typeof msg.systemPrompt === "string" && msg.systemPrompt.trim()
|
||||
? (msg.systemPrompt as string)
|
||||
: undefined,
|
||||
{
|
||||
model: opts.openrouterModel,
|
||||
temperature: opts.temperature,
|
||||
numPredict: opts.numPredict,
|
||||
},
|
||||
apiKey,
|
||||
callbacks,
|
||||
activeAbort.signal,
|
||||
);
|
||||
} else {
|
||||
await streamChat(
|
||||
history,
|
||||
typeof msg.systemPrompt === "string" && msg.systemPrompt.trim()
|
||||
? (msg.systemPrompt as string)
|
||||
: undefined,
|
||||
opts,
|
||||
callbacks,
|
||||
activeAbort.signal,
|
||||
);
|
||||
}
|
||||
dbmod.updateAssistantMessage(assistantRow.id, full.trim(), thinking.trim() || null);
|
||||
socket.send(
|
||||
JSON.stringify({
|
||||
type: "done",
|
||||
messageId: assistantRow.id,
|
||||
aborted: activeAbort.signal.aborted,
|
||||
}),
|
||||
);
|
||||
broadcast({ type: "sessions-changed" });
|
||||
} catch (err) {
|
||||
const aborted =
|
||||
activeAbort.signal.aborted ||
|
||||
(err instanceof Error && err.name === "AbortError");
|
||||
dbmod.updateAssistantMessage(assistantRow.id, full.trim(), thinking.trim() || null);
|
||||
if (aborted) {
|
||||
socket.send(JSON.stringify({ type: "done", messageId: assistantRow.id, aborted: true }));
|
||||
} else {
|
||||
socket.send(
|
||||
JSON.stringify({
|
||||
type: "error",
|
||||
messageId: assistantRow.id,
|
||||
error: err instanceof Error ? err.message : String(err),
|
||||
}),
|
||||
);
|
||||
}
|
||||
} finally {
|
||||
activeAbort = null;
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
const server = app.server;
|
||||
server.on("upgrade", (req, socket, head) => {
|
||||
const url = new URL(req.url ?? "", "http://localhost");
|
||||
if (url.pathname === "/ws") {
|
||||
wss.handleUpgrade(req, socket, head, (ws) => wss.emit("connection", ws, req));
|
||||
} else {
|
||||
socket.destroy();
|
||||
}
|
||||
});
|
||||
|
||||
app.listen({ port: PORT, host: "127.0.0.1" }, () => {
|
||||
console.log(`[oxagenttwo] Server läuft auf http://127.0.0.1:${PORT}`);
|
||||
});
|
||||
84
server/src/ollama.ts
Normal file
84
server/src/ollama.ts
Normal file
|
|
@ -0,0 +1,84 @@
|
|||
export interface OllamaOptions {
|
||||
model: string;
|
||||
think: boolean;
|
||||
temperature: number;
|
||||
numPredict: number;
|
||||
}
|
||||
|
||||
export interface StreamCallbacks {
|
||||
onThinking(text: string): void;
|
||||
onToken(text: string): void;
|
||||
onDone(): void;
|
||||
}
|
||||
|
||||
interface ChatChunk {
|
||||
message?: { content?: string; thinking?: string };
|
||||
done?: boolean;
|
||||
error?: string;
|
||||
}
|
||||
|
||||
export async function streamChat(
|
||||
history: { role: string; content: string }[],
|
||||
systemPrompt: string | undefined,
|
||||
opts: OllamaOptions,
|
||||
cb: StreamCallbacks,
|
||||
signal: AbortSignal,
|
||||
): Promise<void> {
|
||||
const body: Record<string, unknown> = {
|
||||
model: opts.model,
|
||||
messages: [
|
||||
...(systemPrompt ? [{ role: "system", content: systemPrompt }] : []),
|
||||
...history,
|
||||
],
|
||||
stream: true,
|
||||
think: opts.think,
|
||||
options: {
|
||||
temperature: opts.temperature,
|
||||
num_predict: opts.numPredict,
|
||||
},
|
||||
};
|
||||
|
||||
const res = await fetch("http://localhost:11434/api/chat", {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify(body),
|
||||
signal,
|
||||
});
|
||||
|
||||
if (!res.ok || !res.body) {
|
||||
const text = await res.text().catch(() => res.statusText);
|
||||
throw new Error(`Ollama HTTP ${res.status}: ${text}`);
|
||||
}
|
||||
|
||||
const reader = res.body.getReader();
|
||||
const decoder = new TextDecoder();
|
||||
let buffer = "";
|
||||
|
||||
while (true) {
|
||||
const { value, done } = await reader.read();
|
||||
if (done) break;
|
||||
buffer += decoder.decode(value, { stream: true });
|
||||
|
||||
let nlIndex: number;
|
||||
while ((nlIndex = buffer.indexOf("\n")) !== -1) {
|
||||
const line = buffer.slice(0, nlIndex).trim();
|
||||
buffer = buffer.slice(nlIndex + 1);
|
||||
if (!line) continue;
|
||||
|
||||
let chunk: ChatChunk;
|
||||
try {
|
||||
chunk = JSON.parse(line) as ChatChunk;
|
||||
} catch {
|
||||
continue;
|
||||
}
|
||||
if (chunk.error) throw new Error(chunk.error);
|
||||
if (chunk.message?.thinking) cb.onThinking(chunk.message.thinking);
|
||||
if (chunk.message?.content) cb.onToken(chunk.message.content);
|
||||
if (chunk.done) {
|
||||
cb.onDone();
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
cb.onDone();
|
||||
}
|
||||
116
server/src/openrouter.ts
Normal file
116
server/src/openrouter.ts
Normal file
|
|
@ -0,0 +1,116 @@
|
|||
export interface OpenRouterOptions {
|
||||
model: string;
|
||||
temperature: number;
|
||||
numPredict: number;
|
||||
}
|
||||
|
||||
export interface StreamCallbacks {
|
||||
onThinking(text: string): void;
|
||||
onToken(text: string): void;
|
||||
onDone(): void;
|
||||
}
|
||||
|
||||
export function getOpenRouterKey(): string | undefined {
|
||||
return process.env.OPENROUTER_API_KEY;
|
||||
}
|
||||
|
||||
export async function listOpenRouterModels(): Promise<
|
||||
{ id: string; name: string; contextLength: number; promptPrice: number }[]
|
||||
> {
|
||||
const res = await fetch("https://openrouter.ai/api/v1/models");
|
||||
if (!res.ok) throw new Error(`OpenRouter HTTP ${res.status}`);
|
||||
const data = (await res.json()) as {
|
||||
data: {
|
||||
id: string;
|
||||
name: string;
|
||||
context_length: number;
|
||||
pricing: { prompt: string };
|
||||
}[];
|
||||
};
|
||||
return data.data
|
||||
.map((m) => ({
|
||||
id: m.id,
|
||||
name: m.name,
|
||||
contextLength: m.context_length,
|
||||
promptPrice: Number(m.pricing?.prompt ?? 0) * 1_000_000,
|
||||
}))
|
||||
.sort((a, b) => a.name.localeCompare(b.name));
|
||||
}
|
||||
|
||||
export async function streamOpenRouter(
|
||||
history: { role: string; content: string }[],
|
||||
systemPrompt: string | undefined,
|
||||
opts: OpenRouterOptions,
|
||||
apiKey: string,
|
||||
cb: StreamCallbacks,
|
||||
signal: AbortSignal,
|
||||
): Promise<void> {
|
||||
const res = await fetch("https://openrouter.ai/api/v1/chat/completions", {
|
||||
method: "POST",
|
||||
headers: {
|
||||
Authorization: `Bearer ${apiKey}`,
|
||||
"Content-Type": "application/json",
|
||||
"HTTP-Referer": "http://localhost:5173",
|
||||
"X-Title": "oxagenttwo",
|
||||
},
|
||||
body: JSON.stringify({
|
||||
model: opts.model,
|
||||
stream: true,
|
||||
messages: [
|
||||
...(systemPrompt ? [{ role: "system", content: systemPrompt }] : []),
|
||||
...history,
|
||||
],
|
||||
temperature: opts.temperature,
|
||||
max_tokens: opts.numPredict,
|
||||
}),
|
||||
signal,
|
||||
});
|
||||
|
||||
if (!res.ok || !res.body) {
|
||||
const text = await res.text().catch(() => res.statusText);
|
||||
throw new Error(`OpenRouter HTTP ${res.status}: ${text.slice(0, 300)}`);
|
||||
}
|
||||
|
||||
const reader = res.body.getReader();
|
||||
const decoder = new TextDecoder();
|
||||
let buffer = "";
|
||||
|
||||
while (true) {
|
||||
const { value, done } = await reader.read();
|
||||
if (done) break;
|
||||
buffer += decoder.decode(value, { stream: true });
|
||||
|
||||
let nlIndex: number;
|
||||
while ((nlIndex = buffer.indexOf("\n")) !== -1) {
|
||||
const line = buffer.slice(0, nlIndex).trim();
|
||||
buffer = buffer.slice(nlIndex + 1);
|
||||
if (!line.startsWith("data:")) continue;
|
||||
const payload = line.slice(5).trim();
|
||||
if (payload === "[DONE]") {
|
||||
cb.onDone();
|
||||
return;
|
||||
}
|
||||
let chunk: {
|
||||
choices?: {
|
||||
delta?: { content?: string; reasoning?: string };
|
||||
finish_reason?: string | null;
|
||||
}[];
|
||||
error?: { message?: string };
|
||||
};
|
||||
try {
|
||||
chunk = JSON.parse(payload);
|
||||
} catch {
|
||||
continue;
|
||||
}
|
||||
if (chunk.error) throw new Error(chunk.error.message ?? "OpenRouter error");
|
||||
const delta = chunk.choices?.[0]?.delta;
|
||||
if (delta?.reasoning) cb.onThinking(delta.reasoning);
|
||||
if (delta?.content) cb.onToken(delta.content);
|
||||
if (chunk.choices?.[0]?.finish_reason && !delta?.content) {
|
||||
cb.onDone();
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
cb.onDone();
|
||||
}
|
||||
88
server/src/voice.ts
Normal file
88
server/src/voice.ts
Normal file
|
|
@ -0,0 +1,88 @@
|
|||
import { execFile, spawn } from "node:child_process";
|
||||
import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
|
||||
import { tmpdir, homedir } from "node:os";
|
||||
import path from "node:path";
|
||||
import { promisify } from "node:util";
|
||||
|
||||
const pExecFile = promisify(execFile);
|
||||
|
||||
const WHISPER_MODEL =
|
||||
process.env.WHISPER_MODEL ??
|
||||
path.join(homedir(), "whisper-models", "ggml-large-v3-turbo.bin");
|
||||
const PIPER_MODEL =
|
||||
process.env.PIPER_MODEL ??
|
||||
path.join(import.meta.dirname, "..", "voices", "de_DE-thorsten-high.onnx");
|
||||
const WHISPER_LANG = process.env.WHISPER_LANG ?? "de";
|
||||
|
||||
export async function transcribeAudio(input: Buffer): Promise<string> {
|
||||
const dir = await mkdtemp(path.join(tmpdir(), "oxa-voice-"));
|
||||
const rawPath = path.join(dir, "in.raw");
|
||||
const wavPath = path.join(dir, "in.wav");
|
||||
try {
|
||||
await writeFile(rawPath, input);
|
||||
await pExecFile(
|
||||
"ffmpeg",
|
||||
["-y", "-i", rawPath, "-ar", "16000", "-ac", "1", "-c:a", "pcm_s16le", wavPath],
|
||||
{ timeout: 30_000 },
|
||||
);
|
||||
const { stdout } = await pExecFile(
|
||||
"whisper-cli",
|
||||
["-m", WHISPER_MODEL, "-f", wavPath, "-nt", "-np", "-l", WHISPER_LANG],
|
||||
{ timeout: 180_000, maxBuffer: 16 * 1024 * 1024 },
|
||||
);
|
||||
return stdout
|
||||
.split("\n")
|
||||
.map((l) => l.trim())
|
||||
.filter(Boolean)
|
||||
.join(" ")
|
||||
.replace(/\s+/g, " ")
|
||||
.trim();
|
||||
} finally {
|
||||
await rm(dir, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
|
||||
export function stripMarkdownForSpeech(text: string): string {
|
||||
return text
|
||||
.replace(/```[\s\S]*?```/g, " ")
|
||||
.replace(/`([^`]+)`/g, "$1")
|
||||
.replace(/!\[[^\]]*\]\([^)]*\)/g, " ")
|
||||
.replace(/\[([^\]]+)\]\([^)]*\)/g, "$1")
|
||||
.replace(/^\s{0,3}#{1,6}\s+/gm, "")
|
||||
.replace(/(\*\*|__|\*|_|~~)/g, "")
|
||||
.replace(/^\s*[-*+]\s+/gm, "")
|
||||
.replace(/^\s*\|.*\|\s*$/gm, "")
|
||||
.replace(/https?:\/\/\S+/g, "Link")
|
||||
.replace(/\s+/g, " ")
|
||||
.trim();
|
||||
}
|
||||
|
||||
export async function synthesizeSpeech(text: string): Promise<Buffer> {
|
||||
const cleaned = stripMarkdownForSpeech(text);
|
||||
if (!cleaned) throw new Error("Kein sprechbarer Text");
|
||||
|
||||
const dir = await mkdtemp(path.join(tmpdir(), "oxa-tts-"));
|
||||
const wavPath = path.join(dir, "out.wav");
|
||||
try {
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
const proc = spawn(
|
||||
"python3",
|
||||
["-m", "piper", "-m", PIPER_MODEL, "-f", wavPath, "--sentence-silence", "0.2"],
|
||||
{ stdio: ["pipe", "ignore", "pipe"] },
|
||||
);
|
||||
let err = "";
|
||||
proc.stderr.on("data", (d) => (err += d.toString()));
|
||||
proc.on("error", reject);
|
||||
proc.on("close", (code) =>
|
||||
code === 0
|
||||
? resolve()
|
||||
: reject(new Error(`piper exit ${code}: ${err.slice(-400)}`)),
|
||||
);
|
||||
proc.stdin.write(cleaned);
|
||||
proc.stdin.end();
|
||||
});
|
||||
return await readFile(wavPath);
|
||||
} finally {
|
||||
await rm(dir, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
12
server/tsconfig.json
Normal file
12
server/tsconfig.json
Normal file
|
|
@ -0,0 +1,12 @@
|
|||
{
|
||||
"compilerOptions": {
|
||||
"target": "ES2023",
|
||||
"module": "NodeNext",
|
||||
"moduleResolution": "NodeNext",
|
||||
"strict": true,
|
||||
"skipLibCheck": true,
|
||||
"noEmit": true,
|
||||
"types": ["node"]
|
||||
},
|
||||
"include": ["src"]
|
||||
}
|
||||
493
server/voices/de_DE-thorsten-high.onnx.json
Normal file
493
server/voices/de_DE-thorsten-high.onnx.json
Normal file
|
|
@ -0,0 +1,493 @@
|
|||
{
|
||||
"audio": {
|
||||
"sample_rate": 22050,
|
||||
"quality": "high"
|
||||
},
|
||||
"espeak": {
|
||||
"voice": "de"
|
||||
},
|
||||
"inference": {
|
||||
"noise_scale": 0.667,
|
||||
"length_scale": 1,
|
||||
"noise_w": 0.8
|
||||
},
|
||||
"phoneme_type": "espeak",
|
||||
"phoneme_map": {},
|
||||
"phoneme_id_map": {
|
||||
"_": [
|
||||
0
|
||||
],
|
||||
"^": [
|
||||
1
|
||||
],
|
||||
"$": [
|
||||
2
|
||||
],
|
||||
" ": [
|
||||
3
|
||||
],
|
||||
"!": [
|
||||
4
|
||||
],
|
||||
"'": [
|
||||
5
|
||||
],
|
||||
"(": [
|
||||
6
|
||||
],
|
||||
")": [
|
||||
7
|
||||
],
|
||||
",": [
|
||||
8
|
||||
],
|
||||
"-": [
|
||||
9
|
||||
],
|
||||
".": [
|
||||
10
|
||||
],
|
||||
":": [
|
||||
11
|
||||
],
|
||||
";": [
|
||||
12
|
||||
],
|
||||
"?": [
|
||||
13
|
||||
],
|
||||
"a": [
|
||||
14
|
||||
],
|
||||
"b": [
|
||||
15
|
||||
],
|
||||
"c": [
|
||||
16
|
||||
],
|
||||
"d": [
|
||||
17
|
||||
],
|
||||
"e": [
|
||||
18
|
||||
],
|
||||
"f": [
|
||||
19
|
||||
],
|
||||
"h": [
|
||||
20
|
||||
],
|
||||
"i": [
|
||||
21
|
||||
],
|
||||
"j": [
|
||||
22
|
||||
],
|
||||
"k": [
|
||||
23
|
||||
],
|
||||
"l": [
|
||||
24
|
||||
],
|
||||
"m": [
|
||||
25
|
||||
],
|
||||
"n": [
|
||||
26
|
||||
],
|
||||
"o": [
|
||||
27
|
||||
],
|
||||
"p": [
|
||||
28
|
||||
],
|
||||
"q": [
|
||||
29
|
||||
],
|
||||
"r": [
|
||||
30
|
||||
],
|
||||
"s": [
|
||||
31
|
||||
],
|
||||
"t": [
|
||||
32
|
||||
],
|
||||
"u": [
|
||||
33
|
||||
],
|
||||
"v": [
|
||||
34
|
||||
],
|
||||
"w": [
|
||||
35
|
||||
],
|
||||
"x": [
|
||||
36
|
||||
],
|
||||
"y": [
|
||||
37
|
||||
],
|
||||
"z": [
|
||||
38
|
||||
],
|
||||
"æ": [
|
||||
39
|
||||
],
|
||||
"ç": [
|
||||
40
|
||||
],
|
||||
"ð": [
|
||||
41
|
||||
],
|
||||
"ø": [
|
||||
42
|
||||
],
|
||||
"ħ": [
|
||||
43
|
||||
],
|
||||
"ŋ": [
|
||||
44
|
||||
],
|
||||
"œ": [
|
||||
45
|
||||
],
|
||||
"ǀ": [
|
||||
46
|
||||
],
|
||||
"ǁ": [
|
||||
47
|
||||
],
|
||||
"ǂ": [
|
||||
48
|
||||
],
|
||||
"ǃ": [
|
||||
49
|
||||
],
|
||||
"ɐ": [
|
||||
50
|
||||
],
|
||||
"ɑ": [
|
||||
51
|
||||
],
|
||||
"ɒ": [
|
||||
52
|
||||
],
|
||||
"ɓ": [
|
||||
53
|
||||
],
|
||||
"ɔ": [
|
||||
54
|
||||
],
|
||||
"ɕ": [
|
||||
55
|
||||
],
|
||||
"ɖ": [
|
||||
56
|
||||
],
|
||||
"ɗ": [
|
||||
57
|
||||
],
|
||||
"ɘ": [
|
||||
58
|
||||
],
|
||||
"ə": [
|
||||
59
|
||||
],
|
||||
"ɚ": [
|
||||
60
|
||||
],
|
||||
"ɛ": [
|
||||
61
|
||||
],
|
||||
"ɜ": [
|
||||
62
|
||||
],
|
||||
"ɞ": [
|
||||
63
|
||||
],
|
||||
"ɟ": [
|
||||
64
|
||||
],
|
||||
"ɠ": [
|
||||
65
|
||||
],
|
||||
"ɡ": [
|
||||
66
|
||||
],
|
||||
"ɢ": [
|
||||
67
|
||||
],
|
||||
"ɣ": [
|
||||
68
|
||||
],
|
||||
"ɤ": [
|
||||
69
|
||||
],
|
||||
"ɥ": [
|
||||
70
|
||||
],
|
||||
"ɦ": [
|
||||
71
|
||||
],
|
||||
"ɧ": [
|
||||
72
|
||||
],
|
||||
"ɨ": [
|
||||
73
|
||||
],
|
||||
"ɪ": [
|
||||
74
|
||||
],
|
||||
"ɫ": [
|
||||
75
|
||||
],
|
||||
"ɬ": [
|
||||
76
|
||||
],
|
||||
"ɭ": [
|
||||
77
|
||||
],
|
||||
"ɮ": [
|
||||
78
|
||||
],
|
||||
"ɯ": [
|
||||
79
|
||||
],
|
||||
"ɰ": [
|
||||
80
|
||||
],
|
||||
"ɱ": [
|
||||
81
|
||||
],
|
||||
"ɲ": [
|
||||
82
|
||||
],
|
||||
"ɳ": [
|
||||
83
|
||||
],
|
||||
"ɴ": [
|
||||
84
|
||||
],
|
||||
"ɵ": [
|
||||
85
|
||||
],
|
||||
"ɶ": [
|
||||
86
|
||||
],
|
||||
"ɸ": [
|
||||
87
|
||||
],
|
||||
"ɹ": [
|
||||
88
|
||||
],
|
||||
"ɺ": [
|
||||
89
|
||||
],
|
||||
"ɻ": [
|
||||
90
|
||||
],
|
||||
"ɽ": [
|
||||
91
|
||||
],
|
||||
"ɾ": [
|
||||
92
|
||||
],
|
||||
"ʀ": [
|
||||
93
|
||||
],
|
||||
"ʁ": [
|
||||
94
|
||||
],
|
||||
"ʂ": [
|
||||
95
|
||||
],
|
||||
"ʃ": [
|
||||
96
|
||||
],
|
||||
"ʄ": [
|
||||
97
|
||||
],
|
||||
"ʈ": [
|
||||
98
|
||||
],
|
||||
"ʉ": [
|
||||
99
|
||||
],
|
||||
"ʊ": [
|
||||
100
|
||||
],
|
||||
"ʋ": [
|
||||
101
|
||||
],
|
||||
"ʌ": [
|
||||
102
|
||||
],
|
||||
"ʍ": [
|
||||
103
|
||||
],
|
||||
"ʎ": [
|
||||
104
|
||||
],
|
||||
"ʏ": [
|
||||
105
|
||||
],
|
||||
"ʐ": [
|
||||
106
|
||||
],
|
||||
"ʑ": [
|
||||
107
|
||||
],
|
||||
"ʒ": [
|
||||
108
|
||||
],
|
||||
"ʔ": [
|
||||
109
|
||||
],
|
||||
"ʕ": [
|
||||
110
|
||||
],
|
||||
"ʘ": [
|
||||
111
|
||||
],
|
||||
"ʙ": [
|
||||
112
|
||||
],
|
||||
"ʛ": [
|
||||
113
|
||||
],
|
||||
"ʜ": [
|
||||
114
|
||||
],
|
||||
"ʝ": [
|
||||
115
|
||||
],
|
||||
"ʟ": [
|
||||
116
|
||||
],
|
||||
"ʡ": [
|
||||
117
|
||||
],
|
||||
"ʢ": [
|
||||
118
|
||||
],
|
||||
"ʲ": [
|
||||
119
|
||||
],
|
||||
"ˈ": [
|
||||
120
|
||||
],
|
||||
"ˌ": [
|
||||
121
|
||||
],
|
||||
"ː": [
|
||||
122
|
||||
],
|
||||
"ˑ": [
|
||||
123
|
||||
],
|
||||
"˞": [
|
||||
124
|
||||
],
|
||||
"β": [
|
||||
125
|
||||
],
|
||||
"θ": [
|
||||
126
|
||||
],
|
||||
"χ": [
|
||||
127
|
||||
],
|
||||
"ᵻ": [
|
||||
128
|
||||
],
|
||||
"ⱱ": [
|
||||
129
|
||||
],
|
||||
"0": [
|
||||
130
|
||||
],
|
||||
"1": [
|
||||
131
|
||||
],
|
||||
"2": [
|
||||
132
|
||||
],
|
||||
"3": [
|
||||
133
|
||||
],
|
||||
"4": [
|
||||
134
|
||||
],
|
||||
"5": [
|
||||
135
|
||||
],
|
||||
"6": [
|
||||
136
|
||||
],
|
||||
"7": [
|
||||
137
|
||||
],
|
||||
"8": [
|
||||
138
|
||||
],
|
||||
"9": [
|
||||
139
|
||||
],
|
||||
"̧": [
|
||||
140
|
||||
],
|
||||
"̃": [
|
||||
141
|
||||
],
|
||||
"̪": [
|
||||
142
|
||||
],
|
||||
"̯": [
|
||||
143
|
||||
],
|
||||
"̩": [
|
||||
144
|
||||
],
|
||||
"ʰ": [
|
||||
145
|
||||
],
|
||||
"ˤ": [
|
||||
146
|
||||
],
|
||||
"ε": [
|
||||
147
|
||||
],
|
||||
"↓": [
|
||||
148
|
||||
],
|
||||
"#": [
|
||||
149
|
||||
],
|
||||
"\"": [
|
||||
150
|
||||
],
|
||||
"↑": [
|
||||
151
|
||||
],
|
||||
"̺": [
|
||||
152
|
||||
],
|
||||
"̻": [
|
||||
153
|
||||
]
|
||||
},
|
||||
"num_symbols": 256,
|
||||
"num_speakers": 1,
|
||||
"speaker_id_map": {},
|
||||
"piper_version": "1.0.0",
|
||||
"language": {
|
||||
"code": "de_DE",
|
||||
"family": "de",
|
||||
"region": "DE",
|
||||
"name_native": "Deutsch",
|
||||
"name_english": "German",
|
||||
"country_english": "Germany"
|
||||
},
|
||||
"dataset": "thorsten"
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue