mirror of
https://github.com/Jeuners/agenttwo-tools.git
synced 2026-09-09 15:02:31 +02:00
feat: Bilder im Chat (Vision)
qwen3.5 meldet die Fähigkeit "vision", die bisher ungenutzt blieb. Bilder lassen sich jetzt per Button, Zwischenablage (⌘V) oder Drag & Drop anhängen; eine Nachricht darf auch nur aus einem Bild bestehen. - images.ts: Validierung vor dem Speichern — maximal 4 Bilder je Nachricht, je 6 MB, Typ per Magic Bytes statt per Dateiendung (PNG/JPEG/GIF/WebP). Der WebSocket bekommt dazu ein maxPayload von 32 MB, vorher war er unbegrenzt und Bilder machen Nachrichten deutlich größer. - db.ts: Spalte images samt Migration für bestehende Datenbanken aus agenttwo - ollama.ts: images-Feld je Message; openrouter.ts: OpenAI-Format mit image_url, MIME aus den Magic Bytes statt fest image/png - Composer: Vorschau mit Entfernen-Button, Fehlermeldung bei zu großen oder nicht unterstützten Dateien; ChatMessage zeigt Bilder in der Historie - Bilder wandern in die Chat-Historie und werden bei Folgefragen erneut mitgeschickt, damit Rückfragen zum selben Bild funktionieren Getestet gegen das laufende Modell: ein rendertes PNG (roter Kreis auf weiß) wird korrekt als japanische Flagge beschrieben. Validierung geprüft gegen 5 Bilder, Textdatei mit Bildnamen, kaputtes base64 und 7-MB-Blob — alle abgelehnt. Nebenbei: Branding in UI, Titel und Serverlog auf agenttwo-tools umgestellt. Die localStorage-Keys bleiben unverändert, sonst gingen gespeicherte Einstellungen verloren. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01S1NXUpHbxTusqQmsjrFfbU
This commit is contained in:
parent
e48228c570
commit
adb57ae6e3
17 changed files with 616 additions and 71 deletions
|
|
@ -23,6 +23,14 @@ db.exec(`
|
|||
CREATE INDEX IF NOT EXISTS idx_messages_session ON messages(session_id, created_at);
|
||||
`);
|
||||
|
||||
// Bestehende Datenbanken aus agenttwo kennen die Spalte noch nicht.
|
||||
const hasImages = (
|
||||
db.prepare("PRAGMA table_info(messages)").all() as unknown as { name: string }[]
|
||||
).some((c) => c.name === "images");
|
||||
if (!hasImages) {
|
||||
db.exec("ALTER TABLE messages ADD COLUMN images TEXT");
|
||||
}
|
||||
|
||||
export interface SessionRow {
|
||||
id: string;
|
||||
title: string;
|
||||
|
|
@ -35,6 +43,8 @@ export interface MessageRow {
|
|||
role: string;
|
||||
content: string;
|
||||
thinking: string | null;
|
||||
/** JSON-Array mit base64-Bilddaten (ohne data:-Präfix), oder null. */
|
||||
images: string | null;
|
||||
created_at: number;
|
||||
}
|
||||
|
||||
|
|
@ -81,6 +91,7 @@ export function insertMessage(
|
|||
role: string,
|
||||
content: string,
|
||||
thinking?: string | null,
|
||||
images?: string[] | null,
|
||||
): MessageRow {
|
||||
const row: MessageRow = {
|
||||
id: randomUUID(),
|
||||
|
|
@ -88,14 +99,34 @@ export function insertMessage(
|
|||
role,
|
||||
content,
|
||||
thinking: thinking ?? null,
|
||||
images: images?.length ? JSON.stringify(images) : null,
|
||||
created_at: Date.now(),
|
||||
};
|
||||
db.prepare(
|
||||
"INSERT INTO messages (id, session_id, role, content, thinking, created_at) VALUES (?, ?, ?, ?, ?, ?)",
|
||||
).run(row.id, row.session_id, row.role, row.content, row.thinking, row.created_at);
|
||||
"INSERT INTO messages (id, session_id, role, content, thinking, images, created_at) VALUES (?, ?, ?, ?, ?, ?, ?)",
|
||||
).run(
|
||||
row.id,
|
||||
row.session_id,
|
||||
row.role,
|
||||
row.content,
|
||||
row.thinking,
|
||||
row.images,
|
||||
row.created_at,
|
||||
);
|
||||
return row;
|
||||
}
|
||||
|
||||
/** Bilder einer Zeile als Array — leer, wenn keine oder unlesbar. */
|
||||
export function parseImages(row: Pick<MessageRow, "images">): string[] {
|
||||
if (!row.images) return [];
|
||||
try {
|
||||
const parsed: unknown = JSON.parse(row.images);
|
||||
return Array.isArray(parsed) ? parsed.filter((i): i is string => typeof i === "string") : [];
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
export function updateAssistantMessage(
|
||||
id: string,
|
||||
content: string,
|
||||
|
|
|
|||
80
server/src/images.ts
Normal file
80
server/src/images.ts
Normal file
|
|
@ -0,0 +1,80 @@
|
|||
/**
|
||||
* Validierung für Bild-Anhänge.
|
||||
*
|
||||
* Bilder kommen als base64 über den WebSocket herein und gehen von dort an
|
||||
* Ollama bzw. OpenRouter. Weil der Inhalt von außen stammt, wird hier Anzahl,
|
||||
* Größe und Typ begrenzt, bevor irgendetwas gespeichert oder weitergereicht
|
||||
* wird.
|
||||
*/
|
||||
|
||||
export const MAX_IMAGES_PER_MESSAGE = 4;
|
||||
/** Maximale Größe je Bild nach dem Dekodieren. */
|
||||
export const MAX_IMAGE_BYTES = 6 * 1024 * 1024;
|
||||
/** Obergrenze für eine ganze WebSocket-Nachricht inklusive base64-Overhead. */
|
||||
export const MAX_WS_PAYLOAD = 32 * 1024 * 1024;
|
||||
|
||||
const MAGIC: { mime: string; test: (b: Buffer) => boolean }[] = [
|
||||
{ mime: "image/png", test: (b) => b.subarray(0, 8).equals(Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a])) },
|
||||
{ mime: "image/jpeg", test: (b) => b[0] === 0xff && b[1] === 0xd8 && b[2] === 0xff },
|
||||
{ mime: "image/gif", test: (b) => b.subarray(0, 6).toString("latin1").startsWith("GIF8") },
|
||||
{
|
||||
mime: "image/webp",
|
||||
test: (b) =>
|
||||
b.subarray(0, 4).toString("latin1") === "RIFF" &&
|
||||
b.subarray(8, 12).toString("latin1") === "WEBP",
|
||||
},
|
||||
];
|
||||
|
||||
/** Bestimmt den MIME-Typ eines base64-Bildes; Fallback PNG. */
|
||||
export function mimeFromBase64(b64: string): string {
|
||||
const head = Buffer.from(b64.slice(0, 32), "base64");
|
||||
return MAGIC.find((m) => m.test(head))?.mime ?? "image/png";
|
||||
}
|
||||
|
||||
export interface ValidatedImage {
|
||||
/** Reines base64 ohne data:-Präfix — dieses Format erwartet Ollama. */
|
||||
base64: string;
|
||||
mime: string;
|
||||
bytes: number;
|
||||
}
|
||||
|
||||
export class ImageError extends Error {}
|
||||
|
||||
/** Entfernt einen optionalen data:-Präfix und prüft das Ergebnis. */
|
||||
function decode(input: string): Buffer {
|
||||
const comma = input.startsWith("data:") ? input.indexOf(",") : -1;
|
||||
const raw = comma === -1 ? input : input.slice(comma + 1);
|
||||
if (!/^[A-Za-z0-9+/]+={0,2}$/.test(raw.replace(/\s/g, ""))) {
|
||||
throw new ImageError("Bild ist kein gültiges base64");
|
||||
}
|
||||
return Buffer.from(raw, "base64");
|
||||
}
|
||||
|
||||
/**
|
||||
* Prüft eine Liste roher Bild-Strings und gibt die normalisierte Form zurück.
|
||||
* Wirft ImageError, sobald etwas nicht passt — lieber ablehnen als raten.
|
||||
*/
|
||||
export function validateImages(raw: unknown): ValidatedImage[] {
|
||||
if (raw === undefined || raw === null) return [];
|
||||
if (!Array.isArray(raw)) throw new ImageError("images muss ein Array sein");
|
||||
if (raw.length > MAX_IMAGES_PER_MESSAGE) {
|
||||
throw new ImageError(`Maximal ${MAX_IMAGES_PER_MESSAGE} Bilder pro Nachricht`);
|
||||
}
|
||||
|
||||
return raw.map((entry) => {
|
||||
if (typeof entry !== "string" || !entry.trim()) {
|
||||
throw new ImageError("Bild ist leer oder kein String");
|
||||
}
|
||||
const buf = decode(entry);
|
||||
if (buf.length === 0) throw new ImageError("Bild ist leer");
|
||||
if (buf.length > MAX_IMAGE_BYTES) {
|
||||
throw new ImageError(
|
||||
`Bild größer als ${Math.round(MAX_IMAGE_BYTES / 1024 / 1024)} MB`,
|
||||
);
|
||||
}
|
||||
const match = MAGIC.find((m) => m.test(buf));
|
||||
if (!match) throw new ImageError("Nicht unterstütztes Bildformat (PNG, JPEG, GIF, WebP)");
|
||||
|
||||
return { base64: buf.toString("base64"), mime: match.mime, bytes: buf.length };
|
||||
});
|
||||
}
|
||||
|
|
@ -13,6 +13,7 @@ import {
|
|||
getOpenRouterKey,
|
||||
} from "./openrouter.js";
|
||||
import { ALLOWED_ORIGINS, isOriginAllowed, createRateLimiter } from "./security.js";
|
||||
import { validateImages, ImageError, MAX_WS_PAYLOAD } from "./images.js";
|
||||
|
||||
// simple .env loader (project root)
|
||||
try {
|
||||
|
|
@ -186,7 +187,7 @@ function clamp(v: number, min: number, max: number) {
|
|||
return Math.min(Math.max(v, min), max);
|
||||
}
|
||||
|
||||
const wss = new WebSocketServer({ noServer: true });
|
||||
const wss = new WebSocketServer({ noServer: true, maxPayload: MAX_WS_PAYLOAD });
|
||||
|
||||
function broadcast(data: unknown) {
|
||||
for (const client of wss.clients) {
|
||||
|
|
@ -215,16 +216,31 @@ wss.on("connection", (socket: WebSocket, _req: IncomingMessage) => {
|
|||
|
||||
const sessionId = String(msg.sessionId ?? "");
|
||||
const content = String(msg.content ?? "").trim();
|
||||
if (!sessionId || !content) {
|
||||
|
||||
let images: string[];
|
||||
try {
|
||||
images = validateImages(msg.images).map((i) => i.base64);
|
||||
} catch (err) {
|
||||
socket.send(
|
||||
JSON.stringify({
|
||||
type: "error",
|
||||
error: err instanceof ImageError ? err.message : "Bild abgelehnt",
|
||||
}),
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
// Ein Bild allein ist eine gültige Anfrage — Text darf dann fehlen.
|
||||
if (!sessionId || (!content && images.length === 0)) {
|
||||
socket.send(JSON.stringify({ type: "error", error: "sessionId/content fehlt" }));
|
||||
return;
|
||||
}
|
||||
|
||||
let session = dbmod.getSession(sessionId);
|
||||
if (!session) session = dbmod.createSession();
|
||||
dbmod.renameSessionIfDefault(session.id, content);
|
||||
dbmod.renameSessionIfDefault(session.id, content || "Bild");
|
||||
|
||||
const userMsg = dbmod.insertMessage(session.id, "user", content);
|
||||
const userMsg = dbmod.insertMessage(session.id, "user", content, null, images);
|
||||
socket.send(JSON.stringify({ type: "user-message", message: userMsg }));
|
||||
broadcast({ type: "sessions-changed" });
|
||||
|
||||
|
|
@ -232,7 +248,12 @@ wss.on("connection", (socket: WebSocket, _req: IncomingMessage) => {
|
|||
const history = dbmod
|
||||
.listMessages(session.id)
|
||||
.slice(-24)
|
||||
.map((m) => ({ role: m.role, content: m.content }));
|
||||
.map((m) => {
|
||||
const imgs = dbmod.parseImages(m);
|
||||
return imgs.length
|
||||
? { role: m.role, content: m.content, images: imgs }
|
||||
: { role: m.role, content: m.content };
|
||||
});
|
||||
|
||||
const assistantRow = dbmod.insertMessage(session.id, "assistant", "");
|
||||
socket.send(JSON.stringify({ type: "assistant-start", message: assistantRow }));
|
||||
|
|
@ -337,5 +358,5 @@ server.on("upgrade", (req, socket, head) => {
|
|||
});
|
||||
|
||||
app.listen({ port: PORT, host: "127.0.0.1" }, () => {
|
||||
console.log(`[oxagenttwo] Server läuft auf http://127.0.0.1:${PORT}`);
|
||||
console.log(`[agenttwo-tools] Server läuft auf http://127.0.0.1:${PORT}`);
|
||||
});
|
||||
|
|
|
|||
|
|
@ -20,8 +20,15 @@ interface ChatChunk {
|
|||
error?: string;
|
||||
}
|
||||
|
||||
export interface ChatMessage {
|
||||
role: string;
|
||||
content: string;
|
||||
/** base64 ohne data:-Präfix; Ollama erwartet genau dieses Format. */
|
||||
images?: string[];
|
||||
}
|
||||
|
||||
export async function streamChat(
|
||||
history: { role: string; content: string }[],
|
||||
history: ChatMessage[],
|
||||
systemPrompt: string | undefined,
|
||||
opts: OllamaOptions,
|
||||
cb: StreamCallbacks,
|
||||
|
|
@ -31,7 +38,11 @@ export async function streamChat(
|
|||
model: opts.model,
|
||||
messages: [
|
||||
...(systemPrompt ? [{ role: "system", content: systemPrompt }] : []),
|
||||
...history,
|
||||
...history.map((m) =>
|
||||
m.images?.length
|
||||
? { role: m.role, content: m.content, images: m.images }
|
||||
: { role: m.role, content: m.content },
|
||||
),
|
||||
],
|
||||
stream: true,
|
||||
think: opts.think,
|
||||
|
|
|
|||
|
|
@ -1,3 +1,5 @@
|
|||
import { mimeFromBase64 } from "./images.js";
|
||||
|
||||
export interface OpenRouterOptions {
|
||||
model: string;
|
||||
temperature: number;
|
||||
|
|
@ -37,8 +39,26 @@ export async function listOpenRouterModels(): Promise<
|
|||
.sort((a, b) => a.name.localeCompare(b.name));
|
||||
}
|
||||
|
||||
/**
|
||||
* OpenRouter folgt dem OpenAI-Schema: Bilder stecken als data-URL in einem
|
||||
* content-Array, nicht in einem eigenen images-Feld wie bei Ollama.
|
||||
*/
|
||||
function toOpenAIMessage(m: { role: string; content: string; images?: string[] }) {
|
||||
if (!m.images?.length) return { role: m.role, content: m.content };
|
||||
return {
|
||||
role: m.role,
|
||||
content: [
|
||||
...(m.content ? [{ type: "text", text: m.content }] : []),
|
||||
...m.images.map((b64) => ({
|
||||
type: "image_url",
|
||||
image_url: { url: `data:${mimeFromBase64(b64)};base64,${b64}` },
|
||||
})),
|
||||
],
|
||||
};
|
||||
}
|
||||
|
||||
export async function streamOpenRouter(
|
||||
history: { role: string; content: string }[],
|
||||
history: { role: string; content: string; images?: string[] }[],
|
||||
systemPrompt: string | undefined,
|
||||
opts: OpenRouterOptions,
|
||||
apiKey: string,
|
||||
|
|
@ -58,7 +78,7 @@ export async function streamOpenRouter(
|
|||
stream: true,
|
||||
messages: [
|
||||
...(systemPrompt ? [{ role: "system", content: systemPrompt }] : []),
|
||||
...history,
|
||||
...history.map(toOpenAIMessage),
|
||||
],
|
||||
temperature: opts.temperature,
|
||||
max_tokens: opts.numPredict,
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue