mirror of
https://github.com/Jeuners/agenttwo-tools.git
synced 2026-09-09 15:02:31 +02:00
feat: Bilder im Chat (Vision)
qwen3.5 meldet die Fähigkeit "vision", die bisher ungenutzt blieb. Bilder lassen sich jetzt per Button, Zwischenablage (⌘V) oder Drag & Drop anhängen; eine Nachricht darf auch nur aus einem Bild bestehen. - images.ts: Validierung vor dem Speichern — maximal 4 Bilder je Nachricht, je 6 MB, Typ per Magic Bytes statt per Dateiendung (PNG/JPEG/GIF/WebP). Der WebSocket bekommt dazu ein maxPayload von 32 MB, vorher war er unbegrenzt und Bilder machen Nachrichten deutlich größer. - db.ts: Spalte images samt Migration für bestehende Datenbanken aus agenttwo - ollama.ts: images-Feld je Message; openrouter.ts: OpenAI-Format mit image_url, MIME aus den Magic Bytes statt fest image/png - Composer: Vorschau mit Entfernen-Button, Fehlermeldung bei zu großen oder nicht unterstützten Dateien; ChatMessage zeigt Bilder in der Historie - Bilder wandern in die Chat-Historie und werden bei Folgefragen erneut mitgeschickt, damit Rückfragen zum selben Bild funktionieren Getestet gegen das laufende Modell: ein rendertes PNG (roter Kreis auf weiß) wird korrekt als japanische Flagge beschrieben. Validierung geprüft gegen 5 Bilder, Textdatei mit Bildnamen, kaputtes base64 und 7-MB-Blob — alle abgelehnt. Nebenbei: Branding in UI, Titel und Serverlog auf agenttwo-tools umgestellt. Die localStorage-Keys bleiben unverändert, sonst gingen gespeicherte Einstellungen verloren. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01S1NXUpHbxTusqQmsjrFfbU
This commit is contained in:
parent
e48228c570
commit
adb57ae6e3
17 changed files with 616 additions and 71 deletions
26
README.md
26
README.md
|
|
@ -1,7 +1,7 @@
|
|||
# agenttwo-tools
|
||||
|
||||
> Fork von [agenttwo](https://github.com/Jeuners/agenttwo). Die Basis bleibt dort
|
||||
> unverändert; hier kommen Tool-Calling und Vision dazu. Läuft auf eigenen Ports
|
||||
> unverändert; hier kommen Vision (fertig) und Tool-Calling (geplant) dazu. Läuft auf eigenen Ports
|
||||
> (Backend 8788, Frontend 5174), damit beide Projekte parallel laufen können.
|
||||
> Fixes aus der Basis lassen sich per `git cherry-pick` aus dem Remote
|
||||
> `upstream` übernehmen.
|
||||
|
|
@ -91,6 +91,30 @@ Weitere optionale Variablen mit ihren Defaults:
|
|||
| `PIPER_MODEL` | `server/voices/de_DE-thorsten-high.onnx` |
|
||||
| `ALLOWED_ORIGINS` | (leer — siehe Sicherheit) |
|
||||
|
||||
## Bilder (Vision)
|
||||
|
||||
`qwen3.5` bringt die Fähigkeit `vision` mit, deshalb versteht der Chat Bilder.
|
||||
Anhängen geht auf drei Wegen: Button 🖼 im Composer, Einfügen aus der
|
||||
Zwischenablage (⌘V) oder Drag & Drop auf die Eingabezeile. Eine Nachricht darf
|
||||
auch nur aus einem Bild bestehen.
|
||||
|
||||
Bilder werden zusammen mit der Nachricht in der SQLite-Datei abgelegt und bei
|
||||
Folgefragen erneut mitgeschickt, sodass Rückfragen zum selben Bild funktionieren.
|
||||
Für Ollama gehen sie als `images: [base64]` raus, für OpenRouter im
|
||||
OpenAI-Format als `image_url` mit data-URL — der MIME-Typ wird dabei aus den
|
||||
Magic Bytes bestimmt.
|
||||
|
||||
Grenzen (`server/src/images.ts`): maximal 4 Bilder pro Nachricht, je 6 MB,
|
||||
nur PNG, JPEG, GIF und WebP. Der Typ wird an den Magic Bytes geprüft, nicht am
|
||||
angegebenen Dateinamen; der WebSocket hat dafür ein Payload-Limit von 32 MB.
|
||||
|
||||
Ob das eingestellte Modell Bilder kann, verrät:
|
||||
|
||||
```bash
|
||||
curl -s http://localhost:11434/api/show -d '{"name":"qwen3.5:latest"}' \
|
||||
| python3 -c "import json,sys; print(json.load(sys.stdin)['capabilities'])"
|
||||
```
|
||||
|
||||
## Sicherheit
|
||||
|
||||
Der Server hat **keine Authentifizierung** und lauscht deshalb bewusst nur auf
|
||||
|
|
|
|||
|
|
@ -23,6 +23,14 @@ db.exec(`
|
|||
CREATE INDEX IF NOT EXISTS idx_messages_session ON messages(session_id, created_at);
|
||||
`);
|
||||
|
||||
// Bestehende Datenbanken aus agenttwo kennen die Spalte noch nicht.
|
||||
const hasImages = (
|
||||
db.prepare("PRAGMA table_info(messages)").all() as unknown as { name: string }[]
|
||||
).some((c) => c.name === "images");
|
||||
if (!hasImages) {
|
||||
db.exec("ALTER TABLE messages ADD COLUMN images TEXT");
|
||||
}
|
||||
|
||||
export interface SessionRow {
|
||||
id: string;
|
||||
title: string;
|
||||
|
|
@ -35,6 +43,8 @@ export interface MessageRow {
|
|||
role: string;
|
||||
content: string;
|
||||
thinking: string | null;
|
||||
/** JSON-Array mit base64-Bilddaten (ohne data:-Präfix), oder null. */
|
||||
images: string | null;
|
||||
created_at: number;
|
||||
}
|
||||
|
||||
|
|
@ -81,6 +91,7 @@ export function insertMessage(
|
|||
role: string,
|
||||
content: string,
|
||||
thinking?: string | null,
|
||||
images?: string[] | null,
|
||||
): MessageRow {
|
||||
const row: MessageRow = {
|
||||
id: randomUUID(),
|
||||
|
|
@ -88,14 +99,34 @@ export function insertMessage(
|
|||
role,
|
||||
content,
|
||||
thinking: thinking ?? null,
|
||||
images: images?.length ? JSON.stringify(images) : null,
|
||||
created_at: Date.now(),
|
||||
};
|
||||
db.prepare(
|
||||
"INSERT INTO messages (id, session_id, role, content, thinking, created_at) VALUES (?, ?, ?, ?, ?, ?)",
|
||||
).run(row.id, row.session_id, row.role, row.content, row.thinking, row.created_at);
|
||||
"INSERT INTO messages (id, session_id, role, content, thinking, images, created_at) VALUES (?, ?, ?, ?, ?, ?, ?)",
|
||||
).run(
|
||||
row.id,
|
||||
row.session_id,
|
||||
row.role,
|
||||
row.content,
|
||||
row.thinking,
|
||||
row.images,
|
||||
row.created_at,
|
||||
);
|
||||
return row;
|
||||
}
|
||||
|
||||
/** Bilder einer Zeile als Array — leer, wenn keine oder unlesbar. */
|
||||
export function parseImages(row: Pick<MessageRow, "images">): string[] {
|
||||
if (!row.images) return [];
|
||||
try {
|
||||
const parsed: unknown = JSON.parse(row.images);
|
||||
return Array.isArray(parsed) ? parsed.filter((i): i is string => typeof i === "string") : [];
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
export function updateAssistantMessage(
|
||||
id: string,
|
||||
content: string,
|
||||
|
|
|
|||
80
server/src/images.ts
Normal file
80
server/src/images.ts
Normal file
|
|
@ -0,0 +1,80 @@
|
|||
/**
|
||||
* Validierung für Bild-Anhänge.
|
||||
*
|
||||
* Bilder kommen als base64 über den WebSocket herein und gehen von dort an
|
||||
* Ollama bzw. OpenRouter. Weil der Inhalt von außen stammt, wird hier Anzahl,
|
||||
* Größe und Typ begrenzt, bevor irgendetwas gespeichert oder weitergereicht
|
||||
* wird.
|
||||
*/
|
||||
|
||||
export const MAX_IMAGES_PER_MESSAGE = 4;
|
||||
/** Maximale Größe je Bild nach dem Dekodieren. */
|
||||
export const MAX_IMAGE_BYTES = 6 * 1024 * 1024;
|
||||
/** Obergrenze für eine ganze WebSocket-Nachricht inklusive base64-Overhead. */
|
||||
export const MAX_WS_PAYLOAD = 32 * 1024 * 1024;
|
||||
|
||||
const MAGIC: { mime: string; test: (b: Buffer) => boolean }[] = [
|
||||
{ mime: "image/png", test: (b) => b.subarray(0, 8).equals(Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a])) },
|
||||
{ mime: "image/jpeg", test: (b) => b[0] === 0xff && b[1] === 0xd8 && b[2] === 0xff },
|
||||
{ mime: "image/gif", test: (b) => b.subarray(0, 6).toString("latin1").startsWith("GIF8") },
|
||||
{
|
||||
mime: "image/webp",
|
||||
test: (b) =>
|
||||
b.subarray(0, 4).toString("latin1") === "RIFF" &&
|
||||
b.subarray(8, 12).toString("latin1") === "WEBP",
|
||||
},
|
||||
];
|
||||
|
||||
/** Bestimmt den MIME-Typ eines base64-Bildes; Fallback PNG. */
|
||||
export function mimeFromBase64(b64: string): string {
|
||||
const head = Buffer.from(b64.slice(0, 32), "base64");
|
||||
return MAGIC.find((m) => m.test(head))?.mime ?? "image/png";
|
||||
}
|
||||
|
||||
export interface ValidatedImage {
|
||||
/** Reines base64 ohne data:-Präfix — dieses Format erwartet Ollama. */
|
||||
base64: string;
|
||||
mime: string;
|
||||
bytes: number;
|
||||
}
|
||||
|
||||
export class ImageError extends Error {}
|
||||
|
||||
/** Entfernt einen optionalen data:-Präfix und prüft das Ergebnis. */
|
||||
function decode(input: string): Buffer {
|
||||
const comma = input.startsWith("data:") ? input.indexOf(",") : -1;
|
||||
const raw = comma === -1 ? input : input.slice(comma + 1);
|
||||
if (!/^[A-Za-z0-9+/]+={0,2}$/.test(raw.replace(/\s/g, ""))) {
|
||||
throw new ImageError("Bild ist kein gültiges base64");
|
||||
}
|
||||
return Buffer.from(raw, "base64");
|
||||
}
|
||||
|
||||
/**
|
||||
* Prüft eine Liste roher Bild-Strings und gibt die normalisierte Form zurück.
|
||||
* Wirft ImageError, sobald etwas nicht passt — lieber ablehnen als raten.
|
||||
*/
|
||||
export function validateImages(raw: unknown): ValidatedImage[] {
|
||||
if (raw === undefined || raw === null) return [];
|
||||
if (!Array.isArray(raw)) throw new ImageError("images muss ein Array sein");
|
||||
if (raw.length > MAX_IMAGES_PER_MESSAGE) {
|
||||
throw new ImageError(`Maximal ${MAX_IMAGES_PER_MESSAGE} Bilder pro Nachricht`);
|
||||
}
|
||||
|
||||
return raw.map((entry) => {
|
||||
if (typeof entry !== "string" || !entry.trim()) {
|
||||
throw new ImageError("Bild ist leer oder kein String");
|
||||
}
|
||||
const buf = decode(entry);
|
||||
if (buf.length === 0) throw new ImageError("Bild ist leer");
|
||||
if (buf.length > MAX_IMAGE_BYTES) {
|
||||
throw new ImageError(
|
||||
`Bild größer als ${Math.round(MAX_IMAGE_BYTES / 1024 / 1024)} MB`,
|
||||
);
|
||||
}
|
||||
const match = MAGIC.find((m) => m.test(buf));
|
||||
if (!match) throw new ImageError("Nicht unterstütztes Bildformat (PNG, JPEG, GIF, WebP)");
|
||||
|
||||
return { base64: buf.toString("base64"), mime: match.mime, bytes: buf.length };
|
||||
});
|
||||
}
|
||||
|
|
@ -13,6 +13,7 @@ import {
|
|||
getOpenRouterKey,
|
||||
} from "./openrouter.js";
|
||||
import { ALLOWED_ORIGINS, isOriginAllowed, createRateLimiter } from "./security.js";
|
||||
import { validateImages, ImageError, MAX_WS_PAYLOAD } from "./images.js";
|
||||
|
||||
// simple .env loader (project root)
|
||||
try {
|
||||
|
|
@ -186,7 +187,7 @@ function clamp(v: number, min: number, max: number) {
|
|||
return Math.min(Math.max(v, min), max);
|
||||
}
|
||||
|
||||
const wss = new WebSocketServer({ noServer: true });
|
||||
const wss = new WebSocketServer({ noServer: true, maxPayload: MAX_WS_PAYLOAD });
|
||||
|
||||
function broadcast(data: unknown) {
|
||||
for (const client of wss.clients) {
|
||||
|
|
@ -215,16 +216,31 @@ wss.on("connection", (socket: WebSocket, _req: IncomingMessage) => {
|
|||
|
||||
const sessionId = String(msg.sessionId ?? "");
|
||||
const content = String(msg.content ?? "").trim();
|
||||
if (!sessionId || !content) {
|
||||
|
||||
let images: string[];
|
||||
try {
|
||||
images = validateImages(msg.images).map((i) => i.base64);
|
||||
} catch (err) {
|
||||
socket.send(
|
||||
JSON.stringify({
|
||||
type: "error",
|
||||
error: err instanceof ImageError ? err.message : "Bild abgelehnt",
|
||||
}),
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
// Ein Bild allein ist eine gültige Anfrage — Text darf dann fehlen.
|
||||
if (!sessionId || (!content && images.length === 0)) {
|
||||
socket.send(JSON.stringify({ type: "error", error: "sessionId/content fehlt" }));
|
||||
return;
|
||||
}
|
||||
|
||||
let session = dbmod.getSession(sessionId);
|
||||
if (!session) session = dbmod.createSession();
|
||||
dbmod.renameSessionIfDefault(session.id, content);
|
||||
dbmod.renameSessionIfDefault(session.id, content || "Bild");
|
||||
|
||||
const userMsg = dbmod.insertMessage(session.id, "user", content);
|
||||
const userMsg = dbmod.insertMessage(session.id, "user", content, null, images);
|
||||
socket.send(JSON.stringify({ type: "user-message", message: userMsg }));
|
||||
broadcast({ type: "sessions-changed" });
|
||||
|
||||
|
|
@ -232,7 +248,12 @@ wss.on("connection", (socket: WebSocket, _req: IncomingMessage) => {
|
|||
const history = dbmod
|
||||
.listMessages(session.id)
|
||||
.slice(-24)
|
||||
.map((m) => ({ role: m.role, content: m.content }));
|
||||
.map((m) => {
|
||||
const imgs = dbmod.parseImages(m);
|
||||
return imgs.length
|
||||
? { role: m.role, content: m.content, images: imgs }
|
||||
: { role: m.role, content: m.content };
|
||||
});
|
||||
|
||||
const assistantRow = dbmod.insertMessage(session.id, "assistant", "");
|
||||
socket.send(JSON.stringify({ type: "assistant-start", message: assistantRow }));
|
||||
|
|
@ -337,5 +358,5 @@ server.on("upgrade", (req, socket, head) => {
|
|||
});
|
||||
|
||||
app.listen({ port: PORT, host: "127.0.0.1" }, () => {
|
||||
console.log(`[oxagenttwo] Server läuft auf http://127.0.0.1:${PORT}`);
|
||||
console.log(`[agenttwo-tools] Server läuft auf http://127.0.0.1:${PORT}`);
|
||||
});
|
||||
|
|
|
|||
|
|
@ -20,8 +20,15 @@ interface ChatChunk {
|
|||
error?: string;
|
||||
}
|
||||
|
||||
export interface ChatMessage {
|
||||
role: string;
|
||||
content: string;
|
||||
/** base64 ohne data:-Präfix; Ollama erwartet genau dieses Format. */
|
||||
images?: string[];
|
||||
}
|
||||
|
||||
export async function streamChat(
|
||||
history: { role: string; content: string }[],
|
||||
history: ChatMessage[],
|
||||
systemPrompt: string | undefined,
|
||||
opts: OllamaOptions,
|
||||
cb: StreamCallbacks,
|
||||
|
|
@ -31,7 +38,11 @@ export async function streamChat(
|
|||
model: opts.model,
|
||||
messages: [
|
||||
...(systemPrompt ? [{ role: "system", content: systemPrompt }] : []),
|
||||
...history,
|
||||
...history.map((m) =>
|
||||
m.images?.length
|
||||
? { role: m.role, content: m.content, images: m.images }
|
||||
: { role: m.role, content: m.content },
|
||||
),
|
||||
],
|
||||
stream: true,
|
||||
think: opts.think,
|
||||
|
|
|
|||
|
|
@ -1,3 +1,5 @@
|
|||
import { mimeFromBase64 } from "./images.js";
|
||||
|
||||
export interface OpenRouterOptions {
|
||||
model: string;
|
||||
temperature: number;
|
||||
|
|
@ -37,8 +39,26 @@ export async function listOpenRouterModels(): Promise<
|
|||
.sort((a, b) => a.name.localeCompare(b.name));
|
||||
}
|
||||
|
||||
/**
|
||||
* OpenRouter folgt dem OpenAI-Schema: Bilder stecken als data-URL in einem
|
||||
* content-Array, nicht in einem eigenen images-Feld wie bei Ollama.
|
||||
*/
|
||||
function toOpenAIMessage(m: { role: string; content: string; images?: string[] }) {
|
||||
if (!m.images?.length) return { role: m.role, content: m.content };
|
||||
return {
|
||||
role: m.role,
|
||||
content: [
|
||||
...(m.content ? [{ type: "text", text: m.content }] : []),
|
||||
...m.images.map((b64) => ({
|
||||
type: "image_url",
|
||||
image_url: { url: `data:${mimeFromBase64(b64)};base64,${b64}` },
|
||||
})),
|
||||
],
|
||||
};
|
||||
}
|
||||
|
||||
export async function streamOpenRouter(
|
||||
history: { role: string; content: string }[],
|
||||
history: { role: string; content: string; images?: string[] }[],
|
||||
systemPrompt: string | undefined,
|
||||
opts: OpenRouterOptions,
|
||||
apiKey: string,
|
||||
|
|
@ -58,7 +78,7 @@ export async function streamOpenRouter(
|
|||
stream: true,
|
||||
messages: [
|
||||
...(systemPrompt ? [{ role: "system", content: systemPrompt }] : []),
|
||||
...history,
|
||||
...history.map(toOpenAIMessage),
|
||||
],
|
||||
temperature: opts.temperature,
|
||||
max_tokens: opts.numPredict,
|
||||
|
|
|
|||
74
web/dist/assets/index-D8cBv8n7.js
vendored
Normal file
74
web/dist/assets/index-D8cBv8n7.js
vendored
Normal file
File diff suppressed because one or more lines are too long
10
web/dist/assets/index-DFQmRKNP.css
vendored
Normal file
10
web/dist/assets/index-DFQmRKNP.css
vendored
Normal file
File diff suppressed because one or more lines are too long
13
web/dist/index.html
vendored
Normal file
13
web/dist/index.html
vendored
Normal file
|
|
@ -0,0 +1,13 @@
|
|||
<!doctype html>
|
||||
<html lang="de">
|
||||
<head>
|
||||
<meta charset="UTF-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<title>agenttwo-tools — qwen3 mit Vision</title>
|
||||
<script type="module" crossorigin src="/assets/index-D8cBv8n7.js"></script>
|
||||
<link rel="stylesheet" crossorigin href="/assets/index-DFQmRKNP.css">
|
||||
</head>
|
||||
<body>
|
||||
<div id="root"></div>
|
||||
</body>
|
||||
</html>
|
||||
|
|
@ -3,7 +3,7 @@
|
|||
<head>
|
||||
<meta charset="UTF-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<title>oxagenttwo — local qwen3 chat</title>
|
||||
<title>agenttwo-tools — qwen3 mit Vision</title>
|
||||
</head>
|
||||
<body>
|
||||
<div id="root"></div>
|
||||
|
|
|
|||
|
|
@ -130,10 +130,10 @@ export default function App() {
|
|||
}, [chat.streaming, voiceMode, voice]);
|
||||
|
||||
const handleSend = useCallback(
|
||||
(text: string) => {
|
||||
(text: string, images: string[] = []) => {
|
||||
voice.cancelSpeech();
|
||||
pendingSpeechRef.current = "";
|
||||
chat.sendMessage(text);
|
||||
chat.sendMessage(text, images);
|
||||
},
|
||||
[chat, voice],
|
||||
);
|
||||
|
|
@ -296,7 +296,7 @@ export default function App() {
|
|||
<div className="messages" ref={scrollRef}>
|
||||
{chat.messages.length === 0 && (
|
||||
<div className="welcome">
|
||||
<div className="welcome-title">▸ oxagenttwo</div>
|
||||
<div className="welcome-title">▸ agenttwo-tools</div>
|
||||
<p>
|
||||
Echtzeit-Chat mit lokalem Qwen3 über Ollama — per Tastatur oder
|
||||
Stimme (Whisper STT + Piper TTS, alles lokal).
|
||||
|
|
|
|||
|
|
@ -4,9 +4,33 @@ import remarkGfm from "remark-gfm";
|
|||
import rehypeHighlight from "rehype-highlight";
|
||||
import type { Message } from "../types";
|
||||
|
||||
/**
|
||||
* Leitet den MIME-Typ aus den ersten base64-Zeichen ab. "image/*" ist als
|
||||
* data-URL ungültig, deshalb muss der konkrete Typ bestimmt werden.
|
||||
*/
|
||||
function mimeOf(b64: string): string {
|
||||
if (b64.startsWith("iVBORw0KGgo")) return "image/png";
|
||||
if (b64.startsWith("/9j/")) return "image/jpeg";
|
||||
if (b64.startsWith("R0lGOD")) return "image/gif";
|
||||
if (b64.startsWith("UklGR")) return "image/webp";
|
||||
return "image/png";
|
||||
}
|
||||
|
||||
/** Der Server legt Bilder als JSON-Array mit base64-Daten ab. */
|
||||
function imagesOf(message: Message): string[] {
|
||||
if (!message.images) return [];
|
||||
try {
|
||||
const parsed: unknown = JSON.parse(message.images);
|
||||
return Array.isArray(parsed) ? parsed.filter((i): i is string => typeof i === "string") : [];
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
export function ChatMessage({ message }: { message: Message }) {
|
||||
const [showThinking, setShowThinking] = useState(false);
|
||||
const isUser = message.role === "user";
|
||||
const images = imagesOf(message);
|
||||
|
||||
return (
|
||||
<div className={`msg ${isUser ? "msg-user" : "msg-assistant"}`}>
|
||||
|
|
@ -28,8 +52,23 @@ export function ChatMessage({ message }: { message: Message }) {
|
|||
</div>
|
||||
)}
|
||||
|
||||
{images.length > 0 && (
|
||||
<div className="msg-images">
|
||||
{images.map((b64, i) => (
|
||||
<img
|
||||
key={i}
|
||||
src={`data:${mimeOf(b64)};base64,${b64}`}
|
||||
alt={`Anhang ${i + 1}`}
|
||||
loading="lazy"
|
||||
/>
|
||||
))}
|
||||
</div>
|
||||
)}
|
||||
|
||||
{isUser ? (
|
||||
<div className="msg-content user-content">{message.content}</div>
|
||||
message.content ? (
|
||||
<div className="msg-content user-content">{message.content}</div>
|
||||
) : null
|
||||
) : (
|
||||
<div className="msg-content markdown">
|
||||
<ReactMarkdown remarkPlugins={[remarkGfm]} rehypePlugins={[rehypeHighlight]}>
|
||||
|
|
|
|||
|
|
@ -1,5 +1,18 @@
|
|||
import { useEffect, useRef, useState } from "react";
|
||||
|
||||
/** Muss zu MAX_IMAGES_PER_MESSAGE / MAX_IMAGE_BYTES im Server passen. */
|
||||
const MAX_IMAGES = 4;
|
||||
const MAX_IMAGE_BYTES = 6 * 1024 * 1024;
|
||||
const ACCEPTED = ["image/png", "image/jpeg", "image/gif", "image/webp"];
|
||||
|
||||
interface Attachment {
|
||||
id: string;
|
||||
/** reines base64 ohne data:-Präfix — dieses Format erwartet der Server */
|
||||
base64: string;
|
||||
dataUrl: string;
|
||||
name: string;
|
||||
}
|
||||
|
||||
interface Props {
|
||||
streaming: boolean;
|
||||
disabled: boolean;
|
||||
|
|
@ -7,11 +20,29 @@ interface Props {
|
|||
transcribing: boolean;
|
||||
injectedText: string | null;
|
||||
onInjected: () => void;
|
||||
onSend: (text: string) => void;
|
||||
onSend: (text: string, images: string[]) => void;
|
||||
onAbort: () => void;
|
||||
onMicToggle: () => void;
|
||||
}
|
||||
|
||||
function readAsAttachment(file: File): Promise<Attachment> {
|
||||
return new Promise((resolve, reject) => {
|
||||
const reader = new FileReader();
|
||||
reader.onerror = () => reject(new Error(`${file.name} konnte nicht gelesen werden`));
|
||||
reader.onload = () => {
|
||||
const dataUrl = String(reader.result);
|
||||
const comma = dataUrl.indexOf(",");
|
||||
resolve({
|
||||
id: `${file.name}-${file.size}-${Date.now()}-${Math.random()}`,
|
||||
base64: dataUrl.slice(comma + 1),
|
||||
dataUrl,
|
||||
name: file.name || "Bild",
|
||||
});
|
||||
};
|
||||
reader.readAsDataURL(file);
|
||||
});
|
||||
}
|
||||
|
||||
export function Composer({
|
||||
streaming,
|
||||
disabled,
|
||||
|
|
@ -24,7 +55,11 @@ export function Composer({
|
|||
onMicToggle,
|
||||
}: Props) {
|
||||
const [value, setValue] = useState("");
|
||||
const [images, setImages] = useState<Attachment[]>([]);
|
||||
const [imgError, setImgError] = useState<string | null>(null);
|
||||
const [dragging, setDragging] = useState(false);
|
||||
const ref = useRef<HTMLTextAreaElement>(null);
|
||||
const fileRef = useRef<HTMLInputElement>(null);
|
||||
|
||||
useEffect(() => {
|
||||
if (injectedText != null) {
|
||||
|
|
@ -37,63 +72,160 @@ export function Composer({
|
|||
}
|
||||
}, [injectedText, onInjected]);
|
||||
|
||||
const addFiles = async (files: File[]) => {
|
||||
setImgError(null);
|
||||
const usable: File[] = [];
|
||||
for (const f of files) {
|
||||
if (!ACCEPTED.includes(f.type)) {
|
||||
setImgError(`${f.name || "Datei"}: nur PNG, JPEG, GIF oder WebP`);
|
||||
continue;
|
||||
}
|
||||
if (f.size > MAX_IMAGE_BYTES) {
|
||||
setImgError(`${f.name}: größer als ${MAX_IMAGE_BYTES / 1024 / 1024} MB`);
|
||||
continue;
|
||||
}
|
||||
usable.push(f);
|
||||
}
|
||||
if (!usable.length) return;
|
||||
|
||||
try {
|
||||
const added = await Promise.all(usable.map(readAsAttachment));
|
||||
setImages((prev) => {
|
||||
const free = MAX_IMAGES - prev.length;
|
||||
if (added.length > free) setImgError(`Maximal ${MAX_IMAGES} Bilder pro Nachricht`);
|
||||
return [...prev, ...added.slice(0, Math.max(free, 0))];
|
||||
});
|
||||
} catch (err) {
|
||||
setImgError(err instanceof Error ? err.message : "Bild konnte nicht gelesen werden");
|
||||
}
|
||||
};
|
||||
|
||||
const submit = () => {
|
||||
const text = value.trim();
|
||||
if (!text || streaming || disabled) return;
|
||||
onSend(text);
|
||||
if ((!text && images.length === 0) || streaming || disabled) return;
|
||||
onSend(text, images.map((i) => i.base64));
|
||||
setValue("");
|
||||
setImages([]);
|
||||
setImgError(null);
|
||||
requestAnimationFrame(() => ref.current?.focus());
|
||||
};
|
||||
|
||||
const canSend = (value.trim().length > 0 || images.length > 0) && !disabled;
|
||||
|
||||
return (
|
||||
<div className="composer">
|
||||
<button
|
||||
className={`btn-mic ${recording ? "recording" : ""} ${transcribing ? "transcribing" : ""}`}
|
||||
onClick={onMicToggle}
|
||||
title={
|
||||
recording
|
||||
? "Aufnahme stoppen & senden"
|
||||
: transcribing
|
||||
? "Transkribiere …"
|
||||
: "Sprachnachricht aufnehmen"
|
||||
}
|
||||
disabled={transcribing || disabled}
|
||||
>
|
||||
{recording ? "●" : transcribing ? "…" : "🎙"}
|
||||
</button>
|
||||
<textarea
|
||||
ref={ref}
|
||||
value={value}
|
||||
placeholder={
|
||||
recording
|
||||
? "Ich höre zu … (zum Beenden nochmal auf das Mikro klicken)"
|
||||
: disabled
|
||||
? "Keine Session aktiv — neuen Chat starten"
|
||||
: "Nachricht an qwen3 … (Enter = senden, Shift+Enter = Zeilenumbruch)"
|
||||
}
|
||||
rows={1}
|
||||
disabled={disabled}
|
||||
onChange={(e) => setValue(e.target.value)}
|
||||
onKeyDown={(e) => {
|
||||
if (e.key === "Enter" && !e.shiftKey) {
|
||||
e.preventDefault();
|
||||
submit();
|
||||
}
|
||||
}}
|
||||
/>
|
||||
{streaming ? (
|
||||
<button className="btn-send stop" onClick={onAbort}>
|
||||
■ Stop
|
||||
</button>
|
||||
) : (
|
||||
<button
|
||||
className="btn-send"
|
||||
disabled={!value.trim() || disabled}
|
||||
onClick={submit}
|
||||
>
|
||||
Senden ▸
|
||||
</button>
|
||||
<div
|
||||
className={`composer ${dragging ? "dragging" : ""}`}
|
||||
onDragOver={(e) => {
|
||||
if (disabled) return;
|
||||
e.preventDefault();
|
||||
setDragging(true);
|
||||
}}
|
||||
onDragLeave={() => setDragging(false)}
|
||||
onDrop={(e) => {
|
||||
if (disabled) return;
|
||||
e.preventDefault();
|
||||
setDragging(false);
|
||||
void addFiles([...e.dataTransfer.files]);
|
||||
}}
|
||||
>
|
||||
{(images.length > 0 || imgError) && (
|
||||
<div className="attachments">
|
||||
{images.map((img) => (
|
||||
<div className="attachment" key={img.id}>
|
||||
<img src={img.dataUrl} alt={img.name} />
|
||||
<button
|
||||
className="attachment-remove"
|
||||
onClick={() => setImages((p) => p.filter((i) => i.id !== img.id))}
|
||||
title={`${img.name} entfernen`}
|
||||
>
|
||||
×
|
||||
</button>
|
||||
</div>
|
||||
))}
|
||||
{imgError && <span className="attachment-error">{imgError}</span>}
|
||||
</div>
|
||||
)}
|
||||
|
||||
<div className="composer-row">
|
||||
<button
|
||||
className={`btn-mic ${recording ? "recording" : ""} ${transcribing ? "transcribing" : ""}`}
|
||||
onClick={onMicToggle}
|
||||
title={
|
||||
recording
|
||||
? "Aufnahme stoppen & senden"
|
||||
: transcribing
|
||||
? "Transkribiere …"
|
||||
: "Sprachnachricht aufnehmen"
|
||||
}
|
||||
disabled={transcribing || disabled}
|
||||
>
|
||||
{recording ? "●" : transcribing ? "…" : "🎙"}
|
||||
</button>
|
||||
|
||||
<button
|
||||
className="btn-attach"
|
||||
onClick={() => fileRef.current?.click()}
|
||||
disabled={disabled || images.length >= MAX_IMAGES}
|
||||
title={
|
||||
images.length >= MAX_IMAGES
|
||||
? `Maximal ${MAX_IMAGES} Bilder`
|
||||
: "Bild anhängen (auch per Einfügen oder Drag & Drop)"
|
||||
}
|
||||
>
|
||||
🖼
|
||||
</button>
|
||||
<input
|
||||
ref={fileRef}
|
||||
type="file"
|
||||
accept={ACCEPTED.join(",")}
|
||||
multiple
|
||||
hidden
|
||||
onChange={(e) => {
|
||||
void addFiles([...(e.target.files ?? [])]);
|
||||
e.target.value = "";
|
||||
}}
|
||||
/>
|
||||
|
||||
<textarea
|
||||
ref={ref}
|
||||
value={value}
|
||||
placeholder={
|
||||
recording
|
||||
? "Ich höre zu … (zum Beenden nochmal auf das Mikro klicken)"
|
||||
: disabled
|
||||
? "Keine Session aktiv — neuen Chat starten"
|
||||
: "Nachricht an qwen3 … (Enter = senden, Bild einfügen mit ⌘V)"
|
||||
}
|
||||
rows={1}
|
||||
disabled={disabled}
|
||||
onChange={(e) => setValue(e.target.value)}
|
||||
onPaste={(e) => {
|
||||
const files = [...e.clipboardData.files].filter((f) =>
|
||||
f.type.startsWith("image/"),
|
||||
);
|
||||
if (files.length) {
|
||||
e.preventDefault();
|
||||
void addFiles(files);
|
||||
}
|
||||
}}
|
||||
onKeyDown={(e) => {
|
||||
if (e.key === "Enter" && !e.shiftKey) {
|
||||
e.preventDefault();
|
||||
submit();
|
||||
}
|
||||
}}
|
||||
/>
|
||||
|
||||
{streaming ? (
|
||||
<button className="btn-send stop" onClick={onAbort}>
|
||||
■ Stop
|
||||
</button>
|
||||
) : (
|
||||
<button className="btn-send" disabled={!canSend} onClick={submit}>
|
||||
Senden ▸
|
||||
</button>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -13,7 +13,7 @@ export function Sidebar({ sessions, activeId, onSelect, onNew, onDelete }: Props
|
|||
<aside className="sidebar">
|
||||
<div className="sidebar-header">
|
||||
<div className="brand">
|
||||
<span className="brand-prompt">▸</span> oxagenttwo
|
||||
<span className="brand-prompt">▸</span> agenttwo-tools
|
||||
</div>
|
||||
<button className="btn-new" onClick={onNew}>
|
||||
+ Neuer Chat
|
||||
|
|
|
|||
|
|
@ -538,3 +538,87 @@ body {
|
|||
::-webkit-scrollbar-track {
|
||||
background: transparent;
|
||||
}
|
||||
|
||||
/* --- Bild-Anhänge (Vision) --- */
|
||||
.composer.dragging {
|
||||
outline: 2px dashed var(--accent, #3fb950);
|
||||
outline-offset: -4px;
|
||||
}
|
||||
|
||||
.composer-row {
|
||||
display: flex;
|
||||
align-items: flex-end;
|
||||
gap: 10px;
|
||||
width: 100%;
|
||||
}
|
||||
|
||||
.attachments {
|
||||
display: flex;
|
||||
flex-wrap: wrap;
|
||||
align-items: center;
|
||||
gap: 8px;
|
||||
padding: 0 0 8px;
|
||||
}
|
||||
|
||||
.attachment {
|
||||
position: relative;
|
||||
width: 64px;
|
||||
height: 64px;
|
||||
}
|
||||
|
||||
.attachment img {
|
||||
width: 100%;
|
||||
height: 100%;
|
||||
object-fit: cover;
|
||||
border-radius: 6px;
|
||||
border: 1px solid var(--border, #30363d);
|
||||
}
|
||||
|
||||
.attachment-remove {
|
||||
position: absolute;
|
||||
top: -6px;
|
||||
right: -6px;
|
||||
width: 20px;
|
||||
height: 20px;
|
||||
line-height: 18px;
|
||||
padding: 0;
|
||||
border-radius: 50%;
|
||||
border: 1px solid var(--border, #30363d);
|
||||
background: var(--bg, #0d1117);
|
||||
color: var(--fg, #c9d1d9);
|
||||
cursor: pointer;
|
||||
}
|
||||
|
||||
.attachment-error {
|
||||
font-size: 0.85em;
|
||||
color: #f85149;
|
||||
}
|
||||
|
||||
.btn-attach {
|
||||
flex: 0 0 auto;
|
||||
background: transparent;
|
||||
border: 1px solid var(--border, #30363d);
|
||||
border-radius: 8px;
|
||||
padding: 8px 10px;
|
||||
cursor: pointer;
|
||||
color: inherit;
|
||||
}
|
||||
|
||||
.btn-attach:disabled {
|
||||
opacity: 0.4;
|
||||
cursor: default;
|
||||
}
|
||||
|
||||
.msg-images {
|
||||
display: flex;
|
||||
flex-wrap: wrap;
|
||||
gap: 8px;
|
||||
margin: 6px 0;
|
||||
}
|
||||
|
||||
.msg-images img {
|
||||
max-width: min(320px, 100%);
|
||||
max-height: 320px;
|
||||
border-radius: 8px;
|
||||
border: 1px solid var(--border, #30363d);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -10,6 +10,8 @@ export interface Message {
|
|||
role: "user" | "assistant" | "system";
|
||||
content: string;
|
||||
thinking: string | null;
|
||||
/** JSON-Array mit base64-Bilddaten, wie es der Server speichert. */
|
||||
images: string | null;
|
||||
created_at: number;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -138,14 +138,18 @@ export function useChat() {
|
|||
}, []);
|
||||
|
||||
const sendMessage = useCallback(
|
||||
(content: string) => {
|
||||
if (!content.trim() || streamingRef.current || !activeId) return;
|
||||
(content: string, images: string[] = []) => {
|
||||
// Ein Bild ohne Text ist eine gültige Anfrage.
|
||||
if ((!content.trim() && images.length === 0) || streamingRef.current || !activeId) {
|
||||
return;
|
||||
}
|
||||
streamingRef.current = true;
|
||||
setStreaming(true);
|
||||
socketRef.current?.send({
|
||||
type: "chat",
|
||||
sessionId: activeId,
|
||||
content,
|
||||
images,
|
||||
options,
|
||||
systemPrompt,
|
||||
});
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue