mirror of
https://github.com/Jeuners/astra-vision.git
synced 2026-09-09 15:02:35 +02:00
feat: image generation via ComfyUI and PDF/image upload
Zwei neue, sauber getrennte Fähigkeiten, jede in ihrem eigenen Modul: - astra/comfyui.py: async HTTP-Client für einen lokalen ComfyUI-Server (z-image-turbo-Workflow), kennt nichts von der Pipeline. - astra/documents.py: PDF-Textextraktion via pypdf, keine Netzwerkzugriffe. - astra/tools.py: verdrahtet generate_image als natives Ollama-Tool-Call — Qwen 3.5 unterstützt Tools und Vision bereits nativ laut `ollama show`. Dafür wurde astra/services.py so erweitert, dass NativeOllamaService Ollamas native tool_calls im Streaming-Response erkennt und als ChatCompletionChunk-Deltas an Pipecats bereits vorhandene, generische Function-Calling-Maschinerie (_process_context/run_function_calls) weiterreicht — die musste dafür nicht angefasst werden. trim_messages in core.py bewahrt jetzt Tool-Roundtrips und Bild-Anhänge vollständig statt sie auf role/content zu reduzieren. Neuer Upload-Button im UI (Bild oder PDF, während eines laufenden Gesprächs): PDFs gehen als Text, Bilder als Base64 über Qwens Vision in den Gesprächskontext ein. Generierte Bilder werden über /api/media/<id> ausgeliefert und per Datenkanal im Transkript angezeigt. Kompletter Function-Calling-Roundtrip end-to-end gegen echtes Ollama und echtes ComfyUI verifiziert (Modell ruft generate_image korrekt auf, Bild wird erzeugt und im media_store abgelegt). Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01LVgSHNHdRx3UNTBodFmhRA
This commit is contained in:
parent
991f0d1eb5
commit
16786fd3fd
18 changed files with 2602 additions and 21 deletions
51
web/app.js
51
web/app.js
|
|
@ -33,17 +33,60 @@ function addMessage(event) {
|
|||
while ($("messages").children.length > 80) $("messages").firstElementChild.remove();
|
||||
$("messages").scrollTop = $("messages").scrollHeight;
|
||||
}
|
||||
function addImage(url) {
|
||||
$("messages").querySelector(".empty")?.remove();
|
||||
const article = document.createElement("article");
|
||||
article.className = "message assistant";
|
||||
const speaker = document.createElement("span");
|
||||
speaker.className = "speaker";
|
||||
speaker.textContent = "Astra";
|
||||
const img = document.createElement("img");
|
||||
img.className = "generated-image";
|
||||
img.src = url;
|
||||
img.alt = "Von Astra erzeugtes Bild";
|
||||
article.append(speaker, img);
|
||||
$("messages").append(article);
|
||||
$("messages").scrollTop = $("messages").scrollHeight;
|
||||
}
|
||||
function addUploadNote(filename) {
|
||||
$("messages").querySelector(".empty")?.remove();
|
||||
const article = document.createElement("article");
|
||||
article.className = "message user";
|
||||
const speaker = document.createElement("span");
|
||||
speaker.className = "speaker";
|
||||
speaker.textContent = "Du";
|
||||
const text = document.createElement("p");
|
||||
text.textContent = `Hochgeladen: ${filename}`;
|
||||
article.append(speaker, text);
|
||||
$("messages").append(article);
|
||||
$("messages").scrollTop = $("messages").scrollHeight;
|
||||
}
|
||||
function receive(event) {
|
||||
let message;
|
||||
try { message = JSON.parse(event.data); } catch { return; }
|
||||
if (message.type === "state") state(message.state);
|
||||
if (message.type === "partial") $("partial").textContent = message.text;
|
||||
if (message.type === "transcript") addMessage(message);
|
||||
if (message.type === "image") addImage(message.url);
|
||||
if (message.type === "upload") addUploadNote(message.filename);
|
||||
if (message.type === "error") showError(message.message);
|
||||
if (message.type === "metric" && message.name === "llm_ms") {
|
||||
$("latency").textContent = `Erstes Antwortwort · ${(message.value / 1000).toFixed(2)} s`;
|
||||
}
|
||||
}
|
||||
async function uploadFile(file) {
|
||||
if (!pcId) return;
|
||||
const body = new FormData();
|
||||
body.append("pc_id", pcId);
|
||||
body.append("file", file);
|
||||
try {
|
||||
const response = await fetch("/api/upload", {method: "POST", body, signal: AbortSignal.timeout(30000)});
|
||||
const data = await response.json();
|
||||
if (!response.ok) throw new Error(typeof data.detail === "string" ? data.detail : "Upload fehlgeschlagen.");
|
||||
} catch (error) {
|
||||
showError(error.message || "Upload fehlgeschlagen.");
|
||||
}
|
||||
}
|
||||
async function request(path, body, timeout = 20000) {
|
||||
const response = await fetch(path, {method: "POST", headers: {"Content-Type": "application/json"},
|
||||
body: JSON.stringify(body), signal: AbortSignal.timeout(timeout)});
|
||||
|
|
@ -121,6 +164,7 @@ async function connect() {
|
|||
await pc.setRemoteDescription({sdp: answer.sdp, type: answer.type});
|
||||
$("connect").textContent = "Gespräch beenden";
|
||||
$("mute").hidden = false;
|
||||
$("attach").hidden = false;
|
||||
$("clear").textContent = "Neues Gespräch";
|
||||
$("hint").textContent = "Sprich frei. Beim Dazwischenreden hält Astra an.";
|
||||
} catch (error) {
|
||||
|
|
@ -155,6 +199,7 @@ async function disconnect() {
|
|||
$("mute").hidden = true;
|
||||
$("mute").setAttribute("aria-pressed", "false");
|
||||
$("mute").textContent = "Mikrofon pausieren";
|
||||
$("attach").hidden = true;
|
||||
$("connect").textContent = "Gespräch starten";
|
||||
$("clear").textContent = "Verlauf leeren";
|
||||
$("partial").textContent = "";
|
||||
|
|
@ -172,6 +217,12 @@ $("mute").addEventListener("click", () => {
|
|||
$("mute").textContent = muted ? "Mikrofon aktivieren" : "Mikrofon pausieren";
|
||||
state("listening");
|
||||
});
|
||||
$("attach").addEventListener("click", () => $("file-input").click());
|
||||
$("file-input").addEventListener("change", async () => {
|
||||
const file = $("file-input").files[0];
|
||||
$("file-input").value = "";
|
||||
if (file) await uploadFile(file);
|
||||
});
|
||||
$("clear").addEventListener("click", async () => {
|
||||
const reconnect = Boolean(peer);
|
||||
if (reconnect) await disconnect();
|
||||
|
|
|
|||
|
|
@ -22,7 +22,7 @@
|
|||
<label for="voice">Stimme</label>
|
||||
<select id="voice" disabled></select>
|
||||
</div>
|
||||
<div class="controls"><button id="connect" class="primary" disabled><span aria-hidden="true">◉</span> Gespräch starten</button><button id="mute" class="secondary" hidden aria-pressed="false">Mikrofon pausieren</button></div>
|
||||
<div class="controls"><button id="connect" class="primary" disabled><span aria-hidden="true">◉</span> Gespräch starten</button><button id="mute" class="secondary" hidden aria-pressed="false">Mikrofon pausieren</button><button id="attach" class="secondary" hidden>Bild oder PDF hinzufügen</button><input id="file-input" type="file" accept="application/pdf,image/png,image/jpeg,image/webp" hidden></div>
|
||||
<p class="hint" id="hint">Mikrofon wird erst nach dem Start freigegeben.</p>
|
||||
</section>
|
||||
<section class="transcript" aria-labelledby="transcript-title">
|
||||
|
|
|
|||
File diff suppressed because one or more lines are too long
Loading…
Add table
Add a link
Reference in a new issue