大活

art-agent/backend/app/agent/loop.py

@@ -0,0 +1,185 @@
+"""Agent Loop 主循环：LLM 对话 -> 工具调用 -> 结果回传 -> 继续。"""
+
+import json
+from typing import AsyncGenerator, Optional
+
+from openai import AsyncOpenAI
+
+from app.agent.tools import TOOL_DEFINITIONS, execute_tool
+
+
+def _get_client() -> AsyncOpenAI:
+    return AsyncOpenAI()
+
+SYSTEM_PROMPT = """\
+你是一个专业的游戏美术 AI 助手。你的工作是帮助美术人员通过对话生成游戏美术资源。
+
+## 你的能力
+- 根据用户的文字描述生成图片（UI图标、按钮、插画、立绘、概念图等）
+- 理解用户的审美意图，将中文描述转化为高质量的英文生成 prompt
+- 根据用户反馈迭代修改（调整颜色、风格、构图等）
+- 如果用户提供了参考图，将参考图的风格元素融入生成 prompt
+
+## 工作流程
+1. 理解用户需求，必要时追问细节（尺寸、风格、用途等）
+2. 将需求转化为详细的英文 prompt，调用 generate_image 工具生成图片
+3. 向用户展示结果并询问反馈
+4. 根据反馈调整 prompt 并重新生成
+
+## 生成 prompt 要求
+- 必须使用英文
+- 尽量详细描述：主体内容、风格、颜色方案、光照、构图、材质等
+- 如果用户要求游戏 UI 元素，添加相关关键词如 "game UI", "icon", "button" 等
+- 如果用户提供了参考图，在 prompt 中描述参考图的风格特征
+
+## 注意事项
+- 用中文和用户交流
+- 生成图片后简要说明你使用的 prompt 思路
+- 主动建议迭代方向
+"""
+
+
+async def run_agent_loop(
+    messages: list[dict],
+    ref_image_url: Optional[str] = None,
+) -> AsyncGenerator[dict, None]:
+    """
+    运行 Agent Loop，以 SSE 事件流形式 yield 结果。
+
+    事件类型：
+    - text_delta: LLM 文字增量
+    - tool_start: 开始执行工具
+    - image_result: 图片生成结果
+    - done: 完成
+    - error: 错误
+    """
+    # 构建消息列表
+    api_messages = [{"role": "system", "content": SYSTEM_PROMPT}]
+
+    for msg in messages:
+        if msg["role"] == "user" and ref_image_url:
+            # 最后一条用户消息附加参考图（GPT-4o vision）
+            if msg == messages[-1] or (
+                msg.get("role") == "user"
+                and messages.index(msg) == len(messages) - 1
+            ):
+                api_messages.append({
+                    "role": "user",
+                    "content": [
+                        {"type": "text", "text": msg["content"]},
+                        {
+                            "type": "image_url",
+                            "image_url": {"url": ref_image_url},
+                        },
+                    ],
+                })
+                continue
+        api_messages.append({"role": msg["role"], "content": msg["content"]})
+
+    client = _get_client()
+
+    max_iterations = 5
+    for _ in range(max_iterations):
+        try:
+            response = await client.chat.completions.create(
+                model="gpt-4o-mini",
+                messages=api_messages,
+                tools=TOOL_DEFINITIONS,
+                stream=True,
+            )
+        except Exception as e:
+            yield {"type": "error", "data": {"message": str(e)}}
+            return
+
+        collected_text = ""
+        tool_calls_data: dict[int, dict] = {}
+
+        async for chunk in response:
+            delta = chunk.choices[0].delta if chunk.choices else None
+            if not delta:
+                continue
+
+            # 文字内容
+            if delta.content:
+                collected_text += delta.content
+                yield {"type": "text_delta", "data": {"text": delta.content}}
+
+            # 工具调用（流式累积）
+            if delta.tool_calls:
+                for tc in delta.tool_calls:
+                    idx = tc.index
+                    if idx not in tool_calls_data:
+                        tool_calls_data[idx] = {
+                            "id": "",
+                            "name": "",
+                            "arguments": "",
+                        }
+                    if tc.id:
+                        tool_calls_data[idx]["id"] = tc.id
+                    if tc.function and tc.function.name:
+                        tool_calls_data[idx]["name"] = tc.function.name
+                    if tc.function and tc.function.arguments:
+                        tool_calls_data[idx]["arguments"] += tc.function.arguments
+
+        finish_reason = chunk.choices[0].finish_reason if chunk.choices else None
+
+        # 如果没有工具调用，对话结束
+        if not tool_calls_data:
+            yield {"type": "done", "data": {}}
+            return
+
+        # 将 assistant 消息（含 tool_calls）加入历史
+        assistant_msg: dict = {"role": "assistant"}
+        if collected_text:
+            assistant_msg["content"] = collected_text
+        else:
+            assistant_msg["content"] = None
+
+        assistant_msg["tool_calls"] = []
+        for idx in sorted(tool_calls_data.keys()):
+            tc_data = tool_calls_data[idx]
+            assistant_msg["tool_calls"].append({
+                "id": tc_data["id"],
+                "type": "function",
+                "function": {
+                    "name": tc_data["name"],
+                    "arguments": tc_data["arguments"],
+                },
+            })
+        api_messages.append(assistant_msg)
+
+        # 依次执行每个工具调用
+        for idx in sorted(tool_calls_data.keys()):
+            tc_data = tool_calls_data[idx]
+            tool_name = tc_data["name"]
+
+            yield {
+                "type": "tool_start",
+                "data": {"tool": tool_name, "message": "正在生成图片..."},
+            }
+
+            try:
+                arguments = json.loads(tc_data["arguments"])
+            except json.JSONDecodeError:
+                arguments = {}
+
+            result = await execute_tool(tool_name, arguments, ref_image_url)
+
+            # 如果有图片结果，推送给前端
+            if result.get("images"):
+                yield {
+                    "type": "image_result",
+                    "data": {"images": result["images"]},
+                }
+
+            # 工具结果回传给 LLM
+            api_messages.append({
+                "role": "tool",
+                "tool_call_id": tc_data["id"],
+                "content": json.dumps(result, ensure_ascii=False),
+            })
+
+        # 清除 ref_image_url，避免后续轮次重复附加
+        ref_image_url = None
+
+    yield {"type": "done", "data": {}}

art-agent/backend/app/agent/tools.py

@@ -0,0 +1,57 @@
+"""Agent 可调用的工具定义和实现。"""
+
+from app.services.image_gen import generate_images
+
+# OpenAI Function Calling 格式的工具定义
+TOOL_DEFINITIONS = [
+    {
+        "type": "function",
+        "function": {
+            "name": "generate_image",
+            "description": (
+                "根据文字描述生成图片。prompt 必须是英文。"
+                "如果用户提供了参考图，会自动传入 ref_image_url 参数。"
+            ),
+            "parameters": {
+                "type": "object",
+                "properties": {
+                    "prompt": {
+                        "type": "string",
+                        "description": "英文图片描述 prompt，详细描述要生成的图片内容、风格、颜色等",
+                    },
+                    "num_images": {
+                        "type": "integer",
+                        "description": "生成图片数量，1-4 张",
+                        "default": 1,
+                        "minimum": 1,
+                        "maximum": 4,
+                    },
+                },
+                "required": ["prompt"],
+            },
+        },
+    }
+]
+
+
+async def execute_tool(
+    tool_name: str,
+    arguments: dict,
+    ref_image_url: str | None = None,
+) -> dict:
+    """执行工具调用，返回结果。"""
+    if tool_name == "generate_image":
+        prompt = arguments["prompt"]
+        num_images = arguments.get("num_images", 1)
+        image_urls = await generate_images(
+            prompt=prompt,
+            num_images=num_images,
+            ref_image_url=ref_image_url,
+        )
+        return {
+            "success": True,
+            "images": image_urls,
+            "prompt_used": prompt,
+        }
+
+    return {"success": False, "error": f"未知工具: {tool_name}"}

art-agent/backend/app/api/chat.py

@@ -0,0 +1,51 @@
+import json
+import uuid
+from pathlib import Path
+from typing import Optional
+
+from fastapi import APIRouter, File, Form, UploadFile
+from sse_starlette.sse import EventSourceResponse
+
+from app.agent.loop import run_agent_loop
+
+router = APIRouter()
+
+UPLOADS_DIR = Path(__file__).parent.parent.parent / "uploads"
+
+
+async def _save_upload(file: UploadFile) -> str:
+    """保存上传的参考图，返回可访问的 URL 路径。"""
+    ext = Path(file.filename).suffix or ".png"
+    filename = f"{uuid.uuid4().hex}{ext}"
+    filepath = UPLOADS_DIR / filename
+    content = await file.read()
+    filepath.write_bytes(content)
+    return f"/uploads/{filename}"
+
+
+@router.post("/chat")
+async def chat(
+    messages: str = Form(...),
+    ref_image: Optional[UploadFile] = File(None),
+):
+    """
+    主对话端点。
+
+    参数：
+    - messages: JSON 字符串，对话历史 [{role, content}]
+    - ref_image: 可选的参考图文件
+    """
+    parsed_messages = json.loads(messages)
+
+    ref_image_url: Optional[str] = None
+    if ref_image and ref_image.filename:
+        ref_image_url = await _save_upload(ref_image)
+
+    async def event_generator():
+        async for event in run_agent_loop(parsed_messages, ref_image_url):
+            yield {
+                "event": event["type"],
+                "data": json.dumps(event["data"], ensure_ascii=False),
+            }
+
+    return EventSourceResponse(event_generator())

art-agent/backend/app/main.py

@@ -0,0 +1,38 @@
+import os
+from pathlib import Path
+
+from dotenv import load_dotenv
+from fastapi import FastAPI
+from fastapi.middleware.cors import CORSMiddleware
+from fastapi.staticfiles import StaticFiles
+
+from app.api.chat import router as chat_router
+
+load_dotenv()
+
+app = FastAPI(title="Art Agent MVP")
+
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=["*"],
+    allow_credentials=True,
+    allow_methods=["*"],
+    allow_headers=["*"],
+)
+
+# 确保存储目录存在
+UPLOADS_DIR = Path(__file__).parent.parent / "uploads"
+GENERATED_DIR = Path(__file__).parent.parent / "generated"
+UPLOADS_DIR.mkdir(exist_ok=True)
+GENERATED_DIR.mkdir(exist_ok=True)
+
+# 静态文件服务：提供上传的参考图和生成的图片
+app.mount("/uploads", StaticFiles(directory=str(UPLOADS_DIR)), name="uploads")
+app.mount("/generated", StaticFiles(directory=str(GENERATED_DIR)), name="generated")
+
+app.include_router(chat_router, prefix="/api")
+
+
+@app.get("/health")
+async def health():
+    return {"status": "ok"}

art-agent/backend/app/services/image_gen.py

@@ -0,0 +1,76 @@
+"""Replicate 图像生成服务封装。"""
+
+import uuid
+from pathlib import Path
+
+import httpx
+import replicate
+
+GENERATED_DIR = Path(__file__).parent.parent.parent / "generated"
+
+
+async def _download_image(url: str) -> str:
+    """下载远程图片到本地 generated/ 目录，返回本地 URL 路径。"""
+    filename = f"{uuid.uuid4().hex}.png"
+    filepath = GENERATED_DIR / filename
+    async with httpx.AsyncClient() as client:
+        resp = await client.get(url, follow_redirects=True)
+        resp.raise_for_status()
+        filepath.write_bytes(resp.content)
+    return f"/generated/{filename}"
+
+
+async def generate_images(
+    prompt: str,
+    num_images: int = 1,
+    ref_image_url: str | None = None,
+) -> list[str]:
+    """
+    调用 Replicate 生成图片。
+
+    返回本地可访问的图片 URL 列表。
+    """
+    local_urls = []
+
+    for _ in range(num_images):
+        try:
+            if ref_image_url and ref_image_url.startswith("/"):
+                # 本地路径转为 file URI 不适用于 Replicate，
+                # 需要用户上传的图先通过后端 URL 访问
+                # MVP 阶段：参考图作为 prompt 的文字补充，不直接传给模型
+                # 仅使用 flux-schnell 文生图
+                output = await replicate.async_run(
+                    "black-forest-labs/flux-schnell",
+                    input={
+                        "prompt": prompt,
+                        "num_outputs": 1,
+                        "aspect_ratio": "1:1",
+                        "output_format": "png",
+                    },
+                )
+            else:
+                output = await replicate.async_run(
+                    "black-forest-labs/flux-schnell",
+                    input={
+                        "prompt": prompt,
+                        "num_outputs": 1,
+                        "aspect_ratio": "1:1",
+                        "output_format": "png",
+                    },
+                )
+
+            # output 是 FileOutput 列表或单个 URL
+            if isinstance(output, list):
+                for item in output:
+                    url = str(item)
+                    local_url = await _download_image(url)
+                    local_urls.append(local_url)
+            else:
+                url = str(output)
+                local_url = await _download_image(url)
+                local_urls.append(local_url)
+
+        except Exception as e:
+            local_urls.append(f"[生成失败: {e}]")
+
+    return local_urls