小优化，产品愿景，视觉规范，交互和导航大版本
This commit is contained in:
Nostars Developer
@@ -46,6 +46,7 @@ SYSTEM_PROMPT = """\
|
||||
- 如果用户提供了参考图（支持多张），将参考图的风格元素融入生成 prompt
|
||||
- 理解多张参考图各自的角色（如"图1的主体 + 图2的风格/视角"），并在 prompt 中准确传达
|
||||
- 理解用户在图片上的标注（框选区域 + 文字批注），精准定位需要修改的部分
|
||||
- 将一张图片转换为多个不同视角（使用 transform_view 工具），适合建筑、物体等需要从不同角度查看的场景
|
||||
|
||||
## 工作流程
|
||||
1. 理解用户需求，必要时追问细节（尺寸、风格、用途等）
|
||||
@@ -268,9 +269,13 @@ async def run_agent_loop(
|
||||
tc_data = tool_calls_data[idx]
|
||||
tool_name = tc_data["name"]
|
||||
|
||||
tool_message = (
|
||||
"正在进行视角变换..." if tool_name == "transform_view"
|
||||
else "正在生成图片..."
|
||||
)
|
||||
yield {
|
||||
"type": "tool_start",
|
||||
"data": {"tool": tool_name, "message": "正在生成图片..."},
|
||||
"data": {"tool": tool_name, "message": tool_message},
|
||||
}
|
||||
|
||||
try:
|
||||
@@ -292,7 +297,29 @@ async def run_agent_loop(
|
||||
},
|
||||
}
|
||||
|
||||
if result.get("images"):
|
||||
if result.get("error") and not result.get("success"):
|
||||
yield {
|
||||
"type": "tool_error",
|
||||
"data": {
|
||||
"tool": tool_name,
|
||||
"errors": [result["error"]],
|
||||
"model_name": used_model,
|
||||
},
|
||||
}
|
||||
|
||||
if tool_name == "transform_view" and result.get("images"):
|
||||
# transform_view 返回带视角信息的图片列表
|
||||
image_urls = [v["url"] for v in result["images"]]
|
||||
yield {
|
||||
"type": "image_result",
|
||||
"data": {
|
||||
"images": image_urls,
|
||||
"prompt_used": "",
|
||||
"model_name": used_model,
|
||||
"view_info": result["images"],
|
||||
},
|
||||
}
|
||||
elif result.get("images"):
|
||||
yield {
|
||||
"type": "image_result",
|
||||
"data": {
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
"""Agent 可调用的工具定义和实现。"""
|
||||
|
||||
from app.services.image_gen import generate_images
|
||||
from app.services.view_transform import transform_view
|
||||
|
||||
# OpenAI Function Calling 格式的工具定义
|
||||
TOOL_DEFINITIONS = [
|
||||
@@ -32,7 +33,27 @@ TOOL_DEFINITIONS = [
|
||||
"required": ["prompt"],
|
||||
},
|
||||
},
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "transform_view",
|
||||
"description": (
|
||||
"将一张图片转换为多个不同视角。"
|
||||
"使用 Zero123++ 模型从单张图片生成 6 个固定视角的图片。"
|
||||
"适用于建筑、物体等需要从不同角度查看的场景。"
|
||||
"输入图片必须是正方形（或会被自动裁切为正方形），建议分辨率 >= 320x320。"
|
||||
"输出 6 个视角：方位角 30°/90°/150°/210°/270°/330°，"
|
||||
"仰角交替为 30°/-20°（正俯视/微仰视）。"
|
||||
"此工具需要参考图作为输入——必须先有用户上传的图片才能使用。"
|
||||
),
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {},
|
||||
"required": [],
|
||||
},
|
||||
},
|
||||
},
|
||||
]
|
||||
|
||||
|
||||
@@ -65,4 +86,13 @@ async def execute_tool(
|
||||
"model_id": result.model_id,
|
||||
}
|
||||
|
||||
if tool_name == "transform_view":
|
||||
if not ref_image_urls:
|
||||
return {
|
||||
"success": False,
|
||||
"error": "视角变换需要一张输入图片，请先上传参考图",
|
||||
}
|
||||
result = await transform_view(ref_image_urls[0])
|
||||
return result
|
||||
|
||||
return {"success": False, "error": f"未知工具: {tool_name}"}
|
||||
|
||||
@@ -228,6 +228,50 @@ IMAGE_MODELS: dict[str, dict[str, Any]] = {
|
||||
}
|
||||
|
||||
|
||||
# ─── 视角变换模型注册表 ─────────────────────────────────
|
||||
#
|
||||
# 视角变换模型不用于生图，而是将已有图片转换为不同视角。
|
||||
# 输入通常只需一张图，输出为多个固定视角的图片。
|
||||
|
||||
VIEW_TRANSFORM_MODELS: dict[str, dict[str, Any]] = {
|
||||
"zero123plus": {
|
||||
"id": "zero123plus",
|
||||
"name": "Zero123++",
|
||||
"provider": "replicate",
|
||||
"model_id": "jd7h/zero123plusplus:c69c6559a29011b576f1ff0371b3bc1add2856480c60520c7e9ce0b40a6e9052",
|
||||
"description": "单图生成 6 个固定视角，适合建筑/物体的多角度预览",
|
||||
"output_views": [
|
||||
{"azimuth": 30, "elevation": 30},
|
||||
{"azimuth": 90, "elevation": -20},
|
||||
{"azimuth": 150, "elevation": 30},
|
||||
{"azimuth": 210, "elevation": -20},
|
||||
{"azimuth": 270, "elevation": 30},
|
||||
{"azimuth": 330, "elevation": -20},
|
||||
],
|
||||
"grid_layout": {"cols": 3, "rows": 2},
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def get_view_transform_model_config(model_id: str | None = None) -> dict[str, Any]:
|
||||
"""获取视角变换模型配置，默认返回 zero123plus。"""
|
||||
if model_id and model_id in VIEW_TRANSFORM_MODELS:
|
||||
return VIEW_TRANSFORM_MODELS[model_id]
|
||||
return VIEW_TRANSFORM_MODELS["zero123plus"]
|
||||
|
||||
|
||||
def get_view_transform_models_list() -> list[dict]:
|
||||
"""返回前端下拉列表所需的视角变换模型摘要信息。"""
|
||||
return [
|
||||
{
|
||||
"id": cfg["id"],
|
||||
"name": cfg["name"],
|
||||
"description": cfg["description"],
|
||||
}
|
||||
for cfg in VIEW_TRANSFORM_MODELS.values()
|
||||
]
|
||||
|
||||
|
||||
def get_default_image_model_id() -> str:
|
||||
"""返回 .env 中配置的默认模型短 ID，若不在注册表中则回退到 flux-schnell。"""
|
||||
env_model = os.getenv("IMAGE_MODEL", "flux-schnell")
|
||||
|
||||
157
art-agent/backend/app/services/view_transform.py
Normal file
157
art-agent/backend/app/services/view_transform.py
Normal file
@@ -0,0 +1,157 @@
|
||||
"""视角变换服务 — 调用 Zero123++ 等模型，将单张图片转为多视角图片。"""
|
||||
|
||||
import base64
|
||||
import logging
|
||||
import mimetypes
|
||||
import os
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
from PIL import Image
|
||||
|
||||
from app.config import get_view_transform_model_config
|
||||
from app.services.image_gen import _make_replicate_client, GENERATED_DIR, to_data_uri
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_replicate_client = _make_replicate_client()
|
||||
|
||||
|
||||
def _split_grid_image(
|
||||
image_path: Path,
|
||||
cols: int,
|
||||
rows: int,
|
||||
view_labels: list[dict[str, int]],
|
||||
) -> list[dict[str, Any]]:
|
||||
"""将 Zero123++ 输出的 grid 拼接图切分为独立的视角图片。
|
||||
|
||||
返回格式：[{"url": "/generated/xxx.png", "azimuth": 30, "elevation": 30}, ...]
|
||||
Grid 布局从左到右、从上到下依次对应 view_labels 中的视角。
|
||||
"""
|
||||
img = Image.open(image_path)
|
||||
w, h = img.size
|
||||
cell_w = w // cols
|
||||
cell_h = h // rows
|
||||
|
||||
results: list[dict[str, Any]] = []
|
||||
for idx, view in enumerate(view_labels):
|
||||
row_idx = idx // cols
|
||||
col_idx = idx % cols
|
||||
left = col_idx * cell_w
|
||||
upper = row_idx * cell_h
|
||||
right = left + cell_w
|
||||
lower = upper + cell_h
|
||||
|
||||
cropped = img.crop((left, upper, right, lower))
|
||||
filename = f"view_{uuid.uuid4().hex[:8]}_az{view['azimuth']}_el{view['elevation']}.png"
|
||||
filepath = GENERATED_DIR / filename
|
||||
cropped.save(filepath, "PNG")
|
||||
|
||||
results.append({
|
||||
"url": f"/generated/{filename}",
|
||||
"azimuth": view["azimuth"],
|
||||
"elevation": view["elevation"],
|
||||
})
|
||||
|
||||
return results
|
||||
|
||||
|
||||
async def _download_to_local(url: str) -> Path:
|
||||
"""下载远程图片到本地 generated/ 目录，返回本地路径。"""
|
||||
filename = f"grid_{uuid.uuid4().hex}.png"
|
||||
filepath = GENERATED_DIR / filename
|
||||
proxy = os.environ.get("HTTPS_PROXY") or os.environ.get("HTTP_PROXY")
|
||||
async with httpx.AsyncClient(proxy=proxy, timeout=httpx.Timeout(120.0)) as client:
|
||||
resp = await client.get(url, follow_redirects=True)
|
||||
resp.raise_for_status()
|
||||
filepath.write_bytes(resp.content)
|
||||
return filepath
|
||||
|
||||
|
||||
async def transform_view(
|
||||
image_path: str,
|
||||
model_id: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""调用视角变换模型，返回多视角图片列表。
|
||||
|
||||
Args:
|
||||
image_path: 输入图片路径（本地路径、data URI 或 URL）
|
||||
model_id: 视角变换模型 ID，默认 zero123plus
|
||||
|
||||
Returns:
|
||||
{
|
||||
"success": bool,
|
||||
"images": [{"url": str, "azimuth": int, "elevation": int}, ...],
|
||||
"grid_image": str, # 原始拼接图的本地 URL
|
||||
"model_name": str,
|
||||
"error": str | None,
|
||||
}
|
||||
"""
|
||||
config = get_view_transform_model_config(model_id)
|
||||
model_name = config["name"]
|
||||
replicate_model_id = config["model_id"]
|
||||
grid_layout = config["grid_layout"]
|
||||
output_views = config["output_views"]
|
||||
|
||||
GENERATED_DIR.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
try:
|
||||
image_uri = to_data_uri(image_path)
|
||||
input_params = {"image": image_uri}
|
||||
|
||||
logger.info("调用 %s 进行视角变换...", model_name)
|
||||
output = await _replicate_client.async_run(
|
||||
replicate_model_id, input=input_params, wait=False
|
||||
)
|
||||
|
||||
# Zero123++ 返回单张 grid 拼接图
|
||||
items = output if isinstance(output, list) else [output]
|
||||
if not items:
|
||||
return {
|
||||
"success": False,
|
||||
"images": [],
|
||||
"grid_image": None,
|
||||
"model_name": model_name,
|
||||
"error": "模型未返回任何输出",
|
||||
}
|
||||
|
||||
grid_url = str(items[0])
|
||||
if not (grid_url.startswith("http://") or grid_url.startswith("https://")):
|
||||
return {
|
||||
"success": False,
|
||||
"images": [],
|
||||
"grid_image": None,
|
||||
"model_name": model_name,
|
||||
"error": f"模型返回非图片内容: {grid_url[:200]}",
|
||||
}
|
||||
|
||||
grid_local_path = await _download_to_local(grid_url)
|
||||
grid_local_url = f"/generated/{grid_local_path.name}"
|
||||
|
||||
view_images = _split_grid_image(
|
||||
grid_local_path,
|
||||
cols=grid_layout["cols"],
|
||||
rows=grid_layout["rows"],
|
||||
view_labels=output_views,
|
||||
)
|
||||
|
||||
return {
|
||||
"success": True,
|
||||
"images": view_images,
|
||||
"grid_image": grid_local_url,
|
||||
"model_name": model_name,
|
||||
"error": None,
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
detail = str(e) or f"{type(e).__name__}: {repr(e)}"
|
||||
logger.error("视角变换失败: %s", detail, exc_info=True)
|
||||
return {
|
||||
"success": False,
|
||||
"images": [],
|
||||
"grid_image": None,
|
||||
"model_name": model_name,
|
||||
"error": detail,
|
||||
}
|
||||
Reference in New Issue
Block a user