chore: init monorepo snapshot

This commit is contained in:
liaibo
2025-11-23 10:55:04 +08:00
commit c70ff52869
941 changed files with 246586 additions and 0 deletions
+16
View File
@@ -0,0 +1,16 @@
# Python / FastAPI backend ignores
venv/
__pycache__/
*.pyc
.python-version
.env
.env.*
# Editor / tooling
.idea/
.vscode/
.DS_Store
# Local data
storage/
logs/
+59
View File
@@ -0,0 +1,59 @@
# wolai-backendstage0 骨架)
本目录是 design3.0 指南下的 FastAPI + Celery 阶段 0 脚手架。当前目标:
1. 暴露健康检查与占位 API`/api/v1/tasks/ocr``/api/v1/chat`),供前端联调。
2. 预留 Celery 队列、LightRAG、MinerU 集成点,但将重计算逻辑延后到阶段 1。
3. 依赖全部维护在 `requirements.txt`,其中 LightRAG 与 MinerU 通过本地源码安装,使用项目根目录已经下载好的仓库。
## 环境初始化
```powershell
python -m venv venv
.\venv\Scripts\activate
# 如遇代理报错,可在同一终端先运行:
# set HTTP_PROXY=
# set HTTPS_PROXY=
# set ALL_PROXY=
# set NO_PROXY=*
# 安装 PyPI 依赖 + 本地 LightRAG (项目根目录已放置源码)
pip install -r requirements.txt
```
> MinerU 已位于 `services/mineru`,按 README 启动为独立 OCR 服务,无需作为 Python 包安装。
## 运行服务
```powershell
# FastAPI
uvicorn app.main:app --reload --port 8000
# Celery worker(后续阶段会真正处理 MinerU / LightRAG 作业)
celery -A app.workers.celery_app worker --loglevel=info
```
## 目录速览
```
app/
├── config.py # Pydantic 设置,统一读取 .env
├── deps.py # Supabase JWT / 客户端依赖
├── routers/
│ ├── health.py # GET /health
│ ├── tasks.py # POST /api/v1/tasks/ocr、GET /api/v1/tasks/{id}
│ └── chat.py # GET /api/v1/chatSSE 占位)
├── schemas/ # Pydantic 请求/响应模型
├── services/ # LightRAG、MinerU、Storage 等服务占位
└── workers/
├── celery_app.py # Celery 配置
└── tasks.py # 后台任务入口(当前返回占位日志)
```
阶段 0 完成后,stage1 将在此基础上补充:
- MinerU OCR 实装(下载 Supabase Storage 文件 → 解析 → 更新 `documents`
- LightRAG 的增量索引 / 查询
- SSE 流式回答 + 来源引用
- Redis/APScheduler 任务状态回传
+1
View File
@@ -0,0 +1 @@
"""FastAPI 应用初始化模块。"""
+25
View File
@@ -0,0 +1,25 @@
from functools import lru_cache
from pydantic_settings import BaseSettings, SettingsConfigDict
class Settings(BaseSettings):
"""集中管理项目配置,来源于 .env / 环境变量。"""
model_config = SettingsConfigDict(env_file=".env", env_file_encoding="utf-8", extra="ignore")
supabase_url: str
supabase_service_role_key: str
redis_url: str = "redis://localhost:6379/0"
frontend_url: str = "http://localhost:3000"
openai_api_key: str = ""
lightrag_db_url: str
lightrag_collection: str = "wolai-docs"
@lru_cache
def get_settings() -> Settings:
"""惰性实例化设置,避免重复读取文件。"""
return Settings()
settings = get_settings()
+58
View File
@@ -0,0 +1,58 @@
from __future__ import annotations
from dataclasses import dataclass
from typing import Optional
try: # Python 3.9+
from typing import Annotated # type: ignore[attr-defined]
except ImportError: # Python 3.8 fallback
from typing_extensions import Annotated
import httpx
from fastapi import Depends, Header, HTTPException, status
from app.config import settings
@dataclass
class AuthContext:
user_id: str
access_token: str
async def get_current_user(
authorization: Annotated[Optional[str], Header(convert_underscores=False)] = None,
) -> AuthContext:
"""
验证 Supabase JWTstage0 直接依赖 service_role 解析 token。
生产环境应通过 API Gateway 注入 user。
"""
if not authorization or not authorization.startswith("Bearer "):
raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="Missing bearer token")
token = authorization.replace("Bearer ", "", 1).strip()
if not token:
raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="Empty token")
auth_url = f"{settings.supabase_url.rstrip('/')}/auth/v1/user"
headers = {
"Authorization": f"Bearer {token}",
"apikey": settings.supabase_service_role_key,
}
async with httpx.AsyncClient(timeout=10.0) as client:
try:
response = await client.get(auth_url, headers=headers)
response.raise_for_status()
except httpx.HTTPError as exc: # pragma: no cover - 网络异常
raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="Invalid Supabase token") from exc
data = response.json()
user_id = data.get("id") or data.get("user", {}).get("id")
if not user_id:
raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="Invalid Supabase token")
return AuthContext(user_id=str(user_id), access_token=token)
AuthDep = Annotated[AuthContext, Depends(get_current_user)]
+25
View File
@@ -0,0 +1,25 @@
from typing import Dict
from fastapi import FastAPI
from fastapi.middleware.cors import CORSMiddleware
from app.config import settings
from app.routers import api_router, root_router
app = FastAPI(title="Wolai Backend", version="0.1.0-stage0")
app.add_middleware(
CORSMiddleware,
allow_origins=[settings.frontend_url, "http://localhost:3000", "http://localhost:3001"],
allow_credentials=True,
allow_methods=["*"],
allow_headers=["*"],
)
app.include_router(root_router)
app.include_router(api_router)
@app.get("/")
async def root() -> Dict[str, str]:
return {"message": "Wolai backend stage0 up"}
+10
View File
@@ -0,0 +1,10 @@
from fastapi import APIRouter
from . import chat, health, tasks
api_router = APIRouter(prefix="/api/v1")
api_router.include_router(tasks.router, tags=["tasks"])
api_router.include_router(chat.router, tags=["chat"])
root_router = APIRouter()
root_router.include_router(health.router, tags=["health"])
+26
View File
@@ -0,0 +1,26 @@
import asyncio
from fastapi import APIRouter
from fastapi.responses import StreamingResponse
from app.deps import AuthDep
from app.services.lightrag_service import lightrag_service
router = APIRouter(prefix="/chat")
@router.get("")
async def chat(query: str, document_id: str, auth: AuthDep) -> StreamingResponse: # noqa: ARG001
"""
SSE 流式占位。后续会调用 LightRAG + OpenAI。
当前直接返回 mock 文字,确保前端链路可用。
"""
async def event_stream() -> asyncio.AsyncGenerator[str, None]:
reply = await lightrag_service.query(query_text=query, user_id="placeholder-user")
chunks = [reply[: len(reply) // 2 or 1], reply[len(reply) // 2 or 1 :]]
for chunk in chunks:
yield f"data: {chunk}\n\n"
await asyncio.sleep(0.1)
yield "data: [DONE]\n\n"
return StreamingResponse(event_stream(), media_type="text/event-stream")
+11
View File
@@ -0,0 +1,11 @@
from typing import Dict
from fastapi import APIRouter
router = APIRouter()
@router.get("/health")
async def healthcheck() -> Dict[str, str]:
"""简单健康检查,供阶段 0 自检使用。"""
return {"status": "ok"}
+29
View File
@@ -0,0 +1,29 @@
from fastapi import APIRouter, HTTPException, status
from app.deps import AuthDep
from app.schemas.tasks import OcrTaskRequest, TaskStatusResponse
from app.services.task_tracker import task_tracker
from app.workers.tasks import ocr_pipeline
router = APIRouter(prefix="/tasks")
@router.post("/ocr", response_model=TaskStatusResponse)
async def enqueue_ocr_task(payload: OcrTaskRequest, auth: AuthDep) -> TaskStatusResponse:
"""记录任务并投递 Celery,阶段 0 返回占位任务。"""
task = task_tracker.create_task(user_id=auth.user_id, document_id=payload.document_id, task_type="ocr")
ocr_pipeline.delay(
task_id=task.task_id,
document_id=payload.document_id,
file_url=str(payload.file_url),
user_id=auth.user_id,
)
return task
@router.get("/{task_id}", response_model=TaskStatusResponse)
async def get_task_status(task_id: str, auth: AuthDep) -> TaskStatusResponse:
task = task_tracker.get_task(task_id=task_id, user_id=auth.user_id)
if not task:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Task not found")
return task
+17
View File
@@ -0,0 +1,17 @@
from __future__ import annotations
from typing import Literal, Optional
from pydantic import AnyHttpUrl, BaseModel
class OcrTaskRequest(BaseModel):
document_id: str
file_url: AnyHttpUrl
class TaskStatusResponse(BaseModel):
task_id: str
status: Literal["pending", "processing", "completed", "failed"]
progress: int = 0
message: Optional[str] = None
@@ -0,0 +1,19 @@
"""LightRAG 集成占位。阶段 1 会在这里封装真正的查询与索引。"""
from app.config import settings
class LightRAGService:
def __init__(self) -> None:
self.collection = settings.lightrag_collection
async def queue_index(self, document_id: str, raw_text: str) -> None:
"""预留方法:后续调用 LightRAG.update_index。"""
return None
async def query(self, query_text: str, user_id: str) -> str:
"""预留方法:后续调用 LightRAG.query,当前返回占位回答。"""
return f"[mock] {query_text}"
lightrag_service = LightRAGService()
@@ -0,0 +1,13 @@
"""MinerU OCR 占位服务。"""
class MinerUService:
async def extract_markdown(self, file_url: str) -> str:
"""
阶段 0:直接返回固定内容,保证前端流程贯通。
阶段 1:调用 MinerU CLI / SDK,从 Supabase Storage 下载文件后解析。
"""
return f"# OCR Placeholder\n\n源文件:{file_url}"
mineru_service = MinerUService()
@@ -0,0 +1,57 @@
from __future__ import annotations
from typing import Any, Dict, Optional
import httpx
from app.config import settings
class SupabaseRestClient:
"""轻量封装 Supabase RESTful API,兼容本地 sb_secret 密钥。"""
def __init__(self) -> None:
base_url = settings.supabase_url.rstrip("/")
self.client = httpx.Client(
base_url=f"{base_url}/rest/v1",
headers={
"apikey": settings.supabase_service_role_key,
"Authorization": f"Bearer {settings.supabase_service_role_key}",
},
timeout=10.0,
)
def insert(self, table: str, payload: Dict[str, Any]) -> Dict[str, Any]:
response = self.client.post(
f"/{table}",
json=payload,
headers={"Prefer": "return=representation"},
)
response.raise_for_status()
data = response.json()
if isinstance(data, list):
return data[0]
return data
def select_one(self, table: str, filters: Dict[str, Any]) -> Optional[Dict[str, Any]]:
params = {key: f"eq.{value}" for key, value in filters.items()}
params["select"] = "*"
response = self.client.get(f"/{table}", params=params)
response.raise_for_status()
data = response.json()
if isinstance(data, list) and data:
return data[0]
return None
def update(self, table: str, filters: Dict[str, Any], payload: Dict[str, Any]) -> None:
params = {key: f"eq.{value}" for key, value in filters.items()}
response = self.client.patch(
f"/{table}",
params=params,
json=payload,
headers={"Prefer": "return=minimal"},
)
response.raise_for_status()
supabase_rest = SupabaseRestClient()
@@ -0,0 +1,43 @@
from __future__ import annotations
from typing import Any, Dict, Optional
from app.schemas.tasks import TaskStatusResponse
from app.services.supabase_rest import supabase_rest
class SupabaseTaskTracker:
"""利用 Supabase background_tasks 表追踪进度。"""
def create_task(self, *, user_id: str, document_id: str, task_type: str = "ocr") -> TaskStatusResponse:
payload = {
"user_id": user_id,
"document_id": document_id,
"task_type": task_type,
"status": "pending",
"progress": 0,
}
record = supabase_rest.insert("background_tasks", payload)
return self._to_response(record)
def get_task(self, *, task_id: str, user_id: str) -> Optional[TaskStatusResponse]:
record = supabase_rest.select_one("background_tasks", {"id": task_id, "user_id": user_id})
if not record:
return None
return self._to_response(record)
def update_task(self, *, task_id: str, **kwargs: Any) -> None:
supabase_rest.update("background_tasks", {"id": task_id}, kwargs)
def _to_response(self, record: Optional[Dict[str, Any]]) -> TaskStatusResponse:
if not record:
raise ValueError("Task record missing")
return TaskStatusResponse(
task_id=str(record["id"]),
status=record.get("status", "pending"),
progress=record.get("progress", 0),
message=record.get("message"),
)
task_tracker = SupabaseTaskTracker()
+15
View File
@@ -0,0 +1,15 @@
from celery import Celery
from app.config import settings
def create_celery_app() -> Celery:
app = Celery("wolai-backend")
app.conf.broker_url = settings.redis_url
app.conf.result_backend = settings.redis_url
app.conf.task_routes = {"app.workers.tasks.*": {"queue": "wolai-tasks"}}
app.autodiscover_tasks(["app.workers"])
return app
celery_app = create_celery_app()
+51
View File
@@ -0,0 +1,51 @@
from __future__ import annotations
from typing import Dict
from celery import shared_task
from app.services.supabase_rest import supabase_rest
@shared_task(name="app.workers.tasks.ocr_pipeline")
def ocr_pipeline(task_id: str, document_id: str, file_url: str, user_id: str) -> Dict[str, str]:
"""
阶段 0 Celery 任务:模拟 OCR,向 documents+background_tasks 回写占位结果。
阶段 1 将在此处串联 MinerU OCR 与 LightRAG 索引。
"""
supabase_rest.update("background_tasks", {"id": task_id}, {"status": "processing", "progress": 30})
markdown = f"# OCR 结果占位\\n\\n文件地址:{file_url}\\n\\n> 阶段 1 将替换为 MinerU 输出。"
supabase_rest.update(
"documents",
{"id": document_id, "user_id": user_id},
{
"content": {
"blocks": [
{
"type": "paragraph",
"text": markdown,
}
]
},
"raw_text": markdown,
"index_status": "completed",
},
)
supabase_rest.update(
"background_tasks",
{"id": task_id},
{
"status": "completed",
"progress": 100,
"message": "OCR 模拟完成",
},
)
return {
"task_id": task_id,
"document_id": document_id,
"file_url": file_url,
"status": "completed",
}
File diff suppressed because it is too large Load Diff
+19
View File
@@ -0,0 +1,19 @@
-------------- celery@LIX v5.4.0 (opalescent)
--- ***** -----
-- ******* ---- Windows-10-10.0.26100-SP0 2025-11-18 18:34:36
- *** --- * ---
- ** ---------- [config]
- ** ---------- .> app: wolai-backend:0x2b6d01c47d0
- ** ---------- .> transport: redis://localhost:6379/0
- ** ---------- .> results: redis://localhost:6379/0
- *** --- * --- .> concurrency: 24 (prefork)
-- ******* ---- .> task events: OFF (enable -E to monitor tasks in this worker)
--- ***** -----
-------------- [queues]
.> celery exchange=celery(direct) key=celery
[tasks]
. app.workers.tasks.ocr_pipeline
+11
View File
@@ -0,0 +1,11 @@
fastapi==0.115.5
uvicorn[standard]==0.33.0
celery[redis]==5.4.0
redis==5.1.1
httpx==0.27.2
python-dotenv==1.0.1
pgvector==0.3.5
openai==1.59.5
python-multipart==0.0.17
pydantic-settings==2.6.1
typing_extensions==4.12.2
View File
View File
+57
View File
@@ -0,0 +1,57 @@
INFO: Will watch for changes in these directories: ['F:\\SOFT\\MNOTE\\wolai-backend']
INFO: Uvicorn running on http://127.0.0.1:8000 (Press CTRL+C to quit)
INFO: Started reloader process [20384] using WatchFiles
INFO: Started server process [6920]
INFO: Waiting for application startup.
INFO: Application startup complete.
WARNING: WatchFiles detected changes in 'app\main.py'. Reloading...
INFO: Shutting down
INFO: Waiting for application shutdown.
INFO: Application shutdown complete.
INFO: Finished server process [6920]
INFO: Started server process [5608]
INFO: Waiting for application startup.
INFO: Application startup complete.
WARNING: WatchFiles detected changes in 'app\deps.py'. Reloading...
INFO: Shutting down
INFO: Waiting for application shutdown.
INFO: Application shutdown complete.
INFO: Finished server process [5608]
WARNING: WatchFiles detected changes in 'app\deps.py', 'app\routers\tasks.py', 'app\workers\tasks.py', 'app\services\task_tracker.py'. Reloading...
INFO: Started server process [33148]
INFO: Waiting for application startup.
INFO: Application startup complete.
ERROR: Traceback (most recent call last):
File "C:\Users\liaib\AppData\Local\Programs\Python\Python311\Lib\asyncio\runners.py", line 118, in run
return self._loop.run_until_complete(task)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "C:\Users\liaib\AppData\Local\Programs\Python\Python311\Lib\asyncio\base_events.py", line 654, in run_until_complete
return future.result()
^^^^^^^^^^^^^^^
asyncio.exceptions.CancelledError
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "C:\Users\liaib\AppData\Local\Programs\Python\Python311\Lib\asyncio\runners.py", line 190, in run
return runner.run(main)
^^^^^^^^^^^^^^^^
File "C:\Users\liaib\AppData\Local\Programs\Python\Python311\Lib\asyncio\runners.py", line 123, in run
raise KeyboardInterrupt()
KeyboardInterrupt
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "F:\SOFT\MNOTE\wolai-backend\venv\Lib\site-packages\starlette\routing.py", line 700, in lifespan
await receive()
File "F:\SOFT\MNOTE\wolai-backend\venv\Lib\site-packages\uvicorn\lifespan\on.py", line 137, in receive
return await self.receive_queue.get()
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "C:\Users\liaib\AppData\Local\Programs\Python\Python311\Lib\asyncio\queues.py", line 158, in get
await getter
asyncio.exceptions.CancelledError
INFO: Started server process [12156]
INFO: Waiting for application startup.
INFO: Application startup complete.
+1
View File
@@ -0,0 +1 @@
INFO: 127.0.0.1:57108 - "GET /health HTTP/1.1" 200 OK