chore: init monorepo snapshot

services/ingest_service/app/api/routes.py

@@ -0,0 +1,38 @@
+from fastapi import APIRouter, HTTPException
+
+from app.schemas.job import JobCreate, JobResponse, JobStatus
+from app.services.runtime import job_store, worker
+
+router = APIRouter()
+
+
+@router.post("/ingest", response_model=JobResponse)
+async def ingest(job: JobCreate) -> JobResponse:
+    """创建一次全量 ingest 任务并入队"""
+    created = await worker.enqueue(
+        notebook_id=job.notebook_id,
+        doc_id=job.doc_id,
+        source=job.source,
+        blocks=job.blocks or [],
+    )
+    return JobResponse(job=JobStatus(**created.__dict__))
+
+
+@router.post("/ingest/incremental", response_model=JobResponse)
+async def ingest_incremental(job: JobCreate) -> JobResponse:
+    """增量模式目前与全量逻辑一致，后续通过 checkpoint 提升效率"""
+    created = await worker.enqueue(
+        notebook_id=job.notebook_id,
+        doc_id=job.doc_id,
+        source=job.source,
+        blocks=job.blocks or [],
+    )
+    return JobResponse(job=JobStatus(**created.__dict__))
+
+
+@router.get("/jobs/{job_id}", response_model=JobResponse)
+async def get_job(job_id: str) -> JobResponse:
+    job = await job_store.get(job_id)
+    if not job:
+        raise HTTPException(status_code=404, detail="Job not found")
+    return JobResponse(job=JobStatus(**job.__dict__))

services/ingest_service/app/core/config.py

@@ -0,0 +1,28 @@
+import os
+from functools import lru_cache
+from pydantic import BaseSettings, Field
+
+
+class Settings(BaseSettings):
+    """FastAPI 配置，统一读取 .env 或系统环境变量"""
+
+    app_name: str = Field("AIMNOTE Ingest Service", env="INGEST_APP_NAME")
+    environment: str = Field("development", env="INGEST_ENV")
+    api_prefix: str = "/api"
+    supabase_url: str = Field("http://127.0.0.1:54321", env="SUPABASE_URL")
+    supabase_key: str = Field("", env="SUPABASE_SERVICE_ROLE_KEY")
+    lightrag_url: str = Field("http://127.0.0.1:7777", env="LIGHTRAG_URL")
+    ollama_base_url: str = Field("http://127.0.0.1:11434/v1", env="OLLAMA_BASE_URL")
+    ollama_api_key: str = Field("ollama", env="OLLAMA_API_KEY")
+    embeddings_model: str = Field("qwen3-embedding:8b", env="EMBEDDING_MODEL")
+    llm_model: str = Field("qwen3:32b", env="DEFAULT_LLM_MODEL")
+    rerank_model: str = Field("qwen3-reranker-4b", env="RERANK_MODEL")
+
+    class Config:
+        env_file = ".env"
+        case_sensitive = False
+
+
+@lru_cache()
+def get_settings() -> Settings:
+    return Settings()

services/ingest_service/app/core/logging.py

@@ -0,0 +1,27 @@
+import json
+import logging
+import sys
+from typing import Any, Dict
+
+
+class JsonFormatter(logging.Formatter):
+    """将日志输出格式化为JSON"""
+
+    def format(self, record: logging.LogRecord) -> str:  # type: ignore[override]
+        log: Dict[str, Any] = {
+            "ts": self.formatTime(record, datefmt="%Y-%m-%dT%H:%M:%S%z"),
+            "level": record.levelname,
+            "logger": record.name,
+            "message": record.getMessage(),
+        }
+        if record.exc_info:
+            log["exc_info"] = self.formatException(record.exc_info)
+        for key, value in getattr(record, "extra", {}).items():
+            log[key] = value
+        return json.dumps(log, ensure_ascii=False)
+
+
+def setup_logging(level: str = "INFO") -> None:
+    handler = logging.StreamHandler(sys.stdout)
+    handler.setFormatter(JsonFormatter())
+    logging.basicConfig(handlers=[handler], level=level, force=True)

services/ingest_service/app/main.py

@@ -0,0 +1,26 @@
+import asyncio
+from fastapi import FastAPI
+
+from app.api.routes import router as api_router
+from app.core.config import get_settings
+from app.core.logging import setup_logging
+from app.services.runtime import scheduler, worker
+
+settings = get_settings()
+
+setup_logging()
+
+app = FastAPI(title=settings.app_name)
+app.include_router(api_router, prefix=settings.api_prefix)
+
+
+@app.on_event("startup")
+async def _startup() -> None:
+    scheduler.start()
+    await worker.start()
+
+
+@app.on_event("shutdown")
+async def _shutdown() -> None:
+    scheduler.shutdown()
+    await worker.stop()

services/ingest_service/app/models/job.py

@@ -0,0 +1,32 @@
+from dataclasses import dataclass, field
+from datetime import datetime
+from typing import Dict, List, Optional
+
+
+@dataclass
+class IngestJob:
+    """表示一次 ingest 任务，后续可持久化到 Supabase"""
+
+    id: str
+    notebook_id: Optional[str]
+    doc_id: Optional[str]
+    status: str = "queued"
+    source: str = "manual"
+    blocks: List[str] = field(default_factory=list)
+    created_at: datetime = field(default_factory=datetime.utcnow)
+    updated_at: datetime = field(default_factory=datetime.utcnow)
+    error_message: Optional[str] = None
+    payload: Dict[str, str] = field(default_factory=dict)
+
+    def mark_running(self) -> None:
+        self.status = "running"
+        self.updated_at = datetime.utcnow()
+
+    def mark_done(self) -> None:
+        self.status = "succeeded"
+        self.updated_at = datetime.utcnow()
+
+    def mark_failed(self, message: str) -> None:
+        self.status = "failed"
+        self.error_message = message
+        self.updated_at = datetime.utcnow()

services/ingest_service/app/schemas/job.py

@@ -0,0 +1,24 @@
+from datetime import datetime
+from typing import Optional, List
+from pydantic import BaseModel, Field
+
+
+class JobCreate(BaseModel):
+    notebook_id: Optional[str]
+    doc_id: Optional[str]
+    source: str = "manual"
+    blocks: Optional[List[str]] = None
+
+
+class JobStatus(BaseModel):
+    id: str
+    status: str
+    notebook_id: Optional[str]
+    doc_id: Optional[str]
+    created_at: datetime
+    updated_at: datetime
+    error_message: Optional[str]
+
+
+class JobResponse(BaseModel):
+    job: JobStatus

services/ingest_service/app/services/embeddings.py

@@ -0,0 +1,35 @@
+import logging
+from typing import List, Optional
+
+import httpx
+
+logger = logging.getLogger(__name__)
+
+
+class EmbeddingGenerator:
+    """简单的 OpenAI 兼容嵌入生成器"""
+
+    def __init__(self, api_base: str, api_key: str, model: str, timeout: float = 30.0) -> None:
+        self.api_base = api_base.rstrip("/")
+        self.api_key = api_key
+        self.model = model
+        self.timeout = timeout
+
+    async def embed_texts(self, texts: List[str]) -> List[Optional[List[float]]]:
+        if not texts:
+            return []
+        payload = {"input": texts, "model": self.model}
+        headers = {
+            "Authorization": f"Bearer {self.api_key}",
+            "Content-Type": "application/json",
+        }
+        async with httpx.AsyncClient(timeout=self.timeout) as client:
+            resp = await client.post(f"{self.api_base}/embeddings", json=payload, headers=headers)
+            resp.raise_for_status()
+            data = resp.json()
+            embeddings = []
+            for item in data.get("data", []):
+                embeddings.append(item.get("embedding"))
+            if len(embeddings) != len(texts):
+                logger.warning("嵌入结果数量与输入不一致：%s != %s", len(embeddings), len(texts))
+            return embeddings

services/ingest_service/app/services/job_store.py

@@ -0,0 +1,24 @@
+import asyncio
+from typing import Dict, Optional
+
+from app.models.job import IngestJob
+
+
+class InMemoryJobStore:
+    """简单的内存 JobStore，后续可替换为 Supabase 持久化"""
+
+    def __init__(self) -> None:
+        self._jobs: Dict[str, IngestJob] = {}
+        self._lock = asyncio.Lock()
+
+    async def save(self, job: IngestJob) -> None:
+        async with self._lock:
+            self._jobs[job.id] = job
+
+    async def get(self, job_id: str) -> Optional[IngestJob]:
+        async with self._lock:
+            return self._jobs.get(job_id)
+
+    async def all(self) -> Dict[str, IngestJob]:
+        async with self._lock:
+            return dict(self._jobs)

services/ingest_service/app/services/runtime.py

@@ -0,0 +1,42 @@
+import sys
+from pathlib import Path
+
+from app.core.config import get_settings
+from app.models.job import IngestJob
+from app.services.embeddings import EmbeddingGenerator
+from app.services.job_store import InMemoryJobStore
+from app.services.scheduler import SchedulerManager
+from app.services.tasks import IngestWorker
+from app.services.webhooks import LightRAGWebhook
+
+ROOT = Path(__file__).resolve().parents[3]
+sys.path.append(str(ROOT / "packages" / "siyuan_ingest" / "src"))
+from siyuan_ingest import SiYuanClient, SupabaseSync, SupabaseWriter  # type: ignore  # noqa: E402
+
+settings = get_settings()
+
+job_store = InMemoryJobStore()
+scheduler = SchedulerManager()
+siyuan_client = SiYuanClient()
+webhook = LightRAGWebhook()
+embedding_generator = EmbeddingGenerator(
+    api_base=settings.ollama_base_url,
+    api_key=settings.ollama_api_key,
+    model=settings.embeddings_model,
+)
+supabase_sync = SupabaseSync(
+    supabase_url=settings.supabase_url + "/rest/v1",
+    supabase_key=settings.supabase_key,
+)
+supabase_writer = SupabaseWriter(
+    supabase_url=settings.supabase_url + "/rest/v1",
+    supabase_key=settings.supabase_key,
+)
+worker = IngestWorker(
+    job_store=job_store,
+    siyuan_client=siyuan_client,
+    webhook=webhook,
+    supabase_sync=supabase_sync,
+    supabase_writer=supabase_writer,
+    embedding_generator=embedding_generator,
+)

services/ingest_service/app/services/scheduler.py

@@ -0,0 +1,41 @@
+import asyncio
+import logging
+from typing import Awaitable, Callable, Dict, Optional
+
+from apscheduler.schedulers.asyncio import AsyncIOScheduler
+from apscheduler.triggers.interval import IntervalTrigger
+
+logger = logging.getLogger(__name__)
+
+
+class SchedulerManager:
+    """封装 APScheduler，提供周期性增量任务"""
+
+    def __init__(self) -> None:
+        self.scheduler = AsyncIOScheduler()
+        self._jobs: Dict[str, str] = {}
+
+    def start(self) -> None:
+        if not self.scheduler.running:
+            self.scheduler.start()
+            logger.info("APScheduler 启动完成")
+
+    def shutdown(self) -> None:
+        if self.scheduler.running:
+            self.scheduler.shutdown(wait=False)
+            logger.info("APScheduler 已关闭")
+
+    def add_interval_job(
+        self,
+        job_id: str,
+        func: Callable[..., Awaitable[None]],
+        seconds: int,
+        **kwargs,
+    ) -> None:
+        if job_id in self._jobs:
+            logger.info("任务 %s 已存在，跳过重复注册", job_id)
+            return
+        trigger = IntervalTrigger(seconds=seconds)
+        job = self.scheduler.add_job(func, trigger=trigger, id=job_id, kwargs=kwargs, max_instances=1)
+        self._jobs[job_id] = job.id
+        logger.info("已注册周期任务 %s，每隔 %s 秒执行", job_id, seconds)

services/ingest_service/app/services/tasks.py

@@ -0,0 +1,125 @@
+import asyncio
+import contextlib
+import logging
+import sys
+from pathlib import Path
+from typing import Dict, List, Optional
+from uuid import uuid4
+
+from app.models.job import IngestJob
+from app.services.embeddings import EmbeddingGenerator
+from app.services.job_store import InMemoryJobStore
+from app.services.webhooks import LightRAGWebhook
+
+ROOT = Path(__file__).resolve().parents[3]
+sys.path.append(str(ROOT / "packages" / "siyuan_ingest" / "src"))
+from siyuan_ingest import (  # type: ignore  # noqa: E402
+    IngestState,
+    SiYuanClient,
+    SiYuanContentExtractor,
+    SupabaseSync,
+    SupabaseWriter,
+    chunk_blocks,
+)
+
+logger = logging.getLogger(__name__)
+
+
+class IngestWorker:
+    """处理 ingest 队列，串联 SiYuan → Supabase → LightRAG"""
+
+    def __init__(
+        self,
+        job_store: InMemoryJobStore,
+        siyuan_client: SiYuanClient,
+        webhook: LightRAGWebhook,
+        supabase_sync: Optional[SupabaseSync] = None,
+        supabase_writer: Optional[SupabaseWriter] = None,
+        embedding_generator: Optional[EmbeddingGenerator] = None,
+    ) -> None:
+        self.job_store = job_store
+        self.siyuan_client = siyuan_client
+        self.extractor = SiYuanContentExtractor(siyuan_client)
+        self.webhook = webhook
+        self.supabase_sync = supabase_sync
+        self.supabase_writer = supabase_writer
+        self.embedding_generator = embedding_generator
+        self.queue: asyncio.Queue[IngestJob] = asyncio.Queue()
+        self._consumer_task: Optional[asyncio.Task[None]] = None
+
+    async def start(self) -> None:
+        if not self._consumer_task:
+            self._consumer_task = asyncio.create_task(self._consume())
+
+    async def stop(self) -> None:
+        if self._consumer_task:
+            self._consumer_task.cancel()
+            with contextlib.suppress(asyncio.CancelledError):
+                await self._consumer_task
+
+    async def enqueue(self, notebook_id: Optional[str], doc_id: Optional[str], source: str, blocks: List[str]) -> IngestJob:
+        job = IngestJob(
+            id=str(uuid4()),
+            notebook_id=notebook_id,
+            doc_id=doc_id,
+            source=source,
+            blocks=blocks,
+        )
+        await self.job_store.save(job)
+        await self.queue.put(job)
+        logger.info("排队 ingest 任务 %s", job.id)
+        return job
+
+    async def _consume(self) -> None:
+        while True:
+            job = await self.queue.get()
+            await self._handle_job(job)
+            self.queue.task_done()
+
+    async def _handle_job(self, job: IngestJob) -> None:
+        try:
+            job.mark_running()
+            await self.job_store.save(job)
+            blocks = await self.extractor.list_blocks(job.notebook_id or "")
+            chunks = chunk_blocks(blocks)
+            logger.info("任务 %s 产生 %s 个 chunk", job.id, len(chunks))
+            await self._maybe_embed(chunks)
+            await self._persist_to_supabase(job, chunks)
+            await self.webhook.notify_ingest(job.doc_id, [c["block_id"] for c in chunks if c.get("block_id")])
+            job.mark_done()
+        except Exception as exc:  # noqa: BLE001
+            logger.exception("任务 %s 失败: %s", job.id, exc)
+            job.mark_failed(str(exc))
+        finally:
+            await self.job_store.save(job)
+
+    async def _persist_to_supabase(self, job: IngestJob, chunks: List[Dict[str, str]]) -> None:
+        await self._maybe_sync_state(job, chunks)
+        if not self.supabase_writer:
+            logger.debug("未配置 SupabaseWriter，跳过落库")
+            return
+        await self.supabase_writer.ingest_chunks(job.notebook_id, chunks)
+
+    async def _maybe_sync_state(self, job: IngestJob, chunks: List[Dict[str, str]]) -> None:
+        if not self.supabase_sync:
+            return
+        if not chunks:
+            return
+        merged_checksum = chunks[-1]["checksum"]
+        state = await self.supabase_sync.fetch_state(job.notebook_id, job.doc_id)
+        if state and not state.should_ingest(merged_checksum):
+            logger.info("任务 %s 未发生变化，跳过写入 Supabase", job.id)
+            return
+        state = state or IngestState(notebook_id=job.notebook_id, doc_id=job.doc_id)
+        state.update(merged_checksum)
+        await self.supabase_sync.upsert_state(state)
+
+    async def _maybe_embed(self, chunks: List[Dict[str, str]]) -> None:
+        if not self.embedding_generator:
+            return
+        texts = [c.get("text") or "" for c in chunks]
+        if not texts:
+            return
+        embeddings = await self.embedding_generator.embed_texts(texts)
+        for chunk, embedding in zip(chunks, embeddings):
+            chunk["embedding"] = embedding

services/ingest_service/app/services/webhooks.py

@@ -0,0 +1,26 @@
+import logging
+from typing import List, Optional
+
+import httpx
+
+from app.core.config import get_settings
+
+logger = logging.getLogger(__name__)
+
+
+class LightRAGWebhook:
+    """封装与 LightRAG 的交互，默认调用 /ingest"""
+
+    def __init__(self, base_url: Optional[str] = None):
+        settings = get_settings()
+        self.base_url = base_url or settings.lightrag_url.rstrip("/")
+
+    async def notify_ingest(self, doc_id: Optional[str], block_ids: List[str]) -> None:
+        if not block_ids:
+            logger.info("无可用 block 推送给 LightRAG")
+            return
+        payload = {"doc_id": doc_id, "block_ids": block_ids}
+        async with httpx.AsyncClient(timeout=30.0) as client:
+            resp = await client.post(f"{self.base_url}/ingest", json=payload)
+            resp.raise_for_status()
+            logger.info("LightRAG ingest 完成，返回 %s", resp.status_code)