core/rag/src/lib.rs - main - alexefan136/flowstack

core/rag/src/lib.rs
365 строк11 KB
Alexander Efanov
Обновление репозитория
15 июл 2026, 12:19
15 июл 2026, 12:1976704c6
//! # FlowStack RAG Service
//!
//! Production-grade Retrieval-Augmented Generation система с:
//! - **Multi-tenant** изоляцией через workspace filters
//! - **Гибридным поиском** (vector + keyword, TODO)
//! - **Интеграцией с Cloud.ru** Foundation Models (Qwen3-Embedding, Qwen3-Reranker)
//! - **Qdrant** как production vector store
//! - **In-memory** store для тестов и MVP
//!
//! # Architecture
//!
//! ```text
//! ┌─────────────────────────────────────────────────────────────────┐
//! │                      RAG Pipeline                                │
//! ├─────────────────────────────────────────────────────────────────┤
//! │                                                                   │
//! │  INGESTION:  Document → Chunker → Embedder → VectorStore        │
//! │                                                                   │
//! │  RETRIEVAL:  Query → Embedding → Search → Rerank → Context      │
//! │                                                                   │
//! └─────────────────────────────────────────────────────────────────┘
//! ```
//!
//! # Quick Start
//!
//! ```rust,no_run
//! use flowstack_rag::{RagPipeline, Document, DocumentType, Query};
//!
//! #[tokio::main]
//! async fn main() -> Result<(), Box<dyn std::error::Error>> {
//!     // Create pipeline from environment variables
//!     let pipeline = RagPipeline::from_env().await?;
//!     
//!     // Ingest a document
//!     let doc = Document::new(
//!         "Rust is a systems programming language.".to_string(),
//!         "rust-intro.md".into(),
//!         DocumentType::Markdown,
//!         "engineering".into(),
//!     );
//!     let result = pipeline.ingest(doc).await?;
//!     println!("Ingested {} chunks", result.chunks_count);
//!     
//!     // Search
//!     let query = Query::new("What is Rust?", "engineering");
//!     let search = pipeline.search(query).await?;
//!     println!("Found {} results", search.results_count());
//!     println!("Context:\n{}", search.context);
//!     
//!     Ok(())
//! }
//! ```
//!
//! # Modules
//!
//! - [`domain`] — Core types: Document, Chunk, Embedding, Query
//! - [`chunking`] — Document chunking strategies (recursive, paragraph, sentence, fixed_size)
//! - [`embedding`] — Text vectorization (Cloud.ru, Mock)
//! - [`storage`] — Vector storage backends (Qdrant, In-memory)
//! - [`retrieval`] — Search strategies (vector, hybrid, reranking)
//! - [`pipeline`] — High-level orchestration (RagPipeline)
//! - [`api`] — HTTP API (Axum)
//! - [`error`] — Error types
//!
//! # Feature Flags
//!
//! - `pdf` — PDF parsing support (enabled by default)
//! - `linalg` — Linear algebra operations
//! - `full` — All features enabled

#![warn(
    rust_2018_idioms,
    unreachable_pub
)]
#![deny(unsafe_code)]
#![allow(
    missing_docs,
    missing_debug_implementations,
    dead_code,
    unused_imports,
    unused_variables
)]

// ============================================================================
// Core modules
// ============================================================================

/// Error types and Result alias.
pub mod error;

/// Domain types: Document, Chunk, Embedding, Query, SearchFilters.
pub mod domain;

/// Configuration types (LLM, Qdrant, Pipeline).
pub mod config;

/// Chunking strategies for document splitting.
pub mod chunking;

/// Embedding providers (Cloud.ru, Mock).
pub mod embedding;

/// Vector storage backends (Qdrant, In-memory).
pub mod storage;

/// Retrieval strategies (vector search, reranking, hybrid).
pub mod retrieval;

/// High-level RAG pipeline orchestration.
pub mod pipeline;

/// HTTP API (Axum routes and handlers).
pub mod api;

// ============================================================================
// Re-exports: Error
// ============================================================================

pub use error::{Error, Result, ResultExt};

// ============================================================================
// Re-exports: Domain types
// ============================================================================

pub use domain::{
    // Document
    Chunk,
    ChunkId,
    ChunkStrategy,
    Document,
    DocumentId,
    DocumentMetadata,
    DocumentType,
    // Embedding
    Embedding,
    EmbeddingModel,
    // Query & Retrieval
    Query,
    RetrievalResponse,
    RetrievalResult,
    SearchFilters,
};

// ============================================================================
// Re-exports: Chunking
// ============================================================================

pub use chunking::{
    // Chunker implementations
    Chunker,
    ChunkerConfig,
    FixedSizeChunker,
    ParagraphChunker,
    RecursiveChunker,
    SentenceChunker,
    // Factory
    create_chunker,
    default_chunker,
};

// ============================================================================
// Re-exports: Embedding
// ============================================================================

pub use embedding::{
    // Embedder implementations
    CloudRuEmbedder,
    Embedder,
    EmbedderConfig,
    MockEmbedder,
    // Factory
    create_embedder,
    create_mock_embedder,
};

// ============================================================================
// Re-exports: Storage
// ============================================================================

pub use storage::{
    // Store implementations
    MemoryStore,
    QdrantConfig,
    QdrantStore,
    StorageBackend,
    VectorStore,
    // Types
    Distance,
    SearchParams,
    StorageStats,
    // Factories
    create_memory_store,
    create_memory_store_with_size,
    create_store,
    create_store_from_env,
};

// ============================================================================
// Re-exports: Retrieval
// ============================================================================

pub use retrieval::{
    // Retriever implementations
    CloudRuReranker,
    HybridRetriever,
    Retriever,
    RetrieverConfig,
    VectorRetriever,
    // Factories
    create_retriever,
    create_retriever_with_reranker,
};

// ============================================================================
// Re-exports: Pipeline
// ============================================================================

pub use pipeline::{
    // Pipeline
    IngestResult,
    PipelineConfig,
    RagPipeline,
    SearchResult,
    // Functions
    ingest_batch,
    ingest_document,
    search,
    search_batch,
};

// ============================================================================
// Re-exports: Config
// ============================================================================

pub use config::{AppConfig, LlmConfig, ServerConfig};

// ============================================================================
// Re-exports: API
// ============================================================================

pub use api::{
    // Router factory
    create_router,
    // Request/Response models
    models::{
        // Ingest
        IngestBatchRequest,
        IngestBatchResponse,
        IngestResponse,
        IngestTextRequest,
        IngestTiming,
        // Search
        SearchBatchResponse,
        SearchMetadata,
        SearchModels,
        SearchRequest,
        SearchResponse,
        SearchTiming,
        // Document management
        DeleteResponse,
        DocumentInfo,
        DocumentListResponse,
        // System
        ComponentsHealth,
        HealthResponse,
        PipelineConfigResponse,
        ReadyResponse,
        StatsResponse,
    },
};

// ============================================================================
// Version & Service info
// ============================================================================

/// Версия сервиса (из Cargo.toml).
pub const VERSION: &str = env!("CARGO_PKG_VERSION");

/// Название сервиса.
pub const SERVICE_NAME: &str = "flowstack-rag";

/// Получить полную информацию о сервисе.
pub fn service_info() -> ServiceInfo {
    ServiceInfo {
        name: SERVICE_NAME.to_string(),
        version: VERSION.to_string(),
        rust_version: env!("CARGO_PKG_RUST_VERSION").to_string(),
    }
}

/// Информация о сервисе.
#[derive(Debug, Clone, serde::Serialize)]
pub struct ServiceInfo {
    /// Название сервиса.
    pub name: String,
    /// Версия сервиса.
    pub version: String,
    /// Минимальная версия Rust.
    pub rust_version: String,
}

// ============================================================================
// Integration tests
// ============================================================================

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn test_version_info() {
        assert!(!VERSION.is_empty());
        assert_eq!(SERVICE_NAME, "flowstack-rag");
    }

    #[test]
    fn test_service_info() {
        let info = service_info();
        assert_eq!(info.name, "flowstack-rag");
        assert!(!info.version.is_empty());
    }

    #[tokio::test]
    async fn test_full_pipeline_smoke_test() {
        // Smoke test: создаём pipeline, ingest'им документ, ищем
        let pipeline = RagPipeline::for_testing().expect("Failed to create test pipeline");

        // Ingest
        let doc = Document::new(
            "Rust is a systems programming language focused on safety.".to_string(),
            "test.md".into(),
            DocumentType::Markdown,
            "test-workspace".into(),
        );
        let ingest_result = pipeline.ingest(doc).await.expect("Ingest failed");

        assert_eq!(ingest_result.chunks_count, 1);
        assert!(ingest_result.processing_time_ms > 0.0);

        // Search
        let query = Query::new("What is Rust?", "test-workspace");
        let search_result = pipeline.search(query).await.expect("Search failed");

        assert_eq!(search_result.results_count(), 1);
        assert!(search_result.has_results());
        assert!(!search_result.context.is_empty());

        // Stats
        let stats = pipeline.stats().await.expect("Stats failed");
        assert_eq!(stats.total_chunks, 1);
        assert_eq!(stats.total_documents, 1);
        assert_eq!(stats.total_workspaces, 1);

        // Health
        let healthy = pipeline.health_check().await.expect("Health check failed");
        assert!(healthy);

        // Delete
        let deleted = pipeline
            .delete_document(&ingest_result.document_id)
            .await
            .expect("Delete failed");
        assert_eq!(deleted, 1);
    }
}