ragflow_sync_cli/commands/sync_files.py - master - dictator/ragflow_sync

ragflow_sync_cli/commands/sync_files.py
89 строк2 KB
Developer
feat: add sync files command to upload Markdown files from local directory
05 июн 2026, 18:05
05 июн 2026, 18:0569cced8
"""
Sync local Markdown files to RAGFlow command.
"""

import logging
import sys
from typing import Optional

from ragflow_uploader.client import RAGFlowClient
from ragflow_uploader.uploader import DocumentUploader


logger = logging.getLogger(__name__)


def sync_files(
    client: RAGFlowClient,
    directory: str,
    dataset_name: str,
    glob: str = "*.md",
    exclude: Optional[list[str]] = None,
    create_dataset: bool = True,
    cleanup: bool = True,
    parse: bool = True,
    wait_for_parsing: bool = False,
) -> None:
    """Sync Markdown files from a local directory to RAGFlow."""

    logger.info(f"Syncing files from '{directory}' to dataset: {dataset_name}")

    try:
        uploader = DocumentUploader(client)

        # Get or create dataset
        if create_dataset:
            dataset_id = uploader.get_or_create_dataset(
                dataset_name,
                description="Local Markdown files",
            )
        else:
            datasets = client.list_datasets(name=dataset_name)
            if not datasets:
                logger.error(f"Dataset '{dataset_name}' not found")
                sys.exit(1)
            dataset_id = datasets[0]["id"]

        # Sync directory
        result = uploader.sync_directory(
            dataset_id=dataset_id,
            directory=directory,
            glob_pattern=glob,
            exclude=exclude,
            cleanup=cleanup,
            parse=parse,
            wait_for_parsing=wait_for_parsing,
        )

        # Print results
        _print_sync_results(result)

        if not result.success and result.total_processed == 0:
            sys.exit(1)

    except Exception as e:
        logger.error(f"Files sync failed: {e}")
        sys.exit(1)


def _print_sync_results(result) -> None:
    """Print sync results to stdout."""
    print("\n" + "=" * 50)
    print("Sync Results:")
    print("=" * 50)
    print(f"Status: {'✓ Success' if result.success else '✗ Failed'}")
    print(f"Documents created: {result.documents_created}")
    print(f"Documents updated: {result.documents_updated}")
    print(f"Documents failed: {result.documents_failed}")

    if result.documents_parsed > 0:
        print(f"Documents parsed: {result.documents_parsed}")
    if result.documents_parse_failed > 0:
        print(f"Documents parse failed: {result.documents_parse_failed}")

    if result.errors:
        print("\nErrors:")
        for error in result.errors:
            print(f"  - {error}")

    print()