sm/api/api.py - master - MaxIs/savemoney

sm/api/api.py
386 строк19 KB
zalma2006
добавлен flask для работы с selenium в отдельном сервисе, добавлены точки API, добавлена загрузка отчётов из источника, подробнее в HISTORY.md
15 мар 2026, 19:36
15 мар 2026, 19:36ae3ea09
import json
import shutil
from pathlib import Path
from drf_yasg.openapi import (Parameter, TYPE_STRING, Schema,
                              TYPE_FILE, IN_FORM, TYPE_INTEGER, TYPE_OBJECT, TYPE_ARRAY, )
from rest_framework.views import APIView
from rest_framework.parsers import MultiPartParser, JSONParser, FormParser
from rest_framework.response import Response
from api.swagger_auto_actions import CSI
from accounts.auth import CheckApiKey
from drf_yasg.utils import swagger_auto_schema
from rest_framework import status
from api.serializers import ReportFileUploadSerializer, AddResourcesEmitentSerializer, INNSerializer, \
    ReportPathUploadSerializer
from mfiles.models import Report
from savemoney.models import Emitent, EmitentReport, ReportType, ResourcesEmitent, Resources
from sm.redis_sheduler import redis_scheduler
from utils.cust_func import get_simple_json2resp
from utils.get_new_uuid_in_db import get_uuid_from_db
from django.conf import settings
from logging import getLogger
from api import tasks as celery_tasks


class SettingsClassMixin:
    """Чтобы к каждому не добавлять, проверку достоверности и название логгера используется в дочерних классах"""
    permission_classes = [CheckApiKey]

    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        self._logger = getLogger(self.__class__.__name__)

    @property
    def logger(self):
        return self._logger

class ReportData:
    """Класс проверки данных по report view"""
    @staticmethod
    def check_data(serializer, request):
        """Проверка данных"""
        req = get_simple_json2resp()
        req['errors'] = []
        status_code = status.HTTP_200_OK
        serializer = serializer(data=request.data)
        serializer.is_valid(raise_exception=True)
        valid_data = serializer.validated_data
        return req, status_code, valid_data


class UploadFileReportView(SettingsClassMixin, APIView):
    """Загрузка отчёта upload_report/file/"""
    parser_classes = [MultiPartParser]
    serializer_class = [ReportFileUploadSerializer]
    allowed_methods = ['POST']

    @swagger_auto_schema(
        manual_parameters=CSI.MPARAMETERS_POST + [
            CSI.MPARAMETR_INN,
            Parameter(name="file", in_=IN_FORM, type=TYPE_FILE, required=True, default=None,
                      description="загрузка файла"),
        ] + CSI.MPARAMETRS_REPORT,
        operation_description="upload report from file",
        responses={
            status.HTTP_401_UNAUTHORIZED: CSI.RESP_UNAUTHORIZED,
            status.HTTP_200_OK: Schema(type=TYPE_OBJECT,
                                       properties={
                                           "message": Schema(
                                               type=TYPE_STRING, example="success",
                                               description="информация о выполнении команды"),
                                           "errors": Schema(
                                               type=TYPE_STRING,
                                               description="описание ошибки если она есть иначе пустая строка",
                                               default=''
                                           ),
                                       }),
        },
    )
    def post(self, request, *args, **kwargs):
        """Загружаем файл на сервер"""
        req, status_code, valid_data = ReportData.check_data(self.serializer_class[0], request)
        report_type = valid_data.pop("report_type")
        inn = valid_data.pop("inn")
        file = valid_data.pop("file")
        fname = file.name
        uuid_name = get_uuid_from_db()
        path = Path(settings.MEDIA_ROOT) / "mediafiles" / "reports" / Path(f"{uuid_name}{fname.suffix}")
        url = request.build_absolute_uri(settings.MEDIA_URL) + str(uuid_name) + fname.suffix

        # записываем файл
        try:
            with open(path, "wb") as f:
                for chunk in file.chunks(chunk_size=8192):
                    f.write(chunk)
            self.logger.info(f'Файл {path.name} загружен на сервер')
        except Exception as e:
            self.logger.error(e)
            req.append(f'Ошибка записи файла {path}')
            status_code = status.HTTP_500_INTERNAL_SERVER_ERROR
            Response(data=req, status=status_code)

        # добавляем запись
        try:
            size = path.stat().st_size / (1024**2)
            valid_data.update({'size': size, 'uuid_name': uuid_name, 'filepath': str(path), 'url_path': url})
            report = Report(**valid_data)
            report.save()
            emitent = Emitent.objects.filter(emitent_inn=inn).first()
            report_type = ReportType.objects.filter(num_type=report_type).first()
            if all([x is not None for x in [emitent, report_type]]):
                EmitentReport.objects.create(emitent=emitent, report=report, report_type=report_type)
            req['message'] = f'файл = {path} записан, связь Эмитент <-> Отчёт создана'
        except Exception as e:
            self.logger.exception(e)
            req['errors'].append(e)
            status_code = status.HTTP_500_INTERNAL_SERVER_ERROR

        return Response(data=req, status=status_code)


class UploadPathReportView(SettingsClassMixin, APIView):
    """Загрузка отчёта upload_report/path/"""
    parser_classes = [FormParser]
    serializer_class = [ReportPathUploadSerializer]
    allowed_methods = ['POST']

    @swagger_auto_schema(
        manual_parameters=CSI.MPARAMETERS_POST + [
            CSI.MPARAMETR_INN,
            Parameter(name=f"filepath", in_=IN_FORM, type=TYPE_STRING, required=False, default=None,
                      description="путь до файла")
        ] + CSI.MPARAMETRS_REPORT,
        operation_description="upload report from path",
        responses={
            status.HTTP_401_UNAUTHORIZED: CSI.RESP_UNAUTHORIZED,
            status.HTTP_200_OK: Schema(type=TYPE_OBJECT,
                                       properties={
                                           "message": Schema(
                                               type=TYPE_STRING, example="success",
                                               description="информация о выполнении команды"),
                                           "errors": Schema(
                                               type=TYPE_STRING,
                                               description="описание ошибки если она есть иначе пустая строка",
                                               default=''
                                           ),
                                       }),
        },
    )
    def post(self, request, *args, **kwargs):
        """Загружаем файл на сервер"""
        print(request.data)
        req, status_code, valid_data = ReportData.check_data(self.serializer_class[0], request)
        filepath = Path(valid_data.get("filepath"))
        report_type = valid_data.pop("report_type")
        inn = valid_data.pop("inn")
        uuid_name = get_uuid_from_db()
        path = Path(settings.MEDIA_ROOT) / "mediafiles" / "reports" / Path(f"{uuid_name}{filepath.suffix}")

        print(filepath, report_type, inn, uuid_name)
        # записываем файл
        try:
            shutil.move(str(filepath), str(path))
            self.logger.info(f'Файл {path.name} загружен на сервер')
        except Exception as e:
            self.logger.error(e)
            req.append(f'Ошибка записи файла {path}')
            status_code = status.HTTP_500_INTERNAL_SERVER_ERROR
            Response(data=req, status=status_code)
        finally:
            if filepath.exists():
                filepath.unlink()

        # добавляем запись
        try:
            size = path.stat().st_size / (1024**2)
            valid_data.update({'size': size, 'uuid_name': uuid_name, 'filepath': str(path)})
            report = Report(**valid_data)
            report.save()
            emitent = Emitent.objects.filter(emitent_inn=inn).first()
            report_type = ReportType.objects.filter(num_type=report_type).first()
            if all([x is not None for x in [emitent, report_type]]):
                EmitentReport.objects.create(emitent=emitent, report=report, report_type=report_type)
            req['message'] = f'файл = {path} записан, связь Эмитент <-> Отчёт создана'
        except Exception as e:
            self.logger.exception(e)
            req['errors'].append(e)
            status_code = status.HTTP_500_INTERNAL_SERVER_ERROR

        return Response(data=req, status=status_code)

class AddEmitentReportDisclosure(SettingsClassMixin, APIView):
    """Поиск и добавление отчётов Эмитенту find_report/"""
    parser_classes = [FormParser]
    serializer_class = [INNSerializer]
    allowed_methods = ['POST']

    @swagger_auto_schema(
        manual_parameters=CSI.MPARAMETERS_POST + [CSI.MPARAMETR_INN],
        operation_description="find report and download file",
        responses={
            status.HTTP_401_UNAUTHORIZED: CSI.RESP_UNAUTHORIZED,
            status.HTTP_200_OK: Schema(type=TYPE_OBJECT,
                                       properties={
                                           "message": Schema(type=TYPE_STRING, ),
                                           "errors": Schema(
                                               type=TYPE_STRING,
                                               description="описание ошибки если она есть иначе пустая строка",
                                               default=''
                                           ),
                                       }),
        },
    )
    def post(self, request, *args, **kwargs):
        data = get_simple_json2resp()
        data['errors'] = []
        serializer = self.serializer_class[0](data=request.data)
        serializer.is_valid(raise_exception=True)
        inn = request.data.get("inn", None)
        schema = self.request.build_absolute_uri().split(':')[0]
        emitent = None
        idx_company = None
        r_types = None
        if inn is not None:
            emitent = Emitent.objects.filter(emitent_inn=inn).first()
        if emitent is None:
            data['errors'].append(f"for inn {inn} emitent not found")

        if emitent is not None:
            res_em = ResourcesEmitent.objects.filter(emitent=emitent).first()
            if res_em is not None:
                if not res_em.is_exist:
                    data['errors'].append(f"for emitent inn {inn} not exist in source")
                idx_company = res_em.source_sv

        if len(data['errors']) == 0:
            # получим возможные типы отчётности
            r_types = {k.num_type for k in ReportType.objects.all()}
        if r_types is None:
            data['errors'].append(f'r_types not found')
        if len(data['errors']) == 0:
            r_types = '||'.join(list(map(str, r_types)))
            results = celery_tasks.set_report_celery.apply_async(args=[inn, r_types, idx_company, schema,
                                                                       "https://www.e-disclosure.ru/"])
            status_code = status.HTTP_200_OK
        else:
            data['errors'] = '||'.join(data['errors'])
            status_code = status.HTTP_400_BAD_REQUEST

        return Response(data=data, status=status_code)


class AddResourceEmitent(SettingsClassMixin, APIView):
    """resource_emitent/add"""
    parser_classes = [JSONParser]
    serializer_class = [AddResourcesEmitentSerializer]
    allowed_methods = ['POST']

    @swagger_auto_schema(
        manual_parameters=CSI.MPARAMETERS_POST,
        operation_description="Добавляет связь Источник <-> Эмитент",
        request_body=Schema(
            type=TYPE_OBJECT,
            properties={
                "inn": Schema(type=TYPE_STRING, example="123244325242", description="инн эмитента"),
                "source": Schema(type=TYPE_STRING, example="https://www.e-disclosure.ru/",
                                 description="url адрес источника"),
                "source_sv": Schema(type=TYPE_STRING, example="12345",
                                    description="уникальный идентификатор эмитента в источнике данных"),
            },
            required=["inn", "source", "source_sv"],
        ),
        responses={
            status.HTTP_401_UNAUTHORIZED: CSI.RESP_UNAUTHORIZED,
            status.HTTP_200_OK: CSI.RESP_STATUS_OK,
        },
    )
    def post(self, request, *args, **kwargs):
        data = get_simple_json2resp()
        data['errors'] = []

        serializer = self.serializer_class[0](data=request.data)
        serializer.is_valid(raise_exception=True)
        if len(data['errors']) == 0:
            inn = request.data.get("inn", None)
            source_url = request.data.get("source", None)
            source_sv = request.data.get("source_sv", None)
            emitent = Emitent.objects.filter(emitent_inn=inn).first()
            resource = Resources.objects.filter(url=source_url).first()
            conds = resource is not None and emitent is not None and isinstance(source_sv, str) and len(source_sv) > 0
            if conds:
                resem = ResourcesEmitent.objects.filter(emitent=emitent, resource=resource).first()
                if resem is None:
                    is_exist = True if source_sv != '-1' else False
                    ResourcesEmitent.objects.create(emitent=emitent, resource=resource, source_sv=source_sv, is_exist=is_exist)
                    data['message'] = f"{inn} <-> {source_url} created successfully"
                elif resem.source_sv != source_sv:
                    resem.source_sv = source_sv
                    resem.save()
                    data['message'] = f"{inn} <-> {source_url} updated successfully"
                else:
                    data['message'] = f"{inn} <-> {source_url} and {source_sv} exists"
            else:
                if source_sv is None:
                    data['errors'].append(f"source_sv {source_url} does not exist")
                if emitent is None:
                    data['errors'].append(f"emitent {inn} does not exist")
                if not isinstance(source_sv, str):
                    data['errors'].append(f"source_sv {source_sv} must be a string")
                if isinstance(source_sv, str) and len(source_sv) == 0:
                    data['errors'].append(f"source_sv {source_url} cannot be empty")
        if len(data['errors']) == 0:
            status_code = status.HTTP_200_OK
        else:
            data['errors'] = '||'.join(data['errors'])
            status_code = status.HTTP_400_BAD_REQUEST
        return Response(data=data, status=status_code, content_type="application/json; charset=utf-8")


class ReportTypeApiView(SettingsClassMixin, APIView):
    """Получить типы отчётности report_type/get"""
    parser_classes = [FormParser]
    allowed_methods = ['GET']

    @swagger_auto_schema(
        manual_parameters=CSI.MPARAMETERS_GET,
        operation_description="get report type information",
        responses={
            status.HTTP_401_UNAUTHORIZED: CSI.RESP_UNAUTHORIZED,
            status.HTTP_200_OK: Schema(type=TYPE_OBJECT,
                                       properties={
                                           "message": Schema(type=TYPE_STRING, ),
                                           "errors": Schema(
                                               type=TYPE_STRING,
                                               description="описание ошибки если она есть иначе пустая строка",
                                               default=''
                                           ),
                                           "report_type": Schema(
                                               type=TYPE_ARRAY,
                                               items=Schema(type=TYPE_OBJECT,
                                                            properties={
                                                                "num": Schema(type=TYPE_INTEGER,
                                                                              description="номер типа отчётности"),
                                                                "name": Schema(type=TYPE_STRING,
                                                                               description="название типа отчётности",
                                                                               )
                                                            })
                                           ),
                                       }),
        },
    )
    def get(self, request, *args, **kwargs):
        """Получить все типы отчётности из БД"""
        data = get_simple_json2resp()
        report_type = ReportType.objects.all()
        if len(report_type) > 0:
            report_type = [{"num": r.num_type, "name": r.name} for r in report_type]
            data['message'] = 'success'
            data['report_type'] = report_type
        else:
            data['errors'] = 'report type empty'

        return Response(data=data, status=status.HTTP_200_OK)

    # def dispatch(self, request, *args, **kwargs):
    #     """для проверки, здесь можно посмотреть некоторые методы используемые в api"""
    #     print(f"Using permission classes: {self.permission_classes}")
    #     print(f"Using logger: {self.logger}")
    #     return super().dispatch(request, *args, **kwargs)


class CompletedTaskCelery(SettingsClassMixin, APIView):
    """Список выполненных задач Celery в redis"""
    allowed_methods = ['GET']

    @swagger_auto_schema(
        manual_parameters=CSI.MPARAMETERS_GET,

    )
    def get(self, request, *args, **kwargs):
        conn = redis_scheduler.get_connection()
        try:
            tasks = conn.lrange("completed_tasks", 0, -1)
            decoded_tasks = [json.loads(task.decode('utf-8')) for task in tasks]
            return Response(decoded_tasks)
        except Exception as e:
            return Response({"errors": str(e)}, status=status.HTTP_500_INTERNAL_SERVER_ERROR)