scripts/extend_index_yaml.py - develop - githubmirror/oppia

scripts/extend_index_yaml.py
127 строк4 KB
Sean Lip
Make the backend test for scripts/start.py 100 times faster. (#23968)
08 дек 2025, 14:02
08 дек 2025, 14:0287dea72
# Copyright 2021 The Oppia Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the 'License');
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an 'AS-IS' BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Script for extending index.yaml.

This script extracts new kind from
../cloud_datastore_emulator_cache/WEB-INF/index.yaml
and appends it into index.yaml"""

from __future__ import annotations

import os

import xmltodict
import yaml
from typing import Dict, List, Optional, Union

XmlIndexesDict = Dict[
    str,
    Dict[
        str, List[Dict[str, Union[str, Dict[str, str], List[Dict[str, str]]]]]
    ],
]
YamlIndexesDict = Dict[
    str, List[Dict[str, Union[str, Dict[str, str], List[Dict[str, str]]]]]
]

INDEX_YAML_PATH = os.path.join(os.getcwd(), 'index.yaml')
WEB_INF_INDEX_XML_PATH = os.path.join(
    os.getcwd(),
    os.pardir,
    'cloud_datastore_emulator_cache',
    'WEB-INF',
    'appengine-generated',
    'datastore-indexes-auto.xml',
)


def reformat_xml_dict_into_yaml_dict(
    xml_dict: XmlIndexesDict,
) -> Optional[YamlIndexesDict]:
    """Reformats the xml index dict into yaml index dict.

    Args:
        xml_dict: dict. The dict parsed from xml index file.

    Returns:
        dict. The dict in yaml format.
    """
    yaml_index_entries = []
    if (
        'datastore-indexes' not in xml_dict
        or 'datastore-index' not in xml_dict['datastore-indexes']
    ):
        return None

    for xml_index in xml_dict['datastore-indexes']['datastore-index']:
        yaml_index_properties: List[Dict[str, str]] = []
        for xml_index_property in xml_index['property']:
            assert isinstance(xml_index_property, dict)
            yaml_index_property = {
                'name': xml_index_property['@name'],
            }
            if xml_index_property['@direction'] == 'desc':
                yaml_index_property['direction'] = 'desc'
            yaml_index_properties.append(yaml_index_property)

        yaml_index_entries.append(
            {'kind': xml_index['@kind'], 'properties': yaml_index_properties}
        )

    return {'indexes': yaml_index_entries}


def main() -> None:
    """Extends index.yaml file."""
    if not os.path.exists(WEB_INF_INDEX_XML_PATH):
        print('No new index definitions were created during this server run.')
        return

    with open(WEB_INF_INDEX_XML_PATH, 'r', encoding='utf-8') as f:
        web_inf_index_xml_dict = xmltodict.parse(
            f.read(), force_list={'datastore-index', 'property'}
        )
    web_inf_index_yaml_dict = reformat_xml_dict_into_yaml_dict(
        web_inf_index_xml_dict
    )
    if web_inf_index_yaml_dict is None:
        return

    print('\033[94mExtending index.yaml...\033[0m')

    with open(INDEX_YAML_PATH, 'r', encoding='utf-8') as f:
        index_yaml_dict = yaml.safe_load(f)

    new_kinds = [
        kind
        for kind in web_inf_index_yaml_dict['indexes']
        if kind not in index_yaml_dict['indexes']
    ]

    if len(new_kinds) == 0:
        return

    index_yaml_dict['indexes'] += new_kinds
    # The yaml dump function doesn't add new lines between kinds
    # automatically. So we add new lines manually using replace
    # function.
    new_index_yaml_dict = yaml.safe_dump(
        index_yaml_dict, default_flow_style=False, sort_keys=False
    )
    index_yaml = new_index_yaml_dict.replace('- kind', '\n- kind')
    with open(INDEX_YAML_PATH, 'w', encoding='utf-8') as f:
        f.write(index_yaml)

    print('\033[92mDone!\033[0m')