paddlenlp

1
# Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
2
#
3
# Licensed under the Apache License, Version 2.0 (the "License");
4
# you may not use this file except in compliance with the License.
5
# You may obtain a copy of the License at
6
#
7
#     http://www.apache.org/licenses/LICENSE-2.0
8
#
9
# Unless required by applicable law or agreed to in writing, software
10
# distributed under the License is distributed on an "AS IS" BASIS,
11
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
# See the License for the specific language governing permissions and
13
# limitations under the License.
14

15
from . import preprocess
16

17

18
def transform(data, ops=[]):
19
    """transform"""
20
    for op in ops:
21
        data = op(data)
22
    return data
23

24

25
def create_preprocess_operators(params):
26
    """
27
    create operators based on the config
28
    Args:
29
        params(list): a dict list, used to create some operators
30
    """
31
    assert isinstance(params, list), "operator config should be a list"
32
    ops = []
33
    for operator in params:
34
        assert isinstance(operator, dict) and len(operator) == 1, "yaml format error"
35
        op_name = list(operator)[0]
36
        param = {} if operator[op_name] is None else operator[op_name]
37
        op = getattr(preprocess, op_name)(**param)
38
        ops.append(op)
39

40
    return ops
41