google-research

1
# coding=utf-8
2
# Copyright 2024 The Google Research Authors.
3
#
4
# Licensed under the Apache License, Version 2.0 (the "License");
5
# you may not use this file except in compliance with the License.
6
# You may obtain a copy of the License at
7
#
8
#     http://www.apache.org/licenses/LICENSE-2.0
9
#
10
# Unless required by applicable law or agreed to in writing, software
11
# distributed under the License is distributed on an "AS IS" BASIS,
12
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
# See the License for the specific language governing permissions and
14
# limitations under the License.
15

16
"""Helper for running scoring using various LLMs services."""
17

18
from typing import Sequence, Union
19

20
import typing_extensions
21

22

23
class LanguageModelScoringFn(typing_extensions.Protocol):
24
  """Specifcation of LLMs scoring function."""
25

26
  def __call__(self, prompt,
27
               continuations):
28
    """Computes the score of continuations for a given prompt.
29

30
    Arguments:
31
      prompt: prompt to use as context
32
      continuations: continuations to score
33

34
    Returns:
35
      score: the probability normalized by length of each continuation
36
              (in regular scale / not in log scale)
37
    """
38

39

40
def score_with_dummy_llm(
41
    prompt, continuations
42
):
43
  """Retrieves the scores of a LLM - this function is a placeholder for actual calls."""
44
  del prompt
45
  return [0.0] * len(continuations)
46