/
alt3rmann
/
LibreChat
Обзор
Документация
Войти
/
alt3rmann
/
LibreChat
Код
Запросы
0
Задачи
Вики
Пакеты
0
Релизы
0
Аналитика
Безопасность
main
api/server/utils/countTokens.js
37 строк
2 KB
Danny Avila
✨ feat: Assistants API, General File Support, Side Panel, File Explorer (#1696)
14 фев 2024, 04:42
Не верифицирован
14 фев 2024, 04:42
ecd63eb
Код
Авторство
О чём код?
const { Tiktoken } = require('tiktoken/lite'); const p50k_base = require('tiktoken/encoders/p50k_base.json'); const cl100k_base = require('tiktoken/encoders/cl100k_base.json'); const logger = require('~/config/winston'); /** * Counts the number of tokens in a given text using a specified encoding model. * * This function utilizes the 'Tiktoken' library to encode text based on the selected model. * It supports two models, 'text-davinci-003' and 'gpt-3.5-turbo', each with its own encoding strategy. * For 'text-davinci-003', the 'p50k_base' encoder is used, whereas for other models, the 'cl100k_base' encoder is applied. * In case of an error during encoding, the error is logged, and the function returns 0. * * @async * @param {string} text - The text to be tokenized. Defaults to an empty string if not provided. * @param {string} modelName - The name of the model used for tokenizing. Defaults to 'gpt-3.5-turbo'. * @returns {Promise<number>} The number of tokens in the provided text. Returns 0 if an error occurs. * @throws Logs the error to a logger and rethrows if any error occurs during tokenization. */ const countTokens = async (text = '', modelName = 'gpt-3.5-turbo') => { let encoder = null; try { const model = modelName.includes('text-davinci-003') ? p50k_base : cl100k_base; encoder = new Tiktoken(model.bpe_ranks, model.special_tokens, model.pat_str); const tokens = encoder.encode(text); encoder.free(); return tokens.length; } catch (e) { logger.error('[countTokens]', e); if (encoder) { encoder.free(); } return 0; } }; module.exports = countTokens;