AutoGPT

1
"""
2
Test cases for the config class, which handles the configuration settings
3
for the AI and ensures it behaves as a singleton.
4
"""
5
import os
6
from typing import Any
7
from unittest import mock
8
from unittest.mock import patch
9

10
import pytest
11
from openai.pagination import SyncPage
12
from openai.types import Model
13
from pydantic import SecretStr
14

15
from autogpt.app.configurator import GPT_3_MODEL, GPT_4_MODEL, apply_overrides_to_config
16
from autogpt.config import Config, ConfigBuilder
17

18

19
def test_initial_values(config: Config) -> None:
20
    """
21
    Test if the initial values of the config class attributes are set correctly.
22
    """
23
    assert config.continuous_mode is False
24
    assert config.tts_config.speak_mode is False
25
    assert config.fast_llm.startswith("gpt-3.5-turbo")
26
    assert config.smart_llm.startswith("gpt-4")
27

28

29
def test_set_continuous_mode(config: Config) -> None:
30
    """
31
    Test if the set_continuous_mode() method updates the continuous_mode attribute.
32
    """
33
    # Store continuous mode to reset it after the test
34
    continuous_mode = config.continuous_mode
35

36
    config.continuous_mode = True
37
    assert config.continuous_mode is True
38

39
    # Reset continuous mode
40
    config.continuous_mode = continuous_mode
41

42

43
def test_set_speak_mode(config: Config) -> None:
44
    """
45
    Test if the set_speak_mode() method updates the speak_mode attribute.
46
    """
47
    # Store speak mode to reset it after the test
48
    speak_mode = config.tts_config.speak_mode
49

50
    config.tts_config.speak_mode = True
51
    assert config.tts_config.speak_mode is True
52

53
    # Reset speak mode
54
    config.tts_config.speak_mode = speak_mode
55

56

57
def test_set_fast_llm(config: Config) -> None:
58
    """
59
    Test if the set_fast_llm() method updates the fast_llm attribute.
60
    """
61
    # Store model name to reset it after the test
62
    fast_llm = config.fast_llm
63

64
    config.fast_llm = "gpt-3.5-turbo-test"
65
    assert config.fast_llm == "gpt-3.5-turbo-test"
66

67
    # Reset model name
68
    config.fast_llm = fast_llm
69

70

71
def test_set_smart_llm(config: Config) -> None:
72
    """
73
    Test if the set_smart_llm() method updates the smart_llm attribute.
74
    """
75
    # Store model name to reset it after the test
76
    smart_llm = config.smart_llm
77

78
    config.smart_llm = "gpt-4-test"
79
    assert config.smart_llm == "gpt-4-test"
80

81
    # Reset model name
82
    config.smart_llm = smart_llm
83

84

85
@patch("openai.resources.models.Models.list")
86
def test_fallback_to_gpt3_if_gpt4_not_available(
87
    mock_list_models: Any, config: Config
88
) -> None:
89
    """
90
    Test if models update to gpt-3.5-turbo if gpt-4 is not available.
91
    """
92
    fast_llm = config.fast_llm
93
    smart_llm = config.smart_llm
94

95
    config.fast_llm = "gpt-4"
96
    config.smart_llm = "gpt-4"
97

98
    mock_list_models.return_value = SyncPage(
99
        data=[Model(id=GPT_3_MODEL, created=0, object="model", owned_by="AutoGPT")],
100
        object="Models",  # no idea what this should be, but irrelevant
101
    )
102

103
    apply_overrides_to_config(
104
        config=config,
105
        gpt3only=False,
106
        gpt4only=False,
107
    )
108

109
    assert config.fast_llm == "gpt-3.5-turbo"
110
    assert config.smart_llm == "gpt-3.5-turbo"
111

112
    # Reset config
113
    config.fast_llm = fast_llm
114
    config.smart_llm = smart_llm
115

116

117
def test_missing_azure_config(config: Config) -> None:
118
    assert config.openai_credentials is not None
119

120
    config_file = config.app_data_dir / "azure_config.yaml"
121
    with pytest.raises(FileNotFoundError):
122
        config.openai_credentials.load_azure_config(config_file)
123

124
    config_file.write_text("")
125
    with pytest.raises(ValueError):
126
        config.openai_credentials.load_azure_config(config_file)
127

128
    assert config.openai_credentials.api_type != "azure"
129
    assert config.openai_credentials.api_version == ""
130
    assert config.openai_credentials.azure_model_to_deploy_id_map is None
131

132

133
@pytest.fixture
134
def config_with_azure(config: Config):
135
    config_file = config.app_data_dir / "azure_config.yaml"
136
    config_file.write_text(
137
        f"""
138
azure_api_type: azure
139
azure_api_version: 2023-06-01-preview
140
azure_endpoint: https://dummy.openai.azure.com
141
azure_model_map:
142
    {config.fast_llm}: FAST-LLM_ID
143
    {config.smart_llm}: SMART-LLM_ID
144
    {config.embedding_model}: embedding-deployment-id-for-azure
145
"""
146
    )
147
    os.environ["USE_AZURE"] = "True"
148
    os.environ["AZURE_CONFIG_FILE"] = str(config_file)
149
    config_with_azure = ConfigBuilder.build_config_from_env(
150
        project_root=config.project_root
151
    )
152
    yield config_with_azure
153
    del os.environ["USE_AZURE"]
154
    del os.environ["AZURE_CONFIG_FILE"]
155

156

157
def test_azure_config(config_with_azure: Config) -> None:
158
    assert (credentials := config_with_azure.openai_credentials) is not None
159
    assert credentials.api_type == "azure"
160
    assert credentials.api_version == "2023-06-01-preview"
161
    assert credentials.azure_endpoint == SecretStr("https://dummy.openai.azure.com")
162
    assert credentials.azure_model_to_deploy_id_map == {
163
        config_with_azure.fast_llm: "FAST-LLM_ID",
164
        config_with_azure.smart_llm: "SMART-LLM_ID",
165
        config_with_azure.embedding_model: "embedding-deployment-id-for-azure",
166
    }
167

168
    fast_llm = config_with_azure.fast_llm
169
    smart_llm = config_with_azure.smart_llm
170
    assert (
171
        credentials.get_model_access_kwargs(config_with_azure.fast_llm)["model"]
172
        == "FAST-LLM_ID"
173
    )
174
    assert (
175
        credentials.get_model_access_kwargs(config_with_azure.smart_llm)["model"]
176
        == "SMART-LLM_ID"
177
    )
178

179
    # Emulate --gpt4only
180
    config_with_azure.fast_llm = smart_llm
181
    assert (
182
        credentials.get_model_access_kwargs(config_with_azure.fast_llm)["model"]
183
        == "SMART-LLM_ID"
184
    )
185
    assert (
186
        credentials.get_model_access_kwargs(config_with_azure.smart_llm)["model"]
187
        == "SMART-LLM_ID"
188
    )
189

190
    # Emulate --gpt3only
191
    config_with_azure.fast_llm = config_with_azure.smart_llm = fast_llm
192
    assert (
193
        credentials.get_model_access_kwargs(config_with_azure.fast_llm)["model"]
194
        == "FAST-LLM_ID"
195
    )
196
    assert (
197
        credentials.get_model_access_kwargs(config_with_azure.smart_llm)["model"]
198
        == "FAST-LLM_ID"
199
    )
200

201

202
def test_create_config_gpt4only(config: Config) -> None:
203
    with mock.patch("autogpt.llm.api_manager.ApiManager.get_models") as mock_get_models:
204
        mock_get_models.return_value = [
205
            Model(id=GPT_4_MODEL, created=0, object="model", owned_by="AutoGPT")
206
        ]
207
        apply_overrides_to_config(
208
            config=config,
209
            gpt4only=True,
210
        )
211
        assert config.fast_llm == GPT_4_MODEL
212
        assert config.smart_llm == GPT_4_MODEL
213

214

215
def test_create_config_gpt3only(config: Config) -> None:
216
    with mock.patch("autogpt.llm.api_manager.ApiManager.get_models") as mock_get_models:
217
        mock_get_models.return_value = [{"id": GPT_3_MODEL}]
218
        apply_overrides_to_config(
219
            config=config,
220
            gpt3only=True,
221
        )
222
        assert config.fast_llm == GPT_3_MODEL
223
        assert config.smart_llm == GPT_3_MODEL
224