pytorch-lightning

1
# Copyright The Lightning AI team.
2
#
3
# Licensed under the Apache License, Version 2.0 (the "License");
4
# you may not use this file except in compliance with the License.
5
# You may obtain a copy of the License at
6
#
7
#     http://www.apache.org/licenses/LICENSE-2.0
8
#
9
# Unless required by applicable law or agreed to in writing, software
10
# distributed under the License is distributed on an "AS IS" BASIS,
11
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
# See the License for the specific language governing permissions and
13
# limitations under the License.
14

15
import os
16
from pathlib import Path
17
from typing import Optional
18

19
import lightning_cloud.env
20

21

22
def get_lightning_cloud_url() -> str:
23
    # detect local development
24
    if os.getenv("VSCODE_PROXY_URI", "").startswith("http://localhost:9800"):
25
        return "http://localhost:9800"
26
    # DO NOT CHANGE!
27
    return os.getenv("LIGHTNING_CLOUD_URL", "https://lightning.ai")
28

29

30
SUPPORTED_PRIMITIVE_TYPES = (type(None), str, int, float, bool)
31
STATE_UPDATE_TIMEOUT = 0.001
32
STATE_ACCUMULATE_WAIT = 0.15
33
# Duration in seconds of a moving average of a full flow execution
34
# beyond which an exception is raised.
35
FLOW_DURATION_THRESHOLD = 1.0
36
# Number of samples for the moving average of the duration of flow execution
37
FLOW_DURATION_SAMPLES = 5
38

39
APP_SERVER_HOST = os.getenv("LIGHTNING_APP_STATE_URL", "http://127.0.0.1")
40
APP_SERVER_IN_CLOUD = "http://lightningapp" in APP_SERVER_HOST
41
APP_SERVER_PORT = 7501
42
APP_STATE_MAX_SIZE_BYTES = 1024 * 1024  # 1 MB
43

44
WARNING_QUEUE_SIZE = 1000
45
# different flag because queue debug can be very noisy, and almost always not useful unless debugging the queue itself.
46
QUEUE_DEBUG_ENABLED = bool(int(os.getenv("LIGHTNING_QUEUE_DEBUG_ENABLED", "0")))
47

48
REDIS_HOST = os.getenv("REDIS_HOST", "localhost")
49
REDIS_PORT = int(os.getenv("REDIS_PORT", 6379))
50
REDIS_PASSWORD = os.getenv("REDIS_PASSWORD", None)
51
REDIS_QUEUES_READ_DEFAULT_TIMEOUT = 0.005
52

53
HTTP_QUEUE_URL = os.getenv("LIGHTNING_HTTP_QUEUE_URL", "http://localhost:9801")
54
HTTP_QUEUE_REFRESH_INTERVAL = float(os.getenv("LIGHTNING_HTTP_QUEUE_REFRESH_INTERVAL", "1"))
55
HTTP_QUEUE_TOKEN = os.getenv("LIGHTNING_HTTP_QUEUE_TOKEN", None)
56
HTTP_QUEUE_REQUESTS_PER_SECOND = float(os.getenv("LIGHTNING_HTTP_QUEUE_REQUESTS_PER_SECOND", "0.5"))
57

58
USER_ID = os.getenv("USER_ID", "1234")
59
FRONTEND_DIR = str(Path(__file__).parent.parent / "ui")
60
PACKAGE_LIGHTNING = os.getenv("PACKAGE_LIGHTNING", None)
61
CLOUD_UPLOAD_WARNING = int(os.getenv("CLOUD_UPLOAD_WARNING", "2"))
62
DISABLE_DEPENDENCY_CACHE = bool(int(os.getenv("DISABLE_DEPENDENCY_CACHE", "0")))
63
# Project under which the resources need to run in cloud. If this env is not set,
64
# cloud runner will try to get the default project from the cloud
65
LIGHTNING_CLOUD_PROJECT_ID = os.getenv("LIGHTNING_CLOUD_PROJECT_ID")
66
LIGHTNING_CLOUD_PRINT_SPECS = os.getenv("LIGHTNING_CLOUD_PRINT_SPECS")
67
LIGHTNING_DIR = os.getenv("LIGHTNING_DIR", str(Path.home() / ".lightning"))
68
LIGHTNING_CREDENTIAL_PATH = os.getenv("LIGHTNING_CREDENTIAL_PATH", str(Path(LIGHTNING_DIR) / "credentials.json"))
69
DOT_IGNORE_FILENAME = ".lightningignore"
70
LIGHTNING_COMPONENT_PUBLIC_REGISTRY = "https://lightning.ai/v1/components"
71
LIGHTNING_APPS_PUBLIC_REGISTRY = "https://lightning.ai/v1/apps"
72
LIGHTNING_MODELS_PUBLIC_REGISTRY = "https://lightning.ai/v1/models"
73

74
LIGHTNING_CLOUDSPACE_HOST = os.getenv("LIGHTNING_CLOUDSPACE_HOST")
75
LIGHTNING_CLOUDSPACE_EXPOSED_PORT_COUNT = int(os.getenv("LIGHTNING_CLOUDSPACE_EXPOSED_PORT_COUNT", "0"))
76

77
# EXPERIMENTAL: ENV VARIABLES TO ENABLE MULTIPLE WORKS IN THE SAME MACHINE
78
DEFAULT_NUMBER_OF_EXPOSED_PORTS = int(os.getenv("DEFAULT_NUMBER_OF_EXPOSED_PORTS", "50"))
79
ENABLE_MULTIPLE_WORKS_IN_NON_DEFAULT_CONTAINER = bool(
80
    int(os.getenv("ENABLE_MULTIPLE_WORKS_IN_NON_DEFAULT_CONTAINER", "0"))
81
)  # This isn't used in the cloud yet.
82

83
# env var trigger running setup commands in the app
84
ENABLE_APP_COMMENT_COMMAND_EXECUTION = bool(int(os.getenv("ENABLE_APP_COMMENT_COMMAND_EXECUTION", "0")))
85

86

87
DEBUG: bool = lightning_cloud.env.DEBUG
88
DEBUG_ENABLED = bool(int(os.getenv("LIGHTNING_DEBUG", "0")))
89
ENABLE_PULLING_STATE_ENDPOINT = bool(int(os.getenv("ENABLE_PULLING_STATE_ENDPOINT", "1")))
90
ENABLE_PUSHING_STATE_ENDPOINT = ENABLE_PULLING_STATE_ENDPOINT and bool(
91
    int(os.getenv("ENABLE_PUSHING_STATE_ENDPOINT", "1"))
92
)
93
ENABLE_STATE_WEBSOCKET = bool(int(os.getenv("ENABLE_STATE_WEBSOCKET", "1")))
94
ENABLE_UPLOAD_ENDPOINT = bool(int(os.getenv("ENABLE_UPLOAD_ENDPOINT", "1")))
95

96
# directory where system customization sync files stored
97
SYS_CUSTOMIZATIONS_SYNC_ROOT = "/tmp/sys-customizations-sync"  # todo
98
# directory where system customization sync files will be copied to be packed into app tarball
99
SYS_CUSTOMIZATIONS_SYNC_PATH = ".sys-customizations-sync"
100

101
BATCH_DELTA_COUNT = int(os.getenv("BATCH_DELTA_COUNT", "128"))
102

103

104
def enable_multiple_works_in_default_container() -> bool:
105
    return bool(int(os.getenv("ENABLE_MULTIPLE_WORKS_IN_DEFAULT_CONTAINER", "0")))
106

107

108
def get_cloud_queue_type() -> Optional[str]:
109
    value = os.getenv("LIGHTNING_CLOUD_QUEUE_TYPE", None)
110
    if value is None and enable_interruptible_works():
111
        value = "http"
112
    return value
113

114

115
# Number of seconds to wait between filesystem checks when waiting for files in remote storage
116
REMOTE_STORAGE_WAIT = 0.5
117

118

119
# interruptible support
120
def enable_interruptible_works() -> bool:
121
    return bool(int(os.getenv("LIGHTNING_INTERRUPTIBLE_WORKS", "0")))
122

123

124
def get_cluster_driver() -> Optional[str]:
125
    return "direct"
126