gpt-neox

1
# Copyright (c) 2024, EleutherAI
2
#
3
# Licensed under the Apache License, Version 2.0 (the "License");
4
# you may not use this file except in compliance with the License.
5
# You may obtain a copy of the License at
6
#
7
#     http://www.apache.org/licenses/LICENSE-2.0
8
#
9
# Unless required by applicable law or agreed to in writing, software
10
# distributed under the License is distributed on an "AS IS" BASIS,
11
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
# See the License for the specific language governing permissions and
13
# limitations under the License.
14

15
import os
16
import sys
17

18
from huggingface_hub import HfApi, create_repo
19

20
converted_ckpt = sys.argv[1]
21
repo_name = sys.argv[2]
22
branch_name = sys.argv[3]
23
try:
24
    create_repo(repo_name, repo_type="model", private=False)
25
except:
26
    print("repo {repo_name} already exists!")
27
    pass
28

29
files = os.listdir(converted_ckpt)
30

31
api = HfApi()
32
if branch_name != "main":
33
    try:
34
        api.create_branch(
35
            repo_id=repo_name,
36
            repo_type="model",
37
            branch=branch_name,
38
        )
39
    except:
40
        print(f"branch {branch_name} already exists, try again...")
41
print(f"to upload: {files}")
42
for file in files:
43
    print(f"Uploading {file} to branch {branch_name}...")
44
    api.upload_file(
45
        path_or_fileobj=os.path.join(converted_ckpt, file),
46
        path_in_repo=file,
47
        repo_id=repo_name,
48
        repo_type="model",
49
        commit_message=f"Upload {file}",
50
        revision=branch_name,
51
    )
52
    print(f"Successfully uploaded {file} !")
53