pytorch-lightning

1
# Copyright The Lightning AI team.
2
#
3
# Licensed under the Apache License, Version 2.0 (the "License");
4
# you may not use this file except in compliance with the License.
5
# You may obtain a copy of the License at
6
#
7
#     http://www.apache.org/licenses/LICENSE-2.0
8
#
9
# Unless required by applicable law or agreed to in writing, software
10
# distributed under the License is distributed on an "AS IS" BASIS,
11
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
# See the License for the specific language governing permissions and
13
# limitations under the License.
14

15
import hashlib
16
from typing import List
17

18

19
def _get_hash(files: List[str], algorithm: str = "blake2", chunk_num_blocks: int = 128) -> str:
20
    """Hashes the contents of a list of files.
21

22
    Parameters
23
    ----------
24
    files: List[Path]
25
        List of files.
26
    algorithm: str, default "blake2"
27
        Algorithm to hash contents. "blake2" is set by default because it
28
        is faster than "md5". [1]
29
    chunk_num_blocks: int, default 128
30
        Block size to user when iterating over file chunks.
31

32
    References
33
    ----------
34
    [1] https://crypto.stackexchange.com/questions/70101/blake2-vs-md5-for-checksum-file-integrity
35
    [2] https://stackoverflow.com/questions/1131220/get-md5-hash-of-big-files-in-python
36

37
    """
38
    # validate input
39
    if algorithm == "blake2":
40
        h = hashlib.blake2b(digest_size=20)
41
    elif algorithm == "md5":
42
        h = hashlib.md5()
43
    else:
44
        raise ValueError(f"Algorithm {algorithm} not supported")
45

46
    # calculate hash for all files
47
    for file in files:
48
        with open(file, "rb") as f:
49
            for chunk in iter(lambda: f.read(chunk_num_blocks * h.block_size), b""):
50
                h.update(chunk)
51
    return h.hexdigest()
52