cubefs

1
#!/usr/bin/env bash
2

3
# Purpose: plain text tar format
4
# Limitations: - only suitable for text files, directories, and symlinks
5
#              - stores only filename, content, and mode
6
#              - not designed for untrusted input
7
#
8
# Note: must work with bash version 3.2 (macOS)
9

10
# Copyright 2017 Roger Luethi
11
#
12
# Licensed under the Apache License, Version 2.0 (the "License");
13
# you may not use this file except in compliance with the License.
14
# You may obtain a copy of the License at
15
#
16
# http://www.apache.org/licenses/LICENSE-2.0
17
#
18
# Unless required by applicable law or agreed to in writing, software
19
# distributed under the License is distributed on an "AS IS" BASIS,
20
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
21
# See the License for the specific language governing permissions and
22
# limitations under the License.
23

24
set -o errexit -o nounset
25

26
# Sanitize environment (for instance, standard sorting of glob matches)
27
export LC_ALL=C
28

29
path=""
30
CMD=""
31
ARG_STRING="$*"
32

33
#------------------------------------------------------------------------------
34
# Not all sed implementations can work on null bytes. In order to make ttar
35
# work out of the box on macOS, use Python as a stream editor.
36

37
USE_PYTHON=0
38

39
PYTHON_CREATE_FILTER=$(cat << 'PCF'
40
#!/usr/bin/env python
41

42
import re
43
import sys
44

45
for line in sys.stdin:
46
    line = re.sub(r'EOF', r'\EOF', line)
47
    line = re.sub(r'NULLBYTE', r'\NULLBYTE', line)
48
    line = re.sub('\x00', r'NULLBYTE', line)
49
    sys.stdout.write(line)
50
PCF
51
)
52

53
PYTHON_EXTRACT_FILTER=$(cat << 'PEF'
54
#!/usr/bin/env python
55

56
import re
57
import sys
58

59
for line in sys.stdin:
60
    line = re.sub(r'(?<!\\)NULLBYTE', '\x00', line)
61
    line = re.sub(r'\\NULLBYTE', 'NULLBYTE', line)
62
    line = re.sub(r'([^\\])EOF', r'\1', line)
63
    line = re.sub(r'\\EOF', 'EOF', line)
64
    sys.stdout.write(line)
65
PEF
66
)
67

68
function test_environment {
69
    if [[ "$(echo "a" | sed 's/a/\x0/' | wc -c)" -ne 2 ]]; then
70
        echo "WARNING sed unable to handle null bytes, using Python (slow)."
71
        if ! which python >/dev/null; then
72
            echo "ERROR Python not found. Aborting."
73
            exit 2
74
        fi
75
        USE_PYTHON=1
76
    fi
77
}
78

79
#------------------------------------------------------------------------------
80

81
function usage {
82
    bname=$(basename "$0")
83
    cat << USAGE
84
Usage:   $bname [-C <DIR>] -c -f <ARCHIVE> <FILE...> (create archive)
85
         $bname            -t -f <ARCHIVE>           (list archive contents)
86
         $bname [-C <DIR>] -x -f <ARCHIVE>           (extract archive)
87

88
Options:
89
         -C <DIR>           (change directory)
90
         -v                 (verbose)
91
         --recursive-unlink (recursively delete existing directory if path
92
                             collides with file or directory to extract)
93

94
Example: Change to sysfs directory, create ttar file from fixtures directory
95
         $bname -C sysfs -c -f sysfs/fixtures.ttar fixtures/
96
USAGE
97
exit "$1"
98
}
99

100
function vecho {
101
    if [ "${VERBOSE:-}" == "yes" ]; then
102
        echo >&7 "$@"
103
    fi
104
}
105

106
function set_cmd {
107
    if [ -n "$CMD" ]; then
108
        echo "ERROR: more than one command given"
109
        echo
110
        usage 2
111
    fi
112
    CMD=$1
113
}
114

115
unset VERBOSE
116
unset RECURSIVE_UNLINK
117

118
while getopts :cf:-:htxvC: opt; do
119
    case $opt in
120
        c)
121
            set_cmd "create"
122
            ;;
123
        f)
124
            ARCHIVE=$OPTARG
125
            ;;
126
        h)
127
            usage 0
128
            ;;
129
        t)
130
            set_cmd "list"
131
            ;;
132
        x)
133
            set_cmd "extract"
134
            ;;
135
        v)
136
            VERBOSE=yes
137
            exec 7>&1
138
            ;;
139
        C)
140
            CDIR=$OPTARG
141
            ;;
142
        -)
143
            case $OPTARG in
144
                recursive-unlink)
145
                    RECURSIVE_UNLINK="yes"
146
                    ;;
147
                *)
148
                    echo -e "Error: invalid option -$OPTARG"
149
                    echo
150
                    usage 1
151
                    ;;
152
            esac
153
            ;;
154
        *)
155
            echo >&2 "ERROR: invalid option -$OPTARG"
156
            echo
157
            usage 1
158
            ;;
159
    esac
160
done
161

162
# Remove processed options from arguments
163
shift $(( OPTIND - 1 ));
164

165
if [ "${CMD:-}" == "" ]; then
166
    echo >&2 "ERROR: no command given"
167
    echo
168
    usage 1
169
elif [ "${ARCHIVE:-}" == "" ]; then
170
    echo >&2 "ERROR: no archive name given"
171
    echo
172
    usage 1
173
fi
174

175
function list {
176
    local path=""
177
    local size=0
178
    local line_no=0
179
    local ttar_file=$1
180
    if [ -n "${2:-}" ]; then
181
        echo >&2 "ERROR: too many arguments."
182
        echo
183
        usage 1
184
    fi
185
    if [ ! -e "$ttar_file" ]; then
186
        echo >&2 "ERROR: file not found ($ttar_file)"
187
        echo
188
        usage 1
189
    fi
190
    while read -r line; do
191
        line_no=$(( line_no + 1 ))
192
        if [ $size -gt 0 ]; then
193
            size=$(( size - 1 ))
194
            continue
195
        fi
196
        if [[ $line =~ ^Path:\ (.*)$ ]]; then
197
            path=${BASH_REMATCH[1]}
198
        elif [[ $line =~ ^Lines:\ (.*)$ ]]; then
199
            size=${BASH_REMATCH[1]}
200
            echo "$path"
201
        elif [[ $line =~ ^Directory:\ (.*)$ ]]; then
202
            path=${BASH_REMATCH[1]}
203
            echo "$path/"
204
        elif [[ $line =~ ^SymlinkTo:\ (.*)$ ]]; then
205
            echo  "$path -> ${BASH_REMATCH[1]}"
206
        fi
207
    done < "$ttar_file"
208
}
209

210
function extract {
211
    local path=""
212
    local size=0
213
    local line_no=0
214
    local ttar_file=$1
215
    if [ -n "${2:-}" ]; then
216
        echo >&2 "ERROR: too many arguments."
217
        echo
218
        usage 1
219
    fi
220
    if [ ! -e "$ttar_file" ]; then
221
        echo >&2 "ERROR: file not found ($ttar_file)"
222
        echo
223
        usage 1
224
    fi
225
    while IFS= read -r line; do
226
        line_no=$(( line_no + 1 ))
227
        local eof_without_newline
228
        if [ "$size" -gt 0 ]; then
229
            if [[ "$line" =~ [^\\]EOF ]]; then
230
                # An EOF not preceded by a backslash indicates that the line
231
                # does not end with a newline
232
                eof_without_newline=1
233
            else
234
                eof_without_newline=0
235
            fi
236
            # Replace NULLBYTE with null byte if at beginning of line
237
            # Replace NULLBYTE with null byte unless preceded by backslash
238
            # Remove one backslash in front of NULLBYTE (if any)
239
            # Remove EOF unless preceded by backslash
240
            # Remove one backslash in front of EOF
241
            if [ $USE_PYTHON -eq 1 ]; then
242
                echo -n "$line" | python -c "$PYTHON_EXTRACT_FILTER" >> "$path"
243
            else
244
                # The repeated pattern makes up for sed's lack of negative
245
                # lookbehind assertions (for consecutive null bytes).
246
                echo -n "$line" | \
247
                    sed -e 's/^NULLBYTE/\x0/g;
248
                            s/\([^\\]\)NULLBYTE/\1\x0/g;
249
                            s/\([^\\]\)NULLBYTE/\1\x0/g;
250
                            s/\\NULLBYTE/NULLBYTE/g;
251
                            s/\([^\\]\)EOF/\1/g;
252
                            s/\\EOF/EOF/g;
253
                    ' >> "$path"
254
            fi
255
            if [[ "$eof_without_newline" -eq 0 ]]; then
256
                echo >> "$path"
257
            fi
258
            size=$(( size - 1 ))
259
            continue
260
        fi
261
        if [[ $line =~ ^Path:\ (.*)$ ]]; then
262
            path=${BASH_REMATCH[1]}
263
            if [ -L "$path" ]; then
264
                rm "$path"
265
            elif [ -d "$path" ]; then
266
                if [ "${RECURSIVE_UNLINK:-}" == "yes" ]; then
267
                    rm -r "$path"
268
                else
269
                    # Safe because symlinks to directories are dealt with above
270
                    rmdir "$path"
271
                fi
272
            elif [ -e "$path" ]; then
273
                rm "$path"
274
            fi
275
        elif [[ $line =~ ^Lines:\ (.*)$ ]]; then
276
            size=${BASH_REMATCH[1]}
277
            # Create file even if it is zero-length.
278
            touch "$path"
279
            vecho "    $path"
280
        elif [[ $line =~ ^Mode:\ (.*)$ ]]; then
281
            mode=${BASH_REMATCH[1]}
282
            chmod "$mode" "$path"
283
            vecho "$mode"
284
        elif [[ $line =~ ^Directory:\ (.*)$ ]]; then
285
            path=${BASH_REMATCH[1]}
286
            mkdir -p "$path"
287
            vecho "    $path/"
288
        elif [[ $line =~ ^SymlinkTo:\ (.*)$ ]]; then
289
            ln -s "${BASH_REMATCH[1]}" "$path"
290
            vecho "    $path -> ${BASH_REMATCH[1]}"
291
        elif [[ $line =~ ^# ]]; then
292
            # Ignore comments between files
293
            continue
294
        else
295
            echo >&2 "ERROR: Unknown keyword on line $line_no: $line"
296
            exit 1
297
        fi
298
    done < "$ttar_file"
299
}
300

301
function div {
302
    echo "# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -" \
303
         "- - - - - -"
304
}
305

306
function get_mode {
307
    local mfile=$1
308
    if [ -z "${STAT_OPTION:-}" ]; then
309
        if stat -c '%a' "$mfile" >/dev/null 2>&1; then
310
            # GNU stat
311
            STAT_OPTION='-c'
312
            STAT_FORMAT='%a'
313
        else
314
            # BSD stat
315
            STAT_OPTION='-f'
316
            # Octal output, user/group/other (omit file type, sticky bit)
317
            STAT_FORMAT='%OLp'
318
        fi
319
    fi
320
    stat "${STAT_OPTION}" "${STAT_FORMAT}" "$mfile"
321
}
322

323
function _create {
324
    shopt -s nullglob
325
    local mode
326
    local eof_without_newline
327
    while (( "$#" )); do
328
        file=$1
329
        if [ -L "$file" ]; then
330
            echo "Path: $file"
331
            symlinkTo=$(readlink "$file")
332
            echo "SymlinkTo: $symlinkTo"
333
            vecho "    $file -> $symlinkTo"
334
            div
335
        elif [ -d "$file" ]; then
336
            # Strip trailing slash (if there is one)
337
            file=${file%/}
338
            echo "Directory: $file"
339
            mode=$(get_mode "$file")
340
            echo "Mode: $mode"
341
            vecho "$mode $file/"
342
            div
343
            # Find all files and dirs, including hidden/dot files
344
            for x in "$file/"{*,.[^.]*}; do
345
                _create "$x"
346
            done
347
        elif [ -f "$file" ]; then
348
            echo "Path: $file"
349
            lines=$(wc -l "$file"|awk '{print $1}')
350
            eof_without_newline=0
351
            if [[ "$(wc -c "$file"|awk '{print $1}')" -gt 0 ]] && \
352
                    [[ "$(tail -c 1 "$file" | wc -l)" -eq 0 ]]; then
353
                eof_without_newline=1
354
                lines=$((lines+1))
355
            fi
356
            echo "Lines: $lines"
357
            # Add backslash in front of EOF
358
            # Add backslash in front of NULLBYTE
359
            # Replace null byte with NULLBYTE
360
            if [ $USE_PYTHON -eq 1 ]; then
361
                < "$file" python -c "$PYTHON_CREATE_FILTER"
362
            else
363
                < "$file" \
364
                    sed 's/EOF/\\EOF/g;
365
                            s/NULLBYTE/\\NULLBYTE/g;
366
                            s/\x0/NULLBYTE/g;
367
                    '
368
            fi
369
            if [[ "$eof_without_newline" -eq 1 ]]; then
370
                # Finish line with EOF to indicate that the original line did
371
                # not end with a linefeed
372
                echo "EOF"
373
            fi
374
            mode=$(get_mode "$file")
375
            echo "Mode: $mode"
376
            vecho "$mode $file"
377
            div
378
        else
379
            echo >&2 "ERROR: file not found ($file in $(pwd))"
380
            exit 2
381
        fi
382
        shift
383
    done
384
}
385

386
function create {
387
    ttar_file=$1
388
    shift
389
    if [ -z "${1:-}" ]; then
390
        echo >&2 "ERROR: missing arguments."
391
        echo
392
        usage 1
393
    fi
394
    if [ -e "$ttar_file" ]; then
395
        rm "$ttar_file"
396
    fi
397
    exec > "$ttar_file"
398
    echo "# Archive created by ttar $ARG_STRING"
399
    _create "$@"
400
}
401

402
test_environment
403

404
if [ -n "${CDIR:-}" ]; then
405
    if [[ "$ARCHIVE" != /* ]]; then
406
        # Relative path: preserve the archive's location before changing
407
        # directory
408
        ARCHIVE="$(pwd)/$ARCHIVE"
409
    fi
410
    cd "$CDIR"
411
fi
412

413
"$CMD" "$ARCHIVE" "$@"
414