NBash

1
#!/bin/sh
2
# eget - simply shell on wget for loading directories over http (wget does not support wildcard for http)
3
# Use:
4
# eget http://ftp.altlinux.ru/pub/security/ssl/*
5
#
6
# Copyright (C) 2014-2014, 2016, 2020, 2022  Etersoft
7
# Copyright (C) 2014 Daniil Mikhailov <danil@etersoft.ru>
8
# Copyright (C) 2016-2017, 2020, 2022 Vitaly Lipatov <lav@etersoft.ru>
9
#
10
# This program is free software: you can redistribute it and/or modify
11
# it under the terms of the GNU Affero General Public License as published by
12
# the Free Software Foundation, either version 3 of the License, or
13
# (at your option) any later version.
14
#
15
# This program is distributed in the hope that it will be useful,
16
# but WITHOUT ANY WARRANTY; without even the implied warranty of
17
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
18
# GNU Affero General Public License for more details.
19
#
20
# You should have received a copy of the GNU Affero General Public License
21
# along with this program. If not, see <http://www.gnu.org/licenses/>.
22
#
23

24
init_eget()
25
{
26
PROGDIR=$(dirname "$0")
27
PROGNAME=$(basename "$0")
28
CMDSHELL="/bin/sh"
29
[ "$PROGDIR" = "." ] && PROGDIR="$(pwd)"
30
if [ "$0" = "/dev/stdin" ] || [ "$0" = "sh" ] ; then
31
    PROGDIR=""
32
    PROGNAME=""
33
fi
34
}
35
init_eget
36

37

38
fatal()
39
{
40
    echo "FATAL: $*" >&2
41
    exit 1
42
}
43

44
info()
45
{
46
    [ -n "$quiet" ] && return
47
    echo "$*" >&2
48
}
49

50
eget()
51
{
52
	if [ -n "$EPMMODE" ] ; then
53
		# if embedded in epm
54
		(unset EGET_IPFS_GATEWAY; unset EGET_IPFS_API ; unset EGET_IPFS_DB ; EGET_BACKEND=$ORIG_EGET_BACKEND internal_tools_eget "$@" )
55
		return
56
	fi
57

58
	[ -n "$PROGNAME" ] || fatal "pipe mode is not supported"
59

60
	local bashopt=''
61
	#[ -n "$verbose" ] && bashopt='-x'
62

63
	(unset EGET_IPFS_GATEWAY; unset EGET_IPFS_API ; unset EGET_IPFS_DB ; EGET_BACKEND=$ORIG_EGET_BACKEND $CMDSHELL $bashopt $PROGDIR/$PROGNAME "$@" )
64
}
65

66
# TODO:
67
arch="$(uname -m)"
68

69
# copied from eepm project
70

71
# copied from /etc/init.d/outformat (ALT Linux)
72
isatty()
73
{
74
	# Set a sane TERM required for tput
75
	[ -n "$TERM" ] || TERM=dumb
76
	export TERM
77
	test -t 1
78
}
79

80
isatty2()
81
{
82
	# check stderr
83
	test -t 2
84
}
85

86

87
check_tty()
88
{
89
	isatty || return
90
	is_command tput >/dev/null 2>/dev/null || return
91
	# FreeBSD does not support tput -S
92
	echo | a= tput -S >/dev/null 2>/dev/null || return
93
	export USETTY="tput -S"
94
}
95

96
: ${BLACK:=0} ${RED:=1} ${GREEN:=2} ${YELLOW:=3} ${BLUE:=4} ${MAGENTA:=5} ${CYAN:=6} ${WHITE:=7}
97

98
set_boldcolor()
99
{
100
	[ -n "$USETTY" ] || return
101
	{
102
		echo bold
103
		echo setaf $1
104
	} | $USETTY
105
}
106

107
set_color()
108
{
109
	[ -n "$USETTY" ] || return
110
	{
111
		echo setaf $1
112
	} | $USETTY
113
}
114

115
restore_color()
116
{
117
	[ -n "$USETTY" ] || return
118
	{
119
		echo op; # set Original color Pair.
120
		echo sgr0; # turn off all special graphics mode (bold in our case).
121
	} | $USETTY
122
}
123

124

125
echover()
126
{
127
    [ -n "$verbose" ] || return
128
    echo "$*" >&2
129
}
130

131
# Print command line and run command line
132
showcmd()
133
{
134
	if [ -z "$quiet" ] ; then
135
		set_boldcolor $GREEN
136
		local PROMTSIG="\$"
137
		[ "$UID" = 0 ] && PROMTSIG="#"
138
		echo " $PROMTSIG $@"
139
		restore_color
140
	fi >&2
141
}
142

143
# Print command line and run command line
144
docmd()
145
{
146
	showcmd "$@"
147
	"$@"
148
}
149

150
verdocmd()
151
{
152
	[ -n "$verbose" ] && showcmd "$@"
153
	"$@"
154
}
155

156

157
# copied from epm
158
# print a path to the command if exists in $PATH
159
if a= which which 2>/dev/null >/dev/null ; then
160
    # the best case if we have which command (other ways needs checking)
161
    # TODO: don't use which at all, it is binary, not builtin shell command
162
print_command_path()
163
{
164
    a= which -- "$1" 2>/dev/null
165
}
166
elif a= type -a type 2>/dev/null >/dev/null ; then
167
print_command_path()
168
{
169
    a= type -fpP -- "$1" 2>/dev/null
170
}
171
else
172
print_command_path()
173
{
174
    a= type "$1" 2>/dev/null | sed -e 's|.* /|/|'
175
}
176
fi
177

178
# check if <arg> is a real command
179
is_command()
180
{
181
    print_command_path "$1" >/dev/null
182
}
183

184
# add realpath if missed
185
if ! is_command realpath ; then
186
realpath()
187
{
188
    [ -n "$*" ] || return
189
    readlink -f "$@"
190
}
191
fi
192

193

194
# check man glob
195
filter_glob()
196
{
197
	[ -z "$1" ] && cat && return
198
	# translate glob to regexp
199
	grep "$(echo "$1" | sed -e 's|\.|\\.|g' -e 's|\*|.*|g' -e 's|\?|.|g' )$"
200
}
201

202
filter_order()
203
{
204
    if [ -n "$SECONDLATEST" ] ; then
205
        sort -V | tail -n2 | head -n1
206
        return
207
    fi
208
    [ -z "$LATEST" ] && cat && return
209
    sort -V | tail -n1
210
}
211

212

213
is_fileurl()
214
{
215
    echo "$1" | grep -q "^/" && return
216
    echo "$1" | grep -q "^file:/"
217
}
218

219
path_from_url()
220
{
221
    echo "$1" | sed -e 's|^file://*|/|'
222
}
223

224
is_url()
225
{
226
    echo "$1" | grep -q "^[filehtps]*:/"
227
}
228

229
is_strange_url()
230
{
231
    local URL="$1"
232
    is_url "$URL" || return
233
    echo "$URL" | grep -q "[?&]"
234
}
235

236
is_ipfs_hash()
237
{
238
    # If a CID is 46 characters starting with "Qm", it's a CIDv0
239
    echo "$1" | grep -q -E "^Qm[[:alnum:]]{44}$" && return
240
    # TODO: CIDv1 support, see https://github.com/multiformats/cid
241
    return 1
242
}
243

244
is_ipfsurl()
245
{
246
    is_ipfs_hash "$1" && return
247
    echo "$1" | grep -q "^ipfs://"
248
}
249

250
is_httpurl()
251
{
252
    # TODO: improve
253
    echo "$1" | grep -q "^https://" & return
254
    echo "$1" | grep -q "^http://" & return
255
}
256

257
cid_from_url()
258
{
259
    echo "$1" | sed -e 's|^ipfs://*||' -e 's|\?.*||'
260
}
261

262

263
# args: cmd <URL> <options>
264
# will run cmd <options> <URL>
265
download_with_mirroring()
266
{
267
    local CMD="$1"
268
    shift
269
    local URL="$1"
270
    shift
271

272
    local res
273
    $CMD "$@" "$URL" && return
274
    res=$?
275
    [ -n "$CHECKMIRRORS" ] || return $res
276

277
    MIRROR="https://mirror.eterfund.ru"
278
    SECONDURL="$(echo "$URL" | sed -e "s|^.*://|$MIRROR/|")"
279
    $CMD "$@" "$SECONDURL" && URL="$SECONDURL" && return
280

281
    MIRROR="https://mirror.eterfund.org"
282
    SECONDURL="$(echo "$URL" | sed -e "s|^.*://|$MIRROR/|")"
283
    $CMD "$@" "$SECONDURL" && URL="$SECONDURL" && return
284
}
285

286

287

288
check_tty
289

290
quiet=''
291
verbose=''
292
WGETNOSSLCHECK=''
293
CURLNOSSLCHECK=''
294
AXELNOSSLCHECK=''
295
WGETUSERAGENT=''
296
CURLUSERAGENT=''
297
AXELUSERAGENT=''
298
WGETHEADER=''
299
CURLHEADER=''
300
AXELHEADER=''
301
WGETCOMPRESSED=''
302
CURLCOMPRESSED=''
303
AXELCOMPRESSED=''
304
WGETQ='' #-q
305
CURLQ='' #-s
306
AXELQ='' #-q
307
# TODO: aria2c
308
# TODO: wget --trust-server-names
309
# TODO: 
310
WGETNAMEOPTIONS='--content-disposition'
311
CURLNAMEOPTIONS='--remote-name --remote-time --remote-header-name'
312
AXELNAMEOPTIONS=''
313

314

315
LISTONLY=''
316
CHECKURL=''
317
GETRESPONSE=''
318
GETFILENAME=''
319
GETREALURL=''
320
GETIPFSCID=''
321
LATEST=''
322
SECONDLATEST=''
323
CHECKMIRRORS=''
324
TARGETFILE=''
325
FORCEIPV=''
326

327

328
set_quiet()
329
{
330
    WGETQ='-q'
331
    CURLQ='-s'
332
    AXELQ='-q'
333
    quiet=1
334
}
335

336

337
eget_help()
338
{
339
cat <<EOF
340

341
eget - wget like downloader wrapper with wildcard support in filename part of URL
342
Usage: eget [options] http://somesite.ru/dir/na*.log
343

344
Options:
345
    -q|--quiet                - quiet mode
346
    --verbose                 - verbose mode
347
    -k|--no-check-certificate - skip SSL certificate chain support
348
    -H|--header               - use <header> (X-Cache:1 for example)
349
    -U|-A|--user-agent        - send browser like UserAgent
350
    --compressed              - request a compressed response and automatically decompress the content
351
    -4|--ipv4|--inet4-only    - use only IPV4
352
    -6|--ipv6|--inet6-only    - use only IPV6
353
    -O-|-O -                  - output downloaded file to stdout
354
    -O file                   - download to this file
355
    --latest                  - print only latest version of a file
356
    --second-latest           - print only second to latest version of a file
357
    --allow-mirrors           - check mirrors if url is not accessible
358

359
    --list|--list-only        - print only URLs
360
    --check URL               - check if the URL is accessible (returns HTTP 200 OK)
361
    --get-response URL        - get response with all headers (ever if HEAD is not acceptable)
362
    --get-filename URL        - print filename for the URL (via Content-Disposition if applicable)
363
    --get-real-url URL        - print URL after all redirects
364
    --get-ipfs-cid URL        - print CID for URL (after all redirects)
365

366
Supported URLs:
367
  ftp:// http:// https:// file:/ ipfs://
368

369
Supported backends (set like EGET_BACKEND=curl)
370
  wget curl (todo: aria2c)
371

372
Examples:
373
  $ eget http://ftp.somesite.ru/package-*.x64.tar
374
  $ eget http://ftp.somesite.ru/package *.tar
375
  $ eget https://github.com/owner/project package*.ext
376
  $ eget -O myname ipfs://QmVRUjnsnxHWkjq91KreCpUk4D9oZEbMwNQ3rzdjwND5dR
377
  $ eget --list http://ftp.somesite.ru/package-*.tar
378
  $ eget --check http://ftp.somesite.ru/test
379
  $ eget --list http://download.somesite.ru 'package-*.tar.xz'
380
  $ eget --list --latest https://github.com/telegramdesktop/tdesktop/releases 'tsetup.*.tar.xz'
381

382
EOF
383
}
384

385

386
if [ -z "$1" ] ; then
387
    echo "eget - wget like downloader wrapper with wildcard support, uses wget or curl as backend" >&2
388
    echo "Run $0 --help to get help" >&2
389
    exit 1
390
fi
391

392

393
while [ -n "$1" ] ; do
394

395
    case "$1" in
396
        -h|--help)
397
            eget_help
398
            exit
399
            ;;
400
        -q|--quiet)
401
            set_quiet
402
            ;;
403
        --verbose)
404
            verbose="$1"
405
            ;;
406
        -k|--no-check-certificate)
407
            WGETNOSSLCHECK='--no-check-certificate'
408
            CURLNOSSLCHECK='-k'
409
            AXELNOSSLCHECK='--insecure'
410
            ;;
411
        -H|--header)
412
            shift
413
            WGETHEADER="--header=$1"
414
            CURLHEADER="--header $1"
415
            AXELHEADER="--header=$1"
416
            ;;
417
        -U|-A|--user-agent)
418
            user_agent="Mozilla/5.0 (X11; Linux $arch) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/103.0.0.0 Safari/537.36"
419
            WGETUSERAGENT="-U '$user_agent'"
420
            CURLUSERAGENT="-A '$user_agent'"
421
            AXELUSERAGENT="--user-agent='$user_agent'"
422
            ;;
423
        --compressed)
424
            CURLCOMPRESSED='--compressed'
425
            WGETCOMPRESSED='--compression=auto'
426
            ;;
427
        -4|--ipv4|--inet4-only)
428
            FORCEIPV="-4"
429
            ;;
430
        -6|--ipv6|--inet6-only)
431
            FORCEIPV="-6"
432
            ;;
433
        --list|--list-only)
434
            LISTONLY="$1"
435
            set_quiet
436
            ;;
437
        --check)
438
            CHECKURL="$1"
439
            #set_quiet
440
            ;;
441
        --get-filename)
442
            GETFILENAME="$1"
443
            ;;
444
        --get-response)
445
            GETRESPONSE="$1"
446
            ;;
447
        --get-real-url)
448
            GETREALURL="$1"
449
            ;;
450
        --get-ipfs-cid)
451
            GETIPFSCID="$1"
452
            ;;
453
        --latest)
454
            LATEST="$1"
455
            ;;
456
        --second-latest)
457
            SECONDLATEST="$1"
458
            ;;
459
        --check-mirrors)
460
            CHECKMIRRORS="$1"
461
            ;;
462
        -O)
463
            shift
464
            TARGETFILE="$1"
465
            ;;
466
        -O-)
467
            TARGETFILE="-"
468
            ;;
469
        -*)
470
            fatal "Unknown option '$1', check eget --help."
471
            ;;
472
        *)
473
            break
474
            ;;
475
    esac
476
    shift
477
done
478

479

480
#############################3
481
# defaults
482

483
# https://github.com/ipfs/kubo/issues/5541
484
ipfs_diag_timeout='--timeout 60s'
485

486
ipfs_api_local="/ip4/127.0.0.1/tcp/5001"
487
[ -n "$EGET_IPFS_API" ] && ipfs_api_local="$EGET_IPFS_API"
488

489
ipfs_api_brave="/ip4/127.0.0.1/tcp/45005"
490

491
ipfs_gateway="https://cloudflare-ipfs.com/ipfs"
492
[ -n "$EGET_IPFS_GATEWAY" ] && ipfs_gateway="$EGET_IPFS_GATEWAY"
493
IPFS_GATEWAY="$ipfs_gateway"
494

495
# Test data: https://etersoft.ru/templates/etersoft/images/logo.png
496
ipfs_checkQm="QmYwf2GAMvHxfFiUFL2Mr6KUG6QrDiupqGc8ms785ktaYw"
497

498
get_ipfs_brave()
499
{
500
    local ipfs_brave="$(ls ~/.config/BraveSoftware/Brave-Browser/*/*/go-ipfs_* | sort | tail -n1 2>/dev/null)"
501
    [ -n "$ipfs_brave" ] && [ -x "$ipfs_brave" ] || return
502
    echo "$ipfs_brave"
503
}
504

505
ipfs_api_access()
506
{
507
    [ -n "$IPFS_CMD" ] || fatal "IPFS is disabled"
508
    if [ -n "$verbose" ] ; then
509
         verdocmd $IPFS_CMD --api $IPFS_API $ipfs_diag_timeout diag sys >/dev/null
510
    else
511
         verdocmd $IPFS_CMD --api $IPFS_API $ipfs_diag_timeout diag sys >/dev/null 2>/dev/null
512
    fi
513
}
514

515
ipfs_check()
516
{
517
    [ -n "$IPFS_CMD" ] || fatal "IPFS is disabled"
518
    verdocmd $IPFS_CMD --api $IPFS_API $ipfs_diag_timeout cat "$1" >/dev/null
519
}
520

521

522

523
select_ipfs_mode()
524
{
525
    IPFS_CMD="$(print_command_path ipfs)"
526
    if [ -n "$IPFS_CMD" ] ; then
527
        IPFS_API="$ipfs_api_local"
528
        if ipfs_api_access ; then
529
            ipfs_mode="local" && return
530
            #if ipfs_check "$ipfs_checkQm" ; then
531
            #    ipfs_mode="local" && return
532
            #else
533
            #    info "Skipped local: it is accessible via $IPFS_CMD --api $IPFS_API, but can't return shared $ipfs_checkQm"
534
            #fi
535
        fi
536
    fi
537

538
    IPFS_CMD="$(get_ipfs_brave)"
539
    # if no EGET_IPFS_API, check brave
540
    if [ -z "$EGET_IPFS_API" ] && [ -n "$IPFS_CMD" ] ; then
541
        IPFS_API="$ipfs_api_brave"
542
        if ipfs_api_access ; then
543
            ipfs_mode="brave" && return
544
            #if ipfs_check "$ipfs_checkQm" ; then
545
            #    ipfs_mode="brave" && return
546
            #else
547
            #    info "Skipped Brave: it is accessible via $IPFS_CMD --api $IPFS_API, but can't return shared $ipfs_checkQm"
548
            #fi
549
        fi
550
    fi
551

552
    # TODO: check checksum
553
    if docmd eget --check "$ipfs_gateway/$ipfs_checkQm" ; then
554
        ipfs_mode="gateway"
555
        return
556
    fi
557

558
    IPFS_GATEWAY=''
559
    if docmd eget --check "$(dirname $ipfs_gateway)" ; then
560
       info "IPFS gateway $ipfs_gateway is accessible, but can't return shared $ipfs_checkQm"
561
    else
562
       info "IPFS gateway $(dirname $ipfs_gateway) is not accessible"
563
    fi
564

565
    ipfs_mode="disabled"
566
}
567

568

569
# Functions for work with eget ipfs db
570
get_cid_by_url()
571
{
572
    local URL="$1"
573
    [ -r "$EGET_IPFS_DB" ] || return
574
    is_fileurl "$URL" && return 1
575
    grep -F "$URL Qm" "$EGET_IPFS_DB" | cut -f2 -d" " | grep -E "Qm[[:alnum:]]{44}" | head -n1
576
}
577

578
put_cid_and_url()
579
{
580
    local URL="$1"
581
    local CID="$2"
582
    local FN="$3"
583
    [ -w "$EGET_IPFS_DB" ] || return
584

585
    is_fileurl "$URL" && return
586

587
    echo "$URL $CID $FN" >> "$EGET_IPFS_DB"
588
    info "Placed in $EGET_IPFS_DB: $URL $CID $FN"
589
}
590

591
get_filename_by_cid()
592
{
593
    local CID="$1"
594
    [ -z "$EGET_IPFS_DB" ] && basename "$CID" && return
595
    grep -F " $CID " "$EGET_IPFS_DB" | head -n1 | cut -f3 -d" "
596
}
597

598
get_url_by_cid()
599
{
600
    local CID="$1"
601
    [ -z "$EGET_IPFS_DB" ] && echo "$CID" && return
602
    grep -F " $CID " "$EGET_IPFS_DB" | head -n1 | cut -f1 -d" "
603
}
604

605
###################
606

607

608
ipfs_mode="$EGET_IPFS"
609

610
# enable auto mode when set $EGET_IPFS_DB
611
[ -z "$ipfs_mode" ] && [ -n "$EGET_IPFS_DB" ] && ipfs_mode="auto"
612

613
if [ -n "$LISTONLY$CHECKURL" ] ; then
614
    ipfs_mode=""
615
    EGET_IPFS_DB=''
616
fi
617

618

619
if [ -n "$ipfs_mode" ] && [ -n "$EGET_IPFS_DB" ] ; then
620
    ddb="$(dirname "$EGET_IPFS_DB")"
621
    if [ -d "$ddb" ] ; then
622
        info "Using eget IPFS db $EGET_IPFS_DB"
623
        [ -r "$EGET_IPFS_DB" ] || touch "$EGET_IPFS_DB"
624
    else
625
        EGET_IPFS_DB=''
626
    fi
627
fi
628

629

630
# detect if we run with ipfs:// or with auto
631
if is_ipfsurl "$1" && [ -z "$ipfs_mode" ] || [ "$ipfs_mode" = "auto" ] ; then
632
    info "Autodetecting available IPFS relay..."
633
    select_ipfs_mode
634
    info "Auto selected IPFS mode: $ipfs_mode"
635
else
636
    [ -n "$ipfs_mode" ] && info "IPFS mode: $ipfs_mode"
637
fi
638

639
IPFS_CMD=''
640

641
if [ "$ipfs_mode" = "disabled" ] ; then
642

643
ipfs_get()
644
{
645
    fatal "IPFS is disabled"
646
}
647

648
ipfs_put()
649
{
650
    fatal "IPFS is disabled"
651
}
652

653
ipfs_cat()
654
{
655
    fatal "IPFS is disabled"
656
}
657

658

659
elif [ "$ipfs_mode" = "brave" ] ; then
660
    IPFS_CMD="$(get_ipfs_brave)" || fatal "Can't find ipfs command in Brave"
661
    IPFS_PRETTY_CMD="~Brave-Browser/$(basename $IPFS_CMD)"
662
    IPFS_API="$ipfs_api_brave"
663
    ipfs_api_access || fatal "Can't access to Brave IPFS API (Brave browser is not running and IPFS is not activated?)"
664
    info "Will use $IPFS_PRETTY_CMD --api $IPFS_API"
665

666
elif [ "$ipfs_mode" = "local" ] ; then
667
    IPFS_CMD="$(print_command_path ipfs)" || fatal "Can't find ipfs command"
668
    IPFS_PRETTY_CMD="$IPFS_CMD"
669
    IPFS_API="$ipfs_api_local"
670
    ipfs_api_access || fatal "Can't access to IPFS API (ipfs daemon is not running?)"
671
    info "Will use $IPFS_PRETTY_CMD --api $IPFS_API"
672

673
elif [ "$ipfs_mode" = "gateway" ] ; then
674
    info "Will use eget $IPFS_GATEWAY/HASH"
675

676
ipfs_get_real_url()
677
{
678
    [ -n "$IPFS_GATEWAY" ] || fatal "ipfs http gateway is not set"
679
    echo "$IPFS_GATEWAY/$1"
680
}
681

682
ipfs_get()
683
{
684
    if [ -n "$2" ] ; then
685
        docmd eget -O "$2" "$(ipfs_get_real_url "$1")"
686
    else
687
        docmd eget "$(ipfs_get_real_url "$1")"
688
    fi
689
}
690

691
ipfs_cat()
692
{
693
    # FIXME:
694
    ipfs_get "$1" "-"
695
}
696

697
ipfs_put()
698
{
699
    info "IPFS put skipped when a gateway is used"
700
    return 1
701
}
702
elif [ -z "$ipfs_mode" ] ; then
703
    :
704
else
705
    fatal "Unsupported eget ipfs mode $ipfs_mode"
706
fi
707

708
if [ -n "$IPFS_CMD" ] ; then
709

710
ipfs_get_real_url()
711
{
712
    return 1
713
}
714

715
ipfs_get()
716
{
717
    [ -n "$IPFS_CMD" ] || fatal "ipfs api is not usable"
718
    if [ -n "$2" ] ; then
719
        showcmd $IPFS_PRETTY_CMD --api $IPFS_API get -o "$2" "$1"
720
        $IPFS_CMD --api $IPFS_API get -o "$2" "$1"
721
    else
722
        showcmd $IPFS_PRETTY_CMD --api $IPFS_API get "$1"
723
        $IPFS_CMD --api $IPFS_API get "$1"
724
    fi
725
}
726

727
ipfs_put()
728
{
729
    [ -n "$IPFS_CMD" ] || fatal "ipfs api is not usable"
730

731
    # detect if -q is used (will output Qm instead of addded Qm)
732
    local qu="$1"
733
    [ "$qu" = "-q" ] || qu=''
734

735
    showcmd $IPFS_PRETTY_CMD --api $IPFS_API add "$@"
736

737
    local res
738
    res="$($IPFS_CMD --api $IPFS_API add "$@")" || return
739

740
    if [ -z "$qu" ] ; then
741
        res="$(echo "$res" | grep "^added Qm")" || return
742
        res="$(echo "$res" | cut -f2 -d" ")"
743
    fi
744

745
    is_ipfs_hash "$res" && echo "$res" && return
746
    fatal "Can't recognize $res IPFS hash"
747
}
748

749
ipfs_cat()
750
{
751
    [ -n "$IPFS_CMD" ] || fatal "ipfs api is not usable"
752
    showcmd $IPFS_PRETTY_CMD --api $IPFS_API cat "$1"
753
    $IPFS_CMD --api $IPFS_API cat "$1"
754
}
755

756
fi
757
###############################
758

759

760

761
WGET="$(print_command_path wget)"
762
CURL="$(print_command_path curl)"
763

764
ORIG_EGET_BACKEND="$EGET_BACKEND"
765
# override backend
766
if is_fileurl "$1" ; then
767
    EGET_BACKEND="file"
768
elif is_ipfsurl "$1" ; then
769
    EGET_BACKEND="ipfs"
770
fi
771

772

773
case "$EGET_BACKEND" in
774
    file|ipfs)
775
        ;;
776
    wget)
777
        [ -n "$WGET" ] || fatal "There are no wget in the system but you forced using it via EGET_BACKEND. Install it with $ epm install wget"
778
        ;;
779
    curl)
780
        [ -n "$CURL" ] || fatal "There are no curl in the system but you forced using it via EGET_BACKEND. Install it with $ epm install curl"
781
        ;;
782
    '')
783
        [ -n "$WGET" ] && EGET_BACKEND="wget"
784
        [ -z "$EGET_BACKEND" ] && [ -n "$CURL" ] && EGET_BACKEND="curl"
785
        [ -n "$EGET_BACKEND" ] || fatal "There are no wget nor curl in the system. Install something with $ epm install wget"
786
        ;;
787
    *)
788
        fatal "Uknown EGET_BACKEND $EGET_BACKEND"
789
        ;;
790
esac
791

792

793

794
if [ "$EGET_BACKEND" = "file" ] ; then
795

796
# put remote content to stdout
797
url_scat()
798
{
799
    local URL="$1"
800
    cat "$(path_from_url "$URL")"
801
}
802
# download to default name of to $2
803
url_sget()
804
{
805
    local URL="$1"
806
    if [ "$2" = "/dev/stdout" ] || [ "$2" = "-" ] ; then
807
       scat "$URL"
808
       return
809
    elif [ -n "$2" ] ; then
810
       cp -av "$(path_from_url "$URL")" "$2"
811
       return
812
    fi
813
    cp -av "$(path_from_url "$URL")" .
814
}
815

816
url_check()
817
{
818
    local URL="$1"
819
    test -f "$(path_from_url "$URL")"
820
}
821

822
url_get_filename()
823
{
824
    basename "$1"
825
}
826

827
url_get_real_url()
828
{
829
    echo "$1"
830
}
831

832
elif [ "$EGET_BACKEND" = "ipfs" ] ; then
833

834
# put remote content to stdout
835
url_scat()
836
{
837
    local URL="$1"
838
    ipfs_cat "$(cid_from_url "$URL")"
839
}
840
# download to default name of to $2
841
url_sget()
842
{
843
    local URL="$1"
844
    if [ "$2" = "/dev/stdout" ] || [ "$2" = "-" ] ; then
845
       scat "$URL"
846
       return
847
    elif [ -n "$2" ] ; then
848
       ipfs_get "$(cid_from_url "$URL")" "$2"
849
       return
850
    fi
851

852
    local fn="$(url_print_filename_from_url "$URL")"
853
    if [ -n "$fn" ] ; then
854
       ipfs_get "$(cid_from_url "$URL")" "$fn"
855
       return
856
    fi
857

858
    ipfs_get "$(cid_from_url "$URL")"
859
}
860

861
url_check()
862
{
863
    local URL="$1"
864
    # TODO: improve me
865
    scat "$URL" >/dev/null
866
}
867

868
url_print_filename_from_url()
869
{
870
    local URL="$1"
871
    local fn="$(echo "$URL" | sed -e 's|ipfs://.*\?filename=||')"
872
    [ "$URL" != "$fn" ] && echo "$fn" && return
873
}
874

875
url_get_filename()
876
{
877
    local URL="$1"
878
    url_print_filename_from_url "$URL" && return
879
    local CID="$(cid_from_url "$URL")"
880
    get_filename_by_cid "$CID"
881
}
882

883
url_get_real_url()
884
{
885
    local URL="$1"
886
    local CID="$(cid_from_url "$URL")"
887
    # if we use gateway, return URL with gateway
888
    ipfs_get_real_url "$URL" && return
889
    get_url_by_cid "$CID"
890
}
891

892

893
elif [ "$EGET_BACKEND" = "wget" ] ; then
894
__wget()
895
{
896
    if [ -n "$WGETUSERAGENT" ] ; then
897
        docmd $WGET $FORCEIPV $WGETQ $WGETCOMPRESSED $WGETHEADER $WGETNOSSLCHECK "$WGETUSERAGENT" "$@"
898
    else
899
        docmd $WGET $FORCEIPV $WGETQ $WGETCOMPRESSED $WGETHEADER $WGETNOSSLCHECK "$@"
900
    fi
901
}
902

903
# put remote content to stdout
904
url_scat()
905
{
906
    local URL="$1"
907
    download_with_mirroring __wget "$URL" -O-
908
}
909
# download to default name of to $2
910
url_sget()
911
{
912
    local URL="$1"
913
    if [ "$2" = "/dev/stdout" ] || [ "$2" = "-" ] ; then
914
       scat "$URL"
915
       return
916
    elif [ -n "$2" ] ; then
917
       download_with_mirroring __wget "$URL" -O "$2"
918
       return
919
    fi
920
# TODO: поддержка rsync для известных хостов?
921
# Не качать, если одинаковый размер и дата
922
# -nc
923
# TODO: overwrite always
924
    download_with_mirroring __wget "$URL" $WGETNAMEOPTIONS
925
}
926

927
url_get_response()
928
{
929
    local URL="$1"
930
    local answer
931
    answer="$(quiet=1 __wget --spider -S "$URL" 2>&1)"
932
    # HTTP/1.1 405 Method Not Allowed
933
    if echo "$answer" | grep -q "^ *HTTP/[12.]* 405" ; then
934
        (quiet=1 __wget --start-pos=5000G -S "$URL" 2>&1)
935
        return
936
    fi
937
    echo "$answer"
938
}
939

940

941
elif [ "$EGET_BACKEND" = "curl" ] ; then
942

943
__curl()
944
{
945
    if [ -n "$CURLUSERAGENT" ] ; then
946
        docmd $CURL $FORCEIPV --fail -L $CURLQ $CURLCOMPRESSED $CURLHEADER "$CURLUSERAGENT" $CURLNOSSLCHECK "$@"
947
    else
948
        docmd $CURL $FORCEIPV --fail -L $CURLQ $CURLCOMPRESSED $CURLHEADER $CURLNOSSLCHECK "$@"
949
    fi
950
}
951
# put remote content to stdout
952
url_scat()
953
{
954
    local URL="$1"
955
    download_with_mirroring __curl "$URL" --output -
956
}
957
# download to default name of to $2
958
url_sget()
959
{
960
    local URL="$1"
961
    local res
962
    if [ "$2" = "/dev/stdout" ] || [ "$2" = "-" ] ; then
963
       scat "$1"
964
       return
965
    elif [ -n "$2" ] ; then
966
       download_with_mirroring __curl "$URL" --output "$2"
967
       return
968
    fi
969

970
    download_with_mirroring __curl "$URL" $CURLNAMEOPTIONS
971
}
972

973
url_get_response()
974
{
975
    local URL="$1"
976
    local answer
977
    answer="$(quiet=1 __curl -LI "$URL" 2>&1)"
978
    # HTTP/1.1 405 Method Not Allowed
979
    if echo "$answer" | grep -q "^ *HTTP/[12.]* 405" ; then
980
        (quiet=1 __curl -L -i -r0-0 "$URL" 2>&1)
981
        return
982
    fi
983
    echo "$answer"
984
}
985

986
else
987
    fatal "Unknown EGET_BACKEND '$EGET_BACKEND', logical error."
988
fi
989

990

991
# Common code for both wget and curl (http related)
992
if [ "$EGET_BACKEND" = "wget" ] || [ "$EGET_BACKEND" = "curl" ] ; then
993

994
url_get_headers()
995
{
996
    local URL="$1"
997
    url_get_response "$URL" | grep -i "^ *[[:alpha:]].*: " | sed -e 's|^ *||' -e 's|\r$||'
998
}
999

1000
url_check()
1001
{
1002
    local URL="$1"
1003
    url_get_response "$URL" | grep "HTTP/" | tail -n1 | grep -q -w "200\|404"
1004
}
1005

1006
url_get_header()
1007
{
1008
    local URL="$1"
1009
    local HEADER="$2"
1010
    url_get_headers "$URL" | grep -i "^ *$HEADER: " | sed -e "s|^ *$HEADER: ||i"
1011
}
1012

1013
url_get_real_url()
1014
{
1015
    local URL="$1"
1016

1017
    ! is_httpurl "$URL" && echo "$URL" && return
1018

1019
    # don't check location if we have made form of the URL
1020
    [ -n "$MADEURL" ] && [ "$MADEURL" = "$URL" ] && echo "$URL" && return
1021

1022
    local loc
1023
    for loc in $(url_get_header "$URL" "Location" | tac | sed -e 's| .*||') ; do
1024
        # hack for construct full url from related Location
1025
        if echo "$loc" | grep -q "^/" ; then
1026
            loc="$(concatenate_url_and_filename "$(get_host_only "$URL")" "$loc")"
1027
        fi
1028
        if ! is_strange_url "$loc" ; then
1029
            echo "$loc"
1030
            return
1031
        fi
1032
    done
1033

1034
    echo "$URL"
1035
}
1036

1037
url_get_filename()
1038
{
1039
    local URL="$1"
1040

1041
    ! is_httpurl "$URL" && basename "$URL" && return
1042

1043
    # See https://www.cpcwood.com/blog/5-aws-s3-utf-8-content-disposition
1044
    # https://www.rfc-editor.org/rfc/rfc6266
1045
    local cd="$(url_get_header "$URL" "Content-Disposition")"
1046
    if echo "$cd" | grep -qi "filename\*= *UTF-8" ; then
1047
        #Content-Disposition: attachment; filename="unityhub-amd64-3.3.0.deb"; filename*=UTF-8''"unityhub-amd64-3.3.0.deb"
1048
        echo "$cd" | sed -e "s|.*filename\*= *UTF-8''||i" -e 's|^"||' -e 's|";$||' -e 's|"$||'
1049
        return
1050
    fi
1051
    if echo "$cd" | grep -qi "filename=" ; then
1052
        #Content-Disposition: attachment; filename=postman-linux-x64.tar.gz
1053
        #content-disposition: attachment; filename="code-1.77.1-1680651749.el7.x86_64.rpm"
1054
        echo "$cd" | sed -e 's|.*filename= *||i' -e 's|^"||' -e 's|";.*||' -e 's|"$||'
1055
        return
1056
    fi
1057

1058
    basename "$(url_get_real_url "$URL")"
1059
}
1060

1061
fi
1062

1063

1064
if [ -n "$ipfs_mode" ] && [ -n "$EGET_IPFS_DB" ] &&  ! is_ipfsurl "$1"  ; then
1065

1066
download_to_ipfs()
1067
{
1068
    local URL="$1"
1069
    local res
1070
    #res="$(url_scat "$URL" | ipfs_put )" || return
1071
    #res="$(echo "$res" | grep "^added Qm")" || return 1
1072
    #CID="$(echo "$res" | cut -f2 -d" ")"
1073
    # with -q to disable progress (mixed with download progress)
1074
    res="$(url_scat "$URL" | ipfs_put -q)" || return
1075
    is_ipfs_hash "$res" || return 1
1076
    echo "$res"
1077
}
1078

1079
# put remote content to stdout
1080
scat()
1081
{
1082
    local URL="$1"
1083
    url_scat "$URL"
1084

1085
    # It is list only function. Don't save to IPFS
1086
    return
1087

1088
    ###################
1089

1090
    local CID="$(get_cid_by_url "$URL")"
1091
    if [ -n "$CID" ] ; then
1092
        info "$URL -> $CID"
1093
        ipfs_cat "$CID"
1094
        return
1095
    fi
1096

1097
    CID="$(download_to_ipfs "$URL")" || return
1098

1099
    ipfs_cat "$CID" || return
1100

1101
    local FN="$(url_get_filename "$URL")" || return
1102

1103
    put_cid_and_url "$URL" "$CID" "$FN"
1104
}
1105

1106
# download to default name of to $2
1107
sget()
1108
{
1109
    local URL="$1"
1110
    local TARGET="$2"
1111

1112
    if [ -n "$GETFILENAME" ] ; then
1113
        get_filename "$URL"
1114
        exit
1115
    fi
1116

1117
    local REALURL="$(get_real_url "$URL")" || return
1118

1119
    if [ -n "$GETREALURL" ] ; then
1120
        echo "$REALURL"
1121
        exit
1122
    fi
1123

1124
    # skip ipfs for cat
1125
    if [ "$TARGET" = "/dev/stdout" ] || [ "$TARGET" = "-" ] ; then
1126
       url_scat "$URL"
1127
       return
1128
    fi
1129

1130

1131
    #if is_strange_url "$REALURL" ; then
1132
    #    info "Just download strange URL $REALURL, skipping IPFS"
1133
    #    url_sget "$REALURL" "$TARGET"
1134
    #    return
1135
    #fi
1136

1137
    local CID="$(get_cid_by_url "$REALURL")"
1138
    if [ -n "$CID" ] ; then
1139

1140
        if [ -n "$GETIPFSCID" ] ; then
1141
            echo "$CID"
1142
            exit
1143
        fi
1144

1145
        if [ -n "$GETFILENAME" ] ; then
1146
            get_filename_by_cid "$CID"
1147
            exit
1148
        fi
1149

1150
        if [ -n "$GETREALURL" ] ; then
1151
            get_url_by_cid "$CID"
1152
            exit
1153
        fi
1154

1155
        if [ -z "$TARGET" ] ; then
1156
            # TODO: in some cases we can get name from URL...
1157
            TARGET="$(get_filename_by_cid "$CID")"
1158
            if [ -z "$TARGET" ] ; then
1159
                TARGET="$CID"
1160
            fi
1161
        fi
1162
        [ "$URL" = "$REALURL" ] && info "$URL -> $CID -> $TARGET" || info "$URL -> $REALURL -> $CID -> $TARGET"
1163
        ipfs_get "$CID" "$TARGET" && return
1164

1165
        # fail get from IPFS, fallback
1166
        url_sget "$REALURL" "$TARGET"
1167
        return
1168
    fi
1169

1170

1171
    # download and put to IPFS
1172
    local FN="$(url_get_filename "$REALURL")" || return
1173
    if [ -z "$TARGET" ] ; then
1174
        TARGET="$FN"
1175
    fi
1176

1177
    if [ -n "$GETIPFSCID" ] ; then
1178
         # add to IPFS and print out CID
1179
         CID="$(ipfs_put --progress "$REALURL")" || return
1180
         echo "$CID"
1181
         exit
1182
    fi
1183

1184
    # download file and add to IPFS
1185
    url_sget "$REALURL" "$TARGET" || return
1186

1187
    # don't do ipfs put when gateway is using
1188
    [ "$ipfs_mode" = "gateway" ] && return
1189

1190
    CID="$(ipfs_put --progress "$TARGET")" || return
1191

1192
    put_cid_and_url "$REALURL" "$CID" "$FN"
1193
}
1194

1195
check_url_is_accessible()
1196
{
1197
    local URL="$1"
1198
    local REALURL="$(get_real_url "$URL")" || return
1199
    local CID="$(get_cid_by_url "$REALURL")"
1200
    if [ -n "$CID" ] ; then
1201
        [ "$URL" = "$REALURL" ] && info "$URL -> $CID" || info "$URL -> $REALURL -> $CID"
1202
        ipfs_check "$CID"
1203
        return
1204
    fi
1205

1206
    CID="$(download_to_ipfs "$REALURL")" || return
1207

1208
    local FN="$(url_get_filename "$REALURL")" || return
1209
    ipfs_cat "$CID" >/dev/null || return
1210
    put_cid_and_url "$REALURL" "$CID" "$FN"
1211
}
1212

1213
get_filename()
1214
{
1215
    url_get_filename "$1"
1216
}
1217

1218
get_real_url()
1219
{
1220
    url_get_real_url "$1"
1221
}
1222

1223
else
1224
scat()
1225
{
1226
    url_scat "$@"
1227
}
1228

1229
sget()
1230
{
1231
    if [ -n "$GETFILENAME" ] ; then
1232
        get_filename "$1"
1233
        exit
1234
    fi
1235

1236
    if [ -n "$GETREALURL" ] ; then
1237
        get_real_url "$1"
1238
        exit
1239
    fi
1240

1241
    url_sget "$@"
1242
}
1243

1244
check_url_is_accessible()
1245
{
1246
    url_check "$@"
1247
}
1248

1249
get_filename()
1250
{
1251
    url_get_filename "$1"
1252
}
1253

1254
get_real_url()
1255
{
1256
    url_get_real_url "$1"
1257
}
1258

1259
fi
1260

1261

1262
get_github_urls()
1263
{
1264
    # https://github.com/OWNER/PROJECT
1265
    local owner="$(echo "$1" | sed -e "s|^https://github.com/||" -e "s|/.*||")" #"
1266
    local project="$(echo "$1" | sed -e "s|^https://github.com/$owner/||" -e "s|/.*||")" #"
1267
    [ -n "$owner" ] || fatal "Can't get owner from $1"
1268
    [ -n "$project" ] || fatal "Can't get project from $1"
1269
    local URL="https://api.github.com/repos/$owner/$project/releases"
1270
    # api sometime returns unformatted json
1271
    scat $URL | sed -e 's|,\(["{]\)|,\n\1|g' | \
1272
        grep -i -o -E '"browser_download_url": *"https://.*"' | cut -d'"' -f4
1273
}
1274

1275
# drop file path from URL
1276
get_host_only()
1277
{
1278
    echo "$1/" | grep -Eo '(.*://[^/]+)'
1279
}
1280

1281
concatenate_url_and_filename()
1282
{
1283
    local url="$(echo "$1" | sed -e 's|/*$||' )"
1284
    local fn="$(echo "$2" | sed -e 's|^/*||' )"
1285
    echo "$url/$fn"
1286
}
1287

1288
# MADEURL filled with latest made URL as flag it is end form of URL
1289
MADEURL=''
1290

1291
# Args: URL filename
1292
make_fileurl()
1293
{
1294
    local url="$1"
1295
    local fn="$2"
1296

1297
    fn="$(echo "$fn" | sed -e 's|^./||' -e 's|^/+||')"
1298

1299
    if is_fileurl "$url" ; then
1300
        # if it is url
1301
        :
1302
    elif echo "$fn" | grep -q "^/" ; then
1303
        # if there is file path from the root of the site
1304
        url="$(get_host_only "$url")"
1305
    elif echo "$url" | grep -q -v "/$" ; then
1306
        # if there is no slash in the end of URL
1307
        url="$(dirname "$url")"
1308
    fi
1309

1310
    MADEURL="$(concatenate_url_and_filename "$url" "$fn")"
1311
    echo "$MADEURL"
1312
}
1313

1314
get_urls()
1315
{
1316
    if is_fileurl "$URL" ; then
1317
        ls -1 "$(path_from_url "$URL")"
1318
        return
1319
    fi
1320

1321
    # cat html, divide to lines by tags and cut off hrefs only
1322
    scat $URL | sed -e 's|<|<\n|g' -e 's|data-file=|href=|g' -e "s|href=http|href=\"http|g" -e "s|>|\">|g" -e "s|'|\"|g" | \
1323
         grep -i -o -E 'href="(.+)"' | cut -d'"' -f2
1324
}
1325

1326

1327
if [ -n "$CHECKURL" ] ; then
1328
    #set_quiet
1329
    URL="$1"
1330
    check_url_is_accessible "$URL"
1331
    res=$?
1332
    if [ -n "$verbose" ] ; then
1333
        [ "$res" = "0" ] && echo "$URL is accessible via network" || echo "$URL is NOT accessible via network"
1334
    fi
1335
    exit $res
1336
fi
1337

1338
if [ -n "$GETRESPONSE" ] ; then
1339
    url_get_response "$1"
1340
    exit
1341
fi
1342

1343

1344
# separate part for github downloads
1345
if echo "$1" | grep -q "^https://github.com/" && \
1346
   echo "$1" | grep -q -v "/download/" && [ -n "$2" ] ; then
1347
    MASK="$2"
1348

1349
    if [ -n "$LISTONLY" ] ; then
1350
        get_github_urls "$1" | filter_glob "$MASK" | filter_order
1351
        exit
1352
    fi
1353

1354
    ERROR=0
1355
    for fn in $(get_github_urls "$1" | filter_glob "$MASK" | filter_order) ; do
1356
        MADEURL="$fn" # mark it is the end form of the URL
1357
        sget "$fn" "$TARGETFILE" || ERROR=1
1358
        [ -n "$TARGETFILE" ] && [ "$ERROR" = "0" ] && break
1359
    done
1360
    exit
1361
fi
1362

1363
if is_ipfsurl "$1" ; then
1364
    [ -n "$2" ] && fatal "too many args when ipfs://Qm... used: extra '$2' arg"
1365
    sget "$1" "$TARGETFILE"
1366
    exit
1367
fi
1368

1369
# if mask is the second arg
1370
if [ -n "$2" ] ; then
1371
    URL="$1"
1372
    MASK="$2"
1373
else
1374
    # do not support / at the end without separately specified mask
1375
    if echo "$1" | grep -q "/$" ; then
1376
        #fatal "Use http://example.com/e/* to download all files in dir"
1377
        URL="$1"
1378
        MASK=""
1379
    else
1380
        # drop mask part
1381
        URL="$(dirname "$1")/"
1382
        # wildcards allowed only in the last part of path
1383
        MASK=$(basename "$1")
1384
    fi
1385

1386
fi
1387

1388
# https://www.freeoffice.com/download.php?filename=freeoffice-2021-1062.x86_64.rpm
1389
if echo "$URL" | grep -q "[*]" ; then
1390
    fatal "Error: there are globbing symbol (*) in $URL. It is allowed only for mask part"
1391
fi
1392

1393
is_url "$MASK" && fatal "eget supports only one URL as argument"
1394
[ -n "$3" ] && fatal "too many args: extra '$3'. May be you need use quotes for arg with wildcards."
1395

1396
# TODO: curl?
1397
# If ftp protocol, just download
1398
if echo "$URL" | grep -q "^ftp://" ; then
1399
    [ -n "$LISTONLY" ] && fatal "TODO: list files for ftp:// is not supported yet"
1400
    sget "$1" "$TARGETFILE"
1401
    exit
1402
fi
1403

1404

1405
if [ -n "$LISTONLY" ] ; then
1406
    for fn in $(get_urls | filter_glob "$MASK" | filter_order) ; do
1407
        is_url "$fn" && echo "$fn" && continue
1408
        make_fileurl "$URL" "$fn"
1409
    done
1410
    exit
1411
fi
1412

1413
# If there is no wildcard symbol like asterisk, just download
1414
if echo "$MASK" | grep -qv "[*?]" || echo "$MASK" | grep -q "[?].*="; then
1415
    sget "$1" "$TARGETFILE"
1416
    exit
1417
fi
1418

1419
ERROR=0
1420
for fn in $(get_urls | filter_glob "$MASK" | filter_order) ; do
1421
    is_url "$fn" || fn="$(make_fileurl "$URL" "$fn" )" #"
1422
    sget "$fn" "$TARGETFILE" || ERROR=1
1423
    [ -n "$TARGETFILE" ] && [ "$ERROR" = "0" ] && break
1424
done
1425
exit $ERROR
1426

1427