git

1
#!/usr/bin/env python
2
#
3
# git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
4
#
5
# Author: Simon Hausmann <simon@lst.de>
6
# Copyright: 2007 Simon Hausmann <simon@lst.de>
7
#            2007 Trolltech ASA
8
# License: MIT <http://www.opensource.org/licenses/mit-license.php>
9
#
10
# pylint: disable=bad-whitespace
11
# pylint: disable=broad-except
12
# pylint: disable=consider-iterating-dictionary
13
# pylint: disable=disable
14
# pylint: disable=fixme
15
# pylint: disable=invalid-name
16
# pylint: disable=line-too-long
17
# pylint: disable=missing-docstring
18
# pylint: disable=no-self-use
19
# pylint: disable=superfluous-parens
20
# pylint: disable=too-few-public-methods
21
# pylint: disable=too-many-arguments
22
# pylint: disable=too-many-branches
23
# pylint: disable=too-many-instance-attributes
24
# pylint: disable=too-many-lines
25
# pylint: disable=too-many-locals
26
# pylint: disable=too-many-nested-blocks
27
# pylint: disable=too-many-statements
28
# pylint: disable=ungrouped-imports
29
# pylint: disable=unused-import
30
# pylint: disable=wrong-import-order
31
# pylint: disable=wrong-import-position
32
#
33

34
import struct
35
import sys
36
if sys.version_info.major < 3 and sys.version_info.minor < 7:
37
    sys.stderr.write("git-p4: requires Python 2.7 or later.\n")
38
    sys.exit(1)
39

40
import ctypes
41
import errno
42
import functools
43
import glob
44
import marshal
45
import optparse
46
import os
47
import platform
48
import re
49
import shutil
50
import stat
51
import subprocess
52
import tempfile
53
import time
54
import zipfile
55
import zlib
56

57
# On python2.7 where raw_input() and input() are both availble,
58
# we want raw_input's semantics, but aliased to input for python3
59
# compatibility
60
# support basestring in python3
61
try:
62
    if raw_input and input:
63
        input = raw_input
64
except:
65
    pass
66

67
verbose = False
68

69
# Only labels/tags matching this will be imported/exported
70
defaultLabelRegexp = r'[a-zA-Z0-9_\-.]+$'
71

72
# The block size is reduced automatically if required
73
defaultBlockSize = 1 << 20
74

75
defaultMetadataDecodingStrategy = 'passthrough' if sys.version_info.major == 2 else 'fallback'
76
defaultFallbackMetadataEncoding = 'cp1252'
77

78
p4_access_checked = False
79

80
re_ko_keywords = re.compile(br'\$(Id|Header)(:[^$\n]+)?\$')
81
re_k_keywords = re.compile(br'\$(Id|Header|Author|Date|DateTime|Change|File|Revision)(:[^$\n]+)?\$')
82

83

84
def format_size_human_readable(num):
85
    """Returns a number of units (typically bytes) formatted as a
86
       human-readable string.
87
       """
88
    if num < 1024:
89
        return '{:d} B'.format(num)
90
    for unit in ["Ki", "Mi", "Gi", "Ti", "Pi", "Ei", "Zi"]:
91
        num /= 1024.0
92
        if num < 1024.0:
93
            return "{:3.1f} {}B".format(num, unit)
94
    return "{:.1f} YiB".format(num)
95

96

97
def p4_build_cmd(cmd):
98
    """Build a suitable p4 command line.
99

100
       This consolidates building and returning a p4 command line into one
101
       location. It means that hooking into the environment, or other
102
       configuration can be done more easily.
103
       """
104
    real_cmd = ["p4"]
105

106
    user = gitConfig("git-p4.user")
107
    if len(user) > 0:
108
        real_cmd += ["-u", user]
109

110
    password = gitConfig("git-p4.password")
111
    if len(password) > 0:
112
        real_cmd += ["-P", password]
113

114
    port = gitConfig("git-p4.port")
115
    if len(port) > 0:
116
        real_cmd += ["-p", port]
117

118
    host = gitConfig("git-p4.host")
119
    if len(host) > 0:
120
        real_cmd += ["-H", host]
121

122
    client = gitConfig("git-p4.client")
123
    if len(client) > 0:
124
        real_cmd += ["-c", client]
125

126
    retries = gitConfigInt("git-p4.retries")
127
    if retries is None:
128
        # Perform 3 retries by default
129
        retries = 3
130
    if retries > 0:
131
        # Provide a way to not pass this option by setting git-p4.retries to 0
132
        real_cmd += ["-r", str(retries)]
133

134
    real_cmd += cmd
135

136
    # now check that we can actually talk to the server
137
    global p4_access_checked
138
    if not p4_access_checked:
139
        p4_access_checked = True    # suppress access checks in p4_check_access itself
140
        p4_check_access()
141

142
    return real_cmd
143

144

145
def git_dir(path):
146
    """Return TRUE if the given path is a git directory (/path/to/dir/.git).
147
       This won't automatically add ".git" to a directory.
148
       """
149
    d = read_pipe(["git", "--git-dir", path, "rev-parse", "--git-dir"], True).strip()
150
    if not d or len(d) == 0:
151
        return None
152
    else:
153
        return d
154

155

156
def chdir(path, is_client_path=False):
157
    """Do chdir to the given path, and set the PWD environment variable for use
158
       by P4.  It does not look at getcwd() output.  Since we're not using the
159
       shell, it is necessary to set the PWD environment variable explicitly.
160

161
       Normally, expand the path to force it to be absolute.  This addresses
162
       the use of relative path names inside P4 settings, e.g.
163
       P4CONFIG=.p4config.  P4 does not simply open the filename as given; it
164
       looks for .p4config using PWD.
165

166
       If is_client_path, the path was handed to us directly by p4, and may be
167
       a symbolic link.  Do not call os.getcwd() in this case, because it will
168
       cause p4 to think that PWD is not inside the client path.
169
       """
170

171
    os.chdir(path)
172
    if not is_client_path:
173
        path = os.getcwd()
174
    os.environ['PWD'] = path
175

176

177
def calcDiskFree():
178
    """Return free space in bytes on the disk of the given dirname."""
179
    if platform.system() == 'Windows':
180
        free_bytes = ctypes.c_ulonglong(0)
181
        ctypes.windll.kernel32.GetDiskFreeSpaceExW(ctypes.c_wchar_p(os.getcwd()), None, None, ctypes.pointer(free_bytes))
182
        return free_bytes.value
183
    else:
184
        st = os.statvfs(os.getcwd())
185
        return st.f_bavail * st.f_frsize
186

187

188
def die(msg):
189
    """Terminate execution. Make sure that any running child processes have
190
       been wait()ed for before calling this.
191
       """
192
    if verbose:
193
        raise Exception(msg)
194
    else:
195
        sys.stderr.write(msg + "\n")
196
        sys.exit(1)
197

198

199
def prompt(prompt_text):
200
    """Prompt the user to choose one of the choices.
201

202
       Choices are identified in the prompt_text by square brackets around a
203
       single letter option.
204
       """
205
    choices = set(m.group(1) for m in re.finditer(r"\[(.)\]", prompt_text))
206
    while True:
207
        sys.stderr.flush()
208
        sys.stdout.write(prompt_text)
209
        sys.stdout.flush()
210
        response = sys.stdin.readline().strip().lower()
211
        if not response:
212
            continue
213
        response = response[0]
214
        if response in choices:
215
            return response
216

217

218
# We need different encoding/decoding strategies for text data being passed
219
# around in pipes depending on python version
220
if bytes is not str:
221
    # For python3, always encode and decode as appropriate
222
    def decode_text_stream(s):
223
        return s.decode() if isinstance(s, bytes) else s
224

225
    def encode_text_stream(s):
226
        return s.encode() if isinstance(s, str) else s
227
else:
228
    # For python2.7, pass read strings as-is, but also allow writing unicode
229
    def decode_text_stream(s):
230
        return s
231

232
    def encode_text_stream(s):
233
        return s.encode('utf_8') if isinstance(s, unicode) else s
234

235

236
class MetadataDecodingException(Exception):
237
    def __init__(self, input_string):
238
        self.input_string = input_string
239

240
    def __str__(self):
241
        return """Decoding perforce metadata failed!
242
The failing string was:
243
---
244
{}
245
---
246
Consider setting the git-p4.metadataDecodingStrategy config option to
247
'fallback', to allow metadata to be decoded using a fallback encoding,
248
defaulting to cp1252.""".format(self.input_string)
249

250

251
encoding_fallback_warning_issued = False
252
encoding_escape_warning_issued = False
253
def metadata_stream_to_writable_bytes(s):
254
    encodingStrategy = gitConfig('git-p4.metadataDecodingStrategy') or defaultMetadataDecodingStrategy
255
    fallbackEncoding = gitConfig('git-p4.metadataFallbackEncoding') or defaultFallbackMetadataEncoding
256
    if not isinstance(s, bytes):
257
        return s.encode('utf_8')
258
    if encodingStrategy == 'passthrough':
259
        return s
260
    try:
261
        s.decode('utf_8')
262
        return s
263
    except UnicodeDecodeError:
264
        if encodingStrategy == 'fallback' and fallbackEncoding:
265
            global encoding_fallback_warning_issued
266
            global encoding_escape_warning_issued
267
            try:
268
                if not encoding_fallback_warning_issued:
269
                    print("\nCould not decode value as utf-8; using configured fallback encoding %s: %s" % (fallbackEncoding, s))
270
                    print("\n(this warning is only displayed once during an import)")
271
                    encoding_fallback_warning_issued = True
272
                return s.decode(fallbackEncoding).encode('utf_8')
273
            except Exception as exc:
274
                if not encoding_escape_warning_issued:
275
                    print("\nCould not decode value with configured fallback encoding %s; escaping bytes over 127: %s" % (fallbackEncoding, s))
276
                    print("\n(this warning is only displayed once during an import)")
277
                    encoding_escape_warning_issued = True
278
                escaped_bytes = b''
279
                # bytes and strings work very differently in python2 vs python3...
280
                if str is bytes:
281
                    for byte in s:
282
                        byte_number = struct.unpack('>B', byte)[0]
283
                        if byte_number > 127:
284
                            escaped_bytes += b'%'
285
                            escaped_bytes += hex(byte_number)[2:].upper()
286
                        else:
287
                            escaped_bytes += byte
288
                else:
289
                    for byte_number in s:
290
                        if byte_number > 127:
291
                            escaped_bytes += b'%'
292
                            escaped_bytes += hex(byte_number).upper().encode()[2:]
293
                        else:
294
                            escaped_bytes += bytes([byte_number])
295
                return escaped_bytes
296

297
        raise MetadataDecodingException(s)
298

299

300
def decode_path(path):
301
    """Decode a given string (bytes or otherwise) using configured path
302
       encoding options.
303
       """
304

305
    encoding = gitConfig('git-p4.pathEncoding') or 'utf_8'
306
    if bytes is not str:
307
        return path.decode(encoding, errors='replace') if isinstance(path, bytes) else path
308
    else:
309
        try:
310
            path.decode('ascii')
311
        except:
312
            path = path.decode(encoding, errors='replace')
313
            if verbose:
314
                print('Path with non-ASCII characters detected. Used {} to decode: {}'.format(encoding, path))
315
        return path
316

317

318
def run_git_hook(cmd, param=[]):
319
    """Execute a hook if the hook exists."""
320
    args = ['git', 'hook', 'run', '--ignore-missing', cmd]
321
    if param:
322
        args.append("--")
323
        for p in param:
324
            args.append(p)
325
    return subprocess.call(args) == 0
326

327

328
def write_pipe(c, stdin, *k, **kw):
329
    if verbose:
330
        sys.stderr.write('Writing pipe: {}\n'.format(' '.join(c)))
331

332
    p = subprocess.Popen(c, stdin=subprocess.PIPE, *k, **kw)
333
    pipe = p.stdin
334
    val = pipe.write(stdin)
335
    pipe.close()
336
    if p.wait():
337
        die('Command failed: {}'.format(' '.join(c)))
338

339
    return val
340

341

342
def p4_write_pipe(c, stdin, *k, **kw):
343
    real_cmd = p4_build_cmd(c)
344
    if bytes is not str and isinstance(stdin, str):
345
        stdin = encode_text_stream(stdin)
346
    return write_pipe(real_cmd, stdin, *k, **kw)
347

348

349
def read_pipe_full(c, *k, **kw):
350
    """Read output from command. Returns a tuple of the return status, stdout
351
       text and stderr text.
352
       """
353
    if verbose:
354
        sys.stderr.write('Reading pipe: {}\n'.format(' '.join(c)))
355

356
    p = subprocess.Popen(
357
        c, stdout=subprocess.PIPE, stderr=subprocess.PIPE, *k, **kw)
358
    out, err = p.communicate()
359
    return (p.returncode, out, decode_text_stream(err))
360

361

362
def read_pipe(c, ignore_error=False, raw=False, *k, **kw):
363
    """Read output from  command. Returns the output text on success. On
364
       failure, terminates execution, unless ignore_error is True, when it
365
       returns an empty string.
366

367
       If raw is True, do not attempt to decode output text.
368
       """
369
    retcode, out, err = read_pipe_full(c, *k, **kw)
370
    if retcode != 0:
371
        if ignore_error:
372
            out = ""
373
        else:
374
            die('Command failed: {}\nError: {}'.format(' '.join(c), err))
375
    if not raw:
376
        out = decode_text_stream(out)
377
    return out
378

379

380
def read_pipe_text(c, *k, **kw):
381
    """Read output from a command with trailing whitespace stripped. On error,
382
       returns None.
383
       """
384
    retcode, out, err = read_pipe_full(c, *k, **kw)
385
    if retcode != 0:
386
        return None
387
    else:
388
        return decode_text_stream(out).rstrip()
389

390

391
def p4_read_pipe(c, ignore_error=False, raw=False, *k, **kw):
392
    real_cmd = p4_build_cmd(c)
393
    return read_pipe(real_cmd, ignore_error, raw=raw, *k, **kw)
394

395

396
def read_pipe_lines(c, raw=False, *k, **kw):
397
    if verbose:
398
        sys.stderr.write('Reading pipe: {}\n'.format(' '.join(c)))
399

400
    p = subprocess.Popen(c, stdout=subprocess.PIPE, *k, **kw)
401
    pipe = p.stdout
402
    lines = pipe.readlines()
403
    if not raw:
404
        lines = [decode_text_stream(line) for line in lines]
405
    if pipe.close() or p.wait():
406
        die('Command failed: {}'.format(' '.join(c)))
407
    return lines
408

409

410
def p4_read_pipe_lines(c, *k, **kw):
411
    """Specifically invoke p4 on the command supplied."""
412
    real_cmd = p4_build_cmd(c)
413
    return read_pipe_lines(real_cmd, *k, **kw)
414

415

416
def p4_has_command(cmd):
417
    """Ask p4 for help on this command.  If it returns an error, the command
418
       does not exist in this version of p4.
419
       """
420
    real_cmd = p4_build_cmd(["help", cmd])
421
    p = subprocess.Popen(real_cmd, stdout=subprocess.PIPE,
422
                                   stderr=subprocess.PIPE)
423
    p.communicate()
424
    return p.returncode == 0
425

426

427
def p4_has_move_command():
428
    """See if the move command exists, that it supports -k, and that it has not
429
       been administratively disabled.  The arguments must be correct, but the
430
       filenames do not have to exist.  Use ones with wildcards so even if they
431
       exist, it will fail.
432
       """
433

434
    if not p4_has_command("move"):
435
        return False
436
    cmd = p4_build_cmd(["move", "-k", "@from", "@to"])
437
    p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
438
    out, err = p.communicate()
439
    err = decode_text_stream(err)
440
    # return code will be 1 in either case
441
    if err.find("Invalid option") >= 0:
442
        return False
443
    if err.find("disabled") >= 0:
444
        return False
445
    # assume it failed because @... was invalid changelist
446
    return True
447

448

449
def system(cmd, ignore_error=False, *k, **kw):
450
    if verbose:
451
        sys.stderr.write("executing {}\n".format(
452
            ' '.join(cmd) if isinstance(cmd, list) else cmd))
453
    retcode = subprocess.call(cmd, *k, **kw)
454
    if retcode and not ignore_error:
455
        raise subprocess.CalledProcessError(retcode, cmd)
456

457
    return retcode
458

459

460
def p4_system(cmd, *k, **kw):
461
    """Specifically invoke p4 as the system command."""
462
    real_cmd = p4_build_cmd(cmd)
463
    retcode = subprocess.call(real_cmd, *k, **kw)
464
    if retcode:
465
        raise subprocess.CalledProcessError(retcode, real_cmd)
466

467

468
def die_bad_access(s):
469
    die("failure accessing depot: {0}".format(s.rstrip()))
470

471

472
def p4_check_access(min_expiration=1):
473
    """Check if we can access Perforce - account still logged in."""
474

475
    results = p4CmdList(["login", "-s"])
476

477
    if len(results) == 0:
478
        # should never get here: always get either some results, or a p4ExitCode
479
        assert("could not parse response from perforce")
480

481
    result = results[0]
482

483
    if 'p4ExitCode' in result:
484
        # p4 returned non-zero status, e.g. P4PORT invalid, or p4 not in path
485
        die_bad_access("could not run p4")
486

487
    code = result.get("code")
488
    if not code:
489
        # we get here if we couldn't connect and there was nothing to unmarshal
490
        die_bad_access("could not connect")
491

492
    elif code == "stat":
493
        expiry = result.get("TicketExpiration")
494
        if expiry:
495
            expiry = int(expiry)
496
            if expiry > min_expiration:
497
                # ok to carry on
498
                return
499
            else:
500
                die_bad_access("perforce ticket expires in {0} seconds".format(expiry))
501

502
        else:
503
            # account without a timeout - all ok
504
            return
505

506
    elif code == "error":
507
        data = result.get("data")
508
        if data:
509
            die_bad_access("p4 error: {0}".format(data))
510
        else:
511
            die_bad_access("unknown error")
512
    elif code == "info":
513
        return
514
    else:
515
        die_bad_access("unknown error code {0}".format(code))
516

517

518
_p4_version_string = None
519

520

521
def p4_version_string():
522
    """Read the version string, showing just the last line, which hopefully is
523
       the interesting version bit.
524

525
       $ p4 -V
526
       Perforce - The Fast Software Configuration Management System.
527
       Copyright 1995-2011 Perforce Software.  All rights reserved.
528
       Rev. P4/NTX86/2011.1/393975 (2011/12/16).
529
       """
530
    global _p4_version_string
531
    if not _p4_version_string:
532
        a = p4_read_pipe_lines(["-V"])
533
        _p4_version_string = a[-1].rstrip()
534
    return _p4_version_string
535

536

537
def p4_integrate(src, dest):
538
    p4_system(["integrate", "-Dt", wildcard_encode(src), wildcard_encode(dest)])
539

540

541
def p4_sync(f, *options):
542
    p4_system(["sync"] + list(options) + [wildcard_encode(f)])
543

544

545
def p4_add(f):
546
    """Forcibly add file names with wildcards."""
547
    if wildcard_present(f):
548
        p4_system(["add", "-f", f])
549
    else:
550
        p4_system(["add", f])
551

552

553
def p4_delete(f):
554
    p4_system(["delete", wildcard_encode(f)])
555

556

557
def p4_edit(f, *options):
558
    p4_system(["edit"] + list(options) + [wildcard_encode(f)])
559

560

561
def p4_revert(f):
562
    p4_system(["revert", wildcard_encode(f)])
563

564

565
def p4_reopen(type, f):
566
    p4_system(["reopen", "-t", type, wildcard_encode(f)])
567

568

569
def p4_reopen_in_change(changelist, files):
570
    cmd = ["reopen", "-c", str(changelist)] + files
571
    p4_system(cmd)
572

573

574
def p4_move(src, dest):
575
    p4_system(["move", "-k", wildcard_encode(src), wildcard_encode(dest)])
576

577

578
def p4_last_change():
579
    results = p4CmdList(["changes", "-m", "1"], skip_info=True)
580
    return int(results[0]['change'])
581

582

583
def p4_describe(change, shelved=False):
584
    """Make sure it returns a valid result by checking for the presence of
585
       field "time".
586

587
       Return a dict of the results.
588
       """
589

590
    cmd = ["describe", "-s"]
591
    if shelved:
592
        cmd += ["-S"]
593
    cmd += [str(change)]
594

595
    ds = p4CmdList(cmd, skip_info=True)
596
    if len(ds) != 1:
597
        die("p4 describe -s %d did not return 1 result: %s" % (change, str(ds)))
598

599
    d = ds[0]
600

601
    if "p4ExitCode" in d:
602
        die("p4 describe -s %d exited with %d: %s" % (change, d["p4ExitCode"],
603
                                                      str(d)))
604
    if "code" in d:
605
        if d["code"] == "error":
606
            die("p4 describe -s %d returned error code: %s" % (change, str(d)))
607

608
    if "time" not in d:
609
        die("p4 describe -s %d returned no \"time\": %s" % (change, str(d)))
610

611
    return d
612

613

614
def split_p4_type(p4type):
615
    """Canonicalize the p4 type and return a tuple of the base type, plus any
616
       modifiers.  See "p4 help filetypes" for a list and explanation.
617
       """
618

619
    p4_filetypes_historical = {
620
        "ctempobj": "binary+Sw",
621
        "ctext": "text+C",
622
        "cxtext": "text+Cx",
623
        "ktext": "text+k",
624
        "kxtext": "text+kx",
625
        "ltext": "text+F",
626
        "tempobj": "binary+FSw",
627
        "ubinary": "binary+F",
628
        "uresource": "resource+F",
629
        "uxbinary": "binary+Fx",
630
        "xbinary": "binary+x",
631
        "xltext": "text+Fx",
632
        "xtempobj": "binary+Swx",
633
        "xtext": "text+x",
634
        "xunicode": "unicode+x",
635
        "xutf16": "utf16+x",
636
    }
637
    if p4type in p4_filetypes_historical:
638
        p4type = p4_filetypes_historical[p4type]
639
    mods = ""
640
    s = p4type.split("+")
641
    base = s[0]
642
    mods = ""
643
    if len(s) > 1:
644
        mods = s[1]
645
    return (base, mods)
646

647

648
def p4_type(f):
649
    """Return the raw p4 type of a file (text, text+ko, etc)."""
650

651
    results = p4CmdList(["fstat", "-T", "headType", wildcard_encode(f)])
652
    return results[0]['headType']
653

654

655
def p4_keywords_regexp_for_type(base, type_mods):
656
    """Given a type base and modifier, return a regexp matching the keywords
657
       that can be expanded in the file.
658
       """
659

660
    if base in ("text", "unicode", "binary"):
661
        if "ko" in type_mods:
662
            return re_ko_keywords
663
        elif "k" in type_mods:
664
            return re_k_keywords
665
        else:
666
            return None
667
    else:
668
        return None
669

670

671
def p4_keywords_regexp_for_file(file):
672
    """Given a file, return a regexp matching the possible RCS keywords that
673
       will be expanded, or None for files with kw expansion turned off.
674
       """
675

676
    if not os.path.exists(file):
677
        return None
678
    else:
679
        type_base, type_mods = split_p4_type(p4_type(file))
680
        return p4_keywords_regexp_for_type(type_base, type_mods)
681

682

683
def setP4ExecBit(file, mode):
684
    """Reopens an already open file and changes the execute bit to match the
685
       execute bit setting in the passed in mode.
686
       """
687

688
    p4Type = "+x"
689

690
    if not isModeExec(mode):
691
        p4Type = getP4OpenedType(file)
692
        p4Type = re.sub(r'^([cku]?)x(.*)', r'\1\2', p4Type)
693
        p4Type = re.sub(r'(.*?\+.*?)x(.*?)', r'\1\2', p4Type)
694
        if p4Type[-1] == "+":
695
            p4Type = p4Type[0:-1]
696

697
    p4_reopen(p4Type, file)
698

699

700
def getP4OpenedType(file):
701
    """Returns the perforce file type for the given file."""
702

703
    result = p4_read_pipe(["opened", wildcard_encode(file)])
704
    match = re.match(r".*\((.+)\)( \*exclusive\*)?\r?$", result)
705
    if match:
706
        return match.group(1)
707
    else:
708
        die("Could not determine file type for %s (result: '%s')" % (file, result))
709

710

711
def getP4Labels(depotPaths):
712
    """Return the set of all p4 labels."""
713

714
    labels = set()
715
    if not isinstance(depotPaths, list):
716
        depotPaths = [depotPaths]
717

718
    for l in p4CmdList(["labels"] + ["%s..." % p for p in depotPaths]):
719
        label = l['label']
720
        labels.add(label)
721

722
    return labels
723

724

725
def getGitTags():
726
    """Return the set of all git tags."""
727

728
    gitTags = set()
729
    for line in read_pipe_lines(["git", "tag"]):
730
        tag = line.strip()
731
        gitTags.add(tag)
732
    return gitTags
733

734

735
_diff_tree_pattern = None
736

737

738
def parseDiffTreeEntry(entry):
739
    """Parses a single diff tree entry into its component elements.
740

741
       See git-diff-tree(1) manpage for details about the format of the diff
742
       output. This method returns a dictionary with the following elements:
743

744
       src_mode - The mode of the source file
745
       dst_mode - The mode of the destination file
746
       src_sha1 - The sha1 for the source file
747
       dst_sha1 - The sha1 fr the destination file
748
       status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
749
       status_score - The score for the status (applicable for 'C' and 'R'
750
                      statuses). This is None if there is no score.
751
       src - The path for the source file.
752
       dst - The path for the destination file. This is only present for
753
             copy or renames. If it is not present, this is None.
754

755
       If the pattern is not matched, None is returned.
756
       """
757

758
    global _diff_tree_pattern
759
    if not _diff_tree_pattern:
760
        _diff_tree_pattern = re.compile(r':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
761

762
    match = _diff_tree_pattern.match(entry)
763
    if match:
764
        return {
765
            'src_mode': match.group(1),
766
            'dst_mode': match.group(2),
767
            'src_sha1': match.group(3),
768
            'dst_sha1': match.group(4),
769
            'status': match.group(5),
770
            'status_score': match.group(6),
771
            'src': match.group(7),
772
            'dst': match.group(10)
773
        }
774
    return None
775

776

777
def isModeExec(mode):
778
    """Returns True if the given git mode represents an executable file,
779
       otherwise False.
780
       """
781
    return mode[-3:] == "755"
782

783

784
class P4Exception(Exception):
785
    """Base class for exceptions from the p4 client."""
786

787
    def __init__(self, exit_code):
788
        self.p4ExitCode = exit_code
789

790

791
class P4ServerException(P4Exception):
792
    """Base class for exceptions where we get some kind of marshalled up result
793
       from the server.
794
       """
795

796
    def __init__(self, exit_code, p4_result):
797
        super(P4ServerException, self).__init__(exit_code)
798
        self.p4_result = p4_result
799
        self.code = p4_result[0]['code']
800
        self.data = p4_result[0]['data']
801

802

803
class P4RequestSizeException(P4ServerException):
804
    """One of the maxresults or maxscanrows errors."""
805

806
    def __init__(self, exit_code, p4_result, limit):
807
        super(P4RequestSizeException, self).__init__(exit_code, p4_result)
808
        self.limit = limit
809

810

811
class P4CommandException(P4Exception):
812
    """Something went wrong calling p4 which means we have to give up."""
813

814
    def __init__(self, msg):
815
        self.msg = msg
816

817
    def __str__(self):
818
        return self.msg
819

820

821
def isModeExecChanged(src_mode, dst_mode):
822
    return isModeExec(src_mode) != isModeExec(dst_mode)
823

824

825
def p4KeysContainingNonUtf8Chars():
826
    """Returns all keys which may contain non UTF-8 encoded strings
827
       for which a fallback strategy has to be applied.
828
       """
829
    return ['desc', 'client', 'FullName']
830

831

832
def p4KeysContainingBinaryData():
833
    """Returns all keys which may contain arbitrary binary data
834
       """
835
    return ['data']
836

837

838
def p4KeyContainsFilePaths(key):
839
    """Returns True if the key contains file paths. These are handled by decode_path().
840
       Otherwise False.
841
       """
842
    return key.startswith('depotFile') or key in ['path', 'clientFile']
843

844

845
def p4KeyWhichCanBeDirectlyDecoded(key):
846
    """Returns True if the key can be directly decoded as UTF-8 string
847
       Otherwise False.
848

849
       Keys which can not be encoded directly:
850
         - `data` which may contain arbitrary binary data
851
         - `desc` or `client` or `FullName` which may contain non-UTF8 encoded text
852
         - `depotFile[0-9]*`, `path`, or `clientFile` which may contain non-UTF8 encoded text, handled by decode_path()
853
       """
854
    if key in p4KeysContainingNonUtf8Chars() or \
855
       key in p4KeysContainingBinaryData() or  \
856
       p4KeyContainsFilePaths(key):
857
        return False
858
    return True
859

860

861
def p4CmdList(cmd, stdin=None, stdin_mode='w+b', cb=None, skip_info=False,
862
        errors_as_exceptions=False, *k, **kw):
863

864
    cmd = p4_build_cmd(["-G"] + cmd)
865
    if verbose:
866
        sys.stderr.write("Opening pipe: {}\n".format(' '.join(cmd)))
867

868
    # Use a temporary file to avoid deadlocks without
869
    # subprocess.communicate(), which would put another copy
870
    # of stdout into memory.
871
    stdin_file = None
872
    if stdin is not None:
873
        stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode)
874
        if not isinstance(stdin, list):
875
            stdin_file.write(stdin)
876
        else:
877
            for i in stdin:
878
                stdin_file.write(encode_text_stream(i))
879
                stdin_file.write(b'\n')
880
        stdin_file.flush()
881
        stdin_file.seek(0)
882

883
    p4 = subprocess.Popen(
884
        cmd, stdin=stdin_file, stdout=subprocess.PIPE, *k, **kw)
885

886
    result = []
887
    try:
888
        while True:
889
            entry = marshal.load(p4.stdout)
890

891
            if bytes is not str:
892
                # Decode unmarshalled dict to use str keys and values. Special cases are handled below.
893
                decoded_entry = {}
894
                for key, value in entry.items():
895
                    key = key.decode()
896
                    if isinstance(value, bytes) and p4KeyWhichCanBeDirectlyDecoded(key):
897
                        value = value.decode()
898
                    decoded_entry[key] = value
899
                # Parse out data if it's an error response
900
                if decoded_entry.get('code') == 'error' and 'data' in decoded_entry:
901
                    decoded_entry['data'] = decoded_entry['data'].decode()
902
                entry = decoded_entry
903
            if skip_info:
904
                if 'code' in entry and entry['code'] == 'info':
905
                    continue
906
            for key in p4KeysContainingNonUtf8Chars():
907
                if key in entry:
908
                    entry[key] = metadata_stream_to_writable_bytes(entry[key])
909
            if cb is not None:
910
                cb(entry)
911
            else:
912
                result.append(entry)
913
    except EOFError:
914
        pass
915
    exitCode = p4.wait()
916
    if exitCode != 0:
917
        if errors_as_exceptions:
918
            if len(result) > 0:
919
                data = result[0].get('data')
920
                if data:
921
                    m = re.search(r'Too many rows scanned \(over (\d+)\)', data)
922
                    if not m:
923
                        m = re.search(r'Request too large \(over (\d+)\)', data)
924

925
                    if m:
926
                        limit = int(m.group(1))
927
                        raise P4RequestSizeException(exitCode, result, limit)
928

929
                raise P4ServerException(exitCode, result)
930
            else:
931
                raise P4Exception(exitCode)
932
        else:
933
            entry = {}
934
            entry["p4ExitCode"] = exitCode
935
            result.append(entry)
936

937
    return result
938

939

940
def p4Cmd(cmd, *k, **kw):
941
    list = p4CmdList(cmd, *k, **kw)
942
    result = {}
943
    for entry in list:
944
        result.update(entry)
945
    return result
946

947

948
def p4Where(depotPath):
949
    if not depotPath.endswith("/"):
950
        depotPath += "/"
951
    depotPathLong = depotPath + "..."
952
    outputList = p4CmdList(["where", depotPathLong])
953
    output = None
954
    for entry in outputList:
955
        if "depotFile" in entry:
956
            # Search for the base client side depot path, as long as it starts with the branch's P4 path.
957
            # The base path always ends with "/...".
958
            entry_path = decode_path(entry['depotFile'])
959
            if entry_path.find(depotPath) == 0 and entry_path[-4:] == "/...":
960
                output = entry
961
                break
962
        elif "data" in entry:
963
            data = entry.get("data")
964
            space = data.find(" ")
965
            if data[:space] == depotPath:
966
                output = entry
967
                break
968
    if output is None:
969
        return ""
970
    if output["code"] == "error":
971
        return ""
972
    clientPath = ""
973
    if "path" in output:
974
        clientPath = decode_path(output['path'])
975
    elif "data" in output:
976
        data = output.get("data")
977
        lastSpace = data.rfind(b" ")
978
        clientPath = decode_path(data[lastSpace + 1:])
979

980
    if clientPath.endswith("..."):
981
        clientPath = clientPath[:-3]
982
    return clientPath
983

984

985
def currentGitBranch():
986
    return read_pipe_text(["git", "symbolic-ref", "--short", "-q", "HEAD"])
987

988

989
def isValidGitDir(path):
990
    return git_dir(path) is not None
991

992

993
def parseRevision(ref):
994
    return read_pipe(["git", "rev-parse", ref]).strip()
995

996

997
def branchExists(ref):
998
    rev = read_pipe(["git", "rev-parse", "-q", "--verify", ref],
999
                     ignore_error=True)
1000
    return len(rev) > 0
1001

1002

1003
def extractLogMessageFromGitCommit(commit):
1004
    logMessage = ""
1005

1006
    # fixme: title is first line of commit, not 1st paragraph.
1007
    foundTitle = False
1008
    for log in read_pipe_lines(["git", "cat-file", "commit", commit]):
1009
        if not foundTitle:
1010
            if len(log) == 1:
1011
                foundTitle = True
1012
            continue
1013

1014
        logMessage += log
1015
    return logMessage
1016

1017

1018
def extractSettingsGitLog(log):
1019
    values = {}
1020
    for line in log.split("\n"):
1021
        line = line.strip()
1022
        m = re.search(r"^ *\[git-p4: (.*)\]$", line)
1023
        if not m:
1024
            continue
1025

1026
        assignments = m.group(1).split(':')
1027
        for a in assignments:
1028
            vals = a.split('=')
1029
            key = vals[0].strip()
1030
            val = ('='.join(vals[1:])).strip()
1031
            if val.endswith('\"') and val.startswith('"'):
1032
                val = val[1:-1]
1033

1034
            values[key] = val
1035

1036
    paths = values.get("depot-paths")
1037
    if not paths:
1038
        paths = values.get("depot-path")
1039
    if paths:
1040
        values['depot-paths'] = paths.split(',')
1041
    return values
1042

1043

1044
def gitBranchExists(branch):
1045
    proc = subprocess.Popen(["git", "rev-parse", branch],
1046
                            stderr=subprocess.PIPE, stdout=subprocess.PIPE)
1047
    return proc.wait() == 0
1048

1049

1050
def gitUpdateRef(ref, newvalue):
1051
    subprocess.check_call(["git", "update-ref", ref, newvalue])
1052

1053

1054
def gitDeleteRef(ref):
1055
    subprocess.check_call(["git", "update-ref", "-d", ref])
1056

1057

1058
_gitConfig = {}
1059

1060

1061
def gitConfig(key, typeSpecifier=None):
1062
    if key not in _gitConfig:
1063
        cmd = ["git", "config"]
1064
        if typeSpecifier:
1065
            cmd += [typeSpecifier]
1066
        cmd += [key]
1067
        s = read_pipe(cmd, ignore_error=True)
1068
        _gitConfig[key] = s.strip()
1069
    return _gitConfig[key]
1070

1071

1072
def gitConfigBool(key):
1073
    """Return a bool, using git config --bool.  It is True only if the
1074
       variable is set to true, and False if set to false or not present
1075
       in the config.
1076
       """
1077

1078
    if key not in _gitConfig:
1079
        _gitConfig[key] = gitConfig(key, '--bool') == "true"
1080
    return _gitConfig[key]
1081

1082

1083
def gitConfigInt(key):
1084
    if key not in _gitConfig:
1085
        cmd = ["git", "config", "--int", key]
1086
        s = read_pipe(cmd, ignore_error=True)
1087
        v = s.strip()
1088
        try:
1089
            _gitConfig[key] = int(gitConfig(key, '--int'))
1090
        except ValueError:
1091
            _gitConfig[key] = None
1092
    return _gitConfig[key]
1093

1094

1095
def gitConfigList(key):
1096
    if key not in _gitConfig:
1097
        s = read_pipe(["git", "config", "--get-all", key], ignore_error=True)
1098
        _gitConfig[key] = s.strip().splitlines()
1099
        if _gitConfig[key] == ['']:
1100
            _gitConfig[key] = []
1101
    return _gitConfig[key]
1102

1103
def fullP4Ref(incomingRef, importIntoRemotes=True):
1104
    """Standardize a given provided p4 ref value to a full git ref:
1105
         refs/foo/bar/branch -> use it exactly
1106
         p4/branch -> prepend refs/remotes/ or refs/heads/
1107
         branch -> prepend refs/remotes/p4/ or refs/heads/p4/"""
1108
    if incomingRef.startswith("refs/"):
1109
        return incomingRef
1110
    if importIntoRemotes:
1111
        prepend = "refs/remotes/"
1112
    else:
1113
        prepend = "refs/heads/"
1114
    if not incomingRef.startswith("p4/"):
1115
        prepend += "p4/"
1116
    return prepend + incomingRef
1117

1118
def shortP4Ref(incomingRef, importIntoRemotes=True):
1119
    """Standardize to a "short ref" if possible:
1120
         refs/foo/bar/branch -> ignore
1121
         refs/remotes/p4/branch or refs/heads/p4/branch -> shorten
1122
         p4/branch -> shorten"""
1123
    if importIntoRemotes:
1124
        longprefix = "refs/remotes/p4/"
1125
    else:
1126
        longprefix = "refs/heads/p4/"
1127
    if incomingRef.startswith(longprefix):
1128
        return incomingRef[len(longprefix):]
1129
    if incomingRef.startswith("p4/"):
1130
        return incomingRef[3:]
1131
    return incomingRef
1132

1133
def p4BranchesInGit(branchesAreInRemotes=True):
1134
    """Find all the branches whose names start with "p4/", looking
1135
       in remotes or heads as specified by the argument.  Return
1136
       a dictionary of { branch: revision } for each one found.
1137
       The branch names are the short names, without any
1138
       "p4/" prefix.
1139
       """
1140

1141
    branches = {}
1142

1143
    cmdline = ["git", "rev-parse", "--symbolic"]
1144
    if branchesAreInRemotes:
1145
        cmdline.append("--remotes")
1146
    else:
1147
        cmdline.append("--branches")
1148

1149
    for line in read_pipe_lines(cmdline):
1150
        line = line.strip()
1151

1152
        # only import to p4/
1153
        if not line.startswith('p4/'):
1154
            continue
1155
        # special symbolic ref to p4/master
1156
        if line == "p4/HEAD":
1157
            continue
1158

1159
        # strip off p4/ prefix
1160
        branch = line[len("p4/"):]
1161

1162
        branches[branch] = parseRevision(line)
1163

1164
    return branches
1165

1166

1167
def branch_exists(branch):
1168
    """Make sure that the given ref name really exists."""
1169

1170
    cmd = ["git", "rev-parse", "--symbolic", "--verify", branch]
1171
    p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
1172
    out, _ = p.communicate()
1173
    out = decode_text_stream(out)
1174
    if p.returncode:
1175
        return False
1176
    # expect exactly one line of output: the branch name
1177
    return out.rstrip() == branch
1178

1179

1180
def findUpstreamBranchPoint(head="HEAD"):
1181
    branches = p4BranchesInGit()
1182
    # map from depot-path to branch name
1183
    branchByDepotPath = {}
1184
    for branch in branches.keys():
1185
        tip = branches[branch]
1186
        log = extractLogMessageFromGitCommit(tip)
1187
        settings = extractSettingsGitLog(log)
1188
        if "depot-paths" in settings:
1189
            git_branch = "remotes/p4/" + branch
1190
            paths = ",".join(settings["depot-paths"])
1191
            branchByDepotPath[paths] = git_branch
1192
            if "change" in settings:
1193
                paths = paths + ";" + settings["change"]
1194
                branchByDepotPath[paths] = git_branch
1195

1196
    settings = None
1197
    parent = 0
1198
    while parent < 65535:
1199
        commit = head + "~%s" % parent
1200
        log = extractLogMessageFromGitCommit(commit)
1201
        settings = extractSettingsGitLog(log)
1202
        if "depot-paths" in settings:
1203
            paths = ",".join(settings["depot-paths"])
1204
            if "change" in settings:
1205
                expaths = paths + ";" + settings["change"]
1206
                if expaths in branchByDepotPath:
1207
                    return [branchByDepotPath[expaths], settings]
1208
            if paths in branchByDepotPath:
1209
                return [branchByDepotPath[paths], settings]
1210

1211
        parent = parent + 1
1212

1213
    return ["", settings]
1214

1215

1216
def createOrUpdateBranchesFromOrigin(localRefPrefix="refs/remotes/p4/", silent=True):
1217
    if not silent:
1218
        print("Creating/updating branch(es) in %s based on origin branch(es)"
1219
               % localRefPrefix)
1220

1221
    originPrefix = "origin/p4/"
1222

1223
    for line in read_pipe_lines(["git", "rev-parse", "--symbolic", "--remotes"]):
1224
        line = line.strip()
1225
        if (not line.startswith(originPrefix)) or line.endswith("HEAD"):
1226
            continue
1227

1228
        headName = line[len(originPrefix):]
1229
        remoteHead = localRefPrefix + headName
1230
        originHead = line
1231

1232
        original = extractSettingsGitLog(extractLogMessageFromGitCommit(originHead))
1233
        if 'depot-paths' not in original or 'change' not in original:
1234
            continue
1235

1236
        update = False
1237
        if not gitBranchExists(remoteHead):
1238
            if verbose:
1239
                print("creating %s" % remoteHead)
1240
            update = True
1241
        else:
1242
            settings = extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead))
1243
            if 'change' in settings:
1244
                if settings['depot-paths'] == original['depot-paths']:
1245
                    originP4Change = int(original['change'])
1246
                    p4Change = int(settings['change'])
1247
                    if originP4Change > p4Change:
1248
                        print("%s (%s) is newer than %s (%s). "
1249
                               "Updating p4 branch from origin."
1250
                               % (originHead, originP4Change,
1251
                                  remoteHead, p4Change))
1252
                        update = True
1253
                else:
1254
                    print("Ignoring: %s was imported from %s while "
1255
                           "%s was imported from %s"
1256
                           % (originHead, ','.join(original['depot-paths']),
1257
                              remoteHead, ','.join(settings['depot-paths'])))
1258

1259
        if update:
1260
            system(["git", "update-ref", remoteHead, originHead])
1261

1262

1263
def originP4BranchesExist():
1264
    return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
1265

1266

1267
def p4ParseNumericChangeRange(parts):
1268
    changeStart = int(parts[0][1:])
1269
    if parts[1] == '#head':
1270
        changeEnd = p4_last_change()
1271
    else:
1272
        changeEnd = int(parts[1])
1273

1274
    return (changeStart, changeEnd)
1275

1276

1277
def chooseBlockSize(blockSize):
1278
    if blockSize:
1279
        return blockSize
1280
    else:
1281
        return defaultBlockSize
1282

1283

1284
def p4ChangesForPaths(depotPaths, changeRange, requestedBlockSize):
1285
    assert depotPaths
1286

1287
    # Parse the change range into start and end. Try to find integer
1288
    # revision ranges as these can be broken up into blocks to avoid
1289
    # hitting server-side limits (maxrows, maxscanresults). But if
1290
    # that doesn't work, fall back to using the raw revision specifier
1291
    # strings, without using block mode.
1292

1293
    if changeRange is None or changeRange == '':
1294
        changeStart = 1
1295
        changeEnd = p4_last_change()
1296
        block_size = chooseBlockSize(requestedBlockSize)
1297
    else:
1298
        parts = changeRange.split(',')
1299
        assert len(parts) == 2
1300
        try:
1301
            changeStart, changeEnd = p4ParseNumericChangeRange(parts)
1302
            block_size = chooseBlockSize(requestedBlockSize)
1303
        except ValueError:
1304
            changeStart = parts[0][1:]
1305
            changeEnd = parts[1]
1306
            if requestedBlockSize:
1307
                die("cannot use --changes-block-size with non-numeric revisions")
1308
            block_size = None
1309

1310
    changes = set()
1311

1312
    # Retrieve changes a block at a time, to prevent running
1313
    # into a MaxResults/MaxScanRows error from the server. If
1314
    # we _do_ hit one of those errors, turn down the block size
1315

1316
    while True:
1317
        cmd = ['changes']
1318

1319
        if block_size:
1320
            end = min(changeEnd, changeStart + block_size)
1321
            revisionRange = "%d,%d" % (changeStart, end)
1322
        else:
1323
            revisionRange = "%s,%s" % (changeStart, changeEnd)
1324

1325
        for p in depotPaths:
1326
            cmd += ["%s...@%s" % (p, revisionRange)]
1327

1328
        # fetch the changes
1329
        try:
1330
            result = p4CmdList(cmd, errors_as_exceptions=True)
1331
        except P4RequestSizeException as e:
1332
            if not block_size:
1333
                block_size = e.limit
1334
            elif block_size > e.limit:
1335
                block_size = e.limit
1336
            else:
1337
                block_size = max(2, block_size // 2)
1338

1339
            if verbose:
1340
                print("block size error, retrying with block size {0}".format(block_size))
1341
            continue
1342
        except P4Exception as e:
1343
            die('Error retrieving changes description ({0})'.format(e.p4ExitCode))
1344

1345
        # Insert changes in chronological order
1346
        for entry in reversed(result):
1347
            if 'change' not in entry:
1348
                continue
1349
            changes.add(int(entry['change']))
1350

1351
        if not block_size:
1352
            break
1353

1354
        if end >= changeEnd:
1355
            break
1356

1357
        changeStart = end + 1
1358

1359
    changes = sorted(changes)
1360
    return changes
1361

1362

1363
def p4PathStartsWith(path, prefix):
1364
    """This method tries to remedy a potential mixed-case issue:
1365

1366
       If UserA adds  //depot/DirA/file1
1367
       and UserB adds //depot/dira/file2
1368

1369
       we may or may not have a problem. If you have core.ignorecase=true,
1370
       we treat DirA and dira as the same directory.
1371
       """
1372
    if gitConfigBool("core.ignorecase"):
1373
        return path.lower().startswith(prefix.lower())
1374
    return path.startswith(prefix)
1375

1376

1377
def getClientSpec():
1378
    """Look at the p4 client spec, create a View() object that contains
1379
       all the mappings, and return it.
1380
       """
1381

1382
    specList = p4CmdList(["client", "-o"])
1383
    if len(specList) != 1:
1384
        die('Output from "client -o" is %d lines, expecting 1' %
1385
            len(specList))
1386

1387
    # dictionary of all client parameters
1388
    entry = specList[0]
1389

1390
    # the //client/ name
1391
    client_name = entry["Client"]
1392

1393
    # just the keys that start with "View"
1394
    view_keys = [k for k in entry.keys() if k.startswith("View")]
1395

1396
    # hold this new View
1397
    view = View(client_name)
1398

1399
    # append the lines, in order, to the view
1400
    for view_num in range(len(view_keys)):
1401
        k = "View%d" % view_num
1402
        if k not in view_keys:
1403
            die("Expected view key %s missing" % k)
1404
        view.append(entry[k])
1405

1406
    return view
1407

1408

1409
def getClientRoot():
1410
    """Grab the client directory."""
1411

1412
    output = p4CmdList(["client", "-o"])
1413
    if len(output) != 1:
1414
        die('Output from "client -o" is %d lines, expecting 1' % len(output))
1415

1416
    entry = output[0]
1417
    if "Root" not in entry:
1418
        die('Client has no "Root"')
1419

1420
    return entry["Root"]
1421

1422

1423
def wildcard_decode(path):
1424
    """Decode P4 wildcards into %xx encoding
1425

1426
       P4 wildcards are not allowed in filenames.  P4 complains if you simply
1427
       add them, but you can force it with "-f", in which case it translates
1428
       them into %xx encoding internally.
1429
       """
1430

1431
    # Search for and fix just these four characters.  Do % last so
1432
    # that fixing it does not inadvertently create new %-escapes.
1433
    # Cannot have * in a filename in windows; untested as to
1434
    # what p4 would do in such a case.
1435
    if not platform.system() == "Windows":
1436
        path = path.replace("%2A", "*")
1437
    path = path.replace("%23", "#") \
1438
               .replace("%40", "@") \
1439
               .replace("%25", "%")
1440
    return path
1441

1442

1443
def wildcard_encode(path):
1444
    """Encode %xx coded wildcards into P4 coding."""
1445

1446
    # do % first to avoid double-encoding the %s introduced here
1447
    path = path.replace("%", "%25") \
1448
               .replace("*", "%2A") \
1449
               .replace("#", "%23") \
1450
               .replace("@", "%40")
1451
    return path
1452

1453

1454
def wildcard_present(path):
1455
    m = re.search(r"[*#@%]", path)
1456
    return m is not None
1457

1458

1459
class LargeFileSystem(object):
1460
    """Base class for large file system support."""
1461

1462
    def __init__(self, writeToGitStream):
1463
        self.largeFiles = set()
1464
        self.writeToGitStream = writeToGitStream
1465

1466
    def generatePointer(self, cloneDestination, contentFile):
1467
        """Return the content of a pointer file that is stored in Git instead
1468
           of the actual content.
1469
           """
1470
        assert False, "Method 'generatePointer' required in " + self.__class__.__name__
1471

1472
    def pushFile(self, localLargeFile):
1473
        """Push the actual content which is not stored in the Git repository to
1474
           a server.
1475
           """
1476
        assert False, "Method 'pushFile' required in " + self.__class__.__name__
1477

1478
    def hasLargeFileExtension(self, relPath):
1479
        return functools.reduce(
1480
            lambda a, b: a or b,
1481
            [relPath.endswith('.' + e) for e in gitConfigList('git-p4.largeFileExtensions')],
1482
            False
1483
        )
1484

1485
    def generateTempFile(self, contents):
1486
        contentFile = tempfile.NamedTemporaryFile(prefix='git-p4-large-file', delete=False)
1487
        for d in contents:
1488
            contentFile.write(d)
1489
        contentFile.close()
1490
        return contentFile.name
1491

1492
    def exceedsLargeFileThreshold(self, relPath, contents):
1493
        if gitConfigInt('git-p4.largeFileThreshold'):
1494
            contentsSize = sum(len(d) for d in contents)
1495
            if contentsSize > gitConfigInt('git-p4.largeFileThreshold'):
1496
                return True
1497
        if gitConfigInt('git-p4.largeFileCompressedThreshold'):
1498
            contentsSize = sum(len(d) for d in contents)
1499
            if contentsSize <= gitConfigInt('git-p4.largeFileCompressedThreshold'):
1500
                return False
1501
            contentTempFile = self.generateTempFile(contents)
1502
            compressedContentFile = tempfile.NamedTemporaryFile(prefix='git-p4-large-file', delete=True)
1503
            with zipfile.ZipFile(compressedContentFile, mode='w') as zf:
1504
                zf.write(contentTempFile, compress_type=zipfile.ZIP_DEFLATED)
1505
                compressedContentsSize = zf.infolist()[0].compress_size
1506
            os.remove(contentTempFile)
1507
            if compressedContentsSize > gitConfigInt('git-p4.largeFileCompressedThreshold'):
1508
                return True
1509
        return False
1510

1511
    def addLargeFile(self, relPath):
1512
        self.largeFiles.add(relPath)
1513

1514
    def removeLargeFile(self, relPath):
1515
        self.largeFiles.remove(relPath)
1516

1517
    def isLargeFile(self, relPath):
1518
        return relPath in self.largeFiles
1519

1520
    def processContent(self, git_mode, relPath, contents):
1521
        """Processes the content of git fast import. This method decides if a
1522
           file is stored in the large file system and handles all necessary
1523
           steps.
1524
           """
1525
        # symlinks aren't processed by smudge/clean filters
1526
        if git_mode == "120000":
1527
            return (git_mode, contents)
1528

1529
        if self.exceedsLargeFileThreshold(relPath, contents) or self.hasLargeFileExtension(relPath):
1530
            contentTempFile = self.generateTempFile(contents)
1531
            pointer_git_mode, contents, localLargeFile = self.generatePointer(contentTempFile)
1532
            if pointer_git_mode:
1533
                git_mode = pointer_git_mode
1534
            if localLargeFile:
1535
                # Move temp file to final location in large file system
1536
                largeFileDir = os.path.dirname(localLargeFile)
1537
                if not os.path.isdir(largeFileDir):
1538
                    os.makedirs(largeFileDir)
1539
                shutil.move(contentTempFile, localLargeFile)
1540
                self.addLargeFile(relPath)
1541
                if gitConfigBool('git-p4.largeFilePush'):
1542
                    self.pushFile(localLargeFile)
1543
                if verbose:
1544
                    sys.stderr.write("%s moved to large file system (%s)\n" % (relPath, localLargeFile))
1545
        return (git_mode, contents)
1546

1547

1548
class MockLFS(LargeFileSystem):
1549
    """Mock large file system for testing."""
1550

1551
    def generatePointer(self, contentFile):
1552
        """The pointer content is the original content prefixed with "pointer-".
1553
           The local filename of the large file storage is derived from the
1554
           file content.
1555
           """
1556
        with open(contentFile, 'r') as f:
1557
            content = next(f)
1558
            gitMode = '100644'
1559
            pointerContents = 'pointer-' + content
1560
            localLargeFile = os.path.join(os.getcwd(), '.git', 'mock-storage', 'local', content[:-1])
1561
            return (gitMode, pointerContents, localLargeFile)
1562

1563
    def pushFile(self, localLargeFile):
1564
        """The remote filename of the large file storage is the same as the
1565
           local one but in a different directory.
1566
           """
1567
        remotePath = os.path.join(os.path.dirname(localLargeFile), '..', 'remote')
1568
        if not os.path.exists(remotePath):
1569
            os.makedirs(remotePath)
1570
        shutil.copyfile(localLargeFile, os.path.join(remotePath, os.path.basename(localLargeFile)))
1571

1572

1573
class GitLFS(LargeFileSystem):
1574
    """Git LFS as backend for the git-p4 large file system.
1575
       See https://git-lfs.github.com/ for details.
1576
       """
1577

1578
    def __init__(self, *args):
1579
        LargeFileSystem.__init__(self, *args)
1580
        self.baseGitAttributes = []
1581

1582
    def generatePointer(self, contentFile):
1583
        """Generate a Git LFS pointer for the content. Return LFS Pointer file
1584
           mode and content which is stored in the Git repository instead of
1585
           the actual content. Return also the new location of the actual
1586
           content.
1587
           """
1588
        if os.path.getsize(contentFile) == 0:
1589
            return (None, '', None)
1590

1591
        pointerProcess = subprocess.Popen(
1592
            ['git', 'lfs', 'pointer', '--file=' + contentFile],
1593
            stdout=subprocess.PIPE
1594
        )
1595
        pointerFile = decode_text_stream(pointerProcess.stdout.read())
1596
        if pointerProcess.wait():
1597
            os.remove(contentFile)
1598
            die('git-lfs pointer command failed. Did you install the extension?')
1599

1600
        # Git LFS removed the preamble in the output of the 'pointer' command
1601
        # starting from version 1.2.0. Check for the preamble here to support
1602
        # earlier versions.
1603
        # c.f. https://github.com/github/git-lfs/commit/da2935d9a739592bc775c98d8ef4df9c72ea3b43
1604
        if pointerFile.startswith('Git LFS pointer for'):
1605
            pointerFile = re.sub(r'Git LFS pointer for.*\n\n', '', pointerFile)
1606

1607
        oid = re.search(r'^oid \w+:(\w+)', pointerFile, re.MULTILINE).group(1)
1608
        # if someone use external lfs.storage ( not in local repo git )
1609
        lfs_path = gitConfig('lfs.storage')
1610
        if not lfs_path:
1611
            lfs_path = 'lfs'
1612
        if not os.path.isabs(lfs_path):
1613
            lfs_path = os.path.join(os.getcwd(), '.git', lfs_path)
1614
        localLargeFile = os.path.join(
1615
            lfs_path,
1616
            'objects', oid[:2], oid[2:4],
1617
            oid,
1618
        )
1619
        # LFS Spec states that pointer files should not have the executable bit set.
1620
        gitMode = '100644'
1621
        return (gitMode, pointerFile, localLargeFile)
1622

1623
    def pushFile(self, localLargeFile):
1624
        uploadProcess = subprocess.Popen(
1625
            ['git', 'lfs', 'push', '--object-id', 'origin', os.path.basename(localLargeFile)]
1626
        )
1627
        if uploadProcess.wait():
1628
            die('git-lfs push command failed. Did you define a remote?')
1629

1630
    def generateGitAttributes(self):
1631
        return (
1632
            self.baseGitAttributes +
1633
            [
1634
                '\n',
1635
                '#\n',
1636
                '# Git LFS (see https://git-lfs.github.com/)\n',
1637
                '#\n',
1638
            ] +
1639
            ['*.' + f.replace(' ', '[[:space:]]') + ' filter=lfs diff=lfs merge=lfs -text\n'
1640
                for f in sorted(gitConfigList('git-p4.largeFileExtensions'))
1641
            ] +
1642
            ['/' + f.replace(' ', '[[:space:]]') + ' filter=lfs diff=lfs merge=lfs -text\n'
1643
                for f in sorted(self.largeFiles) if not self.hasLargeFileExtension(f)
1644
            ]
1645
        )
1646

1647
    def addLargeFile(self, relPath):
1648
        LargeFileSystem.addLargeFile(self, relPath)
1649
        self.writeToGitStream('100644', '.gitattributes', self.generateGitAttributes())
1650

1651
    def removeLargeFile(self, relPath):
1652
        LargeFileSystem.removeLargeFile(self, relPath)
1653
        self.writeToGitStream('100644', '.gitattributes', self.generateGitAttributes())
1654

1655
    def processContent(self, git_mode, relPath, contents):
1656
        if relPath == '.gitattributes':
1657
            self.baseGitAttributes = contents
1658
            return (git_mode, self.generateGitAttributes())
1659
        else:
1660
            return LargeFileSystem.processContent(self, git_mode, relPath, contents)
1661

1662

1663
class Command:
1664
    delete_actions = ("delete", "move/delete", "purge")
1665
    add_actions = ("add", "branch", "move/add")
1666

1667
    def __init__(self):
1668
        self.usage = "usage: %prog [options]"
1669
        self.needsGit = True
1670
        self.verbose = False
1671

1672
    # This is required for the "append" update_shelve action
1673
    def ensure_value(self, attr, value):
1674
        if not hasattr(self, attr) or getattr(self, attr) is None:
1675
            setattr(self, attr, value)
1676
        return getattr(self, attr)
1677

1678

1679
class P4UserMap:
1680
    def __init__(self):
1681
        self.userMapFromPerforceServer = False
1682
        self.myP4UserId = None
1683

1684
    def p4UserId(self):
1685
        if self.myP4UserId:
1686
            return self.myP4UserId
1687

1688
        results = p4CmdList(["user", "-o"])
1689
        for r in results:
1690
            if 'User' in r:
1691
                self.myP4UserId = r['User']
1692
                return r['User']
1693
        die("Could not find your p4 user id")
1694

1695
    def p4UserIsMe(self, p4User):
1696
        """Return True if the given p4 user is actually me."""
1697
        me = self.p4UserId()
1698
        if not p4User or p4User != me:
1699
            return False
1700
        else:
1701
            return True
1702

1703
    def getUserCacheFilename(self):
1704
        home = os.environ.get("HOME", os.environ.get("USERPROFILE"))
1705
        return home + "/.gitp4-usercache.txt"
1706

1707
    def getUserMapFromPerforceServer(self):
1708
        if self.userMapFromPerforceServer:
1709
            return
1710
        self.users = {}
1711
        self.emails = {}
1712

1713
        for output in p4CmdList(["users"]):
1714
            if "User" not in output:
1715
                continue
1716
            # "FullName" is bytes. "Email" on the other hand might be bytes
1717
            # or unicode string depending on whether we are running under
1718
            # python2 or python3. To support
1719
            # git-p4.metadataDecodingStrategy=fallback, self.users dict values
1720
            # are always bytes, ready to be written to git.
1721
            emailbytes = metadata_stream_to_writable_bytes(output["Email"])
1722
            self.users[output["User"]] = output["FullName"] + b" <" + emailbytes + b">"
1723
            self.emails[output["Email"]] = output["User"]
1724

1725
        mapUserConfigRegex = re.compile(r"^\s*(\S+)\s*=\s*(.+)\s*<(\S+)>\s*$", re.VERBOSE)
1726
        for mapUserConfig in gitConfigList("git-p4.mapUser"):
1727
            mapUser = mapUserConfigRegex.findall(mapUserConfig)
1728
            if mapUser and len(mapUser[0]) == 3:
1729
                user = mapUser[0][0]
1730
                fullname = mapUser[0][1]
1731
                email = mapUser[0][2]
1732
                fulluser = fullname + " <" + email + ">"
1733
                self.users[user] = metadata_stream_to_writable_bytes(fulluser)
1734
                self.emails[email] = user
1735

1736
        s = b''
1737
        for (key, val) in self.users.items():
1738
            keybytes = metadata_stream_to_writable_bytes(key)
1739
            s += b"%s\t%s\n" % (keybytes.expandtabs(1), val.expandtabs(1))
1740

1741
        open(self.getUserCacheFilename(), 'wb').write(s)
1742
        self.userMapFromPerforceServer = True
1743

1744
    def loadUserMapFromCache(self):
1745
        self.users = {}
1746
        self.userMapFromPerforceServer = False
1747
        try:
1748
            cache = open(self.getUserCacheFilename(), 'rb')
1749
            lines = cache.readlines()
1750
            cache.close()
1751
            for line in lines:
1752
                entry = line.strip().split(b"\t")
1753
                self.users[entry[0].decode('utf_8')] = entry[1]
1754
        except IOError:
1755
            self.getUserMapFromPerforceServer()
1756

1757

1758
class P4Submit(Command, P4UserMap):
1759

1760
    conflict_behavior_choices = ("ask", "skip", "quit")
1761

1762
    def __init__(self):
1763
        Command.__init__(self)
1764
        P4UserMap.__init__(self)
1765
        self.options = [
1766
                optparse.make_option("--origin", dest="origin"),
1767
                optparse.make_option("-M", dest="detectRenames", action="store_true"),
1768
                # preserve the user, requires relevant p4 permissions
1769
                optparse.make_option("--preserve-user", dest="preserveUser", action="store_true"),
1770
                optparse.make_option("--export-labels", dest="exportLabels", action="store_true"),
1771
                optparse.make_option("--dry-run", "-n", dest="dry_run", action="store_true"),
1772
                optparse.make_option("--prepare-p4-only", dest="prepare_p4_only", action="store_true"),
1773
                optparse.make_option("--conflict", dest="conflict_behavior",
1774
                                     choices=self.conflict_behavior_choices),
1775
                optparse.make_option("--branch", dest="branch"),
1776
                optparse.make_option("--shelve", dest="shelve", action="store_true",
1777
                                     help="Shelve instead of submit. Shelved files are reverted, "
1778
                                     "restoring the workspace to the state before the shelve"),
1779
                optparse.make_option("--update-shelve", dest="update_shelve", action="append", type="int",
1780
                                     metavar="CHANGELIST",
1781
                                     help="update an existing shelved changelist, implies --shelve, "
1782
                                           "repeat in-order for multiple shelved changelists"),
1783
                optparse.make_option("--commit", dest="commit", metavar="COMMIT",
1784
                                     help="submit only the specified commit(s), one commit or xxx..xxx"),
1785
                optparse.make_option("--disable-rebase", dest="disable_rebase", action="store_true",
1786
                                     help="Disable rebase after submit is completed. Can be useful if you "
1787
                                     "work from a local git branch that is not master"),
1788
                optparse.make_option("--disable-p4sync", dest="disable_p4sync", action="store_true",
1789
                                     help="Skip Perforce sync of p4/master after submit or shelve"),
1790
                optparse.make_option("--no-verify", dest="no_verify", action="store_true",
1791
                                     help="Bypass p4-pre-submit and p4-changelist hooks"),
1792
        ]
1793
        self.description = """Submit changes from git to the perforce depot.\n
1794
    The `p4-pre-submit` hook is executed if it exists and is executable. It
1795
    can be bypassed with the `--no-verify` command line option. The hook takes
1796
    no parameters and nothing from standard input. Exiting with a non-zero status
1797
    from this script prevents `git-p4 submit` from launching.
1798

1799
    One usage scenario is to run unit tests in the hook.
1800

1801
    The `p4-prepare-changelist` hook is executed right after preparing the default
1802
    changelist message and before the editor is started. It takes one parameter,
1803
    the name of the file that contains the changelist text. Exiting with a non-zero
1804
    status from the script will abort the process.
1805

1806
    The purpose of the hook is to edit the message file in place, and it is not
1807
    supressed by the `--no-verify` option. This hook is called even if
1808
    `--prepare-p4-only` is set.
1809

1810
    The `p4-changelist` hook is executed after the changelist message has been
1811
    edited by the user. It can be bypassed with the `--no-verify` option. It
1812
    takes a single parameter, the name of the file that holds the proposed
1813
    changelist text. Exiting with a non-zero status causes the command to abort.
1814

1815
    The hook is allowed to edit the changelist file and can be used to normalize
1816
    the text into some project standard format. It can also be used to refuse the
1817
    Submit after inspect the message file.
1818

1819
    The `p4-post-changelist` hook is invoked after the submit has successfully
1820
    occurred in P4. It takes no parameters and is meant primarily for notification
1821
    and cannot affect the outcome of the git p4 submit action.
1822
    """
1823

1824
        self.usage += " [name of git branch to submit into perforce depot]"
1825
        self.origin = ""
1826
        self.detectRenames = False
1827
        self.preserveUser = gitConfigBool("git-p4.preserveUser")
1828
        self.dry_run = False
1829
        self.shelve = False
1830
        self.update_shelve = list()
1831
        self.commit = ""
1832
        self.disable_rebase = gitConfigBool("git-p4.disableRebase")
1833
        self.disable_p4sync = gitConfigBool("git-p4.disableP4Sync")
1834
        self.prepare_p4_only = False
1835
        self.conflict_behavior = None
1836
        self.isWindows = (platform.system() == "Windows")
1837
        self.exportLabels = False
1838
        self.p4HasMoveCommand = p4_has_move_command()
1839
        self.branch = None
1840
        self.no_verify = False
1841

1842
        if gitConfig('git-p4.largeFileSystem'):
1843
            die("Large file system not supported for git-p4 submit command. Please remove it from config.")
1844

1845
    def check(self):
1846
        if len(p4CmdList(["opened", "..."])) > 0:
1847
            die("You have files opened with perforce! Close them before starting the sync.")
1848

1849
    def separate_jobs_from_description(self, message):
1850
        """Extract and return a possible Jobs field in the commit message.  It
1851
           goes into a separate section in the p4 change specification.
1852

1853
           A jobs line starts with "Jobs:" and looks like a new field in a
1854
           form.  Values are white-space separated on the same line or on
1855
           following lines that start with a tab.
1856

1857
           This does not parse and extract the full git commit message like a
1858
           p4 form.  It just sees the Jobs: line as a marker to pass everything
1859
           from then on directly into the p4 form, but outside the description
1860
           section.
1861

1862
           Return a tuple (stripped log message, jobs string).
1863
           """
1864

1865
        m = re.search(r'^Jobs:', message, re.MULTILINE)
1866
        if m is None:
1867
            return (message, None)
1868

1869
        jobtext = message[m.start():]
1870
        stripped_message = message[:m.start()].rstrip()
1871
        return (stripped_message, jobtext)
1872

1873
    def prepareLogMessage(self, template, message, jobs):
1874
        """Edits the template returned from "p4 change -o" to insert the
1875
           message in the Description field, and the jobs text in the Jobs
1876
           field.
1877
           """
1878
        result = ""
1879

1880
        inDescriptionSection = False
1881

1882
        for line in template.split("\n"):
1883
            if line.startswith("#"):
1884
                result += line + "\n"
1885
                continue
1886

1887
            if inDescriptionSection:
1888
                if line.startswith("Files:") or line.startswith("Jobs:"):
1889
                    inDescriptionSection = False
1890
                    # insert Jobs section
1891
                    if jobs:
1892
                        result += jobs + "\n"
1893
                else:
1894
                    continue
1895
            else:
1896
                if line.startswith("Description:"):
1897
                    inDescriptionSection = True
1898
                    line += "\n"
1899
                    for messageLine in message.split("\n"):
1900
                        line += "\t" + messageLine + "\n"
1901

1902
            result += line + "\n"
1903

1904
        return result
1905

1906
    def patchRCSKeywords(self, file, regexp):
1907
        """Attempt to zap the RCS keywords in a p4 controlled file matching the
1908
           given regex.
1909
           """
1910
        handle, outFileName = tempfile.mkstemp(dir='.')
1911
        try:
1912
            with os.fdopen(handle, "wb") as outFile, open(file, "rb") as inFile:
1913
                for line in inFile.readlines():
1914
                    outFile.write(regexp.sub(br'$\1$', line))
1915
            # Forcibly overwrite the original file
1916
            os.unlink(file)
1917
            shutil.move(outFileName, file)
1918
        except:
1919
            # cleanup our temporary file
1920
            os.unlink(outFileName)
1921
            print("Failed to strip RCS keywords in %s" % file)
1922
            raise
1923

1924
        print("Patched up RCS keywords in %s" % file)
1925

1926
    def p4UserForCommit(self, id):
1927
        """Return the tuple (perforce user,git email) for a given git commit
1928
           id.
1929
           """
1930
        self.getUserMapFromPerforceServer()
1931
        gitEmail = read_pipe(["git", "log", "--max-count=1",
1932
                              "--format=%ae", id])
1933
        gitEmail = gitEmail.strip()
1934
        if gitEmail not in self.emails:
1935
            return (None, gitEmail)
1936
        else:
1937
            return (self.emails[gitEmail], gitEmail)
1938

1939
    def checkValidP4Users(self, commits):
1940
        """Check if any git authors cannot be mapped to p4 users."""
1941
        for id in commits:
1942
            user, email = self.p4UserForCommit(id)
1943
            if not user:
1944
                msg = "Cannot find p4 user for email %s in commit %s." % (email, id)
1945
                if gitConfigBool("git-p4.allowMissingP4Users"):
1946
                    print("%s" % msg)
1947
                else:
1948
                    die("Error: %s\nSet git-p4.allowMissingP4Users to true to allow this." % msg)
1949

1950
    def lastP4Changelist(self):
1951
        """Get back the last changelist number submitted in this client spec.
1952

1953
           This then gets used to patch up the username in the change. If the
1954
           same client spec is being used by multiple processes then this might
1955
           go wrong.
1956
           """
1957
        results = p4CmdList(["client", "-o"])        # find the current client
1958
        client = None
1959
        for r in results:
1960
            if 'Client' in r:
1961
                client = r['Client']
1962
                break
1963
        if not client:
1964
            die("could not get client spec")
1965
        results = p4CmdList(["changes", "-c", client, "-m", "1"])
1966
        for r in results:
1967
            if 'change' in r:
1968
                return r['change']
1969
        die("Could not get changelist number for last submit - cannot patch up user details")
1970

1971
    def modifyChangelistUser(self, changelist, newUser):
1972
        """Fixup the user field of a changelist after it has been submitted."""
1973
        changes = p4CmdList(["change", "-o", changelist])
1974
        if len(changes) != 1:
1975
            die("Bad output from p4 change modifying %s to user %s" %
1976
                (changelist, newUser))
1977

1978
        c = changes[0]
1979
        if c['User'] == newUser:
1980
            # Nothing to do
1981
            return
1982
        c['User'] = newUser
1983
        # p4 does not understand format version 3 and above
1984
        input = marshal.dumps(c, 2)
1985

1986
        result = p4CmdList(["change", "-f", "-i"], stdin=input)
1987
        for r in result:
1988
            if 'code' in r:
1989
                if r['code'] == 'error':
1990
                    die("Could not modify user field of changelist %s to %s:%s" % (changelist, newUser, r['data']))
1991
            if 'data' in r:
1992
                print("Updated user field for changelist %s to %s" % (changelist, newUser))
1993
                return
1994
        die("Could not modify user field of changelist %s to %s" % (changelist, newUser))
1995

1996
    def canChangeChangelists(self):
1997
        """Check to see if we have p4 admin or super-user permissions, either
1998
           of which are required to modify changelists.
1999
           """
2000
        results = p4CmdList(["protects", self.depotPath])
2001
        for r in results:
2002
            if 'perm' in r:
2003
                if r['perm'] == 'admin':
2004
                    return 1
2005
                if r['perm'] == 'super':
2006
                    return 1
2007
        return 0
2008

2009
    def prepareSubmitTemplate(self, changelist=None):
2010
        """Run "p4 change -o" to grab a change specification template.
2011

2012
           This does not use "p4 -G", as it is nice to keep the submission
2013
           template in original order, since a human might edit it.
2014

2015
           Remove lines in the Files section that show changes to files
2016
           outside the depot path we're committing into.
2017
           """
2018

2019
        upstream, settings = findUpstreamBranchPoint()
2020

2021
        template = """\
2022
# A Perforce Change Specification.
2023
#
2024
#  Change:      The change number. 'new' on a new changelist.
2025
#  Date:        The date this specification was last modified.
2026
#  Client:      The client on which the changelist was created.  Read-only.
2027
#  User:        The user who created the changelist.
2028
#  Status:      Either 'pending' or 'submitted'. Read-only.
2029
#  Type:        Either 'public' or 'restricted'. Default is 'public'.
2030
#  Description: Comments about the changelist.  Required.
2031
#  Jobs:        What opened jobs are to be closed by this changelist.
2032
#               You may delete jobs from this list.  (New changelists only.)
2033
#  Files:       What opened files from the default changelist are to be added
2034
#               to this changelist.  You may delete files from this list.
2035
#               (New changelists only.)
2036
"""
2037
        files_list = []
2038
        inFilesSection = False
2039
        change_entry = None
2040
        args = ['change', '-o']
2041
        if changelist:
2042
            args.append(str(changelist))
2043
        for entry in p4CmdList(args):
2044
            if 'code' not in entry:
2045
                continue
2046
            if entry['code'] == 'stat':
2047
                change_entry = entry
2048
                break
2049
        if not change_entry:
2050
            die('Failed to decode output of p4 change -o')
2051
        for key, value in change_entry.items():
2052
            if key.startswith('File'):
2053
                if 'depot-paths' in settings:
2054
                    if not [p for p in settings['depot-paths']
2055
                            if p4PathStartsWith(value, p)]:
2056
                        continue
2057
                else:
2058
                    if not p4PathStartsWith(value, self.depotPath):
2059
                        continue
2060
                files_list.append(value)
2061
                continue
2062
        # Output in the order expected by prepareLogMessage
2063
        for key in ['Change', 'Client', 'User', 'Status', 'Description', 'Jobs']:
2064
            if key not in change_entry:
2065
                continue
2066
            template += '\n'
2067
            template += key + ':'
2068
            if key == 'Description':
2069
                template += '\n'
2070
            for field_line in change_entry[key].splitlines():
2071
                template += '\t'+field_line+'\n'
2072
        if len(files_list) > 0:
2073
            template += '\n'
2074
            template += 'Files:\n'
2075
        for path in files_list:
2076
            template += '\t'+path+'\n'
2077
        return template
2078

2079
    def edit_template(self, template_file):
2080
        """Invoke the editor to let the user change the submission message.
2081

2082
           Return true if okay to continue with the submit.
2083
           """
2084

2085
        # if configured to skip the editing part, just submit
2086
        if gitConfigBool("git-p4.skipSubmitEdit"):
2087
            return True
2088

2089
        # look at the modification time, to check later if the user saved
2090
        # the file
2091
        mtime = os.stat(template_file).st_mtime
2092

2093
        # invoke the editor
2094
        if "P4EDITOR" in os.environ and (os.environ.get("P4EDITOR") != ""):
2095
            editor = os.environ.get("P4EDITOR")
2096
        else:
2097
            editor = read_pipe(["git", "var", "GIT_EDITOR"]).strip()
2098
        system(["sh", "-c", ('%s "$@"' % editor), editor, template_file])
2099

2100
        # If the file was not saved, prompt to see if this patch should
2101
        # be skipped.  But skip this verification step if configured so.
2102
        if gitConfigBool("git-p4.skipSubmitEditCheck"):
2103
            return True
2104

2105
        # modification time updated means user saved the file
2106
        if os.stat(template_file).st_mtime > mtime:
2107
            return True
2108

2109
        response = prompt("Submit template unchanged. Submit anyway? [y]es, [n]o (skip this patch) ")
2110
        if response == 'y':
2111
            return True
2112
        if response == 'n':
2113
            return False
2114

2115
    def get_diff_description(self, editedFiles, filesToAdd, symlinks):
2116
        # diff
2117
        if "P4DIFF" in os.environ:
2118
            del(os.environ["P4DIFF"])
2119
        diff = ""
2120
        for editedFile in editedFiles:
2121
            diff += p4_read_pipe(['diff', '-du',
2122
                                  wildcard_encode(editedFile)])
2123

2124
        # new file diff
2125
        newdiff = ""
2126
        for newFile in filesToAdd:
2127
            newdiff += "==== new file ====\n"
2128
            newdiff += "--- /dev/null\n"
2129
            newdiff += "+++ %s\n" % newFile
2130

2131
            is_link = os.path.islink(newFile)
2132
            expect_link = newFile in symlinks
2133

2134
            if is_link and expect_link:
2135
                newdiff += "+%s\n" % os.readlink(newFile)
2136
            else:
2137
                f = open(newFile, "r")
2138
                try:
2139
                    for line in f.readlines():
2140
                        newdiff += "+" + line
2141
                except UnicodeDecodeError:
2142
                    # Found non-text data and skip, since diff description
2143
                    # should only include text
2144
                    pass
2145
                f.close()
2146

2147
        return (diff + newdiff).replace('\r\n', '\n')
2148

2149
    def applyCommit(self, id):
2150
        """Apply one commit, return True if it succeeded."""
2151

2152
        print("Applying", read_pipe(["git", "show", "-s",
2153
                                     "--format=format:%h %s", id]))
2154

2155
        p4User, gitEmail = self.p4UserForCommit(id)
2156

2157
        diff = read_pipe_lines(
2158
            ["git", "diff-tree", "-r"] + self.diffOpts + ["{}^".format(id), id])
2159
        filesToAdd = set()
2160
        filesToChangeType = set()
2161
        filesToDelete = set()
2162
        editedFiles = set()
2163
        pureRenameCopy = set()
2164
        symlinks = set()
2165
        filesToChangeExecBit = {}
2166
        all_files = list()
2167

2168
        for line in diff:
2169
            diff = parseDiffTreeEntry(line)
2170
            modifier = diff['status']
2171
            path = diff['src']
2172
            all_files.append(path)
2173

2174
            if modifier == "M":
2175
                p4_edit(path)
2176
                if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
2177
                    filesToChangeExecBit[path] = diff['dst_mode']
2178
                editedFiles.add(path)
2179
            elif modifier == "A":
2180
                filesToAdd.add(path)
2181
                filesToChangeExecBit[path] = diff['dst_mode']
2182
                if path in filesToDelete:
2183
                    filesToDelete.remove(path)
2184

2185
                dst_mode = int(diff['dst_mode'], 8)
2186
                if dst_mode == 0o120000:
2187
                    symlinks.add(path)
2188

2189
            elif modifier == "D":
2190
                filesToDelete.add(path)
2191
                if path in filesToAdd:
2192
                    filesToAdd.remove(path)
2193
            elif modifier == "C":
2194
                src, dest = diff['src'], diff['dst']
2195
                all_files.append(dest)
2196
                p4_integrate(src, dest)
2197
                pureRenameCopy.add(dest)
2198
                if diff['src_sha1'] != diff['dst_sha1']:
2199
                    p4_edit(dest)
2200
                    pureRenameCopy.discard(dest)
2201
                if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
2202
                    p4_edit(dest)
2203
                    pureRenameCopy.discard(dest)
2204
                    filesToChangeExecBit[dest] = diff['dst_mode']
2205
                if self.isWindows:
2206
                    # turn off read-only attribute
2207
                    os.chmod(dest, stat.S_IWRITE)
2208
                os.unlink(dest)
2209
                editedFiles.add(dest)
2210
            elif modifier == "R":
2211
                src, dest = diff['src'], diff['dst']
2212
                all_files.append(dest)
2213
                if self.p4HasMoveCommand:
2214
                    p4_edit(src)        # src must be open before move
2215
                    p4_move(src, dest)  # opens for (move/delete, move/add)
2216
                else:
2217
                    p4_integrate(src, dest)
2218
                    if diff['src_sha1'] != diff['dst_sha1']:
2219
                        p4_edit(dest)
2220
                    else:
2221
                        pureRenameCopy.add(dest)
2222
                if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
2223
                    if not self.p4HasMoveCommand:
2224
                        p4_edit(dest)   # with move: already open, writable
2225
                    filesToChangeExecBit[dest] = diff['dst_mode']
2226
                if not self.p4HasMoveCommand:
2227
                    if self.isWindows:
2228
                        os.chmod(dest, stat.S_IWRITE)
2229
                    os.unlink(dest)
2230
                    filesToDelete.add(src)
2231
                editedFiles.add(dest)
2232
            elif modifier == "T":
2233
                filesToChangeType.add(path)
2234
            else:
2235
                die("unknown modifier %s for %s" % (modifier, path))
2236

2237
        diffcmd = "git diff-tree --full-index -p \"%s\"" % (id)
2238
        patchcmd = diffcmd + " | git apply "
2239
        tryPatchCmd = patchcmd + "--check -"
2240
        applyPatchCmd = patchcmd + "--check --apply -"
2241
        patch_succeeded = True
2242

2243
        if verbose:
2244
            print("TryPatch: %s" % tryPatchCmd)
2245

2246
        if os.system(tryPatchCmd) != 0:
2247
            fixed_rcs_keywords = False
2248
            patch_succeeded = False
2249
            print("Unfortunately applying the change failed!")
2250

2251
            # Patch failed, maybe it's just RCS keyword woes. Look through
2252
            # the patch to see if that's possible.
2253
            if gitConfigBool("git-p4.attemptRCSCleanup"):
2254
                file = None
2255
                kwfiles = {}
2256
                for file in editedFiles | filesToDelete:
2257
                    # did this file's delta contain RCS keywords?
2258
                    regexp = p4_keywords_regexp_for_file(file)
2259
                    if regexp:
2260
                        # this file is a possibility...look for RCS keywords.
2261
                        for line in read_pipe_lines(
2262
                                ["git", "diff", "%s^..%s" % (id, id), file],
2263
                                raw=True):
2264
                            if regexp.search(line):
2265
                                if verbose:
2266
                                    print("got keyword match on %s in %s in %s" % (regexp.pattern, line, file))
2267
                                kwfiles[file] = regexp
2268
                                break
2269

2270
                for file, regexp in kwfiles.items():
2271
                    if verbose:
2272
                        print("zapping %s with %s" % (line, regexp.pattern))
2273
                    # File is being deleted, so not open in p4.  Must
2274
                    # disable the read-only bit on windows.
2275
                    if self.isWindows and file not in editedFiles:
2276
                        os.chmod(file, stat.S_IWRITE)
2277
                    self.patchRCSKeywords(file, kwfiles[file])
2278
                    fixed_rcs_keywords = True
2279

2280
            if fixed_rcs_keywords:
2281
                print("Retrying the patch with RCS keywords cleaned up")
2282
                if os.system(tryPatchCmd) == 0:
2283
                    patch_succeeded = True
2284
                    print("Patch succeesed this time with RCS keywords cleaned")
2285

2286
        if not patch_succeeded:
2287
            for f in editedFiles:
2288
                p4_revert(f)
2289
            return False
2290

2291
        #
2292
        # Apply the patch for real, and do add/delete/+x handling.
2293
        #
2294
        system(applyPatchCmd, shell=True)
2295

2296
        for f in filesToChangeType:
2297
            p4_edit(f, "-t", "auto")
2298
        for f in filesToAdd:
2299
            p4_add(f)
2300
        for f in filesToDelete:
2301
            p4_revert(f)
2302
            p4_delete(f)
2303

2304
        # Set/clear executable bits
2305
        for f in filesToChangeExecBit.keys():
2306
            mode = filesToChangeExecBit[f]
2307
            setP4ExecBit(f, mode)
2308

2309
        update_shelve = 0
2310
        if len(self.update_shelve) > 0:
2311
            update_shelve = self.update_shelve.pop(0)
2312
            p4_reopen_in_change(update_shelve, all_files)
2313

2314
        #
2315
        # Build p4 change description, starting with the contents
2316
        # of the git commit message.
2317
        #
2318
        logMessage = extractLogMessageFromGitCommit(id)
2319
        logMessage = logMessage.strip()
2320
        logMessage, jobs = self.separate_jobs_from_description(logMessage)
2321

2322
        template = self.prepareSubmitTemplate(update_shelve)
2323
        submitTemplate = self.prepareLogMessage(template, logMessage, jobs)
2324

2325
        if self.preserveUser:
2326
            submitTemplate += "\n######## Actual user %s, modified after commit\n" % p4User
2327

2328
        if self.checkAuthorship and not self.p4UserIsMe(p4User):
2329
            submitTemplate += "######## git author %s does not match your p4 account.\n" % gitEmail
2330
            submitTemplate += "######## Use option --preserve-user to modify authorship.\n"
2331
            submitTemplate += "######## Variable git-p4.skipUserNameCheck hides this message.\n"
2332

2333
        separatorLine = "######## everything below this line is just the diff #######\n"
2334
        if not self.prepare_p4_only:
2335
            submitTemplate += separatorLine
2336
            submitTemplate += self.get_diff_description(editedFiles, filesToAdd, symlinks)
2337

2338
        handle, fileName = tempfile.mkstemp()
2339
        tmpFile = os.fdopen(handle, "w+b")
2340
        if self.isWindows:
2341
            submitTemplate = submitTemplate.replace("\n", "\r\n")
2342
        tmpFile.write(encode_text_stream(submitTemplate))
2343
        tmpFile.close()
2344

2345
        submitted = False
2346

2347
        try:
2348
            # Allow the hook to edit the changelist text before presenting it
2349
            # to the user.
2350
            if not run_git_hook("p4-prepare-changelist", [fileName]):
2351
                return False
2352

2353
            if self.prepare_p4_only:
2354
                #
2355
                # Leave the p4 tree prepared, and the submit template around
2356
                # and let the user decide what to do next
2357
                #
2358
                submitted = True
2359
                print("")
2360
                print("P4 workspace prepared for submission.")
2361
                print("To submit or revert, go to client workspace")
2362
                print("  " + self.clientPath)
2363
                print("")
2364
                print("To submit, use \"p4 submit\" to write a new description,")
2365
                print("or \"p4 submit -i <%s\" to use the one prepared by"
2366
                      " \"git p4\"." % fileName)
2367
                print("You can delete the file \"%s\" when finished." % fileName)
2368

2369
                if self.preserveUser and p4User and not self.p4UserIsMe(p4User):
2370
                    print("To preserve change ownership by user %s, you must\n"
2371
                          "do \"p4 change -f <change>\" after submitting and\n"
2372
                          "edit the User field.")
2373
                if pureRenameCopy:
2374
                    print("After submitting, renamed files must be re-synced.")
2375
                    print("Invoke \"p4 sync -f\" on each of these files:")
2376
                    for f in pureRenameCopy:
2377
                        print("  " + f)
2378

2379
                print("")
2380
                print("To revert the changes, use \"p4 revert ...\", and delete")
2381
                print("the submit template file \"%s\"" % fileName)
2382
                if filesToAdd:
2383
                    print("Since the commit adds new files, they must be deleted:")
2384
                    for f in filesToAdd:
2385
                        print("  " + f)
2386
                print("")
2387
                sys.stdout.flush()
2388
                return True
2389

2390
            if self.edit_template(fileName):
2391
                if not self.no_verify:
2392
                    if not run_git_hook("p4-changelist", [fileName]):
2393
                        print("The p4-changelist hook failed.")
2394
                        sys.stdout.flush()
2395
                        return False
2396

2397
                # read the edited message and submit
2398
                tmpFile = open(fileName, "rb")
2399
                message = decode_text_stream(tmpFile.read())
2400
                tmpFile.close()
2401
                if self.isWindows:
2402
                    message = message.replace("\r\n", "\n")
2403
                if message.find(separatorLine) != -1:
2404
                    submitTemplate = message[:message.index(separatorLine)]
2405
                else:
2406
                    submitTemplate = message
2407

2408
                if len(submitTemplate.strip()) == 0:
2409
                    print("Changelist is empty, aborting this changelist.")
2410
                    sys.stdout.flush()
2411
                    return False
2412

2413
                if update_shelve:
2414
                    p4_write_pipe(['shelve', '-r', '-i'], submitTemplate)
2415
                elif self.shelve:
2416
                    p4_write_pipe(['shelve', '-i'], submitTemplate)
2417
                else:
2418
                    p4_write_pipe(['submit', '-i'], submitTemplate)
2419
                    # The rename/copy happened by applying a patch that created a
2420
                    # new file.  This leaves it writable, which confuses p4.
2421
                    for f in pureRenameCopy:
2422
                        p4_sync(f, "-f")
2423

2424
                if self.preserveUser:
2425
                    if p4User:
2426
                        # Get last changelist number. Cannot easily get it from
2427
                        # the submit command output as the output is
2428
                        # unmarshalled.
2429
                        changelist = self.lastP4Changelist()
2430
                        self.modifyChangelistUser(changelist, p4User)
2431

2432
                submitted = True
2433

2434
                run_git_hook("p4-post-changelist")
2435
        finally:
2436
            # Revert changes if we skip this patch
2437
            if not submitted or self.shelve:
2438
                if self.shelve:
2439
                    print("Reverting shelved files.")
2440
                else:
2441
                    print("Submission cancelled, undoing p4 changes.")
2442
                sys.stdout.flush()
2443
                for f in editedFiles | filesToDelete:
2444
                    p4_revert(f)
2445
                for f in filesToAdd:
2446
                    p4_revert(f)
2447
                    os.remove(f)
2448

2449
            if not self.prepare_p4_only:
2450
                os.remove(fileName)
2451
        return submitted
2452

2453
    def exportGitTags(self, gitTags):
2454
        """Export git tags as p4 labels. Create a p4 label and then tag with
2455
           that.
2456
           """
2457

2458
        validLabelRegexp = gitConfig("git-p4.labelExportRegexp")
2459
        if len(validLabelRegexp) == 0:
2460
            validLabelRegexp = defaultLabelRegexp
2461
        m = re.compile(validLabelRegexp)
2462

2463
        for name in gitTags:
2464

2465
            if not m.match(name):
2466
                if verbose:
2467
                    print("tag %s does not match regexp %s" % (name, validLabelRegexp))
2468
                continue
2469

2470
            # Get the p4 commit this corresponds to
2471
            logMessage = extractLogMessageFromGitCommit(name)
2472
            values = extractSettingsGitLog(logMessage)
2473

2474
            if 'change' not in values:
2475
                # a tag pointing to something not sent to p4; ignore
2476
                if verbose:
2477
                    print("git tag %s does not give a p4 commit" % name)
2478
                continue
2479
            else:
2480
                changelist = values['change']
2481

2482
            # Get the tag details.
2483
            inHeader = True
2484
            isAnnotated = False
2485
            body = []
2486
            for l in read_pipe_lines(["git", "cat-file", "-p", name]):
2487
                l = l.strip()
2488
                if inHeader:
2489
                    if re.match(r'tag\s+', l):
2490
                        isAnnotated = True
2491
                    elif re.match(r'\s*$', l):
2492
                        inHeader = False
2493
                        continue
2494
                else:
2495
                    body.append(l)
2496

2497
            if not isAnnotated:
2498
                body = ["lightweight tag imported by git p4\n"]
2499

2500
            # Create the label - use the same view as the client spec we are using
2501
            clientSpec = getClientSpec()
2502

2503
            labelTemplate = "Label: %s\n" % name
2504
            labelTemplate += "Description:\n"
2505
            for b in body:
2506
                labelTemplate += "\t" + b + "\n"
2507
            labelTemplate += "View:\n"
2508
            for depot_side in clientSpec.mappings:
2509
                labelTemplate += "\t%s\n" % depot_side
2510

2511
            if self.dry_run:
2512
                print("Would create p4 label %s for tag" % name)
2513
            elif self.prepare_p4_only:
2514
                print("Not creating p4 label %s for tag due to option"
2515
                      " --prepare-p4-only" % name)
2516
            else:
2517
                p4_write_pipe(["label", "-i"], labelTemplate)
2518

2519
                # Use the label
2520
                p4_system(["tag", "-l", name] +
2521
                          ["%s@%s" % (depot_side, changelist) for depot_side in clientSpec.mappings])
2522

2523
                if verbose:
2524
                    print("created p4 label for tag %s" % name)
2525

2526
    def run(self, args):
2527
        if len(args) == 0:
2528
            self.master = currentGitBranch()
2529
        elif len(args) == 1:
2530
            self.master = args[0]
2531
            if not branchExists(self.master):
2532
                die("Branch %s does not exist" % self.master)
2533
        else:
2534
            return False
2535

2536
        for i in self.update_shelve:
2537
            if i <= 0:
2538
                sys.exit("invalid changelist %d" % i)
2539

2540
        if self.master:
2541
            allowSubmit = gitConfig("git-p4.allowSubmit")
2542
            if len(allowSubmit) > 0 and not self.master in allowSubmit.split(","):
2543
                die("%s is not in git-p4.allowSubmit" % self.master)
2544

2545
        upstream, settings = findUpstreamBranchPoint()
2546
        self.depotPath = settings['depot-paths'][0]
2547
        if len(self.origin) == 0:
2548
            self.origin = upstream
2549

2550
        if len(self.update_shelve) > 0:
2551
            self.shelve = True
2552

2553
        if self.preserveUser:
2554
            if not self.canChangeChangelists():
2555
                die("Cannot preserve user names without p4 super-user or admin permissions")
2556

2557
        # if not set from the command line, try the config file
2558
        if self.conflict_behavior is None:
2559
            val = gitConfig("git-p4.conflict")
2560
            if val:
2561
                if val not in self.conflict_behavior_choices:
2562
                    die("Invalid value '%s' for config git-p4.conflict" % val)
2563
            else:
2564
                val = "ask"
2565
            self.conflict_behavior = val
2566

2567
        if self.verbose:
2568
            print("Origin branch is " + self.origin)
2569

2570
        if len(self.depotPath) == 0:
2571
            print("Internal error: cannot locate perforce depot path from existing branches")
2572
            sys.exit(128)
2573

2574
        self.useClientSpec = False
2575
        if gitConfigBool("git-p4.useclientspec"):
2576
            self.useClientSpec = True
2577
        if self.useClientSpec:
2578
            self.clientSpecDirs = getClientSpec()
2579

2580
        # Check for the existence of P4 branches
2581
        branchesDetected = (len(p4BranchesInGit().keys()) > 1)
2582

2583
        if self.useClientSpec and not branchesDetected:
2584
            # all files are relative to the client spec
2585
            self.clientPath = getClientRoot()
2586
        else:
2587
            self.clientPath = p4Where(self.depotPath)
2588

2589
        if self.clientPath == "":
2590
            die("Error: Cannot locate perforce checkout of %s in client view" % self.depotPath)
2591

2592
        print("Perforce checkout for depot path %s located at %s" % (self.depotPath, self.clientPath))
2593
        self.oldWorkingDirectory = os.getcwd()
2594

2595
        # ensure the clientPath exists
2596
        new_client_dir = False
2597
        if not os.path.exists(self.clientPath):
2598
            new_client_dir = True
2599
            os.makedirs(self.clientPath)
2600

2601
        chdir(self.clientPath, is_client_path=True)
2602
        if self.dry_run:
2603
            print("Would synchronize p4 checkout in %s" % self.clientPath)
2604
        else:
2605
            print("Synchronizing p4 checkout...")
2606
            if new_client_dir:
2607
                # old one was destroyed, and maybe nobody told p4
2608
                p4_sync("...", "-f")
2609
            else:
2610
                p4_sync("...")
2611
        self.check()
2612

2613
        commits = []
2614
        if self.master:
2615
            committish = self.master
2616
        else:
2617
            committish = 'HEAD'
2618

2619
        if self.commit != "":
2620
            if self.commit.find("..") != -1:
2621
                limits_ish = self.commit.split("..")
2622
                for line in read_pipe_lines(["git", "rev-list", "--no-merges", "%s..%s" % (limits_ish[0], limits_ish[1])]):
2623
                    commits.append(line.strip())
2624
                commits.reverse()
2625
            else:
2626
                commits.append(self.commit)
2627
        else:
2628
            for line in read_pipe_lines(["git", "rev-list", "--no-merges", "%s..%s" % (self.origin, committish)]):
2629
                commits.append(line.strip())
2630
            commits.reverse()
2631

2632
        if self.preserveUser or gitConfigBool("git-p4.skipUserNameCheck"):
2633
            self.checkAuthorship = False
2634
        else:
2635
            self.checkAuthorship = True
2636

2637
        if self.preserveUser:
2638
            self.checkValidP4Users(commits)
2639

2640
        #
2641
        # Build up a set of options to be passed to diff when
2642
        # submitting each commit to p4.
2643
        #
2644
        if self.detectRenames:
2645
            # command-line -M arg
2646
            self.diffOpts = ["-M"]
2647
        else:
2648
            # If not explicitly set check the config variable
2649
            detectRenames = gitConfig("git-p4.detectRenames")
2650

2651
            if detectRenames.lower() == "false" or detectRenames == "":
2652
                self.diffOpts = []
2653
            elif detectRenames.lower() == "true":
2654
                self.diffOpts = ["-M"]
2655
            else:
2656
                self.diffOpts = ["-M{}".format(detectRenames)]
2657

2658
        # no command-line arg for -C or --find-copies-harder, just
2659
        # config variables
2660
        detectCopies = gitConfig("git-p4.detectCopies")
2661
        if detectCopies.lower() == "false" or detectCopies == "":
2662
            pass
2663
        elif detectCopies.lower() == "true":
2664
            self.diffOpts.append("-C")
2665
        else:
2666
            self.diffOpts.append("-C{}".format(detectCopies))
2667

2668
        if gitConfigBool("git-p4.detectCopiesHarder"):
2669
            self.diffOpts.append("--find-copies-harder")
2670

2671
        num_shelves = len(self.update_shelve)
2672
        if num_shelves > 0 and num_shelves != len(commits):
2673
            sys.exit("number of commits (%d) must match number of shelved changelist (%d)" %
2674
                     (len(commits), num_shelves))
2675

2676
        if not self.no_verify:
2677
            try:
2678
                if not run_git_hook("p4-pre-submit"):
2679
                    print("\nThe p4-pre-submit hook failed, aborting the submit.\n\nYou can skip "
2680
                        "this pre-submission check by adding\nthe command line option '--no-verify', "
2681
                        "however,\nthis will also skip the p4-changelist hook as well.")
2682
                    sys.exit(1)
2683
            except Exception as e:
2684
                print("\nThe p4-pre-submit hook failed, aborting the submit.\n\nThe hook failed "
2685
                    "with the error '{0}'".format(e.message))
2686
                sys.exit(1)
2687

2688
        #
2689
        # Apply the commits, one at a time.  On failure, ask if should
2690
        # continue to try the rest of the patches, or quit.
2691
        #
2692
        if self.dry_run:
2693
            print("Would apply")
2694
        applied = []
2695
        last = len(commits) - 1
2696
        for i, commit in enumerate(commits):
2697
            if self.dry_run:
2698
                print(" ", read_pipe(["git", "show", "-s",
2699
                                      "--format=format:%h %s", commit]))
2700
                ok = True
2701
            else:
2702
                ok = self.applyCommit(commit)
2703
            if ok:
2704
                applied.append(commit)
2705
                if self.prepare_p4_only:
2706
                    if i < last:
2707
                        print("Processing only the first commit due to option"
2708
                                " --prepare-p4-only")
2709
                    break
2710
            else:
2711
                if i < last:
2712
                    # prompt for what to do, or use the option/variable
2713
                    if self.conflict_behavior == "ask":
2714
                        print("What do you want to do?")
2715
                        response = prompt("[s]kip this commit but apply the rest, or [q]uit? ")
2716
                    elif self.conflict_behavior == "skip":
2717
                        response = "s"
2718
                    elif self.conflict_behavior == "quit":
2719
                        response = "q"
2720
                    else:
2721
                        die("Unknown conflict_behavior '%s'" %
2722
                            self.conflict_behavior)
2723

2724
                    if response == "s":
2725
                        print("Skipping this commit, but applying the rest")
2726
                    if response == "q":
2727
                        print("Quitting")
2728
                        break
2729

2730
        chdir(self.oldWorkingDirectory)
2731
        shelved_applied = "shelved" if self.shelve else "applied"
2732
        if self.dry_run:
2733
            pass
2734
        elif self.prepare_p4_only:
2735
            pass
2736
        elif len(commits) == len(applied):
2737
            print("All commits {0}!".format(shelved_applied))
2738

2739
            sync = P4Sync()
2740
            if self.branch:
2741
                sync.branch = self.branch
2742
            if self.disable_p4sync:
2743
                sync.sync_origin_only()
2744
            else:
2745
                sync.run([])
2746

2747
                if not self.disable_rebase:
2748
                    rebase = P4Rebase()
2749
                    rebase.rebase()
2750

2751
        else:
2752
            if len(applied) == 0:
2753
                print("No commits {0}.".format(shelved_applied))
2754
            else:
2755
                print("{0} only the commits marked with '*':".format(shelved_applied.capitalize()))
2756
                for c in commits:
2757
                    if c in applied:
2758
                        star = "*"
2759
                    else:
2760
                        star = " "
2761
                    print(star, read_pipe(["git", "show", "-s",
2762
                                           "--format=format:%h %s",  c]))
2763
                print("You will have to do 'git p4 sync' and rebase.")
2764

2765
        if gitConfigBool("git-p4.exportLabels"):
2766
            self.exportLabels = True
2767

2768
        if self.exportLabels:
2769
            p4Labels = getP4Labels(self.depotPath)
2770
            gitTags = getGitTags()
2771

2772
            missingGitTags = gitTags - p4Labels
2773
            self.exportGitTags(missingGitTags)
2774

2775
        # exit with error unless everything applied perfectly
2776
        if len(commits) != len(applied):
2777
            sys.exit(1)
2778

2779
        return True
2780

2781

2782
class View(object):
2783
    """Represent a p4 view ("p4 help views"), and map files in a repo according
2784
       to the view.
2785
       """
2786

2787
    def __init__(self, client_name):
2788
        self.mappings = []
2789
        self.client_prefix = "//%s/" % client_name
2790
        # cache results of "p4 where" to lookup client file locations
2791
        self.client_spec_path_cache = {}
2792

2793
    def append(self, view_line):
2794
        """Parse a view line, splitting it into depot and client sides.  Append
2795
           to self.mappings, preserving order.  This is only needed for tag
2796
           creation.
2797
           """
2798

2799
        # Split the view line into exactly two words.  P4 enforces
2800
        # structure on these lines that simplifies this quite a bit.
2801
        #
2802
        # Either or both words may be double-quoted.
2803
        # Single quotes do not matter.
2804
        # Double-quote marks cannot occur inside the words.
2805
        # A + or - prefix is also inside the quotes.
2806
        # There are no quotes unless they contain a space.
2807
        # The line is already white-space stripped.
2808
        # The two words are separated by a single space.
2809
        #
2810
        if view_line[0] == '"':
2811
            # First word is double quoted.  Find its end.
2812
            close_quote_index = view_line.find('"', 1)
2813
            if close_quote_index <= 0:
2814
                die("No first-word closing quote found: %s" % view_line)
2815
            depot_side = view_line[1:close_quote_index]
2816
            # skip closing quote and space
2817
            rhs_index = close_quote_index + 1 + 1
2818
        else:
2819
            space_index = view_line.find(" ")
2820
            if space_index <= 0:
2821
                die("No word-splitting space found: %s" % view_line)
2822
            depot_side = view_line[0:space_index]
2823
            rhs_index = space_index + 1
2824

2825
        # prefix + means overlay on previous mapping
2826
        if depot_side.startswith("+"):
2827
            depot_side = depot_side[1:]
2828

2829
        # prefix - means exclude this path, leave out of mappings
2830
        exclude = False
2831
        if depot_side.startswith("-"):
2832
            exclude = True
2833
            depot_side = depot_side[1:]
2834

2835
        if not exclude:
2836
            self.mappings.append(depot_side)
2837

2838
    def convert_client_path(self, clientFile):
2839
        # chop off //client/ part to make it relative
2840
        if not decode_path(clientFile).startswith(self.client_prefix):
2841
            die("No prefix '%s' on clientFile '%s'" %
2842
                (self.client_prefix, clientFile))
2843
        return clientFile[len(self.client_prefix):]
2844

2845
    def update_client_spec_path_cache(self, files):
2846
        """Caching file paths by "p4 where" batch query."""
2847

2848
        # List depot file paths exclude that already cached
2849
        fileArgs = [f['path'] for f in files if decode_path(f['path']) not in self.client_spec_path_cache]
2850

2851
        if len(fileArgs) == 0:
2852
            return  # All files in cache
2853

2854
        where_result = p4CmdList(["-x", "-", "where"], stdin=fileArgs)
2855
        for res in where_result:
2856
            if "code" in res and res["code"] == "error":
2857
                # assume error is "... file(s) not in client view"
2858
                continue
2859
            if "clientFile" not in res:
2860
                die("No clientFile in 'p4 where' output")
2861
            if "unmap" in res:
2862
                # it will list all of them, but only one not unmap-ped
2863
                continue
2864
            depot_path = decode_path(res['depotFile'])
2865
            if gitConfigBool("core.ignorecase"):
2866
                depot_path = depot_path.lower()
2867
            self.client_spec_path_cache[depot_path] = self.convert_client_path(res["clientFile"])
2868

2869
        # not found files or unmap files set to ""
2870
        for depotFile in fileArgs:
2871
            depotFile = decode_path(depotFile)
2872
            if gitConfigBool("core.ignorecase"):
2873
                depotFile = depotFile.lower()
2874
            if depotFile not in self.client_spec_path_cache:
2875
                self.client_spec_path_cache[depotFile] = b''
2876

2877
    def map_in_client(self, depot_path):
2878
        """Return the relative location in the client where this depot file
2879
           should live.
2880

2881
           Returns "" if the file should not be mapped in the client.
2882
           """
2883

2884
        if gitConfigBool("core.ignorecase"):
2885
            depot_path = depot_path.lower()
2886

2887
        if depot_path in self.client_spec_path_cache:
2888
            return self.client_spec_path_cache[depot_path]
2889

2890
        die("Error: %s is not found in client spec path" % depot_path)
2891
        return ""
2892

2893

2894
def cloneExcludeCallback(option, opt_str, value, parser):
2895
    # prepend "/" because the first "/" was consumed as part of the option itself.
2896
    # ("-//depot/A/..." becomes "/depot/A/..." after option parsing)
2897
    parser.values.cloneExclude += ["/" + re.sub(r"\.\.\.$", "", value)]
2898

2899

2900
class P4Sync(Command, P4UserMap):
2901

2902
    def __init__(self):
2903
        Command.__init__(self)
2904
        P4UserMap.__init__(self)
2905
        self.options = [
2906
                optparse.make_option("--branch", dest="branch"),
2907
                optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"),
2908
                optparse.make_option("--changesfile", dest="changesFile"),
2909
                optparse.make_option("--silent", dest="silent", action="store_true"),
2910
                optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"),
2911
                optparse.make_option("--import-labels", dest="importLabels", action="store_true"),
2912
                optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false",
2913
                                     help="Import into refs/heads/ , not refs/remotes"),
2914
                optparse.make_option("--max-changes", dest="maxChanges",
2915
                                     help="Maximum number of changes to import"),
2916
                optparse.make_option("--changes-block-size", dest="changes_block_size", type="int",
2917
                                     help="Internal block size to use when iteratively calling p4 changes"),
2918
                optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true',
2919
                                     help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),
2920
                optparse.make_option("--use-client-spec", dest="useClientSpec", action='store_true',
2921
                                     help="Only sync files that are included in the Perforce Client Spec"),
2922
                optparse.make_option("-/", dest="cloneExclude",
2923
                                     action="callback", callback=cloneExcludeCallback, type="string",
2924
                                     help="exclude depot path"),
2925
        ]
2926
        self.description = """Imports from Perforce into a git repository.\n
2927
    example:
2928
    //depot/my/project/ -- to import the current head
2929
    //depot/my/project/@all -- to import everything
2930
    //depot/my/project/@1,6 -- to import only from revision 1 to 6
2931

2932
    (a ... is not needed in the path p4 specification, it's added implicitly)"""
2933

2934
        self.usage += " //depot/path[@revRange]"
2935
        self.silent = False
2936
        self.createdBranches = set()
2937
        self.committedChanges = set()
2938
        self.branch = ""
2939
        self.detectBranches = False
2940
        self.detectLabels = False
2941
        self.importLabels = False
2942
        self.changesFile = ""
2943
        self.syncWithOrigin = True
2944
        self.importIntoRemotes = True
2945
        self.maxChanges = ""
2946
        self.changes_block_size = None
2947
        self.keepRepoPath = False
2948
        self.depotPaths = None
2949
        self.p4BranchesInGit = []
2950
        self.cloneExclude = []
2951
        self.useClientSpec = False
2952
        self.useClientSpec_from_options = False
2953
        self.clientSpecDirs = None
2954
        self.tempBranches = []
2955
        self.tempBranchLocation = "refs/git-p4-tmp"
2956
        self.largeFileSystem = None
2957
        self.suppress_meta_comment = False
2958

2959
        if gitConfig('git-p4.largeFileSystem'):
2960
            largeFileSystemConstructor = globals()[gitConfig('git-p4.largeFileSystem')]
2961
            self.largeFileSystem = largeFileSystemConstructor(
2962
                lambda git_mode, relPath, contents: self.writeToGitStream(git_mode, relPath, contents)
2963
            )
2964

2965
        if gitConfig("git-p4.syncFromOrigin") == "false":
2966
            self.syncWithOrigin = False
2967

2968
        self.depotPaths = []
2969
        self.changeRange = ""
2970
        self.previousDepotPaths = []
2971
        self.hasOrigin = False
2972

2973
        # map from branch depot path to parent branch
2974
        self.knownBranches = {}
2975
        self.initialParents = {}
2976

2977
        self.tz = "%+03d%02d" % (- time.timezone / 3600, ((- time.timezone % 3600) / 60))
2978
        self.labels = {}
2979

2980
    def checkpoint(self):
2981
        """Force a checkpoint in fast-import and wait for it to finish."""
2982
        self.gitStream.write("checkpoint\n\n")
2983
        self.gitStream.write("progress checkpoint\n\n")
2984
        self.gitStream.flush()
2985
        out = self.gitOutput.readline()
2986
        if self.verbose:
2987
            print("checkpoint finished: " + out)
2988

2989
    def isPathWanted(self, path):
2990
        for p in self.cloneExclude:
2991
            if p.endswith("/"):
2992
                if p4PathStartsWith(path, p):
2993
                    return False
2994
            # "-//depot/file1" without a trailing "/" should only exclude "file1", but not "file111" or "file1_dir/file2"
2995
            elif path.lower() == p.lower():
2996
                return False
2997
        for p in self.depotPaths:
2998
            if p4PathStartsWith(path, decode_path(p)):
2999
                return True
3000
        return False
3001

3002
    def extractFilesFromCommit(self, commit, shelved=False, shelved_cl=0):
3003
        files = []
3004
        fnum = 0
3005
        while "depotFile%s" % fnum in commit:
3006
            path = commit["depotFile%s" % fnum]
3007
            found = self.isPathWanted(decode_path(path))
3008
            if not found:
3009
                fnum = fnum + 1
3010
                continue
3011

3012
            file = {}
3013
            file["path"] = path
3014
            file["rev"] = commit["rev%s" % fnum]
3015
            file["action"] = commit["action%s" % fnum]
3016
            file["type"] = commit["type%s" % fnum]
3017
            if shelved:
3018
                file["shelved_cl"] = int(shelved_cl)
3019
            files.append(file)
3020
            fnum = fnum + 1
3021
        return files
3022

3023
    def extractJobsFromCommit(self, commit):
3024
        jobs = []
3025
        jnum = 0
3026
        while "job%s" % jnum in commit:
3027
            job = commit["job%s" % jnum]
3028
            jobs.append(job)
3029
            jnum = jnum + 1
3030
        return jobs
3031

3032
    def stripRepoPath(self, path, prefixes):
3033
        """When streaming files, this is called to map a p4 depot path to where
3034
           it should go in git.  The prefixes are either self.depotPaths, or
3035
           self.branchPrefixes in the case of branch detection.
3036
           """
3037

3038
        if self.useClientSpec:
3039
            # branch detection moves files up a level (the branch name)
3040
            # from what client spec interpretation gives
3041
            path = decode_path(self.clientSpecDirs.map_in_client(path))
3042
            if self.detectBranches:
3043
                for b in self.knownBranches:
3044
                    if p4PathStartsWith(path, b + "/"):
3045
                        path = path[len(b)+1:]
3046

3047
        elif self.keepRepoPath:
3048
            # Preserve everything in relative path name except leading
3049
            # //depot/; just look at first prefix as they all should
3050
            # be in the same depot.
3051
            depot = re.sub(r"^(//[^/]+/).*", r'\1', prefixes[0])
3052
            if p4PathStartsWith(path, depot):
3053
                path = path[len(depot):]
3054

3055
        else:
3056
            for p in prefixes:
3057
                if p4PathStartsWith(path, p):
3058
                    path = path[len(p):]
3059
                    break
3060

3061
        path = wildcard_decode(path)
3062
        return path
3063

3064
    def splitFilesIntoBranches(self, commit):
3065
        """Look at each depotFile in the commit to figure out to what branch it
3066
           belongs.
3067
           """
3068

3069
        if self.clientSpecDirs:
3070
            files = self.extractFilesFromCommit(commit)
3071
            self.clientSpecDirs.update_client_spec_path_cache(files)
3072

3073
        branches = {}
3074
        fnum = 0
3075
        while "depotFile%s" % fnum in commit:
3076
            raw_path = commit["depotFile%s" % fnum]
3077
            path = decode_path(raw_path)
3078
            found = self.isPathWanted(path)
3079
            if not found:
3080
                fnum = fnum + 1
3081
                continue
3082

3083
            file = {}
3084
            file["path"] = raw_path
3085
            file["rev"] = commit["rev%s" % fnum]
3086
            file["action"] = commit["action%s" % fnum]
3087
            file["type"] = commit["type%s" % fnum]
3088
            fnum = fnum + 1
3089

3090
            # start with the full relative path where this file would
3091
            # go in a p4 client
3092
            if self.useClientSpec:
3093
                relPath = decode_path(self.clientSpecDirs.map_in_client(path))
3094
            else:
3095
                relPath = self.stripRepoPath(path, self.depotPaths)
3096

3097
            for branch in self.knownBranches.keys():
3098
                # add a trailing slash so that a commit into qt/4.2foo
3099
                # doesn't end up in qt/4.2, e.g.
3100
                if p4PathStartsWith(relPath, branch + "/"):
3101
                    if branch not in branches:
3102
                        branches[branch] = []
3103
                    branches[branch].append(file)
3104
                    break
3105

3106
        return branches
3107

3108
    def writeToGitStream(self, gitMode, relPath, contents):
3109
        self.gitStream.write(encode_text_stream(u'M {} inline {}\n'.format(gitMode, relPath)))
3110
        self.gitStream.write('data %d\n' % sum(len(d) for d in contents))
3111
        for d in contents:
3112
            self.gitStream.write(d)
3113
        self.gitStream.write('\n')
3114

3115
    def encodeWithUTF8(self, path):
3116
        try:
3117
            path.decode('ascii')
3118
        except:
3119
            encoding = 'utf8'
3120
            if gitConfig('git-p4.pathEncoding'):
3121
                encoding = gitConfig('git-p4.pathEncoding')
3122
            path = path.decode(encoding, 'replace').encode('utf8', 'replace')
3123
            if self.verbose:
3124
                print('Path with non-ASCII characters detected. Used %s to encode: %s ' % (encoding, path))
3125
        return path
3126

3127
    def streamOneP4File(self, file, contents):
3128
        """Output one file from the P4 stream.
3129

3130
           This is a helper for streamP4Files().
3131
           """
3132

3133
        file_path = file['depotFile']
3134
        relPath = self.stripRepoPath(decode_path(file_path), self.branchPrefixes)
3135

3136
        if verbose:
3137
            if 'fileSize' in self.stream_file:
3138
                size = int(self.stream_file['fileSize'])
3139
            else:
3140
                # Deleted files don't get a fileSize apparently
3141
                size = 0
3142
            sys.stdout.write('\r%s --> %s (%s)\n' % (
3143
                file_path, relPath, format_size_human_readable(size)))
3144
            sys.stdout.flush()
3145

3146
        type_base, type_mods = split_p4_type(file["type"])
3147

3148
        git_mode = "100644"
3149
        if "x" in type_mods:
3150
            git_mode = "100755"
3151
        if type_base == "symlink":
3152
            git_mode = "120000"
3153
            # p4 print on a symlink sometimes contains "target\n";
3154
            # if it does, remove the newline
3155
            data = ''.join(decode_text_stream(c) for c in contents)
3156
            if not data:
3157
                # Some version of p4 allowed creating a symlink that pointed
3158
                # to nothing.  This causes p4 errors when checking out such
3159
                # a change, and errors here too.  Work around it by ignoring
3160
                # the bad symlink; hopefully a future change fixes it.
3161
                print("\nIgnoring empty symlink in %s" % file_path)
3162
                return
3163
            elif data[-1] == '\n':
3164
                contents = [data[:-1]]
3165
            else:
3166
                contents = [data]
3167

3168
        if type_base == "utf16":
3169
            # p4 delivers different text in the python output to -G
3170
            # than it does when using "print -o", or normal p4 client
3171
            # operations.  utf16 is converted to ascii or utf8, perhaps.
3172
            # But ascii text saved as -t utf16 is completely mangled.
3173
            # Invoke print -o to get the real contents.
3174
            #
3175
            # On windows, the newlines will always be mangled by print, so put
3176
            # them back too.  This is not needed to the cygwin windows version,
3177
            # just the native "NT" type.
3178
            #
3179
            try:
3180
                text = p4_read_pipe(['print', '-q', '-o', '-', '%s@%s' % (decode_path(file['depotFile']), file['change'])], raw=True)
3181
            except Exception as e:
3182
                if 'Translation of file content failed' in str(e):
3183
                    type_base = 'binary'
3184
                else:
3185
                    raise e
3186
            else:
3187
                if p4_version_string().find('/NT') >= 0:
3188
                    text = text.replace(b'\x0d\x00\x0a\x00', b'\x0a\x00')
3189
                contents = [text]
3190

3191
        if type_base == "apple":
3192
            # Apple filetype files will be streamed as a concatenation of
3193
            # its appledouble header and the contents.  This is useless
3194
            # on both macs and non-macs.  If using "print -q -o xx", it
3195
            # will create "xx" with the data, and "%xx" with the header.
3196
            # This is also not very useful.
3197
            #
3198
            # Ideally, someday, this script can learn how to generate
3199
            # appledouble files directly and import those to git, but
3200
            # non-mac machines can never find a use for apple filetype.
3201
            print("\nIgnoring apple filetype file %s" % file['depotFile'])
3202
            return
3203

3204
        if type_base == "utf8":
3205
            # The type utf8 explicitly means utf8 *with BOM*. These are
3206
            # streamed just like regular text files, however, without
3207
            # the BOM in the stream.
3208
            # Therefore, to accurately import these files into git, we
3209
            # need to explicitly re-add the BOM before writing.
3210
            # 'contents' is a set of bytes in this case, so create the
3211
            # BOM prefix as a b'' literal.
3212
            contents = [b'\xef\xbb\xbf' + contents[0]] + contents[1:]
3213

3214
        # Note that we do not try to de-mangle keywords on utf16 files,
3215
        # even though in theory somebody may want that.
3216
        regexp = p4_keywords_regexp_for_type(type_base, type_mods)
3217
        if regexp:
3218
            contents = [regexp.sub(br'$\1$', c) for c in contents]
3219

3220
        if self.largeFileSystem:
3221
            git_mode, contents = self.largeFileSystem.processContent(git_mode, relPath, contents)
3222

3223
        self.writeToGitStream(git_mode, relPath, contents)
3224

3225
    def streamOneP4Deletion(self, file):
3226
        relPath = self.stripRepoPath(decode_path(file['path']), self.branchPrefixes)
3227
        if verbose:
3228
            sys.stdout.write("delete %s\n" % relPath)
3229
            sys.stdout.flush()
3230
        self.gitStream.write(encode_text_stream(u'D {}\n'.format(relPath)))
3231

3232
        if self.largeFileSystem and self.largeFileSystem.isLargeFile(relPath):
3233
            self.largeFileSystem.removeLargeFile(relPath)
3234

3235
    def streamP4FilesCb(self, marshalled):
3236
        """Handle another chunk of streaming data."""
3237

3238
        # catch p4 errors and complain
3239
        err = None
3240
        if "code" in marshalled:
3241
            if marshalled["code"] == "error":
3242
                if "data" in marshalled:
3243
                    err = marshalled["data"].rstrip()
3244

3245
        if not err and 'fileSize' in self.stream_file:
3246
            required_bytes = int((4 * int(self.stream_file["fileSize"])) - calcDiskFree())
3247
            if required_bytes > 0:
3248
                err = 'Not enough space left on %s! Free at least %s.' % (
3249
                    os.getcwd(), format_size_human_readable(required_bytes))
3250

3251
        if err:
3252
            f = None
3253
            if self.stream_have_file_info:
3254
                if "depotFile" in self.stream_file:
3255
                    f = self.stream_file["depotFile"]
3256
            try:
3257
                # force a failure in fast-import, else an empty
3258
                # commit will be made
3259
                self.gitStream.write("\n")
3260
                self.gitStream.write("die-now\n")
3261
                self.gitStream.close()
3262
                # ignore errors, but make sure it exits first
3263
                self.importProcess.wait()
3264
            finally:
3265
                if f:
3266
                    die("Error from p4 print for %s: %s" % (f, err))
3267
                else:
3268
                    die("Error from p4 print: %s" % err)
3269

3270
        if 'depotFile' in marshalled and self.stream_have_file_info:
3271
            # start of a new file - output the old one first
3272
            self.streamOneP4File(self.stream_file, self.stream_contents)
3273
            self.stream_file = {}
3274
            self.stream_contents = []
3275
            self.stream_have_file_info = False
3276

3277
        # pick up the new file information... for the
3278
        # 'data' field we need to append to our array
3279
        for k in marshalled.keys():
3280
            if k == 'data':
3281
                if 'streamContentSize' not in self.stream_file:
3282
                    self.stream_file['streamContentSize'] = 0
3283
                self.stream_file['streamContentSize'] += len(marshalled['data'])
3284
                self.stream_contents.append(marshalled['data'])
3285
            else:
3286
                self.stream_file[k] = marshalled[k]
3287

3288
        if (verbose and
3289
                'streamContentSize' in self.stream_file and
3290
                'fileSize' in self.stream_file and
3291
                'depotFile' in self.stream_file):
3292
            size = int(self.stream_file["fileSize"])
3293
            if size > 0:
3294
                progress = 100*self.stream_file['streamContentSize']/size
3295
                sys.stdout.write('\r%s %d%% (%s)' % (
3296
                    self.stream_file['depotFile'], progress,
3297
                    format_size_human_readable(size)))
3298
                sys.stdout.flush()
3299

3300
        self.stream_have_file_info = True
3301

3302
    def streamP4Files(self, files):
3303
        """Stream directly from "p4 files" into "git fast-import."""
3304

3305
        filesForCommit = []
3306
        filesToRead = []
3307
        filesToDelete = []
3308

3309
        for f in files:
3310
            filesForCommit.append(f)
3311
            if f['action'] in self.delete_actions:
3312
                filesToDelete.append(f)
3313
            else:
3314
                filesToRead.append(f)
3315

3316
        # deleted files...
3317
        for f in filesToDelete:
3318
            self.streamOneP4Deletion(f)
3319

3320
        if len(filesToRead) > 0:
3321
            self.stream_file = {}
3322
            self.stream_contents = []
3323
            self.stream_have_file_info = False
3324

3325
            # curry self argument
3326
            def streamP4FilesCbSelf(entry):
3327
                self.streamP4FilesCb(entry)
3328

3329
            fileArgs = []
3330
            for f in filesToRead:
3331
                if 'shelved_cl' in f:
3332
                    # Handle shelved CLs using the "p4 print file@=N" syntax to print
3333
                    # the contents
3334
                    fileArg = f['path'] + encode_text_stream('@={}'.format(f['shelved_cl']))
3335
                else:
3336
                    fileArg = f['path'] + encode_text_stream('#{}'.format(f['rev']))
3337

3338
                fileArgs.append(fileArg)
3339

3340
            p4CmdList(["-x", "-", "print"],
3341
                      stdin=fileArgs,
3342
                      cb=streamP4FilesCbSelf)
3343

3344
            # do the last chunk
3345
            if 'depotFile' in self.stream_file:
3346
                self.streamOneP4File(self.stream_file, self.stream_contents)
3347

3348
    def make_email(self, userid):
3349
        if userid in self.users:
3350
            return self.users[userid]
3351
        else:
3352
            userid_bytes = metadata_stream_to_writable_bytes(userid)
3353
            return b"%s <a@b>" % userid_bytes
3354

3355
    def streamTag(self, gitStream, labelName, labelDetails, commit, epoch):
3356
        """Stream a p4 tag.
3357

3358
           Commit is either a git commit, or a fast-import mark, ":<p4commit>".
3359
           """
3360

3361
        if verbose:
3362
            print("writing tag %s for commit %s" % (labelName, commit))
3363
        gitStream.write("tag %s\n" % labelName)
3364
        gitStream.write("from %s\n" % commit)
3365

3366
        if 'Owner' in labelDetails:
3367
            owner = labelDetails["Owner"]
3368
        else:
3369
            owner = None
3370

3371
        # Try to use the owner of the p4 label, or failing that,
3372
        # the current p4 user id.
3373
        if owner:
3374
            email = self.make_email(owner)
3375
        else:
3376
            email = self.make_email(self.p4UserId())
3377

3378
        gitStream.write("tagger ")
3379
        gitStream.write(email)
3380
        gitStream.write(" %s %s\n" % (epoch, self.tz))
3381

3382
        print("labelDetails=", labelDetails)
3383
        if 'Description' in labelDetails:
3384
            description = labelDetails['Description']
3385
        else:
3386
            description = 'Label from git p4'
3387

3388
        gitStream.write("data %d\n" % len(description))
3389
        gitStream.write(description)
3390
        gitStream.write("\n")
3391

3392
    def inClientSpec(self, path):
3393
        if not self.clientSpecDirs:
3394
            return True
3395
        inClientSpec = self.clientSpecDirs.map_in_client(path)
3396
        if not inClientSpec and self.verbose:
3397
            print('Ignoring file outside of client spec: {0}'.format(path))
3398
        return inClientSpec
3399

3400
    def hasBranchPrefix(self, path):
3401
        if not self.branchPrefixes:
3402
            return True
3403
        hasPrefix = [p for p in self.branchPrefixes
3404
                        if p4PathStartsWith(path, p)]
3405
        if not hasPrefix and self.verbose:
3406
            print('Ignoring file outside of prefix: {0}'.format(path))
3407
        return hasPrefix
3408

3409
    def findShadowedFiles(self, files, change):
3410
        """Perforce allows you commit files and directories with the same name,
3411
           so you could have files //depot/foo and //depot/foo/bar both checked
3412
           in.  A p4 sync of a repository in this state fails.  Deleting one of
3413
           the files recovers the repository.
3414

3415
           Git will not allow the broken state to exist and only the most
3416
           recent of the conflicting names is left in the repository.  When one
3417
           of the conflicting files is deleted we need to re-add the other one
3418
           to make sure the git repository recovers in the same way as
3419
           perforce.
3420
           """
3421

3422
        deleted = [f for f in files if f['action'] in self.delete_actions]
3423
        to_check = set()
3424
        for f in deleted:
3425
            path = decode_path(f['path'])
3426
            to_check.add(path + '/...')
3427
            while True:
3428
                path = path.rsplit("/", 1)[0]
3429
                if path == "/" or path in to_check:
3430
                    break
3431
                to_check.add(path)
3432
        to_check = ['%s@%s' % (wildcard_encode(p), change) for p in to_check
3433
            if self.hasBranchPrefix(p)]
3434
        if to_check:
3435
            stat_result = p4CmdList(["-x", "-", "fstat", "-T",
3436
                "depotFile,headAction,headRev,headType"], stdin=to_check)
3437
            for record in stat_result:
3438
                if record['code'] != 'stat':
3439
                    continue
3440
                if record['headAction'] in self.delete_actions:
3441
                    continue
3442
                files.append({
3443
                    'action': 'add',
3444
                    'path': record['depotFile'],
3445
                    'rev': record['headRev'],
3446
                    'type': record['headType']})
3447

3448
    def commit(self, details, files, branch, parent="", allow_empty=False):
3449
        epoch = details["time"]
3450
        author = details["user"]
3451
        jobs = self.extractJobsFromCommit(details)
3452

3453
        if self.verbose:
3454
            print('commit into {0}'.format(branch))
3455

3456
        files = [f for f in files
3457
            if self.hasBranchPrefix(decode_path(f['path']))]
3458
        self.findShadowedFiles(files, details['change'])
3459

3460
        if self.clientSpecDirs:
3461
            self.clientSpecDirs.update_client_spec_path_cache(files)
3462

3463
        files = [f for f in files if self.inClientSpec(decode_path(f['path']))]
3464

3465
        if gitConfigBool('git-p4.keepEmptyCommits'):
3466
            allow_empty = True
3467

3468
        if not files and not allow_empty:
3469
            print('Ignoring revision {0} as it would produce an empty commit.'
3470
                .format(details['change']))
3471
            return
3472

3473
        self.gitStream.write("commit %s\n" % branch)
3474
        self.gitStream.write("mark :%s\n" % details["change"])
3475
        self.committedChanges.add(int(details["change"]))
3476
        if author not in self.users:
3477
            self.getUserMapFromPerforceServer()
3478

3479
        self.gitStream.write("committer ")
3480
        self.gitStream.write(self.make_email(author))
3481
        self.gitStream.write(" %s %s\n" % (epoch, self.tz))
3482

3483
        self.gitStream.write("data <<EOT\n")
3484
        self.gitStream.write(details["desc"])
3485
        if len(jobs) > 0:
3486
            self.gitStream.write("\nJobs: %s" % (' '.join(jobs)))
3487

3488
        if not self.suppress_meta_comment:
3489
            self.gitStream.write("\n[git-p4: depot-paths = \"%s\": change = %s" %
3490
                                (','.join(self.branchPrefixes), details["change"]))
3491
            if len(details['options']) > 0:
3492
                self.gitStream.write(": options = %s" % details['options'])
3493
            self.gitStream.write("]\n")
3494

3495
        self.gitStream.write("EOT\n\n")
3496

3497
        if len(parent) > 0:
3498
            if self.verbose:
3499
                print("parent %s" % parent)
3500
            self.gitStream.write("from %s\n" % parent)
3501

3502
        self.streamP4Files(files)
3503
        self.gitStream.write("\n")
3504

3505
        change = int(details["change"])
3506

3507
        if change in self.labels:
3508
            label = self.labels[change]
3509
            labelDetails = label[0]
3510
            labelRevisions = label[1]
3511
            if self.verbose:
3512
                print("Change %s is labelled %s" % (change, labelDetails))
3513

3514
            files = p4CmdList(["files"] + ["%s...@%s" % (p, change)
3515
                                                for p in self.branchPrefixes])
3516

3517
            if len(files) == len(labelRevisions):
3518

3519
                cleanedFiles = {}
3520
                for info in files:
3521
                    if info["action"] in self.delete_actions:
3522
                        continue
3523
                    cleanedFiles[info["depotFile"]] = info["rev"]
3524

3525
                if cleanedFiles == labelRevisions:
3526
                    self.streamTag(self.gitStream, 'tag_%s' % labelDetails['label'], labelDetails, branch, epoch)
3527

3528
                else:
3529
                    if not self.silent:
3530
                        print("Tag %s does not match with change %s: files do not match."
3531
                               % (labelDetails["label"], change))
3532

3533
            else:
3534
                if not self.silent:
3535
                    print("Tag %s does not match with change %s: file count is different."
3536
                           % (labelDetails["label"], change))
3537

3538
    def getLabels(self):
3539
        """Build a dictionary of changelists and labels, for "detect-labels"
3540
           option.
3541
           """
3542

3543
        self.labels = {}
3544

3545
        l = p4CmdList(["labels"] + ["%s..." % p for p in self.depotPaths])
3546
        if len(l) > 0 and not self.silent:
3547
            print("Finding files belonging to labels in %s" % self.depotPaths)
3548

3549
        for output in l:
3550
            label = output["label"]
3551
            revisions = {}
3552
            newestChange = 0
3553
            if self.verbose:
3554
                print("Querying files for label %s" % label)
3555
            for file in p4CmdList(["files"] +
3556
                                      ["%s...@%s" % (p, label)
3557
                                          for p in self.depotPaths]):
3558
                revisions[file["depotFile"]] = file["rev"]
3559
                change = int(file["change"])
3560
                if change > newestChange:
3561
                    newestChange = change
3562

3563
            self.labels[newestChange] = [output, revisions]
3564

3565
        if self.verbose:
3566
            print("Label changes: %s" % self.labels.keys())
3567

3568
    def importP4Labels(self, stream, p4Labels):
3569
        """Import p4 labels as git tags. A direct mapping does not exist, so
3570
           assume that if all the files are at the same revision then we can
3571
           use that, or it's something more complicated we should just ignore.
3572
           """
3573

3574
        if verbose:
3575
            print("import p4 labels: " + ' '.join(p4Labels))
3576

3577
        ignoredP4Labels = gitConfigList("git-p4.ignoredP4Labels")
3578
        validLabelRegexp = gitConfig("git-p4.labelImportRegexp")
3579
        if len(validLabelRegexp) == 0:
3580
            validLabelRegexp = defaultLabelRegexp
3581
        m = re.compile(validLabelRegexp)
3582

3583
        for name in p4Labels:
3584
            commitFound = False
3585

3586
            if not m.match(name):
3587
                if verbose:
3588
                    print("label %s does not match regexp %s" % (name, validLabelRegexp))
3589
                continue
3590

3591
            if name in ignoredP4Labels:
3592
                continue
3593

3594
            labelDetails = p4CmdList(['label', "-o", name])[0]
3595

3596
            # get the most recent changelist for each file in this label
3597
            change = p4Cmd(["changes", "-m", "1"] + ["%s...@%s" % (p, name)
3598
                                for p in self.depotPaths])
3599

3600
            if 'change' in change:
3601
                # find the corresponding git commit; take the oldest commit
3602
                changelist = int(change['change'])
3603
                if changelist in self.committedChanges:
3604
                    gitCommit = ":%d" % changelist       # use a fast-import mark
3605
                    commitFound = True
3606
                else:
3607
                    gitCommit = read_pipe(["git", "rev-list", "--max-count=1",
3608
                        "--reverse", r":/\[git-p4:.*change = %d\]" % changelist], ignore_error=True)
3609
                    if len(gitCommit) == 0:
3610
                        print("importing label %s: could not find git commit for changelist %d" % (name, changelist))
3611
                    else:
3612
                        commitFound = True
3613
                        gitCommit = gitCommit.strip()
3614

3615
                if commitFound:
3616
                    # Convert from p4 time format
3617
                    try:
3618
                        tmwhen = time.strptime(labelDetails['Update'], "%Y/%m/%d %H:%M:%S")
3619
                    except ValueError:
3620
                        print("Could not convert label time %s" % labelDetails['Update'])
3621
                        tmwhen = 1
3622

3623
                    when = int(time.mktime(tmwhen))
3624
                    self.streamTag(stream, name, labelDetails, gitCommit, when)
3625
                    if verbose:
3626
                        print("p4 label %s mapped to git commit %s" % (name, gitCommit))
3627
            else:
3628
                if verbose:
3629
                    print("Label %s has no changelists - possibly deleted?" % name)
3630

3631
            if not commitFound:
3632
                # We can't import this label; don't try again as it will get very
3633
                # expensive repeatedly fetching all the files for labels that will
3634
                # never be imported. If the label is moved in the future, the
3635
                # ignore will need to be removed manually.
3636
                system(["git", "config", "--add", "git-p4.ignoredP4Labels", name])
3637

3638
    def guessProjectName(self):
3639
        for p in self.depotPaths:
3640
            if p.endswith("/"):
3641
                p = p[:-1]
3642
            p = p[p.strip().rfind("/") + 1:]
3643
            if not p.endswith("/"):
3644
                p += "/"
3645
            return p
3646

3647
    def getBranchMapping(self):
3648
        lostAndFoundBranches = set()
3649

3650
        user = gitConfig("git-p4.branchUser")
3651

3652
        for info in p4CmdList(
3653
            ["branches"] + (["-u", user] if len(user) > 0 else [])):
3654
            details = p4Cmd(["branch", "-o", info["branch"]])
3655
            viewIdx = 0
3656
            while "View%s" % viewIdx in details:
3657
                paths = details["View%s" % viewIdx].split(" ")
3658
                viewIdx = viewIdx + 1
3659
                # require standard //depot/foo/... //depot/bar/... mapping
3660
                if len(paths) != 2 or not paths[0].endswith("/...") or not paths[1].endswith("/..."):
3661
                    continue
3662
                source = paths[0]
3663
                destination = paths[1]
3664
                # HACK
3665
                if p4PathStartsWith(source, self.depotPaths[0]) and p4PathStartsWith(destination, self.depotPaths[0]):
3666
                    source = source[len(self.depotPaths[0]):-4]
3667
                    destination = destination[len(self.depotPaths[0]):-4]
3668

3669
                    if destination in self.knownBranches:
3670
                        if not self.silent:
3671
                            print("p4 branch %s defines a mapping from %s to %s" % (info["branch"], source, destination))
3672
                            print("but there exists another mapping from %s to %s already!" % (self.knownBranches[destination], destination))
3673
                        continue
3674

3675
                    self.knownBranches[destination] = source
3676

3677
                    lostAndFoundBranches.discard(destination)
3678

3679
                    if source not in self.knownBranches:
3680
                        lostAndFoundBranches.add(source)
3681

3682
        # Perforce does not strictly require branches to be defined, so we also
3683
        # check git config for a branch list.
3684
        #
3685
        # Example of branch definition in git config file:
3686
        # [git-p4]
3687
        #   branchList=main:branchA
3688
        #   branchList=main:branchB
3689
        #   branchList=branchA:branchC
3690
        configBranches = gitConfigList("git-p4.branchList")
3691
        for branch in configBranches:
3692
            if branch:
3693
                source, destination = branch.split(":")
3694
                self.knownBranches[destination] = source
3695

3696
                lostAndFoundBranches.discard(destination)
3697

3698
                if source not in self.knownBranches:
3699
                    lostAndFoundBranches.add(source)
3700

3701
        for branch in lostAndFoundBranches:
3702
            self.knownBranches[branch] = branch
3703

3704
    def getBranchMappingFromGitBranches(self):
3705
        branches = p4BranchesInGit(self.importIntoRemotes)
3706
        for branch in branches.keys():
3707
            if branch == "master":
3708
                branch = "main"
3709
            else:
3710
                branch = branch[len(self.projectName):]
3711
            self.knownBranches[branch] = branch
3712

3713
    def updateOptionDict(self, d):
3714
        option_keys = {}
3715
        if self.keepRepoPath:
3716
            option_keys['keepRepoPath'] = 1
3717

3718
        d["options"] = ' '.join(sorted(option_keys.keys()))
3719

3720
    def readOptions(self, d):
3721
        self.keepRepoPath = ('options' in d
3722
                             and ('keepRepoPath' in d['options']))
3723

3724
    def gitRefForBranch(self, branch):
3725
        if branch == "main":
3726
            return self.refPrefix + "master"
3727

3728
        if len(branch) <= 0:
3729
            return branch
3730

3731
        return self.refPrefix + self.projectName + branch
3732

3733
    def gitCommitByP4Change(self, ref, change):
3734
        if self.verbose:
3735
            print("looking in ref " + ref + " for change %s using bisect..." % change)
3736

3737
        earliestCommit = ""
3738
        latestCommit = parseRevision(ref)
3739

3740
        while True:
3741
            if self.verbose:
3742
                print("trying: earliest %s latest %s" % (earliestCommit, latestCommit))
3743
            next = read_pipe(["git", "rev-list", "--bisect",
3744
                latestCommit, earliestCommit]).strip()
3745
            if len(next) == 0:
3746
                if self.verbose:
3747
                    print("argh")
3748
                return ""
3749
            log = extractLogMessageFromGitCommit(next)
3750
            settings = extractSettingsGitLog(log)
3751
            currentChange = int(settings['change'])
3752
            if self.verbose:
3753
                print("current change %s" % currentChange)
3754

3755
            if currentChange == change:
3756
                if self.verbose:
3757
                    print("found %s" % next)
3758
                return next
3759

3760
            if currentChange < change:
3761
                earliestCommit = "^%s" % next
3762
            else:
3763
                if next == latestCommit:
3764
                    die("Infinite loop while looking in ref %s for change %s. Check your branch mappings" % (ref, change))
3765
                latestCommit = "%s^@" % next
3766

3767
        return ""
3768

3769
    def importNewBranch(self, branch, maxChange):
3770
        # make fast-import flush all changes to disk and update the refs using the checkpoint
3771
        # command so that we can try to find the branch parent in the git history
3772
        self.gitStream.write("checkpoint\n\n")
3773
        self.gitStream.flush()
3774
        branchPrefix = self.depotPaths[0] + branch + "/"
3775
        range = "@1,%s" % maxChange
3776
        changes = p4ChangesForPaths([branchPrefix], range, self.changes_block_size)
3777
        if len(changes) <= 0:
3778
            return False
3779
        firstChange = changes[0]
3780
        sourceBranch = self.knownBranches[branch]
3781
        sourceDepotPath = self.depotPaths[0] + sourceBranch
3782
        sourceRef = self.gitRefForBranch(sourceBranch)
3783

3784
        branchParentChange = int(p4Cmd(["changes", "-m", "1", "%s...@1,%s" % (sourceDepotPath, firstChange)])["change"])
3785
        gitParent = self.gitCommitByP4Change(sourceRef, branchParentChange)
3786
        if len(gitParent) > 0:
3787
            self.initialParents[self.gitRefForBranch(branch)] = gitParent
3788

3789
        self.importChanges(changes)
3790
        return True
3791

3792
    def searchParent(self, parent, branch, target):
3793
        targetTree = read_pipe(["git", "rev-parse",
3794
                                "{}^{{tree}}".format(target)]).strip()
3795
        for line in read_pipe_lines(["git", "rev-list", "--format=%H %T",
3796
                                     "--no-merges", parent]):
3797
            if line.startswith("commit "):
3798
                continue
3799
            commit, tree = line.strip().split(" ")
3800
            if tree == targetTree:
3801
                if self.verbose:
3802
                    print("Found parent of %s in commit %s" % (branch, commit))
3803
                return commit
3804
        return None
3805

3806
    def importChanges(self, changes, origin_revision=0):
3807
        cnt = 1
3808
        for change in changes:
3809
            description = p4_describe(change)
3810
            self.updateOptionDict(description)
3811

3812
            if not self.silent:
3813
                sys.stdout.write("\rImporting revision %s (%d%%)" % (
3814
                    change, (cnt * 100) // len(changes)))
3815
                sys.stdout.flush()
3816
            cnt = cnt + 1
3817

3818
            try:
3819
                if self.detectBranches:
3820
                    branches = self.splitFilesIntoBranches(description)
3821
                    for branch in branches.keys():
3822
                        # HACK  --hwn
3823
                        branchPrefix = self.depotPaths[0] + branch + "/"
3824
                        self.branchPrefixes = [branchPrefix]
3825

3826
                        parent = ""
3827

3828
                        filesForCommit = branches[branch]
3829

3830
                        if self.verbose:
3831
                            print("branch is %s" % branch)
3832

3833
                        self.updatedBranches.add(branch)
3834

3835
                        if branch not in self.createdBranches:
3836
                            self.createdBranches.add(branch)
3837
                            parent = self.knownBranches[branch]
3838
                            if parent == branch:
3839
                                parent = ""
3840
                            else:
3841
                                fullBranch = self.projectName + branch
3842
                                if fullBranch not in self.p4BranchesInGit:
3843
                                    if not self.silent:
3844
                                        print("\n    Importing new branch %s" % fullBranch)
3845
                                    if self.importNewBranch(branch, change - 1):
3846
                                        parent = ""
3847
                                        self.p4BranchesInGit.append(fullBranch)
3848
                                    if not self.silent:
3849
                                        print("\n    Resuming with change %s" % change)
3850

3851
                                if self.verbose:
3852
                                    print("parent determined through known branches: %s" % parent)
3853

3854
                        branch = self.gitRefForBranch(branch)
3855
                        parent = self.gitRefForBranch(parent)
3856

3857
                        if self.verbose:
3858
                            print("looking for initial parent for %s; current parent is %s" % (branch, parent))
3859

3860
                        if len(parent) == 0 and branch in self.initialParents:
3861
                            parent = self.initialParents[branch]
3862
                            del self.initialParents[branch]
3863

3864
                        blob = None
3865
                        if len(parent) > 0:
3866
                            tempBranch = "%s/%d" % (self.tempBranchLocation, change)
3867
                            if self.verbose:
3868
                                print("Creating temporary branch: " + tempBranch)
3869
                            self.commit(description, filesForCommit, tempBranch)
3870
                            self.tempBranches.append(tempBranch)
3871
                            self.checkpoint()
3872
                            blob = self.searchParent(parent, branch, tempBranch)
3873
                        if blob:
3874
                            self.commit(description, filesForCommit, branch, blob)
3875
                        else:
3876
                            if self.verbose:
3877
                                print("Parent of %s not found. Committing into head of %s" % (branch, parent))
3878
                            self.commit(description, filesForCommit, branch, parent)
3879
                else:
3880
                    files = self.extractFilesFromCommit(description)
3881
                    self.commit(description, files, self.branch,
3882
                                self.initialParent)
3883
                    # only needed once, to connect to the previous commit
3884
                    self.initialParent = ""
3885
            except IOError:
3886
                print(self.gitError.read())
3887
                sys.exit(1)
3888

3889
    def sync_origin_only(self):
3890
        if self.syncWithOrigin:
3891
            self.hasOrigin = originP4BranchesExist()
3892
            if self.hasOrigin:
3893
                if not self.silent:
3894
                    print('Syncing with origin first, using "git fetch origin"')
3895
                system(["git", "fetch", "origin"])
3896

3897
    def importHeadRevision(self, revision):
3898
        print("Doing initial import of %s from revision %s into %s" % (' '.join(self.depotPaths), revision, self.branch))
3899

3900
        details = {}
3901
        details["user"] = "git perforce import user"
3902
        details["desc"] = ("Initial import of %s from the state at revision %s\n"
3903
                           % (' '.join(self.depotPaths), revision))
3904
        details["change"] = revision
3905
        newestRevision = 0
3906

3907
        fileCnt = 0
3908
        fileArgs = ["%s...%s" % (p, revision) for p in self.depotPaths]
3909

3910
        for info in p4CmdList(["files"] + fileArgs):
3911

3912
            if 'code' in info and info['code'] == 'error':
3913
                sys.stderr.write("p4 returned an error: %s\n"
3914
                                 % info['data'])
3915
                if info['data'].find("must refer to client") >= 0:
3916
                    sys.stderr.write("This particular p4 error is misleading.\n")
3917
                    sys.stderr.write("Perhaps the depot path was misspelled.\n")
3918
                    sys.stderr.write("Depot path:  %s\n" % " ".join(self.depotPaths))
3919
                sys.exit(1)
3920
            if 'p4ExitCode' in info:
3921
                sys.stderr.write("p4 exitcode: %s\n" % info['p4ExitCode'])
3922
                sys.exit(1)
3923

3924
            change = int(info["change"])
3925
            if change > newestRevision:
3926
                newestRevision = change
3927

3928
            if info["action"] in self.delete_actions:
3929
                continue
3930

3931
            for prop in ["depotFile", "rev", "action", "type"]:
3932
                details["%s%s" % (prop, fileCnt)] = info[prop]
3933

3934
            fileCnt = fileCnt + 1
3935

3936
        details["change"] = newestRevision
3937

3938
        # Use time from top-most change so that all git p4 clones of
3939
        # the same p4 repo have the same commit SHA1s.
3940
        res = p4_describe(newestRevision)
3941
        details["time"] = res["time"]
3942

3943
        self.updateOptionDict(details)
3944
        try:
3945
            self.commit(details, self.extractFilesFromCommit(details), self.branch)
3946
        except IOError as err:
3947
            print("IO error with git fast-import. Is your git version recent enough?")
3948
            print("IO error details: {}".format(err))
3949
            print(self.gitError.read())
3950

3951
    def importRevisions(self, args, branch_arg_given):
3952
        changes = []
3953

3954
        if len(self.changesFile) > 0:
3955
            with open(self.changesFile) as f:
3956
                output = f.readlines()
3957
            changeSet = set()
3958
            for line in output:
3959
                changeSet.add(int(line))
3960

3961
            for change in changeSet:
3962
                changes.append(change)
3963

3964
            changes.sort()
3965
        else:
3966
            # catch "git p4 sync" with no new branches, in a repo that
3967
            # does not have any existing p4 branches
3968
            if len(args) == 0:
3969
                if not self.p4BranchesInGit:
3970
                    raise P4CommandException("No remote p4 branches.  Perhaps you never did \"git p4 clone\" in here.")
3971

3972
                # The default branch is master, unless --branch is used to
3973
                # specify something else.  Make sure it exists, or complain
3974
                # nicely about how to use --branch.
3975
                if not self.detectBranches:
3976
                    if not branch_exists(self.branch):
3977
                        if branch_arg_given:
3978
                            raise P4CommandException("Error: branch %s does not exist." % self.branch)
3979
                        else:
3980
                            raise P4CommandException("Error: no branch %s; perhaps specify one with --branch." %
3981
                                self.branch)
3982

3983
            if self.verbose:
3984
                print("Getting p4 changes for %s...%s" % (', '.join(self.depotPaths),
3985
                                                          self.changeRange))
3986
            changes = p4ChangesForPaths(self.depotPaths, self.changeRange, self.changes_block_size)
3987

3988
            if len(self.maxChanges) > 0:
3989
                changes = changes[:min(int(self.maxChanges), len(changes))]
3990

3991
        if len(changes) == 0:
3992
            if not self.silent:
3993
                print("No changes to import!")
3994
        else:
3995
            if not self.silent and not self.detectBranches:
3996
                print("Import destination: %s" % self.branch)
3997

3998
            self.updatedBranches = set()
3999

4000
            if not self.detectBranches:
4001
                if args:
4002
                    # start a new branch
4003
                    self.initialParent = ""
4004
                else:
4005
                    # build on a previous revision
4006
                    self.initialParent = parseRevision(self.branch)
4007

4008
            self.importChanges(changes)
4009

4010
            if not self.silent:
4011
                print("")
4012
                if len(self.updatedBranches) > 0:
4013
                    sys.stdout.write("Updated branches: ")
4014
                    for b in self.updatedBranches:
4015
                        sys.stdout.write("%s " % b)
4016
                    sys.stdout.write("\n")
4017

4018
    def openStreams(self):
4019
        self.importProcess = subprocess.Popen(["git", "fast-import"],
4020
                                              stdin=subprocess.PIPE,
4021
                                              stdout=subprocess.PIPE,
4022
                                              stderr=subprocess.PIPE)
4023
        self.gitOutput = self.importProcess.stdout
4024
        self.gitStream = self.importProcess.stdin
4025
        self.gitError = self.importProcess.stderr
4026

4027
        if bytes is not str:
4028
            # Wrap gitStream.write() so that it can be called using `str` arguments
4029
            def make_encoded_write(write):
4030
                def encoded_write(s):
4031
                    return write(s.encode() if isinstance(s, str) else s)
4032
                return encoded_write
4033

4034
            self.gitStream.write = make_encoded_write(self.gitStream.write)
4035

4036
    def closeStreams(self):
4037
        if self.gitStream is None:
4038
            return
4039
        self.gitStream.close()
4040
        if self.importProcess.wait() != 0:
4041
            die("fast-import failed: %s" % self.gitError.read())
4042
        self.gitOutput.close()
4043
        self.gitError.close()
4044
        self.gitStream = None
4045

4046
    def run(self, args):
4047
        if self.importIntoRemotes:
4048
            self.refPrefix = "refs/remotes/p4/"
4049
        else:
4050
            self.refPrefix = "refs/heads/p4/"
4051

4052
        self.sync_origin_only()
4053

4054
        branch_arg_given = bool(self.branch)
4055
        if len(self.branch) == 0:
4056
            self.branch = self.refPrefix + "master"
4057
            if gitBranchExists("refs/heads/p4") and self.importIntoRemotes:
4058
                system(["git", "update-ref", self.branch, "refs/heads/p4"])
4059
                system(["git", "branch", "-D", "p4"])
4060

4061
        # accept either the command-line option, or the configuration variable
4062
        if self.useClientSpec:
4063
            # will use this after clone to set the variable
4064
            self.useClientSpec_from_options = True
4065
        else:
4066
            if gitConfigBool("git-p4.useclientspec"):
4067
                self.useClientSpec = True
4068
        if self.useClientSpec:
4069
            self.clientSpecDirs = getClientSpec()
4070

4071
        # TODO: should always look at previous commits,
4072
        # merge with previous imports, if possible.
4073
        if args == []:
4074
            if self.hasOrigin:
4075
                createOrUpdateBranchesFromOrigin(self.refPrefix, self.silent)
4076

4077
            # branches holds mapping from branch name to sha1
4078
            branches = p4BranchesInGit(self.importIntoRemotes)
4079

4080
            # restrict to just this one, disabling detect-branches
4081
            if branch_arg_given:
4082
                short = shortP4Ref(self.branch, self.importIntoRemotes)
4083
                if short in branches:
4084
                    self.p4BranchesInGit = [short]
4085
                elif self.branch.startswith('refs/') and \
4086
                        branchExists(self.branch) and \
4087
                        '[git-p4:' in extractLogMessageFromGitCommit(self.branch):
4088
                    self.p4BranchesInGit = [self.branch]
4089
            else:
4090
                self.p4BranchesInGit = branches.keys()
4091

4092
            if len(self.p4BranchesInGit) > 1:
4093
                if not self.silent:
4094
                    print("Importing from/into multiple branches")
4095
                self.detectBranches = True
4096
                for branch in branches.keys():
4097
                    self.initialParents[self.refPrefix + branch] = \
4098
                        branches[branch]
4099

4100
            if self.verbose:
4101
                print("branches: %s" % self.p4BranchesInGit)
4102

4103
            p4Change = 0
4104
            for branch in self.p4BranchesInGit:
4105
                logMsg = extractLogMessageFromGitCommit(fullP4Ref(branch,
4106
                                                        self.importIntoRemotes))
4107

4108
                settings = extractSettingsGitLog(logMsg)
4109

4110
                self.readOptions(settings)
4111
                if 'depot-paths' in settings and 'change' in settings:
4112
                    change = int(settings['change']) + 1
4113
                    p4Change = max(p4Change, change)
4114

4115
                    depotPaths = sorted(settings['depot-paths'])
4116
                    if self.previousDepotPaths == []:
4117
                        self.previousDepotPaths = depotPaths
4118
                    else:
4119
                        paths = []
4120
                        for (prev, cur) in zip(self.previousDepotPaths, depotPaths):
4121
                            prev_list = prev.split("/")
4122
                            cur_list = cur.split("/")
4123
                            for i in range(0, min(len(cur_list), len(prev_list))):
4124
                                if cur_list[i] != prev_list[i]:
4125
                                    i = i - 1
4126
                                    break
4127

4128
                            paths.append("/".join(cur_list[:i + 1]))
4129

4130
                        self.previousDepotPaths = paths
4131

4132
            if p4Change > 0:
4133
                self.depotPaths = sorted(self.previousDepotPaths)
4134
                self.changeRange = "@%s,#head" % p4Change
4135
                if not self.silent and not self.detectBranches:
4136
                    print("Performing incremental import into %s git branch" % self.branch)
4137

4138
        self.branch = fullP4Ref(self.branch, self.importIntoRemotes)
4139

4140
        if len(args) == 0 and self.depotPaths:
4141
            if not self.silent:
4142
                print("Depot paths: %s" % ' '.join(self.depotPaths))
4143
        else:
4144
            if self.depotPaths and self.depotPaths != args:
4145
                print("previous import used depot path %s and now %s was specified. "
4146
                       "This doesn't work!" % (' '.join(self.depotPaths),
4147
                                               ' '.join(args)))
4148
                sys.exit(1)
4149

4150
            self.depotPaths = sorted(args)
4151

4152
        revision = ""
4153
        self.users = {}
4154

4155
        # Make sure no revision specifiers are used when --changesfile
4156
        # is specified.
4157
        bad_changesfile = False
4158
        if len(self.changesFile) > 0:
4159
            for p in self.depotPaths:
4160
                if p.find("@") >= 0 or p.find("#") >= 0:
4161
                    bad_changesfile = True
4162
                    break
4163
        if bad_changesfile:
4164
            die("Option --changesfile is incompatible with revision specifiers")
4165

4166
        newPaths = []
4167
        for p in self.depotPaths:
4168
            if p.find("@") != -1:
4169
                atIdx = p.index("@")
4170
                self.changeRange = p[atIdx:]
4171
                if self.changeRange == "@all":
4172
                    self.changeRange = ""
4173
                elif ',' not in self.changeRange:
4174
                    revision = self.changeRange
4175
                    self.changeRange = ""
4176
                p = p[:atIdx]
4177
            elif p.find("#") != -1:
4178
                hashIdx = p.index("#")
4179
                revision = p[hashIdx:]
4180
                p = p[:hashIdx]
4181
            elif self.previousDepotPaths == []:
4182
                # pay attention to changesfile, if given, else import
4183
                # the entire p4 tree at the head revision
4184
                if len(self.changesFile) == 0:
4185
                    revision = "#head"
4186

4187
            p = re.sub(r"\.\.\.$", "", p)
4188
            if not p.endswith("/"):
4189
                p += "/"
4190

4191
            newPaths.append(p)
4192

4193
        self.depotPaths = newPaths
4194

4195
        # --detect-branches may change this for each branch
4196
        self.branchPrefixes = self.depotPaths
4197

4198
        self.loadUserMapFromCache()
4199
        self.labels = {}
4200
        if self.detectLabels:
4201
            self.getLabels()
4202

4203
        if self.detectBranches:
4204
            # FIXME - what's a P4 projectName ?
4205
            self.projectName = self.guessProjectName()
4206

4207
            if self.hasOrigin:
4208
                self.getBranchMappingFromGitBranches()
4209
            else:
4210
                self.getBranchMapping()
4211
            if self.verbose:
4212
                print("p4-git branches: %s" % self.p4BranchesInGit)
4213
                print("initial parents: %s" % self.initialParents)
4214
            for b in self.p4BranchesInGit:
4215
                if b != "master":
4216

4217
                    # FIXME
4218
                    b = b[len(self.projectName):]
4219
                self.createdBranches.add(b)
4220

4221
        p4_check_access()
4222

4223
        self.openStreams()
4224

4225
        err = None
4226

4227
        try:
4228
            if revision:
4229
                self.importHeadRevision(revision)
4230
            else:
4231
                self.importRevisions(args, branch_arg_given)
4232

4233
            if gitConfigBool("git-p4.importLabels"):
4234
                self.importLabels = True
4235

4236
            if self.importLabels:
4237
                p4Labels = getP4Labels(self.depotPaths)
4238
                gitTags = getGitTags()
4239

4240
                missingP4Labels = p4Labels - gitTags
4241
                self.importP4Labels(self.gitStream, missingP4Labels)
4242

4243
        except P4CommandException as e:
4244
            err = e
4245

4246
        finally:
4247
            self.closeStreams()
4248

4249
        if err:
4250
            die(str(err))
4251

4252
        # Cleanup temporary branches created during import
4253
        if self.tempBranches != []:
4254
            for branch in self.tempBranches:
4255
                read_pipe(["git", "update-ref", "-d", branch])
4256
            if len(read_pipe(["git", "for-each-ref", self.tempBranchLocation])) > 0:
4257
                   die("There are unexpected temporary branches")
4258

4259
        # Create a symbolic ref p4/HEAD pointing to p4/<branch> to allow
4260
        # a convenient shortcut refname "p4".
4261
        if self.importIntoRemotes:
4262
            head_ref = self.refPrefix + "HEAD"
4263
            if not gitBranchExists(head_ref) and gitBranchExists(self.branch):
4264
                system(["git", "symbolic-ref", head_ref, self.branch])
4265

4266
        return True
4267

4268

4269
class P4Rebase(Command):
4270
    def __init__(self):
4271
        Command.__init__(self)
4272
        self.options = [
4273
                optparse.make_option("--import-labels", dest="importLabels", action="store_true"),
4274
        ]
4275
        self.importLabels = False
4276
        self.description = ("Fetches the latest revision from perforce and "
4277
                            + "rebases the current work (branch) against it")
4278

4279
    def run(self, args):
4280
        sync = P4Sync()
4281
        sync.importLabels = self.importLabels
4282
        sync.run([])
4283

4284
        return self.rebase()
4285

4286
    def rebase(self):
4287
        if os.system("git update-index --refresh") != 0:
4288
            die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up to date or stash away all your changes with git stash.")
4289
        if len(read_pipe(["git", "diff-index", "HEAD", "--"])) > 0:
4290
            die("You have uncommitted changes. Please commit them before rebasing or stash them away with git stash.")
4291

4292
        upstream, settings = findUpstreamBranchPoint()
4293
        if len(upstream) == 0:
4294
            die("Cannot find upstream branchpoint for rebase")
4295

4296
        # the branchpoint may be p4/foo~3, so strip off the parent
4297
        upstream = re.sub(r"~[0-9]+$", "", upstream)
4298

4299
        print("Rebasing the current branch onto %s" % upstream)
4300
        oldHead = read_pipe(["git", "rev-parse", "HEAD"]).strip()
4301
        system(["git", "rebase", upstream])
4302
        system(["git", "diff-tree", "--stat", "--summary", "-M", oldHead,
4303
            "HEAD", "--"])
4304
        return True
4305

4306

4307
class P4Clone(P4Sync):
4308
    def __init__(self):
4309
        P4Sync.__init__(self)
4310
        self.description = "Creates a new git repository and imports from Perforce into it"
4311
        self.usage = "usage: %prog [options] //depot/path[@revRange]"
4312
        self.options += [
4313
            optparse.make_option("--destination", dest="cloneDestination",
4314
                                 action='store', default=None,
4315
                                 help="where to leave result of the clone"),
4316
            optparse.make_option("--bare", dest="cloneBare",
4317
                                 action="store_true", default=False),
4318
        ]
4319
        self.cloneDestination = None
4320
        self.needsGit = False
4321
        self.cloneBare = False
4322

4323
    def defaultDestination(self, args):
4324
        # TODO: use common prefix of args?
4325
        depotPath = args[0]
4326
        depotDir = re.sub(r"(@[^@]*)$", "", depotPath)
4327
        depotDir = re.sub(r"(#[^#]*)$", "", depotDir)
4328
        depotDir = re.sub(r"\.\.\.$", "", depotDir)
4329
        depotDir = re.sub(r"/$", "", depotDir)
4330
        return os.path.split(depotDir)[1]
4331

4332
    def run(self, args):
4333
        if len(args) < 1:
4334
            return False
4335

4336
        if self.keepRepoPath and not self.cloneDestination:
4337
            sys.stderr.write("Must specify destination for --keep-path\n")
4338
            sys.exit(1)
4339

4340
        depotPaths = args
4341

4342
        if not self.cloneDestination and len(depotPaths) > 1:
4343
            self.cloneDestination = depotPaths[-1]
4344
            depotPaths = depotPaths[:-1]
4345

4346
        for p in depotPaths:
4347
            if not p.startswith("//"):
4348
                sys.stderr.write('Depot paths must start with "//": %s\n' % p)
4349
                return False
4350

4351
        if not self.cloneDestination:
4352
            self.cloneDestination = self.defaultDestination(args)
4353

4354
        print("Importing from %s into %s" % (', '.join(depotPaths), self.cloneDestination))
4355

4356
        if not os.path.exists(self.cloneDestination):
4357
            os.makedirs(self.cloneDestination)
4358
        chdir(self.cloneDestination)
4359

4360
        init_cmd = ["git", "init"]
4361
        if self.cloneBare:
4362
            init_cmd.append("--bare")
4363
        retcode = subprocess.call(init_cmd)
4364
        if retcode:
4365
            raise subprocess.CalledProcessError(retcode, init_cmd)
4366

4367
        if not P4Sync.run(self, depotPaths):
4368
            return False
4369

4370
        # create a master branch and check out a work tree
4371
        if gitBranchExists(self.branch):
4372
            system(["git", "branch", currentGitBranch(), self.branch])
4373
            if not self.cloneBare:
4374
                system(["git", "checkout", "-f"])
4375
        else:
4376
            print('Not checking out any branch, use '
4377
                  '"git checkout -q -b master <branch>"')
4378

4379
        # auto-set this variable if invoked with --use-client-spec
4380
        if self.useClientSpec_from_options:
4381
            system(["git", "config", "--bool", "git-p4.useclientspec", "true"])
4382

4383
        # persist any git-p4 encoding-handling config options passed in for clone:
4384
        if gitConfig('git-p4.metadataDecodingStrategy'):
4385
            system(["git", "config", "git-p4.metadataDecodingStrategy", gitConfig('git-p4.metadataDecodingStrategy')])
4386
        if gitConfig('git-p4.metadataFallbackEncoding'):
4387
            system(["git", "config", "git-p4.metadataFallbackEncoding", gitConfig('git-p4.metadataFallbackEncoding')])
4388
        if gitConfig('git-p4.pathEncoding'):
4389
            system(["git", "config", "git-p4.pathEncoding", gitConfig('git-p4.pathEncoding')])
4390

4391
        return True
4392

4393

4394
class P4Unshelve(Command):
4395
    def __init__(self):
4396
        Command.__init__(self)
4397
        self.options = []
4398
        self.origin = "HEAD"
4399
        self.description = "Unshelve a P4 changelist into a git commit"
4400
        self.usage = "usage: %prog [options] changelist"
4401
        self.options += [
4402
                optparse.make_option("--origin", dest="origin",
4403
                    help="Use this base revision instead of the default (%s)" % self.origin),
4404
        ]
4405
        self.verbose = False
4406
        self.noCommit = False
4407
        self.destbranch = "refs/remotes/p4-unshelved"
4408

4409
    def renameBranch(self, branch_name):
4410
        """Rename the existing branch to branch_name.N ."""
4411

4412
        for i in range(0, 1000):
4413
            backup_branch_name = "{0}.{1}".format(branch_name, i)
4414
            if not gitBranchExists(backup_branch_name):
4415
                # Copy ref to backup
4416
                gitUpdateRef(backup_branch_name, branch_name)
4417
                gitDeleteRef(branch_name)
4418
                print("renamed old unshelve branch to {0}".format(backup_branch_name))
4419
                break
4420
        else:
4421
            sys.exit("gave up trying to rename existing branch {0}".format(branch_name))
4422

4423
    def findLastP4Revision(self, starting_point):
4424
        """Look back from starting_point for the first commit created by git-p4
4425
           to find the P4 commit we are based on, and the depot-paths.
4426
           """
4427

4428
        for parent in (range(65535)):
4429
            log = extractLogMessageFromGitCommit("{0}~{1}".format(starting_point, parent))
4430
            settings = extractSettingsGitLog(log)
4431
            if 'change' in settings:
4432
                return settings
4433

4434
        sys.exit("could not find git-p4 commits in {0}".format(self.origin))
4435

4436
    def createShelveParent(self, change, branch_name, sync, origin):
4437
        """Create a commit matching the parent of the shelved changelist
4438
           'change'.
4439
           """
4440
        parent_description = p4_describe(change, shelved=True)
4441
        parent_description['desc'] = 'parent for shelved changelist {}\n'.format(change)
4442
        files = sync.extractFilesFromCommit(parent_description, shelved=False, shelved_cl=change)
4443

4444
        parent_files = []
4445
        for f in files:
4446
            # if it was added in the shelved changelist, it won't exist in the parent
4447
            if f['action'] in self.add_actions:
4448
                continue
4449

4450
            # if it was deleted in the shelved changelist it must not be deleted
4451
            # in the parent - we might even need to create it if the origin branch
4452
            # does not have it
4453
            if f['action'] in self.delete_actions:
4454
                f['action'] = 'add'
4455

4456
            parent_files.append(f)
4457

4458
        sync.commit(parent_description, parent_files, branch_name,
4459
                parent=origin, allow_empty=True)
4460
        print("created parent commit for {0} based on {1} in {2}".format(
4461
            change, self.origin, branch_name))
4462

4463
    def run(self, args):
4464
        if len(args) != 1:
4465
            return False
4466

4467
        if not gitBranchExists(self.origin):
4468
            sys.exit("origin branch {0} does not exist".format(self.origin))
4469

4470
        sync = P4Sync()
4471
        changes = args
4472

4473
        # only one change at a time
4474
        change = changes[0]
4475

4476
        # if the target branch already exists, rename it
4477
        branch_name = "{0}/{1}".format(self.destbranch, change)
4478
        if gitBranchExists(branch_name):
4479
            self.renameBranch(branch_name)
4480
        sync.branch = branch_name
4481

4482
        sync.verbose = self.verbose
4483
        sync.suppress_meta_comment = True
4484

4485
        settings = self.findLastP4Revision(self.origin)
4486
        sync.depotPaths = settings['depot-paths']
4487
        sync.branchPrefixes = sync.depotPaths
4488

4489
        sync.openStreams()
4490
        sync.loadUserMapFromCache()
4491
        sync.silent = True
4492

4493
        # create a commit for the parent of the shelved changelist
4494
        self.createShelveParent(change, branch_name, sync, self.origin)
4495

4496
        # create the commit for the shelved changelist itself
4497
        description = p4_describe(change, True)
4498
        files = sync.extractFilesFromCommit(description, True, change)
4499

4500
        sync.commit(description, files, branch_name, "")
4501
        sync.closeStreams()
4502

4503
        print("unshelved changelist {0} into {1}".format(change, branch_name))
4504

4505
        return True
4506

4507

4508
class P4Branches(Command):
4509
    def __init__(self):
4510
        Command.__init__(self)
4511
        self.options = []
4512
        self.description = ("Shows the git branches that hold imports and their "
4513
                            + "corresponding perforce depot paths")
4514
        self.verbose = False
4515

4516
    def run(self, args):
4517
        if originP4BranchesExist():
4518
            createOrUpdateBranchesFromOrigin()
4519

4520
        for line in read_pipe_lines(["git", "rev-parse", "--symbolic", "--remotes"]):
4521
            line = line.strip()
4522

4523
            if not line.startswith('p4/') or line == "p4/HEAD":
4524
                continue
4525
            branch = line
4526

4527
            log = extractLogMessageFromGitCommit("refs/remotes/%s" % branch)
4528
            settings = extractSettingsGitLog(log)
4529

4530
            print("%s <= %s (%s)" % (branch, ",".join(settings["depot-paths"]), settings["change"]))
4531
        return True
4532

4533

4534
class HelpFormatter(optparse.IndentedHelpFormatter):
4535
    def __init__(self):
4536
        optparse.IndentedHelpFormatter.__init__(self)
4537

4538
    def format_description(self, description):
4539
        if description:
4540
            return description + "\n"
4541
        else:
4542
            return ""
4543

4544

4545
def printUsage(commands):
4546
    print("usage: %s <command> [options]" % sys.argv[0])
4547
    print("")
4548
    print("valid commands: %s" % ", ".join(commands))
4549
    print("")
4550
    print("Try %s <command> --help for command specific help." % sys.argv[0])
4551
    print("")
4552

4553

4554
commands = {
4555
    "submit": P4Submit,
4556
    "commit": P4Submit,
4557
    "sync": P4Sync,
4558
    "rebase": P4Rebase,
4559
    "clone": P4Clone,
4560
    "branches": P4Branches,
4561
    "unshelve": P4Unshelve,
4562
}
4563

4564

4565
def main():
4566
    if len(sys.argv[1:]) == 0:
4567
        printUsage(commands.keys())
4568
        sys.exit(2)
4569

4570
    cmdName = sys.argv[1]
4571
    try:
4572
        klass = commands[cmdName]
4573
        cmd = klass()
4574
    except KeyError:
4575
        print("unknown command %s" % cmdName)
4576
        print("")
4577
        printUsage(commands.keys())
4578
        sys.exit(2)
4579

4580
    options = cmd.options
4581
    cmd.gitdir = os.environ.get("GIT_DIR", None)
4582

4583
    args = sys.argv[2:]
4584

4585
    options.append(optparse.make_option("--verbose", "-v", dest="verbose", action="store_true"))
4586
    if cmd.needsGit:
4587
        options.append(optparse.make_option("--git-dir", dest="gitdir"))
4588

4589
    parser = optparse.OptionParser(cmd.usage.replace("%prog", "%prog " + cmdName),
4590
                                   options,
4591
                                   description=cmd.description,
4592
                                   formatter=HelpFormatter())
4593

4594
    try:
4595
        cmd, args = parser.parse_args(sys.argv[2:], cmd)
4596
    except:
4597
        parser.print_help()
4598
        raise
4599

4600
    global verbose
4601
    verbose = cmd.verbose
4602
    if cmd.needsGit:
4603
        if cmd.gitdir is None:
4604
            cmd.gitdir = os.path.abspath(".git")
4605
            if not isValidGitDir(cmd.gitdir):
4606
                # "rev-parse --git-dir" without arguments will try $PWD/.git
4607
                cmd.gitdir = read_pipe(["git", "rev-parse", "--git-dir"]).strip()
4608
                if os.path.exists(cmd.gitdir):
4609
                    cdup = read_pipe(["git", "rev-parse", "--show-cdup"]).strip()
4610
                    if len(cdup) > 0:
4611
                        chdir(cdup)
4612

4613
        if not isValidGitDir(cmd.gitdir):
4614
            if isValidGitDir(cmd.gitdir + "/.git"):
4615
                cmd.gitdir += "/.git"
4616
            else:
4617
                die("fatal: cannot locate git repository at %s" % cmd.gitdir)
4618

4619
        # so git commands invoked from the P4 workspace will succeed
4620
        os.environ["GIT_DIR"] = cmd.gitdir
4621

4622
    if not cmd.run(args):
4623
        parser.print_help()
4624
        sys.exit(2)
4625

4626

4627
if __name__ == '__main__':
4628
    main()
4629