pytorch
/
c2_defs.bzl
527 строк · 19.1 Кб
1load("@bazel_skylib//lib:collections.bzl", "collections")
2load("@bazel_skylib//lib:paths.bzl", "paths")
3load("@fbcode_macros//build_defs:native_rules.bzl", "buck_genrule")
4load("@fbsource//tools/build_defs:default_platform_defs.bzl", "compose_platform_setting_list")
5load("@fbsource//tools/build_defs:dict_defs.bzl", "dict_defs")
6load("@fbsource//tools/build_defs:expect.bzl", "expect")
7load("@fbsource//tools/build_defs:fb_xplat_cxx_library.bzl", "fb_xplat_cxx_library")
8load("@fbsource//tools/build_defs:fbsource_utils.bzl", "is_arvr_mode", "is_fbcode_mode_mac")
9load("@fbsource//tools/build_defs:platform_defs.bzl", "ANDROID", "APPLE", "CXX", "IOS", "MACOSX", "WINDOWS")
10load("@fbsource//tools/build_defs/apple:build_mode_defs.bzl", "is_production_build")
11load("@fbsource//tools/build_defs/apple:config_utils_defs.bzl", "STATIC_LIBRARY_IOS_CONFIG", "STATIC_LIBRARY_MAC_CONFIG", "fbobjc_configs")
12load("@fbsource//xplat/caffe2:buckbuild.bzl", "read_bool")
13load("@fbsource//xplat/pfh/Msgr/Mobile/ProductInfra:DEFS.bzl", "Msgr_Mobile_ProductInfra")
14
15def get_c2_expose_op_to_c10():
16c2_op_to_c10 = native.read_config("caffe2", "expose_op_to_c10", "0")
17
18expect(
19c2_op_to_c10 in ("0", "1"),
20c2_op_to_c10,
21)
22
23return bool(int(c2_op_to_c10))
24
25def get_c2_mpscnn():
26c2_mpscnn = native.read_config("caffe2", "enable_mpscnn", "1")
27
28expect(
29c2_mpscnn in ("0", "1"),
30c2_mpscnn,
31)
32
33return bool(int(c2_mpscnn))
34
35def get_c2_mpscnn_test():
36c2_mpscnn_test = native.read_config("caffe2", "enable_mpscnn_test", "0")
37
38expect(
39c2_mpscnn_test in ("0", "1"),
40c2_mpscnn_test,
41)
42
43return bool(int(c2_mpscnn_test))
44
45def get_c2_nomnigraph():
46c2_nomnigraph = native.read_config("caffe2", "enable_nomnigraph", "1")
47
48expect(
49c2_nomnigraph in ("0", "1"),
50c2_nomnigraph,
51)
52
53return bool(int(c2_nomnigraph))
54
55def get_c2_qpl():
56c2_qpl = native.read_config("caffe2", "enable_qpl", "1")
57
58expect(
59c2_qpl in ("0", "1"),
60c2_qpl,
61)
62
63return bool(int(c2_qpl))
64
65def get_c2_strip_debug_info():
66c2_strip_debug_info = native.read_config("caffe2", "strip_debug_info", "0")
67
68expect(
69c2_strip_debug_info in ("0", "1"),
70c2_strip_debug_info,
71)
72
73return bool(int(c2_strip_debug_info))
74
75def get_c2_strip_glog():
76c2_strip_glog = native.read_config("caffe2", "strip_glog", "1")
77
78expect(
79c2_strip_glog in ("0", "1"),
80c2_strip_glog,
81)
82
83return bool(int(c2_strip_glog))
84
85def get_c2_tvm():
86c2_tvm = native.read_config("caffe2", "enable_tvm", "1")
87
88expect(
89c2_tvm in ("0", "1"),
90c2_tvm,
91)
92
93return bool(int(c2_tvm))
94
95_C2_XPLAT_NO_HPTT_PREPROCESSOR_FLAGS = [
96"-Icaffe2",
97"-Imodules",
98"-DEIGEN_NO_DEBUG",
99"-DCAFFE2_USE_LITE_PROTO",
100"-DCAFFE2_USE_GOOGLE_GLOG",
101"-DCAFFE2_RNN_NO_TEXT_FORMAT",
102"-DGEMMLOWP_ALLOW_SLOW_SCALAR_FALLBACK=1",
103"-DCAFFE2_IS_XPLAT_BUILD",
104"-DSTRIP_ERROR_MESSAGES",
105"-DUSE_INTERNAL_PTHREADPOOL_IMPL",
106]
107
108def get_c2_xplat_no_hptt_preprocessor_flags():
109flags = []
110flags += _C2_XPLAT_NO_HPTT_PREPROCESSOR_FLAGS
111if is_arvr_mode() and get_c2_strip_glog():
112flags += ["-UGOOGLE_STRIP_LOG", "-DGOOGLE_STRIP_LOG=1"]
113if get_c2_expose_op_to_c10():
114flags += ["-DEXPOSE_C2_OPS", "-frtti"]
115return flags
116
117C2_XPLAT_SERVER_PREPROCESSOR_FLAGS = [
118"-DCAFFE2_USE_EIGEN_FOR_BLAS",
119"-DC10_DISABLE_SIGNAL_HANDLERS",
120"-DCAFFE2_DISABLE_NUMA",
121]
122
123C2_XPLAT_HPTT_PREPROCESSOR_FLAGS = [
124"-DCAFFE2_USE_HPTT",
125]
126
127def get_c2_xplat_preprocessor_flags():
128flags = get_c2_xplat_no_hptt_preprocessor_flags() + C2_XPLAT_HPTT_PREPROCESSOR_FLAGS
129if get_c2_nomnigraph():
130flags.append("-DCAFFE2_OPTIMIZER")
131return flags
132
133def get_c2_xplat_no_hptt_compiler_flags():
134return [
135"-Os",
136"-fexceptions",
137"-frtti",
138"-Wno-shadow",
139"-Wno-unknown-pragmas",
140"-Wno-unused-variable",
141"-Wno-sign-compare",
142]
143
144def get_c2_xplat_compiler_flags():
145return get_c2_xplat_no_hptt_compiler_flags() + C2_XPLAT_HPTT_PREPROCESSOR_FLAGS
146
147def get_c2_fbobjc_xplat_compiler_flags():
148flags = []
149
150if is_production_build():
151flags.append("-DCAFFE2_NO_OPERATOR_SCHEMA")
152
153flags.append("-DCAFFE2_NO_GRADIENT_OPS")
154
155# For iOS production builds (and all Android builds), strip GLOG logging to
156# save size. We can disable by setting caffe2.strip_glog=0 in .buckconfig.local.
157if is_production_build() or get_c2_strip_glog():
158flags += ["-UGOOGLE_STRIP_LOG", "-DGOOGLE_STRIP_LOG=3"]
159else:
160flags.append("-UGOOGLE_STRIP_LOG")
161
162return flags
163
164def get_c2_fbandroid_xplat_compiler_flags():
165flags = [
166"-Wno-unused-but-set-variable",
167"-DHAVE_MMAP",
168]
169
170if get_c2_strip_glog():
171flags += ["-UGOOGLE_STRIP_LOG", "-DGOOGLE_STRIP_LOG=1"]
172
173if get_c2_strip_debug_info():
174flags.append("-g0")
175
176return flags
177
178_C2_FBOBJC_COMPILER_FLAGS = [
179"-Wno-missing-prototypes",
180"-Wno-global-constructors",
181"-Wno-unknown-pragmas",
182"-Wno-invalid-partial-specialization",
183"-Wno-missing-braces",
184"-Wno-range-loop-analysis",
185]
186
187def get_c2_fbobjc_compiler_flags():
188flags = list(_C2_FBOBJC_COMPILER_FLAGS)
189
190# Avoid linking Accelerate on MacOS because we have
191# inconsistent LAPACK headers (see problems in D19257077).
192flags.append("-DCAFFE2_USE_ACCELERATE" if not is_arvr_mode() else "-DCAFFE2_USE_EIGEN_FOR_BLAS")
193if get_c2_mpscnn():
194flags.append(
195# TODO(t19120552) - fix this. MPSCNNConvolutionDescriptor.strideInPixelsX
196# is marked as iOS 11+, but it's been available since iOS 10.
197"-Wno-unguarded-availability",
198)
199return flags
200
201C2_FBOBJC_MACOSX_COMPILER_FLAGS = [
202"-msse4.2",
203]
204
205C2_FBOBJC_IPHONE_COMPILER_FLAGS = [
206"-mfpu=neon-fp16",
207]
208
209def get_c2_fbobjc_frameworks():
210frameworks = []
211if not is_arvr_mode():
212frameworks.append(
213# On iOS, presumably Accelerate is a faster BLAS
214"$SDKROOT/System/Library/Frameworks/Accelerate.framework",
215)
216return frameworks
217
218def get_c2_fbobjc_ios_frameworks():
219frameworks = []
220
221if get_c2_mpscnn():
222frameworks.extend([
223"$SDKROOT/System/Library/Frameworks/Metal.framework",
224"$SDKROOT/System/Library/Frameworks/MetalPerformanceShaders.framework",
225])
226
227return frameworks
228
229def get_c2_fbobjc_exported_preprocessor_flags():
230flags = []
231
232if get_c2_mpscnn():
233flags.append("-DCAFFE2_USE_MPSCNN")
234
235if get_c2_mpscnn_test():
236flags.append("-DCAFFE2_USE_MPSCNN_TEST")
237
238return flags
239
240def get_c2_fbandroid_exported_preprocessor_flags():
241flags = []
242
243BUILD_MODE_DO_NOT_USE_WITHOUT_ASKING_SERIOUSLY = native.read_config(
244"fbandroid",
245"build_mode",
246"dev",
247)
248if BUILD_MODE_DO_NOT_USE_WITHOUT_ASKING_SERIOUSLY == "opt":
249flags.append("-DCAFFE2_NO_OPERATOR_SCHEMA")
250
251flags.append("-DCAFFE2_NO_GRADIENT_OPS")
252
253return flags
254
255C2_FBANDROID_COMPILER_FLAGS = [
256"-DCAFFE2_USE_EIGEN_FOR_BLAS",
257"-Wno-unknown-pragmas",
258"-Wno-deprecated-declarations",
259"-Wno-invalid-partial-specialization",
260"-Wno-missing-braces",
261]
262
263C2_FBANDROID_ARMV7_COMPILER_FLAGS = [
264"-mfpu=neon-fp16",
265]
266
267C2_FBANDROID_X86_COMPILER_FLAGS = [
268"-mssse3",
269]
270
271C2_FBANDROID_LINKER_FLAGS = []
272
273C2_FBOBJC_EXTRA_TARGET_CONFIG = {
274"MTL_LANGUAGE_REVISION": "Metal12",
275}
276
277def get_c2_default_cxx_args():
278return dict(
279header_namespace = "",
280apple_sdks = (IOS, MACOSX),
281compiler_flags = get_c2_xplat_compiler_flags(),
282fbandroid_compiler_flags = C2_FBANDROID_COMPILER_FLAGS + get_c2_fbandroid_xplat_compiler_flags(),
283fbandroid_exported_platform_preprocessor_flags = [
284(
285"android-armv7",
286get_c2_fbandroid_exported_preprocessor_flags(),
287),
288],
289fbandroid_linker_flags = C2_FBANDROID_LINKER_FLAGS,
290fbandroid_platform_compiler_flags = [
291("android-armv7", C2_FBANDROID_ARMV7_COMPILER_FLAGS),
292(".*x86.*", C2_FBANDROID_X86_COMPILER_FLAGS),
293],
294fbobjc_compiler_flags = get_c2_fbobjc_compiler_flags() + get_c2_fbobjc_xplat_compiler_flags(),
295fbobjc_configs = fbobjc_configs(
296STATIC_LIBRARY_IOS_CONFIG,
297extra_target_config = C2_FBOBJC_EXTRA_TARGET_CONFIG,
298),
299fbobjc_exported_platform_preprocessor_flags = [
300(
301"iphoneos",
302get_c2_fbobjc_exported_preprocessor_flags(),
303),
304],
305fbobjc_frameworks = get_c2_fbobjc_frameworks() + get_c2_fbobjc_ios_frameworks(),
306fbobjc_platform_compiler_flags = [
307("iphoneos", C2_FBOBJC_IPHONE_COMPILER_FLAGS),
308],
309macosx_compiler_flags = C2_FBOBJC_MACOSX_COMPILER_FLAGS,
310fbobjc_macosx_configs_override = fbobjc_configs(
311STATIC_LIBRARY_MAC_CONFIG,
312),
313macosx_frameworks_override = get_c2_fbobjc_frameworks(),
314preprocessor_flags = [
315# Use the internal pthreadpool impl for all Caffe2 targets on all
316# platforms but do not export the preprocessor flag downstream.
317"-DUSE_INTERNAL_PTHREADPOOL_IMPL",
318],
319visibility = ["PUBLIC"],
320windows_preferred_linkage = "static" if is_arvr_mode() else None,
321xcode_public_headers_symlinks = True,
322)
323
324def get_c2_aten_cpu_fbobjc_macosx_deps():
325return select({
326"DEFAULT": [],
327"ovr_config//os:macos-x86_64": ["fbsource//xplat/deeplearning/fbgemm:fbgemm"],
328}) if is_arvr_mode() else []
329
330def build_cpukernel_avx2():
331return read_bool("caffe2", "build_cpukernel_avx2", not is_arvr_mode())
332
333def get_c2_aten_cpu_fbobjc_macosx_platform_deps():
334return compose_platform_setting_list([
335{
336"cpu": "x86_64",
337"flags": [
338"fbsource//xplat/deeplearning/fbgemm:fbgemmAppleMac",
339] + ([
340"fbsource//xplat/caffe2:cpukernel_avx2AppleMac",
341] if build_cpukernel_avx2() else []),
342"os": "macosx",
343},
344{
345"cpu": "arm64",
346"flags": ["fbsource//xplat/third-party/XNNPACK:XNNPACKAppleMac"],
347"os": "macosx",
348},
349])
350
351def using_protobuf_v3():
352# Consider migrating this to `read_config("protobuf", "use_v3")`
353# The `is_fbcode_mode_mac()` clause was added rather than changing to `read_config` to minimize changes in behavior
354return is_arvr_mode() or is_fbcode_mode_mac()
355
356def get_c2_protobuf_dep():
357return "fbsource//third-party/protobuf:libprotobuf" if using_protobuf_v3() else "fbsource//xplat/third-party/protobuf:fb-protobuf-lite"
358
359def c2_cxx_library(fbobjc_compiler_flags = [], **kwargs):
360args = get_c2_default_cxx_args()
361args.update(kwargs)
362args.setdefault("platforms", (ANDROID, APPLE, CXX, WINDOWS))
363
364# Make sure we don't overwrite custom `fbobjc_compiler_flags`
365args["fbobjc_compiler_flags"] = args.pop("fbobjc_compiler_flags", []) + fbobjc_compiler_flags
366
367fb_xplat_cxx_library(
368labels = [
369"supermodule:android/default/caffe2",
370"supermodule:ios/default/public.caffe2",
371],
372feature = Msgr_Mobile_ProductInfra,
373**args
374)
375
376def c2_protobuf_rule(protos):
377cpps = []
378headers = {}
379raw_headers = {}
380for p in protos:
381proto = paths.basename(p)
382protocexe = "$(exe fbsource//third-party/protobuf:protoc-host)" if is_arvr_mode() else "$(location fbsource//xplat/third-party/protobuf:protoc.Windows)"
383protocmd_exe = "powershell.exe -file $(location fbsource//xplat/caffe2/scripts:proto)\\proto.ps1 -Protoc {} -Unprocessed $SRCDIR/{} -Processed $SRCDIR/{} -out $OUT -srcdir $SRCDIR".format(protocexe, p, proto)
384protocmd = ("cp $SRCDIR/{} $SRCDIR/{} && chmod +w $SRCDIR/{} && echo \"option optimize_for = LITE_RUNTIME;\" >> $SRCDIR/{} && ".format(p, proto, proto, proto) +
385"cp $SRCDIR/caffe2/proto/caffe2.proto $SRCDIR/caffe2.proto && chmod +w $SRCDIR/caffe2.proto && echo \"option optimize_for = LITE_RUNTIME;\" >> $SRCDIR/caffe2.proto && " +
386"sed -i -e 's/caffe2\\/proto\\/caffe2.proto/caffe2.proto/g' $SRCDIR/{} && ".format(proto) +
387("$(exe fbsource//third-party/protobuf:protoc-host) " if using_protobuf_v3() else "$(exe fbsource//xplat/third-party/protobuf:protoc) --osx $(location fbsource//xplat/third-party/protobuf:protoc.Darwin) --linux $(location fbsource//xplat/third-party/protobuf:protoc.Linux) ") +
388"-I $SRCDIR --cpp_out=$OUT $SRCDIR/{}".format(proto))
389buck_genrule(
390name = proto,
391srcs = sorted(collections.uniq([p, "caffe2/proto/caffe2.proto"])),
392cmd_exe = protocmd_exe,
393bash = protocmd,
394out = ".",
395)
396(name, _) = paths.split_extension(proto)
397cpp = name + ".pb.cc"
398h = name + ".pb.h"
399buck_genrule(
400name = h,
401cmd_exe = "@powershell -Command \" & { " + "(Get-Content $(location :{})\\{}".format(proto, h) + ") -replace \\\"caffe2.pb.h\\\", \\\"caffe2/proto/caffe2.pb.h\\\" | Set-Content $OUT } \"",
402bash = "cp -f $(location :{})/{} $OUT && ".format(proto, h) +
403"sed -i -e 's/caffe2.pb.h/caffe2\\/proto\\/caffe2.pb.h/g' $OUT",
404out = h,
405)
406headers["caffe2/proto/" + h] = ":{}".format(h)
407raw_headers[h] = ":{}".format(h)
408buck_genrule(
409name = cpp,
410cmd_exe = "@powershell -Command copy $(location :{})/{} $OUT".format(proto, cpp),
411bash = "cp -f $(location :{})/{} $OUT".format(proto, cpp),
412out = cpp,
413)
414cpps.append(":{}".format(cpp))
415return (cpps, headers, raw_headers)
416
417# C2 uses lite version of protobuf while torch/jit uses some method only exists
418# in full protobuf. This is a temporary workaround to enable experiment build.
419# DO NOT USE IT IN PRODUCTION BUILD!
420def c2_full_protobuf_rule(protos):
421prefix = "full_"
422cpps = []
423headers = {}
424raw_headers = {}
425for p in protos:
426proto = paths.basename(p)
427protocexe = "$(exe fbsource//third-party/protobuf:protoc-host)" if is_arvr_mode() else "$(location fbsource//xplat/third-party/protobuf:protoc.Windows)"
428protocmd_exe = "powershell.exe -file $(location fbsource//xplat/caffe2/scripts:proto)\\proto.ps1 -Protoc {} -Unprocessed $SRCDIR/{} -Processed $SRCDIR/{} -out $OUT -srcdir $SRCDIR".format(protocexe, p, proto)
429protocmd = ("cp $SRCDIR/{} $SRCDIR/{} && ".format(p, proto) +
430"cp $SRCDIR/caffe2/proto/caffe2.proto $SRCDIR/caffe2.proto && " +
431"sed -i -e 's/caffe2\\/proto\\/caffe2.proto/caffe2.proto/g' $SRCDIR/{} && ".format(proto) +
432("$(exe fbsource//third-party/protobuf:protoc-host) " if using_protobuf_v3() else "$(exe fbsource//xplat/third-party/protobuf:protoc) --osx $(location fbsource//xplat/third-party/protobuf:protoc.Darwin) --linux $(location fbsource//xplat/third-party/protobuf:protoc.Linux) ") +
433"-I $SRCDIR --cpp_out=$OUT $SRCDIR/{}".format(proto))
434buck_genrule(
435name = prefix + proto,
436srcs = sorted(collections.uniq([p, "caffe2/proto/caffe2.proto"])),
437cmd = protocmd,
438cmd_exe = protocmd_exe,
439out = ".",
440)
441(name, _) = paths.split_extension(proto)
442cpp = name + ".pb.cc"
443h = name + ".pb.h"
444buck_genrule(
445name = prefix + h,
446cmd_exe = "@powershell -Command \" & { " + "(Get-Content $(location :{})\\{}".format(prefix + proto, h) + ") -replace \\\"caffe2.pb.h\\\", \\\"caffe2/proto/caffe2.pb.h\\\" | Set-Content $OUT } \"",
447bash = "cp -f $(location :{})/{} $OUT && ".format(prefix + proto, h) +
448"sed -i -e 's/caffe2.pb.h/caffe2\\/proto\\/caffe2.pb.h/g' $OUT",
449out = h,
450)
451headers["caffe2/proto/" + h] = ":{}".format(prefix + h)
452raw_headers[h] = ":{}".format(prefix + h)
453buck_genrule(
454name = prefix + cpp,
455cmd_exe = "@powershell -Command copy $(location :{})/{} $OUT".format(prefix + proto, cpp),
456bash = "cp -f $(location :{})/{} $OUT".format(prefix + proto, cpp),
457out = cpp,
458)
459cpps.append(":{}".format(prefix + cpp))
460return (cpps, headers, raw_headers)
461
462def libcaffe2_cxx_library(name, use_hptt, **kwargs):
463c2_cxx_library(
464name = name,
465exported_deps = [
466"fbsource//xplat/caffe2/c10:c10",
467get_c2_protobuf_dep(),
468":caffe2_protobuf_headers",
469":pthreadpool",
470":common_core",
471":caffe2_proto_types",
472],
473compiler_flags = get_c2_xplat_compiler_flags() if use_hptt else get_c2_xplat_no_hptt_compiler_flags(),
474exported_preprocessor_flags = get_c2_xplat_preprocessor_flags() if use_hptt else get_c2_xplat_no_hptt_preprocessor_flags(),
475cxx_preprocessor_flags = C2_XPLAT_SERVER_PREPROCESSOR_FLAGS,
476fbandroid_exported_preprocessor_flags = get_c2_fbandroid_xplat_compiler_flags(),
477fbobjc_exported_preprocessor_flags = get_c2_fbobjc_xplat_compiler_flags(),
478# Hack to work around lack of platform_srcs support in Xcode project generation.
479macosx_extra_xcode_sources_override = [],
480link_whole = True,
481**kwargs
482)
483
484def c2_operator_library(name, **kwargs):
485dict_defs.key_extend(
486kwargs,
487"deps",
488[
489"fbsource//xplat/folly:molly",
490"fbsource//third-party/glog:glog",
491":caffe2",
492] + ([":aten_cpu"] if get_c2_expose_op_to_c10() else []),
493)
494
495# NOTE: Currently operators can "depend" on other operators, which is used
496# so that loading one will implicitly load the dependencies. So, make sure
497# that no `--as-needed` flags pulled in from dependencies cause these
498# operator deps to get dropped.
499linker_flags = [] if (read_config("caffe2", "link_as_needed", "0") == "1") else ["-Wl,--no-as-needed"]
500c2_cxx_library(
501name = name,
502soname = "lib" + name + ".$(ext)",
503fbandroid_compiler_flags = get_c2_default_cxx_args()["fbandroid_compiler_flags"] + ["-Os"],
504fbobjc_compiler_flags = get_c2_default_cxx_args()["fbobjc_compiler_flags"] + ["-Oz", "-DCOMPILING_FOR_MIN_SIZE=1"],
505link_whole = True,
506cxx_exported_linker_flags = linker_flags,
507fbandroid_exported_linker_flags = linker_flags,
508exported_deps = [
509":caffe2",
510],
511**kwargs
512)
513
514def c2_genrule(genrule, genfiles, prefix = "", src_path = "", header_namespace = ""):
515headers = {}
516srcs = []
517for generated_filename in genfiles:
518buck_genrule(
519name = prefix + generated_filename,
520bash = "cp -f $(location :{})/{} $OUT".format(genrule, src_path + generated_filename),
521cmd_exe = "@powershell -Command copy $(location :{})/{} $OUT".format(genrule, src_path + generated_filename),
522out = generated_filename,
523)
524rule = ":{}{}".format(prefix, generated_filename)
525headers[header_namespace + generated_filename] = rule
526srcs.append(rule)
527return {"headers": headers, "srcs": srcs}
528