sentencepiece

1
#!/usr/bin/env python
2

3
# Copyright 2018 Google Inc.
4
#
5
# Licensed under the Apache License, Version 2.0 (the "License");
6
# you may not use this file except in compliance with the License.
7
# You may obtain a copy of the License at
8
#
9
#     http://www.apache.org/licenses/LICENSE-2.0
10
#
11
# Unless required by applicable law or agreed to in writing, software
12
# distributed under the License is distributed on an "AS IS" BASIS,
13
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14
# See the License for the specific language governing permissions and
15
# limitations under the License.!
16

17
import codecs
18
import os
19
import string
20
import subprocess
21
import sys
22
from setuptools import Extension, setup
23
from setuptools.command.build_ext import build_ext as _build_ext
24
from setuptools.command.build_py import build_py as _build_py
25

26
sys.path.append(os.path.join('.', 'test'))
27

28

29
def long_description():
30
  with codecs.open('README.md', 'r', 'utf-8') as f:
31
    long_description = f.read()
32
  return long_description
33

34

35
exec(open('src/sentencepiece/_version.py').read())
36

37

38
def run_pkg_config(section, pkg_config_path=None):
39
  try:
40
    cmd = 'pkg-config sentencepiece --{}'.format(section)
41
    if pkg_config_path:
42
      cmd = 'env PKG_CONFIG_PATH={} {}'.format(pkg_config_path, cmd)
43
    output = subprocess.check_output(cmd, shell=True)
44
    if sys.version_info >= (3, 0, 0):
45
      output = output.decode('utf-8')
46
  except subprocess.CalledProcessError:
47
    sys.stderr.write('Failed to find sentencepiece pkg-config\n')
48
    sys.exit(1)
49
  return output.strip().split()
50

51

52
def is_sentencepiece_installed():
53
  try:
54
    subprocess.check_call('pkg-config sentencepiece --libs', shell=True)
55
    return True
56
  except subprocess.CalledProcessError:
57
    return False
58

59

60
def get_cflags_and_libs(root):
61
  cflags = ['-std=c++17', '-I' + os.path.join(root, 'include')]
62
  libs = []
63
  if os.path.exists(os.path.join(root, 'lib/pkgconfig/sentencepiece.pc')):
64
    libs = [
65
        os.path.join(root, 'lib/libsentencepiece.a'),
66
        os.path.join(root, 'lib/libsentencepiece_train.a'),
67
    ]
68
  elif os.path.exists(os.path.join(root, 'lib64/pkgconfig/sentencepiece.pc')):
69
    libs = [
70
        os.path.join(root, 'lib64/libsentencepiece.a'),
71
        os.path.join(root, 'lib64/libsentencepiece_train.a'),
72
    ]
73
  return cflags, libs
74

75

76
class build_ext(_build_ext):
77
  """Override build_extension to run cmake."""
78

79
  def build_extension(self, ext):
80
    cflags, libs = get_cflags_and_libs('../build/root')
81

82
    if len(libs) == 0:
83
      if is_sentencepiece_installed():
84
        cflags = cflags + run_pkg_config('cflags')
85
        libs = run_pkg_config('libs')
86
      else:
87
        subprocess.check_call(['./build_bundled.sh', __version__])
88
        cflags, libs = get_cflags_and_libs('./build/root')
89

90
    # Fix compile on some versions of Mac OSX
91
    # See: https://github.com/neulab/xnmt/issues/199
92
    if sys.platform == 'darwin':
93
      cflags.append('-mmacosx-version-min=10.9')
94
    else:
95
      cflags.append('-Wl,-strip-all')
96
      libs.append('-Wl,-strip-all')
97
    if sys.platform == 'linux':
98
      libs.append('-Wl,-Bsymbolic')
99
    print('## cflags={}'.format(' '.join(cflags)))
100
    print('## libs={}'.format(' '.join(libs)))
101
    ext.extra_compile_args = cflags
102
    ext.extra_link_args = libs
103
    _build_ext.build_extension(self, ext)
104

105

106
if os.name == 'nt':
107
  # Must pre-install sentencepice into build directory.
108
  arch = 'win32'
109
  if sys.maxsize > 2**32:
110
    arch = 'amd64'
111
  if os.path.exists('..\\build\\root_{}\\lib'.format(arch)):
112
    cflags = ['/std:c++17', '/I..\\build\\root_{}\\include'.format(arch)]
113
    libs = [
114
        '..\\build\\root_{}\\lib\\sentencepiece.lib'.format(arch),
115
        '..\\build\\root_{}\\lib\\sentencepiece_train.lib'.format(arch),
116
    ]
117
  elif os.path.exists('..\\build\\root\\lib'):
118
    cflags = ['/std:c++17', '/I..\\build\\root\\include']
119
    libs = [
120
        '..\\build\\root\\lib\\sentencepiece.lib',
121
        '..\\build\\root\\lib\\sentencepiece_train.lib',
122
    ]
123
  else:
124
    # build library locally with cmake and vc++.
125
    cmake_arch = 'Win32'
126
    if arch == 'amd64':
127
      cmake_arch = 'x64'
128
    subprocess.check_call([
129
        'cmake',
130
        'sentencepiece',
131
        '-A',
132
        cmake_arch,
133
        '-B',
134
        'build',
135
        '-DSPM_ENABLE_SHARED=OFF',
136
        '-DCMAKE_INSTALL_PREFIX=build\\root',
137
    ])
138
    subprocess.check_call([
139
        'cmake',
140
        '--build',
141
        'build',
142
        '--config',
143
        'Release',
144
        '--target',
145
        'install',
146
        '--parallel',
147
        '8',
148
    ])
149
    cflags = ['/std:c++17', '/I.\\build\\root\\include']
150
    libs = [
151
        '.\\build\\root\\lib\\sentencepiece.lib',
152
        '.\\build\\root\\lib\\sentencepiece_train.lib',
153
    ]
154

155
  SENTENCEPIECE_EXT = Extension(
156
      'sentencepiece._sentencepiece',
157
      sources=['src/sentencepiece/sentencepiece_wrap.cxx'],
158
      extra_compile_args=cflags,
159
      extra_link_args=libs,
160
  )
161
  cmdclass = {}
162
else:
163
  SENTENCEPIECE_EXT = Extension(
164
      'sentencepiece._sentencepiece',
165
      sources=['src/sentencepiece/sentencepiece_wrap.cxx'],
166
  )
167
  cmdclass = {'build_ext': build_ext}
168

169
setup(
170
    name='sentencepiece',
171
    author='Taku Kudo',
172
    author_email='taku@google.com',
173
    description='SentencePiece python wrapper',
174
    long_description=long_description(),
175
    long_description_content_type='text/markdown',
176
    version=__version__,
177
    package_dir={'': 'src'},
178
    url='https://github.com/google/sentencepiece',
179
    license='Apache',
180
    platforms='Unix',
181
    py_modules=[
182
        'sentencepiece/__init__',
183
        'sentencepiece/_version',
184
        'sentencepiece/sentencepiece_model_pb2',
185
        'sentencepiece/sentencepiece_pb2',
186
    ],
187
    ext_modules=[SENTENCEPIECE_EXT],
188
    cmdclass=cmdclass,
189
    classifiers=[
190
        'Development Status :: 5 - Production/Stable',
191
        'Environment :: Console',
192
        'Intended Audience :: Developers',
193
        'Intended Audience :: Science/Research',
194
        'License :: OSI Approved :: Apache Software License',
195
        'Operating System :: Unix',
196
        'Programming Language :: Python',
197
        'Topic :: Text Processing :: Linguistic',
198
        'Topic :: Software Development :: Libraries :: Python Modules',
199
    ],
200
    test_suite='sentencepiece_test.suite',
201
)
202