hyperopt

1
# This file has been taken from Keras' `docs` module found here:
2
# https://github.com/keras-team/keras/blob/master/docs/autogen.py
3
#
4

5
import re
6
import inspect
7
import os
8
import shutil
9

10

11
EXCLUDE = {}
12

13

14
PAGES = [
15
    # {
16
    #     'page': 'target.md',
17
    #     'classes': [
18
    #     ],
19
    #     'functions': [
20
    #     ],
21
    # },
22
    # {
23
    #     'page': 'other_target.md',
24
    #     'all_module_functions': [],
25
    # },
26
]
27

28
ROOT = "http://hyperopt.github.io/hyperopt"
29

30

31
def get_function_signature(function, method=True):
32
    wrapped = getattr(function, "_original_function", None)
33
    if wrapped is None:
34
        signature = inspect.getargspec(function)
35
    else:
36
        signature = inspect.getargspec(wrapped)
37
    defaults = signature.defaults
38
    if method:
39
        args = signature.args[1:]
40
    else:
41
        args = signature.args
42
    if defaults:
43
        kwargs = zip(args[-len(defaults) :], defaults)
44
        args = args[: -len(defaults)]
45
    else:
46
        kwargs = []
47

48
    signature = [f"{clean_module_name(function.__module__)}.{function.__name__}("]
49

50
    for arg in args:
51
        signature.append(str(arg))
52
    for key, value in kwargs:
53
        if isinstance(value, str):
54
            value = f"'{value}'"
55
        signature.append(f"{key}={value}")
56
    return ", ".join(signature) + ")"
57

58

59
def get_class_signature(cls):
60
    try:
61
        class_signature = get_function_signature(cls.__init__)
62
        class_signature = class_signature.replace("__init__", cls.__name__)
63
    except (TypeError, AttributeError):
64
        # in case the class inherits from object and does not
65
        # define __init__
66
        class_signature = "{clean_module_name}.{cls_name}()".format(
67
            clean_module_name=clean_module_name(cls.__module__), cls_name=cls.__name__
68
        )
69
    return class_signature
70

71

72
def clean_module_name(name):
73
    assert name[:8] == "hyperopt.", "Invalid module name: %s" % name
74
    return name
75

76

77
def class_to_docs_link(cls):
78
    module_name = clean_module_name(cls.__module__)
79
    module_name = module_name[6:]
80
    link = ROOT + module_name.replace(".", "/") + "#" + cls.__name__.lower()
81
    return link
82

83

84
def class_to_source_link(cls):
85
    module_name = clean_module_name(cls.__module__)
86
    path = module_name.replace(".", "/")
87
    path += ".py"
88
    line = inspect.getsourcelines(cls)[-1]
89
    link = "https://github.com/hyperopt/" "hyperopt/blob/master/" + path + "#L" + str(
90
        line
91
    )
92
    return "[[source]](" + link + ")"
93

94

95
def code_snippet(snippet):
96
    result = "```python\n"
97
    result += snippet + "\n"
98
    result += "```\n"
99
    return result
100

101

102
def count_leading_spaces(s):
103
    ws = re.search(r"\S", s)
104
    if ws:
105
        return ws.start()
106
    else:
107
        return 0
108

109

110
def process_list_block(docstring, starting_point, leading_spaces, marker):
111
    ending_point = docstring.find("\n\n", starting_point)
112
    block = docstring[
113
        starting_point : (None if ending_point == -1 else ending_point - 1)
114
    ]
115
    # Place marker for later reinjection.
116
    docstring = docstring.replace(block, marker)
117
    lines = block.split("\n")
118
    # Remove the computed number of leading white spaces from each line.
119
    lines = [re.sub("^" + " " * leading_spaces, "", line) for line in lines]
120
    # Usually lines have at least 4 additional leading spaces.
121
    # These have to be removed, but first the list roots have to be detected.
122
    top_level_regex = r"^    ([^\s\\\(]+):(.*)"
123
    top_level_replacement = r"- __\1__:\2"
124
    lines = [re.sub(top_level_regex, top_level_replacement, line) for line in lines]
125
    # All the other lines get simply the 4 leading space (if present) removed
126
    lines = [re.sub(r"^    ", "", line) for line in lines]
127
    # Fix text lines after lists
128
    indent = 0
129
    text_block = False
130
    for i in range(len(lines)):
131
        line = lines[i]
132
        spaces = re.search(r"\S", line)
133
        if spaces:
134
            # If it is a list element
135
            if line[spaces.start()] == "-":
136
                indent = spaces.start() + 1
137
                if text_block:
138
                    text_block = False
139
                    lines[i] = "\n" + line
140
            elif spaces.start() < indent:
141
                text_block = True
142
                indent = spaces.start()
143
                lines[i] = "\n" + line
144
        else:
145
            text_block = False
146
            indent = 0
147
    block = "\n".join(lines)
148
    return docstring, block
149

150

151
def process_docstring(docstring):
152
    # First, extract code blocks and process them.
153
    code_blocks = []
154
    if "```" in docstring:
155
        tmp = docstring[:]
156
        while "```" in tmp:
157
            tmp = tmp[tmp.find("```") :]
158
            index = tmp[3:].find("```") + 6
159
            snippet = tmp[:index]
160
            # Place marker in docstring for later reinjection.
161
            docstring = docstring.replace(snippet, "$CODE_BLOCK_%d" % len(code_blocks))
162
            snippet_lines = snippet.split("\n")
163
            # Remove leading spaces.
164
            num_leading_spaces = snippet_lines[-1].find("`")
165
            snippet_lines = [snippet_lines[0]] + [
166
                line[num_leading_spaces:] for line in snippet_lines[1:]
167
            ]
168
            # Most code snippets have 3 or 4 more leading spaces
169
            # on inner lines, but not all. Remove them.
170
            inner_lines = snippet_lines[1:-1]
171
            leading_spaces = None
172
            for line in inner_lines:
173
                if not line or line[0] == "\n":
174
                    continue
175
                spaces = count_leading_spaces(line)
176
                if leading_spaces is None:
177
                    leading_spaces = spaces
178
                if spaces < leading_spaces:
179
                    leading_spaces = spaces
180
            if leading_spaces:
181
                snippet_lines = (
182
                    [snippet_lines[0]]
183
                    + [line[leading_spaces:] for line in snippet_lines[1:-1]]
184
                    + [snippet_lines[-1]]
185
                )
186
            snippet = "\n".join(snippet_lines)
187
            code_blocks.append(snippet)
188
            tmp = tmp[index:]
189

190
    # Format docstring lists.
191
    section_regex = r"\n( +)# (.*)\n"
192
    section_idx = re.search(section_regex, docstring)
193
    shift = 0
194
    sections = {}
195
    while section_idx and section_idx.group(2):
196
        anchor = section_idx.group(2)
197
        leading_spaces = len(section_idx.group(1))
198
        shift += section_idx.end()
199
        marker = "$" + anchor.replace(" ", "_") + "$"
200
        docstring, content = process_list_block(
201
            docstring, shift, leading_spaces, marker
202
        )
203
        sections[marker] = content
204
        section_idx = re.search(section_regex, docstring[shift:])
205

206
    # Format docstring section titles.
207
    docstring = re.sub(r"\n(\s+)# (.*)\n", r"\n\1__\2__\n\n", docstring)
208

209
    # Strip all remaining leading spaces.
210
    lines = docstring.split("\n")
211
    docstring = "\n".join([line.lstrip(" ") for line in lines])
212

213
    # Reinject list blocks.
214
    for marker, content in sections.items():
215
        docstring = docstring.replace(marker, content)
216

217
    # Reinject code blocks.
218
    for i, code_block in enumerate(code_blocks):
219
        docstring = docstring.replace("$CODE_BLOCK_%d" % i, code_block)
220
    return docstring
221

222

223
print("Cleaning up existing sources directory.")
224
if os.path.exists("sources"):
225
    shutil.rmtree("sources")
226

227
print("Populating sources directory with templates.")
228
for subdir, dirs, fnames in os.walk("templates"):
229
    for fname in fnames:
230
        new_subdir = subdir.replace("templates", "sources")
231
        if not os.path.exists(new_subdir):
232
            os.makedirs(new_subdir)
233
        if fname[-3:] == ".md":
234
            fpath = os.path.join(subdir, fname)
235
            new_fpath = fpath.replace("templates", "sources")
236
            shutil.copy(fpath, new_fpath)
237

238

239
def read_file(path):
240
    with open(path) as f:
241
        return f.read()
242

243

244
def collect_class_methods(cls, methods):
245
    if isinstance(methods, (list, tuple)):
246
        return [getattr(cls, m) if isinstance(m, str) else m for m in methods]
247
    methods = []
248
    for _, method in inspect.getmembers(cls, predicate=inspect.isroutine):
249
        if method.__name__[0] == "_" or method.__name__ in EXCLUDE:
250
            continue
251
        methods.append(method)
252
    return methods
253

254

255
def render_function(function, method=True):
256
    subblocks = []
257
    signature = get_function_signature(function, method=method)
258
    if method:
259
        signature = signature.replace(clean_module_name(function.__module__) + ".", "")
260
    subblocks.append("### " + function.__name__ + "\n")
261
    subblocks.append(code_snippet(signature))
262
    docstring = function.__doc__
263
    if docstring:
264
        subblocks.append(process_docstring(docstring))
265
    return "\n\n".join(subblocks)
266

267

268
def read_page_data(page_data, type):
269
    assert type in ["classes", "functions", "methods"]
270
    data = page_data.get(type, [])
271
    for module in page_data.get(f"all_module_{type}", []):
272
        module_data = []
273
        for name in dir(module):
274
            if name[0] == "_" or name in EXCLUDE:
275
                continue
276
            module_member = getattr(module, name)
277
            if (
278
                inspect.isclass(module_member)
279
                and type == "classes"
280
                or inspect.isfunction(module_member)
281
                and type == "functions"
282
            ):
283
                instance = module_member
284
                if module.__name__ in instance.__module__:
285
                    if instance not in module_data:
286
                        module_data.append(instance)
287
        module_data.sort(key=lambda x: id(x))
288
        data += module_data
289
    return data
290

291

292
if __name__ == "__main__":
293
    readme = read_file("../README.md")
294
    index = read_file("templates/index.md")
295
    index = index.replace("{{autogenerated}}", readme[readme.find("##") :])
296
    with open("sources/index.md", "w") as f:
297
        f.write(index)
298

299
    print("Generating Hyperopt docs")
300
    for page_data in PAGES:
301
        classes = read_page_data(page_data, "classes")
302

303
        blocks = []
304
        for element in classes:
305
            if not isinstance(element, (list, tuple)):
306
                element = (element, [])
307
            cls = element[0]
308
            subblocks = []
309
            signature = get_class_signature(cls)
310
            subblocks.append(
311
                '<span style="float:right;">' + class_to_source_link(cls) + "</span>"
312
            )
313
            if element[1]:
314
                subblocks.append("## " + cls.__name__ + " class\n")
315
            else:
316
                subblocks.append("### " + cls.__name__ + "\n")
317
            subblocks.append(code_snippet(signature))
318
            docstring = cls.__doc__
319
            if docstring:
320
                subblocks.append(process_docstring(docstring))
321
            methods = collect_class_methods(cls, element[1])
322
            if methods:
323
                subblocks.append("\n---")
324
                subblocks.append("## " + cls.__name__ + " methods\n")
325
                subblocks.append(
326
                    "\n---\n".join(
327
                        [render_function(method, method=True) for method in methods]
328
                    )
329
                )
330
            blocks.append("\n".join(subblocks))
331

332
        methods = read_page_data(page_data, "methods")
333

334
        for method in methods:
335
            blocks.append(render_function(method, method=True))
336

337
        functions = read_page_data(page_data, "functions")
338

339
        for function in functions:
340
            blocks.append(render_function(function, method=False))
341

342
        if not blocks:
343
            raise RuntimeError("Found no content for page " + page_data["page"])
344

345
        mkdown = "\n----\n\n".join(blocks)
346
        # save module page.
347
        # Either insert content into existing page,
348
        # or create page otherwise
349
        page_name = page_data["page"]
350
        path = os.path.join("sources", page_name)
351
        if os.path.exists(path):
352
            template = read_file(path)
353
            assert "{{autogenerated}}" in template, (
354
                "Template found for " + path + " but missing {{autogenerated}}" " tag."
355
            )
356
            mkdown = template.replace("{{autogenerated}}", mkdown)
357
            print("...inserting autogenerated content into template:", path)
358
        else:
359
            print("...creating new page with autogenerated content:", path)
360
        subdir = os.path.dirname(path)
361
        if not os.path.exists(subdir):
362
            os.makedirs(subdir)
363
        with open(path, "w") as f:
364
            f.write(mkdown)
365