/
githubmirror
/
libxmlb
Обзор
Документация
Войти
/
githubmirror
/
libxmlb
Код
Запросы
0
Пакеты
0
Релизы
0
Аналитика
Безопасность
0.3.27
src/xb-builder.c
1 289 строк
37 KB
Richard Hughes
Reject XML with more than 65535 unique element names
15 май 2026, 16:07
15 май 2026, 16:07
7cc04c8
Код
Авторство
О чём код?
/* * Copyright 2018 Richard Hughes <richard@hughsie.com> * * SPDX-License-Identifier: LGPL-2.1-or-later */ #define G_LOG_DOMAIN "XbSilo" #include "config.h" #include <gio/gio.h> #include <string.h> #include "xb-builder-fixup-private.h" #include "xb-builder-node-private.h" #include "xb-builder-source-private.h" #include "xb-builder.h" #include "xb-opcode-private.h" #include "xb-silo-private.h" #include "xb-string-private.h" #include "xb-version.h" #define XB_BUILDER_MAX_DEPTH 100 typedef struct { GPtrArray *sources; /* of XbBuilderSource */ GPtrArray *nodes; /* of XbBuilderNode */ GPtrArray *fixups; /* of XbBuilderFixup */ GPtrArray *locales; /* of str */ XbSiloProfileFlags profile_flags; GString *guid; } XbBuilderPrivate; G_DEFINE_TYPE_WITH_PRIVATE(XbBuilder, xb_builder, G_TYPE_OBJECT) #define GET_PRIVATE(o) (xb_builder_get_instance_private(o)) typedef struct { XbSilo *silo; XbBuilderNode *root; /* transfer full */ XbBuilderNode *current; /* transfer none */ XbBuilderCompileFlags compile_flags; XbBuilderSourceFlags source_flags; GHashTable *strtab_hash; GByteArray *strtab; GPtrArray *locales; gboolean elem_closed; guint depth; GError *error; } XbBuilderCompileHelper; static guint32 xb_builder_compile_add_to_strtab(XbBuilderCompileHelper *helper, const gchar *str) { gpointer val; guint32 idx; gsize str_len; /* already exists */ if (g_hash_table_lookup_extended(helper->strtab_hash, str, NULL, &val)) return GPOINTER_TO_UINT(val); /* check for integer truncation */ if (helper->strtab->len > G_MAXINT32) { g_set_error(&helper->error, G_IO_ERROR, G_IO_ERROR_INVALID_DATA, "string table too large"); return XB_SILO_UNSET; } /* check if adding this string would exceed the limit */ str_len = strlen(str) + 1; if (helper->strtab->len + str_len > G_MAXUINT32) { g_set_error(&helper->error, G_IO_ERROR, G_IO_ERROR_INVALID_DATA, "string table too large"); return XB_SILO_UNSET; } /* new */ idx = helper->strtab->len; g_byte_array_append(helper->strtab, (const guint8 *)str, str_len); g_hash_table_insert(helper->strtab_hash, g_strdup(str), GUINT_TO_POINTER(idx)); return idx; } static gint xb_builder_get_locale_priority(XbBuilderCompileHelper *helper, const gchar *locale) { for (guint i = 0; i < helper->locales->len; i++) { const gchar *locale_tmp = g_ptr_array_index(helper->locales, i); if (g_strcmp0(locale_tmp, locale) == 0) return helper->locales->len - i; } return -1; } static void xb_builder_compile_start_element_cb(GMarkupParseContext *context, const gchar *element_name, const gchar **attr_names, const gchar **attr_values, gpointer user_data, GError **error) { XbBuilderCompileHelper *helper = (XbBuilderCompileHelper *)user_data; g_autoptr(XbBuilderNode) bn = xb_builder_node_new(element_name); /* check recursion depth to prevent stack exhaustion */ if (helper->depth >= XB_BUILDER_MAX_DEPTH) { g_set_error(error, G_IO_ERROR, G_IO_ERROR_INVALID_DATA, "nesting deeper than %u levels not supported", (guint)XB_BUILDER_MAX_DEPTH); return; } helper->depth++; /* parent node is being ignored */ if (helper->current != NULL && xb_builder_node_has_flag(helper->current, XB_BUILDER_NODE_FLAG_IGNORE)) xb_builder_node_add_flag(bn, XB_BUILDER_NODE_FLAG_IGNORE); /* check if we should ignore the locale */ if (!xb_builder_node_has_flag(bn, XB_BUILDER_NODE_FLAG_IGNORE) && helper->compile_flags & XB_BUILDER_COMPILE_FLAG_NATIVE_LANGS) { const gchar *xml_lang = NULL; for (guint i = 0; attr_names[i] != NULL; i++) { if (g_strcmp0(attr_names[i], "xml:lang") == 0) { xml_lang = attr_values[i]; break; } } if (xml_lang == NULL) { if (helper->current != NULL) { gint prio = xb_builder_node_get_priority(helper->current); xb_builder_node_set_priority(bn, prio); } } else { gint prio = xb_builder_get_locale_priority(helper, xml_lang); if (prio < 0) xb_builder_node_add_flag(bn, XB_BUILDER_NODE_FLAG_IGNORE); xb_builder_node_set_priority(bn, prio); } } /* add attributes */ if (!xb_builder_node_has_flag(bn, XB_BUILDER_NODE_FLAG_IGNORE)) { for (guint i = 0; attr_names[i] != NULL; i++) xb_builder_node_set_attr(bn, attr_names[i], attr_values[i]); } /* add to tree */ xb_builder_node_add_child(helper->current, bn); helper->current = bn; helper->elem_closed = FALSE; } static void xb_builder_compile_end_element_cb(GMarkupParseContext *context, const gchar *element_name, gpointer user_data, GError **error) { XbBuilderCompileHelper *helper = (XbBuilderCompileHelper *)user_data; g_autoptr(XbBuilderNode) parent = xb_builder_node_get_parent(helper->current); if (parent == NULL) { g_set_error_literal(error, G_IO_ERROR, G_IO_ERROR_INVALID_DATA, "Mismatched XML; no parent"); return; } helper->current = parent; helper->depth--; helper->elem_closed = TRUE; } static void xb_builder_compile_text_cb(GMarkupParseContext *context, const gchar *text, gsize text_len, gpointer user_data, GError **error) { XbBuilderCompileHelper *helper = (XbBuilderCompileHelper *)user_data; XbBuilderNode *bn = helper->current; XbBuilderNode *bc = xb_builder_node_get_last_child(bn); /* unimportant */ if (xb_builder_node_has_flag(bn, XB_BUILDER_NODE_FLAG_IGNORE)) return; /* repair text unless we know it's valid */ if (helper->source_flags & XB_BUILDER_SOURCE_FLAG_LITERAL_TEXT) xb_builder_node_add_flag(bn, XB_BUILDER_NODE_FLAG_LITERAL_TEXT); /* text or tail */ if (!helper->elem_closed) { if (xb_builder_node_has_flag(bn, XB_BUILDER_NODE_FLAG_HAS_TEXT)) { g_autoptr(GString) str = g_string_new(xb_builder_node_get_text(bn)); g_string_append_len(str, text, text_len); xb_builder_node_set_text(bn, str->str, str->len); } else { xb_builder_node_set_text(bn, text, text_len); } return; } /* does this node have a child */ if (bc != NULL) { xb_builder_node_set_tail(bc, text, text_len); return; } /* always set a tail, even if already set */ xb_builder_node_set_tail(bn, text, text_len); } /** * xb_builder_import_source: * @self: a #XbSilo * @source: a #XbBuilderSource * * Adds a #XbBuilderSource to the #XbBuilder. * * Since: 0.1.0 **/ void xb_builder_import_source(XbBuilder *self, XbBuilderSource *source) { XbBuilderPrivate *priv = GET_PRIVATE(self); g_autofree gchar *guid = NULL; g_return_if_fail(XB_IS_BUILDER(self)); g_return_if_fail(XB_IS_BUILDER_SOURCE(source)); /* get latest GUID */ guid = xb_builder_source_get_guid(source); xb_builder_append_guid(self, guid); g_ptr_array_add(priv->sources, g_object_ref(source)); } static gboolean xb_builder_compile_source(XbBuilderCompileHelper *helper, XbBuilderSource *source, XbBuilderNode *root, GCancellable *cancellable, GError **error) { GPtrArray *children; XbBuilderNode *info; gsize chunk_size = 32 * 1024; gssize len; g_autofree gchar *data = NULL; g_autofree gchar *guid = xb_builder_source_get_guid(source); g_autoptr(GPtrArray) children_copy = NULL; g_autoptr(GInputStream) istream = NULL; g_autoptr(GMarkupParseContext) ctx = NULL; g_autoptr(GTimer) timer = xb_silo_start_profile(helper->silo); g_autoptr(XbBuilderNode) root_tmp = xb_builder_node_new(NULL); const GMarkupParser parser = {xb_builder_compile_start_element_cb, xb_builder_compile_end_element_cb, xb_builder_compile_text_cb, NULL, NULL}; /* add the source to a fake root in case it fails during processing */ helper->current = root_tmp; helper->source_flags = xb_builder_source_get_flags(source); /* decompress */ istream = xb_builder_source_get_istream(source, cancellable, error); if (istream == NULL) return FALSE; /* parse */ ctx = g_markup_parse_context_new(&parser, G_MARKUP_PREFIX_ERROR_POSITION | G_MARKUP_TREAT_CDATA_AS_TEXT, helper, NULL); data = g_malloc(chunk_size); while ((len = g_input_stream_read(istream, data, chunk_size, cancellable, error)) > 0) { if (!g_markup_parse_context_parse(ctx, data, len, error)) return FALSE; } if (len < 0) return FALSE; /* more opening than closing */ if (root_tmp != helper->current) { g_set_error_literal(error, G_IO_ERROR, G_IO_ERROR_INVALID_DATA, "Mismatched XML"); return FALSE; } /* run any node functions */ if (!xb_builder_source_fixup(source, root_tmp, error)) return FALSE; /* check to see if the root was ignored */ if (xb_builder_node_has_flag(root_tmp, XB_BUILDER_NODE_FLAG_IGNORE)) { g_set_error_literal(error, G_IO_ERROR, G_IO_ERROR_INVALID_DATA, "root node cannot be ignored"); return FALSE; } /* a single root with no siblings was required */ if (helper->compile_flags & XB_BUILDER_COMPILE_FLAG_SINGLE_ROOT) { if (xb_builder_node_get_children(root_tmp)->len > 1) { g_set_error_literal(error, G_IO_ERROR, G_IO_ERROR_INVALID_DATA, "A root node without siblings was required"); return FALSE; } } /* this is something we can query with later */ info = xb_builder_source_get_info(source); if (info != NULL) { children = xb_builder_node_get_children(helper->current); for (guint i = 0; i < children->len; i++) { XbBuilderNode *bn = g_ptr_array_index(children, i); if (!xb_builder_node_has_flag(bn, XB_BUILDER_NODE_FLAG_IGNORE)) xb_builder_node_add_child(bn, info); } } /* add the children to the main document */ children = xb_builder_node_get_children(root_tmp); children_copy = g_ptr_array_new_with_free_func((GDestroyNotify)g_object_unref); for (guint i = 0; i < children->len; i++) { XbBuilderNode *bn = g_ptr_array_index(children, i); g_ptr_array_add(children_copy, g_object_ref(bn)); } for (guint i = 0; i < children_copy->len; i++) { XbBuilderNode *bn = g_ptr_array_index(children_copy, i); xb_builder_node_unlink(bn); xb_builder_node_add_child(root, bn); } /* success */ xb_silo_add_profile(helper->silo, timer, "compile %s", guid); return TRUE; } static gboolean xb_builder_strtab_element_names_cb(XbBuilderNode *bn, gpointer user_data) { XbBuilderCompileHelper *helper = (XbBuilderCompileHelper *)user_data; const gchar *tmp; guint32 strtab_idx; /* root node */ if (xb_builder_node_get_element(bn) == NULL) return FALSE; if (xb_builder_node_has_flag(bn, XB_BUILDER_NODE_FLAG_IGNORE)) return FALSE; tmp = xb_builder_node_get_element(bn); strtab_idx = xb_builder_compile_add_to_strtab(helper, tmp); if (strtab_idx == XB_SILO_UNSET) return TRUE; xb_builder_node_set_element_idx(bn, strtab_idx); return FALSE; } static gboolean xb_builder_strtab_attr_name_cb(XbBuilderNode *bn, gpointer user_data) { GPtrArray *attrs; XbBuilderCompileHelper *helper = (XbBuilderCompileHelper *)user_data; /* root node */ if (xb_builder_node_get_element(bn) == NULL) return FALSE; if (xb_builder_node_has_flag(bn, XB_BUILDER_NODE_FLAG_IGNORE)) return FALSE; attrs = xb_builder_node_get_attrs(bn); for (guint i = 0; attrs != NULL && i < attrs->len; i++) { XbBuilderNodeAttr *attr = g_ptr_array_index(attrs, i); attr->name_idx = xb_builder_compile_add_to_strtab(helper, attr->name); if (attr->name_idx == XB_SILO_UNSET) return TRUE; } return FALSE; } static gboolean xb_builder_strtab_attr_value_cb(XbBuilderNode *bn, gpointer user_data) { GPtrArray *attrs; XbBuilderCompileHelper *helper = (XbBuilderCompileHelper *)user_data; /* root node */ if (xb_builder_node_get_element(bn) == NULL) return FALSE; if (xb_builder_node_has_flag(bn, XB_BUILDER_NODE_FLAG_IGNORE)) return FALSE; attrs = xb_builder_node_get_attrs(bn); for (guint i = 0; attrs != NULL && i < attrs->len; i++) { XbBuilderNodeAttr *attr = g_ptr_array_index(attrs, i); attr->value_idx = xb_builder_compile_add_to_strtab(helper, attr->value); if (attr->value_idx == XB_SILO_UNSET) return TRUE; } return FALSE; } static gboolean xb_builder_strtab_text_cb(XbBuilderNode *bn, gpointer user_data) { XbBuilderCompileHelper *helper = (XbBuilderCompileHelper *)user_data; const gchar *tmp; guint32 strtab_idx; /* root node */ if (xb_builder_node_get_element(bn) == NULL) return FALSE; if (xb_builder_node_has_flag(bn, XB_BUILDER_NODE_FLAG_IGNORE)) return FALSE; if (xb_builder_node_get_text(bn) != NULL) { tmp = xb_builder_node_get_text(bn); strtab_idx = xb_builder_compile_add_to_strtab(helper, tmp); if (strtab_idx == XB_SILO_UNSET) return TRUE; xb_builder_node_set_text_idx(bn, strtab_idx); } if (xb_builder_node_get_tail(bn) != NULL) { tmp = xb_builder_node_get_tail(bn); strtab_idx = xb_builder_compile_add_to_strtab(helper, tmp); if (strtab_idx == XB_SILO_UNSET) return TRUE; xb_builder_node_set_tail_idx(bn, strtab_idx); } return FALSE; } static gboolean xb_builder_strtab_tokens_cb(XbBuilderNode *bn, gpointer user_data) { XbBuilderCompileHelper *helper = (XbBuilderCompileHelper *)user_data; GPtrArray *tokens = xb_builder_node_get_tokens(bn); guint32 strtab_idx; /* root node */ if (xb_builder_node_get_element(bn) == NULL) return FALSE; if (xb_builder_node_has_flag(bn, XB_BUILDER_NODE_FLAG_IGNORE)) return FALSE; if (tokens == NULL) return FALSE; for (guint i = 0; i < MIN(tokens->len, XB_OPCODE_TOKEN_MAX); i++) { const gchar *tmp = g_ptr_array_index(tokens, i); if (tmp == NULL) continue; strtab_idx = xb_builder_compile_add_to_strtab(helper, tmp); if (strtab_idx == XB_SILO_UNSET) return TRUE; xb_builder_node_add_token_idx(bn, strtab_idx); } return FALSE; } static gboolean xb_builder_xml_lang_prio_cb(XbBuilderNode *bn, gpointer user_data) { GPtrArray *nodes_to_destroy = (GPtrArray *)user_data; gint prio_best = 0; g_autoptr(GPtrArray) nodes = g_ptr_array_new(); GPtrArray *siblings; g_autoptr(XbBuilderNode) parent = xb_builder_node_get_parent(bn); /* root node */ if (xb_builder_node_get_element(bn) == NULL) return FALSE; /* already ignored */ if (xb_builder_node_get_priority(bn) == -2) return FALSE; /* get all the siblings with the same name */ siblings = xb_builder_node_get_children(parent); for (guint i = 0; i < siblings->len; i++) { XbBuilderNode *bn2 = g_ptr_array_index(siblings, i); if (g_strcmp0(xb_builder_node_get_element(bn), xb_builder_node_get_element(bn2)) == 0) g_ptr_array_add(nodes, bn2); } /* only one thing, so bail early */ if (nodes->len == 1) return FALSE; /* find the best locale */ for (guint i = 0; i < nodes->len; i++) { XbBuilderNode *bn2 = g_ptr_array_index(nodes, i); if (xb_builder_node_get_priority(bn2) > prio_best) prio_best = xb_builder_node_get_priority(bn2); } /* add any nodes not as good as the bext locale to the kill list */ for (guint i = 0; i < nodes->len; i++) { XbBuilderNode *bn2 = g_ptr_array_index(nodes, i); if (xb_builder_node_get_priority(bn2) < prio_best) g_ptr_array_add(nodes_to_destroy, g_object_ref(bn2)); /* never visit this node again */ xb_builder_node_set_priority(bn2, -2); } return FALSE; } static gboolean xb_builder_nodetab_size_cb(XbBuilderNode *bn, gpointer user_data) { guint32 *sz = (guint32 *)user_data; /* root node */ if (xb_builder_node_get_element(bn) == NULL) return FALSE; if (xb_builder_node_has_flag(bn, XB_BUILDER_NODE_FLAG_IGNORE)) return FALSE; *sz += xb_builder_node_size(bn) + 1; /* +1 for the sentinel */ return FALSE; } typedef struct { GByteArray *buf; } XbBuilderNodetabHelper; static void xb_builder_nodetab_write_sentinel(XbBuilderNodetabHelper *helper) { XbSiloNode sn = { .flags = XB_SILO_NODE_FLAG_NONE, .attr_count = 0, }; // g_debug ("SENT @%u", (guint) helper->buf->len); g_byte_array_append(helper->buf, (const guint8 *)&sn, xb_silo_node_get_size(&sn)); } #define XB_BUILDER_ATTR_MAX ((1 << 6) - 1) static gboolean xb_builder_nodetab_write_node(XbBuilderNodetabHelper *helper, XbBuilderNode *bn, GError **error) { GPtrArray *attrs = xb_builder_node_get_attrs(bn); GArray *token_idxs = xb_builder_node_get_token_idxs(bn); XbSiloNode sn = { .flags = XB_SILO_NODE_FLAG_IS_ELEMENT, .attr_count = (attrs != NULL) ? attrs->len : 0, .element_name = xb_builder_node_get_element_idx(bn), .next = 0x0, .parent = 0x0, .text = xb_builder_node_get_text_idx(bn), .tail = xb_builder_node_get_tail_idx(bn), .token_count = 0, }; /* sanity check */ if (attrs != NULL && attrs->len > XB_BUILDER_ATTR_MAX) { g_set_error(error, G_IO_ERROR, G_IO_ERROR_NOT_SUPPORTED, "too many attributes: %u", attrs->len); return FALSE; } /* add tokens */ if (token_idxs != NULL) sn.flags |= XB_SILO_NODE_FLAG_IS_TOKENIZED; /* if the node had no children and the text is just whitespace then * remove it even in literal mode */ if (xb_builder_node_has_flag(bn, XB_BUILDER_NODE_FLAG_LITERAL_TEXT)) { if (xb_string_isspace(xb_builder_node_get_text(bn), -1)) sn.text = XB_SILO_UNSET; if (xb_string_isspace(xb_builder_node_get_tail(bn), -1)) sn.tail = XB_SILO_UNSET; } /* save this so we can set up the ->next pointers correctly */ xb_builder_node_set_offset(bn, helper->buf->len); // g_debug ("NODE @%u (%s)", (guint) helper->buf->len, xb_builder_node_get_element //(bn)); /* there is no point adding more tokens than we can match */ if (token_idxs != NULL) sn.token_count = MIN(token_idxs->len, XB_OPCODE_TOKEN_MAX); /* add to the buf */ g_byte_array_append(helper->buf, (const guint8 *)&sn, sizeof(sn)); /* add to the buf */ for (guint i = 0; attrs != NULL && i < attrs->len; i++) { XbBuilderNodeAttr *ba = g_ptr_array_index(attrs, i); XbSiloNodeAttr attr = { .attr_name = ba->name_idx, .attr_value = ba->value_idx, }; g_byte_array_append(helper->buf, (const guint8 *)&attr, sizeof(attr)); } /* add tokens */ for (guint i = 0; i < sn.token_count; i++) { guint32 idx = g_array_index(token_idxs, guint32, i); g_byte_array_append(helper->buf, (const guint8 *)&idx, sizeof(idx)); } /* success */ return TRUE; } static gboolean xb_builder_nodetab_write(XbBuilderNodetabHelper *helper, XbBuilderNode *bn, GError **error) { GPtrArray *children; /* ignore this */ if (xb_builder_node_has_flag(bn, XB_BUILDER_NODE_FLAG_IGNORE)) return TRUE; /* element */ if (xb_builder_node_get_element(bn) != NULL) { if (!xb_builder_nodetab_write_node(helper, bn, error)) return FALSE; } /* children */ children = xb_builder_node_get_children(bn); for (guint i = 0; i < children->len; i++) { XbBuilderNode *bc = g_ptr_array_index(children, i); if (!xb_builder_nodetab_write(helper, bc, error)) return FALSE; } /* sentinel */ if (xb_builder_node_get_element(bn) != NULL) xb_builder_nodetab_write_sentinel(helper); /* success */ return TRUE; } static XbSiloNode * xb_builder_get_node(GByteArray *str, guint32 off) { return (XbSiloNode *)(str->data + off); } static gboolean xb_builder_nodetab_fix_cb(XbBuilderNode *bn, gpointer user_data) { GPtrArray *siblings; XbBuilderNodetabHelper *helper = (XbBuilderNodetabHelper *)user_data; XbSiloNode *sn; gboolean found = FALSE; g_autoptr(XbBuilderNode) parent = xb_builder_node_get_parent(bn); /* root node */ if (xb_builder_node_get_element(bn) == NULL) return FALSE; if (xb_builder_node_has_flag(bn, XB_BUILDER_NODE_FLAG_IGNORE)) return FALSE; /* get the position in the buffer */ sn = xb_builder_get_node(helper->buf, xb_builder_node_get_offset(bn)); if (sn == NULL) return FALSE; /* set the parent if the node has one */ if (xb_builder_node_get_element(parent) != NULL) sn->parent = xb_builder_node_get_offset(parent); /* set ->next if the node has one */ siblings = xb_builder_node_get_children(parent); for (guint i = 0; i < siblings->len; i++) { XbBuilderNode *bn2 = g_ptr_array_index(siblings, i); if (bn2 == bn) { found = TRUE; continue; } if (!found) continue; if (!xb_builder_node_has_flag(bn2, XB_BUILDER_NODE_FLAG_IGNORE)) { sn->next = xb_builder_node_get_offset(bn2); break; } } return FALSE; } static void xb_builder_compile_helper_free(XbBuilderCompileHelper *helper) { g_hash_table_unref(helper->strtab_hash); g_byte_array_unref(helper->strtab); g_clear_error(&helper->error); g_clear_object(&helper->silo); g_object_unref(helper->root); g_free(helper); } G_DEFINE_AUTOPTR_CLEANUP_FUNC(XbBuilderCompileHelper, xb_builder_compile_helper_free) static gchar * xb_builder_generate_guid(XbBuilder *self) { XbBuilderPrivate *priv = GET_PRIVATE(self); XbGuid guid = {0x0}; if (priv->guid->len > 0) { xb_guid_compute_for_data(&guid, (const guint8 *)priv->guid->str, priv->guid->len); } return xb_guid_to_string(&guid); } /** * xb_builder_import_node: * @self: a #XbSilo * @bn: a #XbBuilderNode * * Adds a node tree to the builder. * * If you are manually adding dynamic data sourced from a non-static source then you MUST use * xb_builder_append_guid() with the appropriate GUID value, e.g. the file name and mtime. * * If no appropriate value is available, the caller can use something like: * * g_autofree gchar *tmp = xb_builder_node_export(bn, XB_NODE_EXPORT_FLAG_NONE, NULL); * xb_builder_append_guid(builder, tmp); * * Failure to include an appropriate GUID value would allow an out-of-data silo to be used. * * Since: 0.1.0 **/ void xb_builder_import_node(XbBuilder *self, XbBuilderNode *bn) { XbBuilderPrivate *priv = GET_PRIVATE(self); g_return_if_fail(XB_IS_BUILDER(self)); g_return_if_fail(XB_IS_BUILDER_NODE(bn)); g_ptr_array_add(priv->nodes, g_object_ref(bn)); } /** * xb_builder_add_locale: * @self: a #XbSilo * @locale: a locale, e.g. "en_US" * * Adds a locale to the builder. Locales added first will be prioritised over * locales added later. * * Since: 0.1.0 **/ void xb_builder_add_locale(XbBuilder *self, const gchar *locale) { XbBuilderPrivate *priv = GET_PRIVATE(self); g_return_if_fail(XB_IS_BUILDER(self)); g_return_if_fail(locale != NULL); if (g_str_has_suffix(locale, ".UTF-8")) return; for (guint i = 0; i < priv->locales->len; i++) { const gchar *locale_tmp = g_ptr_array_index(priv->locales, i); if (g_strcmp0(locale_tmp, locale) == 0) return; } g_ptr_array_add(priv->locales, g_strdup(locale)); /* if the user changes LANG, the blob is no longer valid */ xb_builder_append_guid(self, locale); } static gboolean xb_builder_watch_source(XbBuilder *self, XbBuilderSource *source, XbSilo *silo, GCancellable *cancellable, GError **error) { GFile *file = xb_builder_source_get_file(source); g_autoptr(GFile) watched_file = NULL; if (file == NULL) return TRUE; if ((xb_builder_source_get_flags(source) & (XB_BUILDER_SOURCE_FLAG_WATCH_FILE | XB_BUILDER_SOURCE_FLAG_WATCH_DIRECTORY)) == 0) return TRUE; if (xb_builder_source_get_flags(source) & XB_BUILDER_SOURCE_FLAG_WATCH_DIRECTORY) watched_file = g_file_get_parent(file); else watched_file = g_object_ref(file); return xb_silo_watch_file(silo, watched_file, cancellable, error); } static gboolean xb_builder_watch_sources(XbBuilder *self, XbSilo *silo, GCancellable *cancellable, GError **error) { XbBuilderPrivate *priv = GET_PRIVATE(self); for (guint i = 0; i < priv->sources->len; i++) { XbBuilderSource *source = g_ptr_array_index(priv->sources, i); if (!xb_builder_watch_source(self, source, silo, cancellable, error)) return FALSE; } return TRUE; } /** * xb_builder_compile: * @self: a #XbSilo * @flags: some #XbBuilderCompileFlags, e.g. %XB_BUILDER_SOURCE_FLAG_LITERAL_TEXT * @cancellable: a #GCancellable, or %NULL * @error: the #GError, or %NULL * * Compiles a #XbSilo. * * Returns: (transfer full): a #XbSilo, or %NULL for error * * Since: 0.1.0 **/ XbSilo * xb_builder_compile(XbBuilder *self, XbBuilderCompileFlags flags, GCancellable *cancellable, GError **error) { XbBuilderPrivate *priv = GET_PRIVATE(self); guint32 nodetabsz = sizeof(XbSiloHeader); g_autoptr(GByteArray) buf = NULL; g_autoptr(GBytes) blob = NULL; XbSiloHeader *hdrptr; XbSiloHeader hdr = { .magic = XB_SILO_MAGIC_BYTES, .version = XB_SILO_VERSION, .strtab = 0, .strtab_ntags = 0, .padding = {0x0}, .guid = {0x0}, .filesz = 0x0, }; XbBuilderNodetabHelper nodetab_helper = { .buf = NULL, }; g_autoptr(GPtrArray) nodes_to_destroy = g_ptr_array_new_with_free_func((GDestroyNotify)g_object_unref); g_autoptr(GTimer) timer = NULL; g_autoptr(XbBuilderCompileHelper) helper = NULL; g_return_val_if_fail(XB_IS_BUILDER(self), NULL); g_return_val_if_fail(cancellable == NULL || G_IS_CANCELLABLE(cancellable), NULL); g_return_val_if_fail(error == NULL || *error == NULL, NULL); /* this is inferred */ if (flags & XB_BUILDER_COMPILE_FLAG_SINGLE_LANG) flags |= XB_BUILDER_COMPILE_FLAG_NATIVE_LANGS; /* the builder needs to know the locales */ if (priv->locales->len == 0 && (flags & XB_BUILDER_COMPILE_FLAG_NATIVE_LANGS)) { g_set_error_literal(error, G_IO_ERROR, G_IO_ERROR_INVALID_DATA, "No locales set and using NATIVE_LANGS"); return NULL; } /* create helper used for compiling */ helper = g_new0(XbBuilderCompileHelper, 1); helper->compile_flags = flags; helper->root = xb_builder_node_new(NULL); helper->silo = xb_silo_new(); helper->locales = priv->locales; helper->strtab = g_byte_array_new(); helper->strtab_hash = g_hash_table_new_full(g_str_hash, g_str_equal, g_free, NULL); helper->elem_closed = FALSE; /* for profiling */ xb_silo_set_profile_flags(helper->silo, priv->profile_flags); timer = xb_silo_start_profile(helper->silo); /* build node tree */ for (guint i = 0; i < priv->sources->len; i++) { XbBuilderSource *source = g_ptr_array_index(priv->sources, i); const gchar *prefix = xb_builder_source_get_prefix(source); g_autofree gchar *source_guid = xb_builder_source_get_guid(source); g_autoptr(XbBuilderNode) root = NULL; g_autoptr(GError) error_local = NULL; /* find, or create the prefix */ if (prefix != NULL) { root = xb_builder_node_get_child(helper->root, prefix, NULL); if (root == NULL) root = xb_builder_node_insert(helper->root, prefix, NULL); } else { /* don't allow damaged XML files to ruin all the next ones */ root = g_object_ref(helper->root); } /* watch the source */ if (!xb_builder_watch_source(self, source, helper->silo, cancellable, error)) return NULL; if (priv->profile_flags & XB_SILO_PROFILE_FLAG_DEBUG) g_debug("compiling %s…", source_guid); if (!xb_builder_compile_source(helper, source, root, cancellable, &error_local)) { if (flags & XB_BUILDER_COMPILE_FLAG_IGNORE_INVALID) { g_debug("ignoring invalid file %s: %s", source_guid, error_local->message); continue; } g_propagate_prefixed_error(error, g_steal_pointer(&error_local), "failed to compile %s: ", source_guid); return NULL; } } /* run any node functions */ for (guint i = 0; i < priv->fixups->len; i++) { XbBuilderFixup *fixup = g_ptr_array_index(priv->fixups, i); if (!xb_builder_fixup_node(fixup, helper->root, error)) return NULL; } /* only include the highest priority translation */ if (flags & XB_BUILDER_COMPILE_FLAG_SINGLE_LANG) { xb_builder_node_traverse(helper->root, G_PRE_ORDER, G_TRAVERSE_ALL, -1, xb_builder_xml_lang_prio_cb, nodes_to_destroy); for (guint i = 0; i < nodes_to_destroy->len; i++) { XbBuilderNode *bn = g_ptr_array_index(nodes_to_destroy, i); xb_builder_node_unlink(bn); } xb_silo_add_profile(helper->silo, timer, "filter single-lang"); } /* add any manually build nodes */ for (guint i = 0; i < priv->nodes->len; i++) { XbBuilderNode *bn = g_ptr_array_index(priv->nodes, i); xb_builder_node_add_child(helper->root, bn); } /* get the size of the nodetab */ xb_builder_node_traverse(helper->root, G_PRE_ORDER, G_TRAVERSE_ALL, -1, xb_builder_nodetab_size_cb, &nodetabsz); buf = g_byte_array_sized_new(nodetabsz); xb_silo_add_profile(helper->silo, timer, "get size nodetab"); /* add everything to the strtab */ xb_builder_node_traverse(helper->root, G_PRE_ORDER, G_TRAVERSE_ALL, -1, xb_builder_strtab_element_names_cb, helper); if (helper->error != NULL) { g_propagate_error(error, g_steal_pointer(&helper->error)); return NULL; } if (g_hash_table_size(helper->strtab_hash) > G_MAXUINT16) { g_set_error_literal(error, G_IO_ERROR, G_IO_ERROR_INVALID_DATA, "too many unique element names for strtab"); return NULL; } hdr.strtab_ntags = (guint16)g_hash_table_size(helper->strtab_hash); xb_silo_add_profile(helper->silo, timer, "adding strtab element"); xb_builder_node_traverse(helper->root, G_PRE_ORDER, G_TRAVERSE_ALL, -1, xb_builder_strtab_attr_name_cb, helper); if (helper->error != NULL) { g_propagate_error(error, g_steal_pointer(&helper->error)); return NULL; } xb_silo_add_profile(helper->silo, timer, "adding strtab attr name"); xb_builder_node_traverse(helper->root, G_PRE_ORDER, G_TRAVERSE_ALL, -1, xb_builder_strtab_attr_value_cb, helper); if (helper->error != NULL) { g_propagate_error(error, g_steal_pointer(&helper->error)); return NULL; } xb_silo_add_profile(helper->silo, timer, "adding strtab attr value"); xb_builder_node_traverse(helper->root, G_PRE_ORDER, G_TRAVERSE_ALL, -1, xb_builder_strtab_text_cb, helper); if (helper->error != NULL) { g_propagate_error(error, g_steal_pointer(&helper->error)); return NULL; } xb_silo_add_profile(helper->silo, timer, "adding strtab text"); xb_builder_node_traverse(helper->root, G_PRE_ORDER, G_TRAVERSE_ALL, -1, xb_builder_strtab_tokens_cb, helper); if (helper->error != NULL) { g_propagate_error(error, g_steal_pointer(&helper->error)); return NULL; } xb_silo_add_profile(helper->silo, timer, "adding strtab tokens"); /* add the initial header */ hdr.strtab = nodetabsz; if (priv->guid->len > 0) { XbGuid guid_tmp; xb_guid_compute_for_data(&guid_tmp, (const guint8 *)priv->guid->str, priv->guid->len); memcpy(&hdr.guid, &guid_tmp, sizeof(guid_tmp)); } g_byte_array_append(buf, (const guint8 *)&hdr, sizeof(hdr)); /* write nodes to the nodetab */ nodetab_helper.buf = buf; if (!xb_builder_nodetab_write(&nodetab_helper, helper->root, error)) return NULL; xb_silo_add_profile(helper->silo, timer, "writing nodetab"); /* set all the ->next and ->parent offsets */ xb_builder_node_traverse(helper->root, G_PRE_ORDER, G_TRAVERSE_ALL, -1, xb_builder_nodetab_fix_cb, &nodetab_helper); xb_silo_add_profile(helper->silo, timer, "fixing ->parent and ->next"); /* append the string table */ g_byte_array_append(buf, (const guint8 *)helper->strtab->data, helper->strtab->len); xb_silo_add_profile(helper->silo, timer, "appending strtab"); /* update the file size */ hdrptr = (XbSiloHeader *)buf->data; hdrptr->filesz = buf->len; /* create data */ blob = g_byte_array_free_to_bytes(g_steal_pointer(&buf)); if (!xb_silo_load_from_bytes(helper->silo, blob, XB_SILO_LOAD_FLAG_NONE, error)) return NULL; /* success */ return g_steal_pointer(&helper->silo); } /** * xb_builder_ensure: * @self: a #XbSilo * @file: a #GFile * @flags: some #XbBuilderCompileFlags, e.g. %XB_BUILDER_COMPILE_FLAG_IGNORE_INVALID * @cancellable: a #GCancellable, or %NULL * @error: the #GError, or %NULL * * Ensures @file is up to date, and returns a compiled #XbSilo. * * If @silo is being used by a query (e.g. in another thread) then all node * data is immediately invalid. * * The returned #XbSilo will use the thread-default main context at the time of * calling this function for its future signal emissions. * * Returns: (transfer full): a #XbSilo, or %NULL for error * * Since: 0.1.0 **/ XbSilo * xb_builder_ensure(XbBuilder *self, GFile *file, XbBuilderCompileFlags flags, GCancellable *cancellable, GError **error) { XbBuilderPrivate *priv = GET_PRIVATE(self); XbSiloLoadFlags load_flags = XB_SILO_LOAD_FLAG_NONE; g_autofree gchar *fn = NULL; g_autoptr(XbSilo) silo_tmp = xb_silo_new(); g_autoptr(XbSilo) silo = NULL; g_autoptr(GError) error_local = NULL; g_return_val_if_fail(XB_IS_BUILDER(self), NULL); g_return_val_if_fail(G_IS_FILE(file), NULL); g_return_val_if_fail(cancellable == NULL || G_IS_CANCELLABLE(cancellable), NULL); g_return_val_if_fail(error == NULL || *error == NULL, NULL); /* watch the blob, so propagate flags */ if (flags & XB_BUILDER_COMPILE_FLAG_WATCH_BLOB) { load_flags |= XB_SILO_LOAD_FLAG_WATCH_BLOB; if (!xb_silo_watch_file(silo_tmp, file, cancellable, error)) return NULL; } /* ensure all the sources are watched */ if (!xb_builder_watch_sources(self, silo_tmp, cancellable, error)) return NULL; /* profile new silo if needed */ xb_silo_set_profile_flags(silo_tmp, priv->profile_flags); /* load the file and peek at the GUIDs */ fn = g_file_get_path(file); g_debug("attempting to load %s", fn); if (!xb_silo_load_from_file(silo_tmp, file, XB_SILO_LOAD_FLAG_NONE, cancellable, &error_local)) { g_debug("failed to load silo: %s", error_local->message); } else { g_autofree gchar *guid = xb_builder_generate_guid(self); if (priv->profile_flags & XB_SILO_PROFILE_FLAG_DEBUG) g_debug("GUID string: %s", priv->guid->str); g_debug("file: %s, current:%s", xb_silo_get_guid(silo_tmp), guid); /* no compile required */ if (g_strcmp0(xb_silo_get_guid(silo_tmp), guid) == 0 || (flags & XB_BUILDER_COMPILE_FLAG_IGNORE_GUID) > 0) { g_debug("loading silo with existing file contents"); return g_steal_pointer(&silo_tmp); } } /* fallback to just creating a new file */ silo = xb_builder_compile(self, flags, cancellable, error); if (silo == NULL) return NULL; /* this might seem unnecessary, but windows cannot do _chsize() (introduced in GLib commit * https://gitlab.gnome.org/GNOME/glib/-/commit/3f705ffa1230757b910a06a705104d4b0fee2c05) * on a mmap'd file -- so manually tear that down before writing the new file */ g_clear_object(&silo_tmp); if (!xb_silo_save_to_file(silo, file, NULL, error)) return NULL; /* load from a file to re-mmap it */ if (!xb_silo_load_from_file(silo, file, load_flags, cancellable, error)) return NULL; /* ensure all the sources are watched on the reloaded silo */ if (!xb_builder_watch_sources(self, silo, cancellable, error)) return NULL; /* success */ return g_steal_pointer(&silo); } /** * xb_builder_append_guid: * @self: a #XbSilo * @guid: any text, typcically a filename or GUID * * Adds the GUID to the internal correctness hash. * * Since: 0.1.0 **/ void xb_builder_append_guid(XbBuilder *self, const gchar *guid) { XbBuilderPrivate *priv = GET_PRIVATE(self); if (priv->guid->len > 0) g_string_append(priv->guid, "&"); g_string_append(priv->guid, guid); } /** * xb_builder_set_profile_flags: * @self: a #XbBuilder * @profile_flags: some #XbSiloProfileFlags, e.g. %XB_SILO_PROFILE_FLAG_DEBUG * * Enables or disables the collection of profiling data. * * Since: 0.1.1 **/ void xb_builder_set_profile_flags(XbBuilder *self, XbSiloProfileFlags profile_flags) { XbBuilderPrivate *priv = GET_PRIVATE(self); g_return_if_fail(XB_IS_BUILDER(self)); priv->profile_flags = profile_flags; } /** * xb_builder_add_fixup: * @self: a #XbBuilder * @fixup: a #XbBuilderFixup * * Adds a function that will get run on every #XbBuilderNode compile creates * for the silo. This is run after all the #XbBuilderSource fixups have been * run. * * Since: 0.1.3 **/ void xb_builder_add_fixup(XbBuilder *self, XbBuilderFixup *fixup) { XbBuilderPrivate *priv = GET_PRIVATE(self); g_autofree gchar *guid = NULL; g_return_if_fail(XB_IS_BUILDER(self)); g_return_if_fail(XB_IS_BUILDER_FIXUP(fixup)); /* append function IDs */ guid = xb_builder_fixup_get_guid(fixup); xb_builder_append_guid(self, guid); g_ptr_array_add(priv->fixups, g_object_ref(fixup)); } static void xb_builder_finalize(GObject *obj) { XbBuilder *self = XB_BUILDER(obj); XbBuilderPrivate *priv = GET_PRIVATE(self); g_ptr_array_unref(priv->sources); g_ptr_array_unref(priv->nodes); g_ptr_array_unref(priv->locales); g_ptr_array_unref(priv->fixups); g_string_free(priv->guid, TRUE); G_OBJECT_CLASS(xb_builder_parent_class)->finalize(obj); } static void xb_builder_class_init(XbBuilderClass *klass) { GObjectClass *object_class = G_OBJECT_CLASS(klass); object_class->finalize = xb_builder_finalize; } static void xb_builder_init(XbBuilder *self) { XbBuilderPrivate *priv = GET_PRIVATE(self); priv->sources = g_ptr_array_new_with_free_func((GDestroyNotify)g_object_unref); priv->nodes = g_ptr_array_new_with_free_func((GDestroyNotify)g_object_unref); priv->fixups = g_ptr_array_new_with_free_func((GDestroyNotify)g_object_unref); priv->locales = g_ptr_array_new_with_free_func(g_free); priv->guid = g_string_new(xb_version_string()); } /** * xb_builder_new: * * Creates a new builder. * * The #XbSilo returned by the methods of this #XbBuilder will use the * thread-default main context at the time of calling this function for its * future signal emissions. * * Returns: a new #XbBuilder * * Since: 0.1.0 **/ XbBuilder * xb_builder_new(void) { return g_object_new(XB_TYPE_BUILDER, NULL); }