From: Lennart Poettering Date: Tue, 20 Nov 2012 23:28:00 +0000 (+0100) Subject: journal: by default do not decompress dat objects larger than 64K X-Git-Tag: v196~4 X-Git-Url: http://www.chiark.greenend.org.uk/ucgi/~ianmdlvl/git?p=elogind.git;a=commitdiff_plain;h=93b73b064c663d6248bebfbbbd82989b5ca10fc5 journal: by default do not decompress dat objects larger than 64K This introduces a new data threshold setting for sd_journal objects which controls the maximum size of objects to decompress. This is relieves the library from having to decompress full data objects even if a client program is only interested in the initial part of them. This speeds up "systemd-coredumpctl" drastically when invoked without parameters. --- diff --git a/man/sd_journal_get_data.xml b/man/sd_journal_get_data.xml index 6470f19cc..1259b0cdb 100644 --- a/man/sd_journal_get_data.xml +++ b/man/sd_journal_get_data.xml @@ -47,6 +47,8 @@ sd_journal_enumerate_data sd_journal_restart_data SD_JOURNAL_FOREACH_DATA + sd_journal_set_data_threshold + sd_journal_get_data_threshold Read data fields from the current journal entry @@ -81,6 +83,17 @@ size_t length + + int sd_journal_set_data_threshold + sd_journal* j + size_t sz + + + + int sd_journal_get_data_threshold + sd_journal* j + size_t* sz + @@ -102,7 +115,11 @@ sd_journal_enumerate_data(), or the read pointer is altered. Note that the data returned will be prefixed with the field name and - '='. + '='. Also note that by default data fields larger than + 64K might get truncated to 64K. This threshold may be + changed and turned off with + sd_journal_set_data_threshold() (see + below). sd_journal_enumerate_data() may be used to iterate through all fields of the @@ -128,6 +145,32 @@ sd_journal_next3 (or related call) has been called at least once, in order to position the read pointer at a valid entry. + + sd_journal_set_data_threshold() + may be used to change the data field size threshold + for data returned by + sd_journal_get_data(), + sd_journal_enumerate_data() and + sd_journal_enumerate_unique(). This + threshold is a hint only: it indicates that the client + program is interested only in the initial parts of the + data fields, up to the threshold in size -- but the + library might still return larger data objects. That + means applications should not rely exclusively on this + setting to limit the size of the data fields returned, + but need to apply a explicit size limit on the + returned data as well. This threshold defaults to 64K + by default. To retrieve the complete data fields this + threshold should be turned off by setting it to 0, so + that the library always returns the complete data + objects. It is recommended to set this threshold as + low as possible since this relieves the library from + having to decompress large compressed data objects in + full. + + sd_journal_get_data_threshold() + returns the currently configured data field size + threshold. @@ -144,15 +187,22 @@ read, 0 when no more fields are known, or a negative errno-style error code. sd_journal_restart_data() - returns nothing. + returns + nothing. sd_journal_set_data_threshold() + and sd_journal_get_threshold() + return 0 on success or a negative errno-style error + code. Notes The sd_journal_get_data(), - sd_journal_enumerate_data() and - sd_journal_restart_data() + sd_journal_enumerate_data(), + sd_journal_restart_data(), + sd_journal_set_data_threshold() + and + sd_journal_get_data_threshold() interfaces are available as shared library, which can be compiled and linked to with the libsystemd-journal diff --git a/man/sd_journal_query_unique.xml b/man/sd_journal_query_unique.xml index f2f8af0eb..502a7e08c 100644 --- a/man/sd_journal_query_unique.xml +++ b/man/sd_journal_query_unique.xml @@ -113,7 +113,9 @@ invocation of sd_journal_enumerate_unique(). Note that the data returned will be prefixed with the field - name and '='. + name and '='. Note that this call is subject to the + data field size threshold as controlled by + sd_journal_set_data_threshold(). sd_journal_restart_unique() resets the data enumeration index to the beginning of diff --git a/src/journal/compress.c b/src/journal/compress.c index 75e70c5ff..a4427be75 100644 --- a/src/journal/compress.c +++ b/src/journal/compress.c @@ -24,6 +24,7 @@ #include #include +#include "macro.h" #include "compress.h" bool compress_blob(const void *src, uint64_t src_size, void *dst, uint64_t *dst_size) { @@ -66,10 +67,11 @@ fail: } bool uncompress_blob(const void *src, uint64_t src_size, - void **dst, uint64_t *dst_alloc_size, uint64_t* dst_size) { + void **dst, uint64_t *dst_alloc_size, uint64_t* dst_size, uint64_t dst_max) { lzma_stream s = LZMA_STREAM_INIT; lzma_ret ret; + uint64_t space; bool b = false; assert(src); @@ -98,7 +100,8 @@ bool uncompress_blob(const void *src, uint64_t src_size, s.avail_in = src_size; s.next_out = *dst; - s.avail_out = *dst_alloc_size; + space = dst_max > 0 ? MIN(*dst_alloc_size, dst_max) : *dst_alloc_size; + s.avail_out = space; for (;;) { void *p; @@ -111,18 +114,23 @@ bool uncompress_blob(const void *src, uint64_t src_size, if (ret != LZMA_OK) goto fail; - p = realloc(*dst, *dst_alloc_size*2); + if (dst_max > 0 && (space - s.avail_out) >= dst_max) + break; + + p = realloc(*dst, space*2); if (!p) goto fail; s.next_out = (uint8_t*) p + ((uint8_t*) s.next_out - (uint8_t*) *dst); - s.avail_out += *dst_alloc_size; + s.avail_out += space; + + space *= 2; *dst = p; - *dst_alloc_size *= 2; + *dst_alloc_size = space; } - *dst_size = *dst_alloc_size - s.avail_out; + *dst_size = space - s.avail_out; b = true; fail: diff --git a/src/journal/compress.h b/src/journal/compress.h index b6f1aa0ed..2b87e73a2 100644 --- a/src/journal/compress.h +++ b/src/journal/compress.h @@ -27,7 +27,7 @@ bool compress_blob(const void *src, uint64_t src_size, void *dst, uint64_t *dst_size); bool uncompress_blob(const void *src, uint64_t src_size, - void **dst, uint64_t *dst_alloc_size, uint64_t* dst_size); + void **dst, uint64_t *dst_alloc_size, uint64_t* dst_size, uint64_t dst_max); bool uncompress_startswith(const void *src, uint64_t src_size, void **buffer, uint64_t *buffer_size, diff --git a/src/journal/coredumpctl.c b/src/journal/coredumpctl.c index 4adc9236f..f9cebb7a4 100644 --- a/src/journal/coredumpctl.c +++ b/src/journal/coredumpctl.c @@ -342,6 +342,11 @@ static int dump_list(sd_journal *j) { assert(j); + /* The coredumps are likely to compressed, and for just + * listing them we don#t need to decompress them, so let's + * pick a fairly low data threshold here */ + sd_journal_set_data_threshold(j, 4096); + SD_JOURNAL_FOREACH(j) { if (field) print_field(stdout, j); @@ -381,6 +386,9 @@ static int dump_core(sd_journal* j) { assert(j); + /* We want full data, nothing truncated. */ + sd_journal_set_data_threshold(j, 0); + r = focus(j); if (r < 0) return r; @@ -428,6 +436,8 @@ static int run_gdb(sd_journal *j) { assert(j); + sd_journal_set_data_threshold(j, 0); + r = focus(j); if (r < 0) return r; diff --git a/src/journal/journal-file.c b/src/journal/journal-file.c index 3df099dbd..13fc8edea 100644 --- a/src/journal/journal-file.c +++ b/src/journal/journal-file.c @@ -780,7 +780,7 @@ int journal_file_find_data_object_with_hash( l -= offsetof(Object, data.payload); - if (!uncompress_blob(o->data.payload, l, &f->compress_buffer, &f->compress_buffer_size, &rsize)) + if (!uncompress_blob(o->data.payload, l, &f->compress_buffer, &f->compress_buffer_size, &rsize, 0)) return -EBADMSG; if (rsize == size && @@ -2591,7 +2591,6 @@ int journal_file_open_reliably( metrics, mmap_cache, template, ret); } - int journal_file_copy_entry(JournalFile *from, JournalFile *to, Object *o, uint64_t p, uint64_t *seqnum, Object **ret, uint64_t *offset) { uint64_t i, n; uint64_t q, xor_hash = 0; @@ -2645,7 +2644,7 @@ int journal_file_copy_entry(JournalFile *from, JournalFile *to, Object *o, uint6 #ifdef HAVE_XZ uint64_t rsize; - if (!uncompress_blob(o->data.payload, l, &from->compress_buffer, &from->compress_buffer_size, &rsize)) + if (!uncompress_blob(o->data.payload, l, &from->compress_buffer, &from->compress_buffer_size, &rsize, 0)) return -EBADMSG; data = from->compress_buffer; diff --git a/src/journal/journal-internal.h b/src/journal/journal-internal.h index 75a4129e5..97de0e75f 100644 --- a/src/journal/journal-internal.h +++ b/src/journal/journal-internal.h @@ -121,6 +121,8 @@ struct sd_journal { uint64_t unique_offset; bool on_network; + + size_t data_threshold; }; char *journal_make_match_string(sd_journal *j); diff --git a/src/journal/journal-verify.c b/src/journal/journal-verify.c index 1a67d5a04..ed28b4573 100644 --- a/src/journal/journal-verify.c +++ b/src/journal/journal-verify.c @@ -69,7 +69,7 @@ static int journal_file_object_verify(JournalFile *f, Object *o) { if (!uncompress_blob(o->data.payload, le64toh(o->object.size) - offsetof(Object, data.payload), - &b, &alloc, &b_size)) + &b, &alloc, &b_size, 0)) return -EBADMSG; h2 = hash64(b, b_size); diff --git a/src/journal/journald-server.c b/src/journal/journald-server.c index 3eaf0d3ee..43ffe7556 100644 --- a/src/journal/journald-server.c +++ b/src/journal/journald-server.c @@ -934,6 +934,8 @@ int server_flush_to_var(Server *s) { return r; } + sd_journal_set_data_threshold(j, 0); + SD_JOURNAL_FOREACH(j) { Object *o = NULL; JournalFile *f; diff --git a/src/journal/libsystemd-journal.sym b/src/journal/libsystemd-journal.sym index 17b5bf80d..7b602f59c 100644 --- a/src/journal/libsystemd-journal.sym +++ b/src/journal/libsystemd-journal.sym @@ -86,4 +86,6 @@ global: sd_journal_fd_reliable; sd_journal_get_catalog; sd_journal_get_catalog_for_message_id; + sd_journal_set_data_threshold; + sd_journal_get_data_threshold; } LIBSYSTEMD_JOURNAL_195; diff --git a/src/journal/sd-journal.c b/src/journal/sd-journal.c index fe0478f16..095fbb249 100644 --- a/src/journal/sd-journal.c +++ b/src/journal/sd-journal.c @@ -47,6 +47,8 @@ #define REPLACE_VAR_MAX 256 +#define DEFAULT_DATA_THRESHOLD (64*1024) + static void detach_location(sd_journal *j) { Iterator i; JournalFile *f; @@ -1560,6 +1562,7 @@ static sd_journal *journal_new(int flags, const char *path) { j->inotify_fd = -1; j->flags = flags; + j->data_threshold = DEFAULT_DATA_THRESHOLD; if (path) { j->path = strdup(path); @@ -1838,7 +1841,8 @@ _public_ int sd_journal_get_data(sd_journal *j, const char *field, const void ** uint64_t rsize; if (!uncompress_blob(o->data.payload, l, - &f->compress_buffer, &f->compress_buffer_size, &rsize)) + &f->compress_buffer, &f->compress_buffer_size, &rsize, + j->data_threshold)) return -EBADMSG; *data = f->compress_buffer; @@ -1862,7 +1866,7 @@ _public_ int sd_journal_get_data(sd_journal *j, const char *field, const void ** *data = o->data.payload; *size = t; - return 0; + return 1; } r = journal_file_move_to_object(f, OBJECT_ENTRY, f->current_offset, &o); @@ -1873,7 +1877,7 @@ _public_ int sd_journal_get_data(sd_journal *j, const char *field, const void ** return -ENOENT; } -static int return_data(JournalFile *f, Object *o, const void **data, size_t *size) { +static int return_data(sd_journal *j, JournalFile *f, Object *o, const void **data, size_t *size) { size_t t; uint64_t l; @@ -1888,7 +1892,7 @@ static int return_data(JournalFile *f, Object *o, const void **data, size_t *siz #ifdef HAVE_XZ uint64_t rsize; - if (!uncompress_blob(o->data.payload, l, &f->compress_buffer, &f->compress_buffer_size, &rsize)) + if (!uncompress_blob(o->data.payload, l, &f->compress_buffer, &f->compress_buffer_size, &rsize, j->data_threshold)) return -EBADMSG; *data = f->compress_buffer; @@ -1942,7 +1946,7 @@ _public_ int sd_journal_enumerate_data(sd_journal *j, const void **data, size_t if (le_hash != o->data.hash) return -EBADMSG; - r = return_data(f, o, data, size); + r = return_data(j, f, o, data, size); if (r < 0) return r; @@ -2339,7 +2343,7 @@ _public_ int sd_journal_enumerate_unique(sd_journal *j, const void **data, size_ if (o->object.type != OBJECT_DATA) return -EBADMSG; - r = return_data(j->unique_file, o, &odata, &ol); + r = return_data(j, j->unique_file, o, &odata, &ol); if (r < 0) return r; @@ -2371,7 +2375,7 @@ _public_ int sd_journal_enumerate_unique(sd_journal *j, const void **data, size_ if (found) continue; - r = return_data(j->unique_file, o, data, l); + r = return_data(j, j->unique_file, o, data, l); if (r < 0) return r; @@ -2456,3 +2460,21 @@ _public_ int sd_journal_get_catalog_for_message_id(sd_id128_t id, char **ret) { return catalog_get(id, ret); } + +_public_ int sd_journal_set_data_threshold(sd_journal *j, size_t sz) { + if (!j) + return -EINVAL; + + j->data_threshold = sz; + return 0; +} + +_public_ int sd_journal_get_data_threshold(sd_journal *j, size_t *sz) { + if (!j) + return -EINVAL; + if (!sz) + return -EINVAL; + + *sz = j->data_threshold; + return 0; +} diff --git a/src/shared/logs-show.c b/src/shared/logs-show.c index cb93761bd..ca5ad43b6 100644 --- a/src/shared/logs-show.c +++ b/src/shared/logs-show.c @@ -91,7 +91,7 @@ static bool shall_print(const char *p, size_t l, OutputFlags flags) { if (flags & OUTPUT_SHOW_ALL) return true; - if (l > PRINT_THRESHOLD) + if (l >= PRINT_THRESHOLD) return false; if (!utf8_is_printable_n(p, l)) @@ -119,6 +119,8 @@ static int output_short( assert(f); assert(j); + sd_journal_set_data_threshold(j, flags & OUTPUT_SHOW_ALL ? 0 : PRINT_THRESHOLD); + SD_JOURNAL_FOREACH_DATA(j, data, length) { r = parse_field(data, length, "PRIORITY=", &priority, &priority_len); @@ -308,6 +310,8 @@ static int output_verbose( assert(f); assert(j); + sd_journal_set_data_threshold(j, 0); + r = sd_journal_get_realtime_usec(j, &realtime); if (r < 0) { log_error("Failed to get realtime timestamp: %s", strerror(-r)); @@ -368,6 +372,8 @@ static int output_export( assert(j); + sd_journal_set_data_threshold(j, 0); + r = sd_journal_get_realtime_usec(j, &realtime); if (r < 0) { log_error("Failed to get realtime timestamp: %s", strerror(-r)); @@ -441,7 +447,7 @@ void json_escape( assert(f); assert(p); - if (!(flags & OUTPUT_SHOW_ALL) && l > JSON_THRESHOLD) + if (!(flags & OUTPUT_SHOW_ALL) && l >= JSON_THRESHOLD) fputs("null", f); @@ -502,6 +508,8 @@ static int output_json( assert(j); + sd_journal_set_data_threshold(j, flags & OUTPUT_SHOW_ALL ? 0 : JSON_THRESHOLD); + r = sd_journal_get_realtime_usec(j, &realtime); if (r < 0) { log_error("Failed to get realtime timestamp: %s", strerror(-r)); @@ -714,6 +722,8 @@ static int output_cat( assert(j); assert(f); + sd_journal_set_data_threshold(j, 0); + r = sd_journal_get_data(j, "MESSAGE", &data, &l); if (r < 0) { /* An entry without MESSAGE=? */ diff --git a/src/systemd/sd-journal.h b/src/systemd/sd-journal.h index fd9c0f562..2e8d2d882 100644 --- a/src/systemd/sd-journal.h +++ b/src/systemd/sd-journal.h @@ -97,6 +97,9 @@ int sd_journal_next_skip(sd_journal *j, uint64_t skip); int sd_journal_get_realtime_usec(sd_journal *j, uint64_t *ret); int sd_journal_get_monotonic_usec(sd_journal *j, uint64_t *ret, sd_id128_t *ret_boot_id); +int sd_journal_set_data_threshold(sd_journal *j, size_t sz); +int sd_journal_get_data_threshold(sd_journal *j, size_t *sz); + int sd_journal_get_data(sd_journal *j, const char *field, const void **data, size_t *l); int sd_journal_enumerate_data(sd_journal *j, const void **data, size_t *l); void sd_journal_restart_data(sd_journal *j);