From fd53fee04b1f1c1ca1e30e8d470d7416900a35dc Mon Sep 17 00:00:00 2001 From: =?utf8?q?Zbigniew=20J=C4=99drzejewski-Szmek?= Date: Sat, 5 Jul 2014 14:29:56 -0400 Subject: [PATCH] compress: add benchmark-style test This is useful to test the behaviour of the compressor for various buffer sizes. Time is limited to a minute per compression, since otherwise, when LZ4 takes more than a second which is necessary to reduce the noise, XZ takes more than 10 minutes. % build/test-compress-benchmark (without time limit) XZ: compressed & decompressed 2535300963 bytes in 794.57s (3.04MiB/s), mean compresion 99.95%, skipped 3570 bytes LZ4: compressed & decompressed 2535303543 bytes in 1.56s (1550.07MiB/s), mean compresion 99.60%, skipped 990 bytes % build/test-compress-benchmark (with time limit) XZ: compressed & decompressed 174321481 bytes in 60.02s (2.77MiB/s), mean compresion 99.76%, skipped 3570 bytes LZ4: compressed & decompressed 2535303543 bytes in 1.63s (1480.83MiB/s), mean compresion 99.60%, skipped 990 bytes It appears that there's a bug in lzma_end where it leaks 32 bytes. --- .gitignore | 1 + Makefile.am | 11 ++- src/journal/test-compress-benchmark.c | 113 ++++++++++++++++++++++++++ 3 files changed, 124 insertions(+), 1 deletion(-) create mode 100644 src/journal/test-compress-benchmark.c diff --git a/.gitignore b/.gitignore index 9523ea027..822f5a6ff 100644 --- a/.gitignore +++ b/.gitignore @@ -142,6 +142,7 @@ /test-cgroup-mask /test-cgroup-util /test-compress +/test-compress-benchmark /test-conf-files /test-coredump-vacuum /test-daemon diff --git a/Makefile.am b/Makefile.am index 01afbe3a2..e9be6facc 100644 --- a/Makefile.am +++ b/Makefile.am @@ -3537,6 +3537,13 @@ test_compress_LDADD = \ libsystemd-journal-internal.la \ libsystemd-shared.la +test_compress_benchmark_SOURCES = \ + src/journal/test-compress-benchmark.c + +test_compress_benchmark_LDADD = \ + libsystemd-journal-internal.la \ + libsystemd-shared.la + libsystemd_journal_core_la_SOURCES = \ src/journal/journald-kmsg.c \ src/journal/journald-kmsg.h \ @@ -3619,7 +3626,9 @@ tests += \ test-mmap-cache \ test-catalog -tests += test-compress +tests += \ + test-compress \ + test-compress-benchmark pkginclude_HEADERS += \ src/systemd/sd-journal.h \ diff --git a/src/journal/test-compress-benchmark.c b/src/journal/test-compress-benchmark.c new file mode 100644 index 000000000..0a23bd1e1 --- /dev/null +++ b/src/journal/test-compress-benchmark.c @@ -0,0 +1,113 @@ +/*** + This file is part of systemd + + Copyright 2014 Zbigniew Jędrzejewski-Szmek + + systemd is free software; you can redistribute it and/or modify it + under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation; either version 2.1 of the License, or + (at your option) any later version. + + systemd is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with systemd; If not, see . +***/ + +#include "compress.h" +#include "util.h" +#include "macro.h" + +typedef int (compress_t)(const void *src, uint64_t src_size, void *dst, uint64_t *dst_size); +typedef int (decompress_t)(const void *src, uint64_t src_size, + void **dst, uint64_t *dst_alloc_size, uint64_t* dst_size, uint64_t dst_max); + +#define MAX_SIZE (1024*1024LU) + +static char* make_buf(size_t count) { + char *buf; + size_t i; + + buf = malloc(count); + assert(buf); + + for (i = 0; i < count; i++) + buf[i] = 'a' + i % ('z' - 'a' + 1); + + return buf; +} + +static void test_compress_decompress(const char* label, + compress_t compress, decompress_t decompress) { + usec_t n, n2; + float dt; + + _cleanup_free_ char *text, *buf; + _cleanup_free_ void *buf2 = NULL; + size_t buf2_allocated = 0; + size_t skipped = 0, compressed = 0, total = 0; + + text = make_buf(MAX_SIZE); + buf = calloc(MAX_SIZE + 1, 1); + assert(text && buf); + + n = now(CLOCK_MONOTONIC); + + for (size_t i = 1; i <= MAX_SIZE; i += (i < 2048 ? 1 : 217)) { + size_t j = 0, k = 0; + int r; + + r = compress(text, i, buf, &j); + /* assume compresion must be succesful except for small inputs */ + assert(r == 0 || (i < 2048 && r == -ENOBUFS)); + /* check for overwrites */ + assert(buf[i] == 0); + if (r != 0) { + skipped += i; + continue; + } + + assert(j > 0); + if (j >= i) + log_error("%s \"compressed\" %zu -> %zu", label, i, j); + + r = decompress(buf, j, &buf2, &buf2_allocated, &k, 0); + assert(r == 0); + assert(buf2_allocated >= k); + assert(k == i); + + assert(memcmp(text, buf2, i) == 0); + + total += i; + compressed += j; + + n2 = now(CLOCK_MONOTONIC); + if (n2 - n > 60 * USEC_PER_SEC) + break; + } + + dt = (n2-n) / 1e6; + + log_info("%s: compressed & decompressed %zu bytes in %.2fs (%.2fMiB/s), " + "mean compresion %.2f%%, skipped %zu bytes", + label, total, dt, + total / 1024. / 1024 / dt, + 100 - compressed * 100. / total, + skipped); +} + +int main(int argc, char *argv[]) { + + log_set_max_level(LOG_DEBUG); + +#ifdef HAVE_XZ + test_compress_decompress("XZ", compress_blob_xz, decompress_blob_xz); +#endif +#ifdef HAVE_LZ4 + test_compress_decompress("LZ4", compress_blob_lz4, decompress_blob_lz4); +#endif + return 0; +} -- 2.30.2