chiark / gitweb /
journal: add LZ4 as optional compressor
authorZbigniew Jędrzejewski-Szmek <zbyszek@in.waw.pl>
Fri, 4 Jul 2014 02:42:22 +0000 (22:42 -0400)
committerZbigniew Jędrzejewski-Szmek <zbyszek@in.waw.pl>
Sun, 6 Jul 2014 23:06:03 +0000 (19:06 -0400)
Add liblz4 as an optional dependency when requested with --enable-lz4,
and use it in preference to liblzma for journal blob and coredump
compression. To retain backwards compatibility, XZ is used to
decompress old blobs.

Things will function correctly only with lz4-119.

Based on the benchmarks found on the web, lz4 seems to be the best
choice for "quick" compressors atm.

For pkg-config status, see http://code.google.com/p/lz4/issues/detail?id=135.

12 files changed:
Makefile.am
configure.ac
src/journal/compress.c
src/journal/compress.h
src/journal/coredump.c
src/journal/coredumpctl.c
src/journal/journal-def.h
src/journal/journal-file.c
src/journal/journal-file.h
src/journal/journal-verify.c
src/journal/sd-journal.c
src/journal/test-compress.c

index e238cde..01afbe3 100644 (file)
@@ -3530,14 +3530,12 @@ test_catalog_CPPFLAGS = \
 test_catalog_LDADD = \
        libsystemd-journal-core.la
 
-if HAVE_XZ
 test_compress_SOURCES = \
        src/journal/test-compress.c
 
 test_compress_LDADD = \
        libsystemd-journal-internal.la \
        libsystemd-shared.la
-endif
 
 libsystemd_journal_core_la_SOURCES = \
        src/journal/journald-kmsg.c \
@@ -3621,9 +3619,7 @@ tests += \
        test-mmap-cache \
        test-catalog
 
-if HAVE_XZ
 tests += test-compress
-endif
 
 pkginclude_HEADERS += \
        src/systemd/sd-journal.h \
@@ -3656,10 +3652,10 @@ libsystemd_journal_internal_la_CFLAGS = \
 
 libsystemd_journal_internal_la_LIBADD =
 
-if HAVE_XZ
 libsystemd_journal_internal_la_SOURCES += \
        src/journal/compress.c
 
+if HAVE_XZ
 libsystemd_journal_internal_la_CFLAGS += \
        $(XZ_CFLAGS)
 
@@ -3667,6 +3663,11 @@ libsystemd_journal_internal_la_LIBADD += \
        $(XZ_LIBS)
 endif
 
+if HAVE_LZ4
+libsystemd_journal_internal_la_LIBADD += \
+       -llz4
+endif
+
 if HAVE_GCRYPT
 libsystemd_journal_internal_la_SOURCES += \
        src/journal/journal-authenticate.c \
index ae88382..2ee73ca 100644 (file)
@@ -503,14 +503,24 @@ have_xz=no
 AC_ARG_ENABLE(xz, AS_HELP_STRING([--disable-xz], [Disable optional XZ support]))
 if test "x$enable_xz" != "xno"; then
         PKG_CHECK_MODULES(XZ, [ liblzma ],
-                [AC_DEFINE(HAVE_XZ, 1, [Define if XZ is available]) have_xz=yes], have_xz=no)
+                [AC_DEFINE(HAVE_XZ, 1, [Define if XZ is available]) have_xz=yes])
         if test "x$have_xz" = xno -a "x$enable_xz" = xyes; then
-                AC_MSG_ERROR([*** Xz support requested but libraries not found])
+                AC_MSG_ERROR([*** XZ support requested but libraries not found])
         fi
 fi
 AM_CONDITIONAL(HAVE_XZ, [test "$have_xz" = "yes"])
 
 # ------------------------------------------------------------------------------
+have_lz4=no
+AC_ARG_ENABLE(lz4, AS_HELP_STRING([--enable-lz4], [Enable optional LZ4 support]))
+AS_IF([test "x$enable_lz4" == "xyes"], [
+        AC_CHECK_HEADERS(lz4.h,
+               [AC_DEFINE(HAVE_LZ4, 1, [Define in LZ4 is available]) have_lz4=yes],
+               [AC_MSG_ERROR([*** LZ4 support requested but headers not found])])
+])
+AM_CONDITIONAL(HAVE_LZ4, [test "$have_lz4" = "yes"])
+
+# ------------------------------------------------------------------------------
 AC_ARG_ENABLE([pam],
         AS_HELP_STRING([--disable-pam],[Disable optional PAM support]),
                 [case "${enableval}" in
@@ -1266,6 +1276,7 @@ AC_MSG_RESULT([
         SECCOMP:                 ${have_seccomp}
         SMACK:                   ${have_smack}
         XZ:                      ${have_xz}
+        LZ4:                     ${have_lz4}
         ACL:                     ${have_acl}
         GCRYPT:                  ${have_gcrypt}
         QRENCODE:                ${have_qrencode}
index 37c55a8..49d694a 100644 (file)
 #include <stdlib.h>
 #include <string.h>
 #include <unistd.h>
-#include <lzma.h>
+
+#ifdef HAVE_XZ
+#  include <lzma.h>
+#endif
+
+#ifdef HAVE_LZ4
+#  include <lz4.h>
+#endif
 
 #include "compress.h"
 #include "macro.h"
 #include "util.h"
+#include "sparse-endian.h"
+#include "journal-def.h"
+
+#define ALIGN_8(l) ALIGN_TO(l, sizeof(size_t))
+
+static const char* const object_compressed_table[_OBJECT_COMPRESSED_MAX] = {
+        [OBJECT_COMPRESSED_XZ] = "XZ",
+        [OBJECT_COMPRESSED_LZ4] = "LZ4",
+};
 
-bool compress_blob(const void *src, uint64_t src_size, void *dst, uint64_t *dst_size) {
+DEFINE_STRING_TABLE_LOOKUP(object_compressed, int);
+
+int compress_blob_xz(const void *src, uint64_t src_size, void *dst, uint64_t *dst_size) {
+#ifdef HAVE_XZ
         lzma_ret ret;
         size_t out_pos = 0;
 
@@ -38,25 +57,54 @@ bool compress_blob(const void *src, uint64_t src_size, void *dst, uint64_t *dst_
         assert(dst);
         assert(dst_size);
 
-        /* Returns false if we couldn't compress the data or the
+        /* Returns < 0 if we couldn't compress the data or the
          * compressed result is longer than the original */
 
         ret = lzma_easy_buffer_encode(LZMA_PRESET_DEFAULT, LZMA_CHECK_NONE, NULL,
-                                      src, src_size, dst, &out_pos, src_size);
+                                      src, src_size, dst, &out_pos, src_size - 1);
         if (ret != LZMA_OK)
-                return false;
-
-        /* Is it actually shorter? */
-        if (out_pos == src_size)
-                return false;
+                return -ENOBUFS;
 
         *dst_size = out_pos;
-        return true;
+        return 0;
+#else
+        return -EPROTONOSUPPORT;
+#endif
 }
 
-bool uncompress_blob(const void *src, uint64_t src_size,
-                     void **dst, uint64_t *dst_alloc_size, uint64_t* dst_size, uint64_t dst_max) {
+int compress_blob_lz4(const void *src, uint64_t src_size, void *dst, uint64_t *dst_size) {
+#ifdef HAVE_LZ4
+        int r;
+
+        assert(src);
+        assert(src_size > 0);
+        assert(dst);
+        assert(dst_size);
+
+        /* Returns < 0 if we couldn't compress the data or the
+         * compressed result is longer than the original */
+
+        if (src_size < 9)
+                return -ENOBUFS;
 
+        r = LZ4_compress_limitedOutput(src, dst + 8, src_size, src_size - 8 - 1);
+        if (r <= 0)
+                return -ENOBUFS;
+
+        *(le64_t*) dst = htole64(src_size);
+        *dst_size = r + 8;
+
+        return 0;
+#else
+        return -EPROTONOSUPPORT;
+#endif
+}
+
+
+int decompress_blob_xz(const void *src, uint64_t src_size,
+                       void **dst, uint64_t *dst_alloc_size, uint64_t* dst_size, uint64_t dst_max) {
+
+#ifdef HAVE_XZ
         _cleanup_(lzma_end) lzma_stream s = LZMA_STREAM_INIT;
         lzma_ret ret;
         uint64_t space;
@@ -70,7 +118,7 @@ bool uncompress_blob(const void *src, uint64_t src_size,
 
         ret = lzma_stream_decoder(&s, UINT64_MAX, 0);
         if (ret != LZMA_OK)
-                return false;
+                return -ENOMEM;
 
         space = MIN(src_size * 2, dst_max ?: (uint64_t) -1);
         if (!greedy_realloc(dst, dst_alloc_size, space, 1))
@@ -89,15 +137,13 @@ bool uncompress_blob(const void *src, uint64_t src_size,
 
                 if (ret == LZMA_STREAM_END)
                         break;
-
-                if (ret != LZMA_OK)
-                        return false;
+                else if (ret != LZMA_OK)
+                        return -ENOMEM;
 
                 if (dst_max > 0 && (space - s.avail_out) >= dst_max)
                         break;
-
-                if (dst_max > 0 && space == dst_max)
-                        return false;
+                else if (dst_max > 0 && space == dst_max)
+                        return -ENOBUFS;
 
                 used = space - s.avail_out;
                 space = MIN(2 * space, dst_max ?: (uint64_t) -1);
@@ -109,18 +155,75 @@ bool uncompress_blob(const void *src, uint64_t src_size,
         }
 
         *dst_size = space - s.avail_out;
-        return true;
+        return 0;
+#else
+        return -EPROTONOSUPPORT;
+#endif
 }
 
-bool uncompress_startswith(const void *src, uint64_t src_size,
-                           void **buffer, uint64_t *buffer_size,
-                           const void *prefix, uint64_t prefix_len,
-                           uint8_t extra) {
+int decompress_blob_lz4(const void *src, uint64_t src_size,
+                        void **dst, uint64_t *dst_alloc_size, uint64_t* dst_size, uint64_t dst_max) {
+
+#ifdef HAVE_LZ4
+        char* out;
+        uint64_t size;
+        int r;
 
+        assert(src);
+        assert(src_size > 0);
+        assert(dst);
+        assert(dst_alloc_size);
+        assert(dst_size);
+        assert(*dst_alloc_size == 0 || *dst);
+
+        if (src_size <= 8)
+                return -EBADMSG;
+
+        size = le64toh( *(le64_t*)src );
+        if (size > *dst_alloc_size) {
+                out = realloc(*dst, size);
+                if (!out)
+                        return -ENOMEM;
+                *dst = out;
+                *dst_alloc_size = size;
+        } else
+                out = *dst;
+
+        r = LZ4_decompress_safe(src + 8, out, src_size - 8, size);
+        if (r < 0 || (uint64_t) r != size)
+                return -EBADMSG;
+
+        *dst_size = size;
+        return 0;
+#else
+        return -EPROTONOSUPPORT;
+#endif
+}
+
+int decompress_blob(int compression,
+                    const void *src, uint64_t src_size,
+                    void **dst, uint64_t *dst_alloc_size, uint64_t* dst_size, uint64_t dst_max) {
+        if (compression == OBJECT_COMPRESSED_XZ)
+                return decompress_blob_xz(src, src_size,
+                                          dst, dst_alloc_size, dst_size, dst_max);
+        else if (compression == OBJECT_COMPRESSED_LZ4)
+                return decompress_blob_lz4(src, src_size,
+                                           dst, dst_alloc_size, dst_size, dst_max);
+        else
+                return -EBADMSG;
+}
+
+
+int decompress_startswith_xz(const void *src, uint64_t src_size,
+                             void **buffer, uint64_t *buffer_size,
+                             const void *prefix, uint64_t prefix_len,
+                             uint8_t extra) {
+
+#ifdef HAVE_XZ
         _cleanup_(lzma_end) lzma_stream s = LZMA_STREAM_INIT;
         lzma_ret ret;
 
-        /* Checks whether the uncompressed blob starts with the
+        /* Checks whether the decompressed blob starts with the
          * mentioned prefix. The byte extra needs to follow the
          * prefix */
 
@@ -133,10 +236,10 @@ bool uncompress_startswith(const void *src, uint64_t src_size,
 
         ret = lzma_stream_decoder(&s, UINT64_MAX, 0);
         if (ret != LZMA_OK)
-                return false;
+                return -EBADMSG;
 
-        if (!(greedy_realloc(buffer, buffer_size, prefix_len + 1, 1)))
-                return false;
+        if (!(greedy_realloc(buffer, buffer_size, ALIGN_8(prefix_len + 1), 1)))
+                return -ENOMEM;
 
         s.next_in = src;
         s.avail_in = src_size;
@@ -148,25 +251,88 @@ bool uncompress_startswith(const void *src, uint64_t src_size,
                 ret = lzma_code(&s, LZMA_FINISH);
 
                 if (ret != LZMA_STREAM_END && ret != LZMA_OK)
-                        return false;
+                        return -EBADMSG;
 
                 if (*buffer_size - s.avail_out >= prefix_len + 1)
                         return memcmp(*buffer, prefix, prefix_len) == 0 &&
                                 ((const uint8_t*) *buffer)[prefix_len] == extra;
 
                 if (ret == LZMA_STREAM_END)
-                        return false;
+                        return 0;
 
                 s.avail_out += *buffer_size;
 
                 if (!(greedy_realloc(buffer, buffer_size, *buffer_size * 2, 1)))
-                        return false;
+                        return -ENOMEM;
 
                 s.next_out = *buffer + *buffer_size - s.avail_out;
         }
+
+#else
+        return -EPROTONOSUPPORT;
+#endif
+}
+
+int decompress_startswith_lz4(const void *src, uint64_t src_size,
+                              void **buffer, uint64_t *buffer_size,
+                              const void *prefix, uint64_t prefix_len,
+                              uint8_t extra) {
+#ifdef HAVE_LZ4
+        /* Checks whether the decompressed blob starts with the
+         * mentioned prefix. The byte extra needs to follow the
+         * prefix */
+
+        int r;
+
+        assert(src);
+        assert(src_size > 0);
+        assert(buffer);
+        assert(buffer_size);
+        assert(prefix);
+        assert(*buffer_size == 0 || *buffer);
+
+        if (src_size <= 8)
+                return -EBADMSG;
+
+        if (!(greedy_realloc(buffer, buffer_size, ALIGN_8(prefix_len + 1), 1)))
+                return -ENOMEM;
+
+        r = LZ4_decompress_safe_partial(src + 8, *buffer, src_size - 8,
+                                        prefix_len + 1, *buffer_size);
+
+        if (r < 0)
+                return -EBADMSG;
+        if ((unsigned) r >= prefix_len + 1)
+                return memcmp(*buffer, prefix, prefix_len) == 0 &&
+                        ((const uint8_t*) *buffer)[prefix_len] == extra;
+        else
+                return 0;
+
+#else
+        return -EPROTONOSUPPORT;
+#endif
 }
 
-int compress_stream(int fdf, int fdt, uint32_t preset, off_t max_bytes) {
+int decompress_startswith(int compression,
+                          const void *src, uint64_t src_size,
+                          void **buffer, uint64_t *buffer_size,
+                          const void *prefix, uint64_t prefix_len,
+                          uint8_t extra) {
+        if (compression == OBJECT_COMPRESSED_XZ)
+                return decompress_startswith_xz(src, src_size,
+                                                buffer, buffer_size,
+                                                prefix, prefix_len,
+                                                extra);
+        else if (compression == OBJECT_COMPRESSED_LZ4)
+                return decompress_startswith_lz4(src, src_size,
+                                                 buffer, buffer_size,
+                                                 prefix, prefix_len,
+                                                 extra);
+        else
+                return -EBADMSG;
+}
+
+int compress_stream_xz(int fdf, int fdt, off_t max_bytes) {
         _cleanup_(lzma_end) lzma_stream s = LZMA_STREAM_INIT;
         lzma_ret ret;
 
@@ -176,7 +342,7 @@ int compress_stream(int fdf, int fdt, uint32_t preset, off_t max_bytes) {
         assert(fdf >= 0);
         assert(fdt >= 0);
 
-        ret = lzma_easy_encoder(&s, preset, LZMA_CHECK_CRC64);
+        ret = lzma_easy_encoder(&s, LZMA_PRESET_DEFAULT, LZMA_CHECK_CRC64);
         if (ret != LZMA_OK) {
                 log_error("Failed to initialize XZ encoder: code %d", ret);
                 return -EINVAL;
@@ -230,7 +396,7 @@ int compress_stream(int fdf, int fdt, uint32_t preset, off_t max_bytes) {
                                 return errno ? -errno : -EIO;
 
                         if (ret == LZMA_STREAM_END) {
-                                log_debug("Compression finished (%zu -> %zu bytes, %.1f%%)",
+                                log_debug("XZ compression finished (%zu -> %zu bytes, %.1f%%)",
                                           s.total_in, s.total_out,
                                           (double) s.total_out / s.total_in * 100);
 
@@ -240,7 +406,91 @@ int compress_stream(int fdf, int fdt, uint32_t preset, off_t max_bytes) {
         }
 }
 
-int decompress_stream(int fdf, int fdt, off_t max_bytes) {
+#define LZ4_BUFSIZE (512*1024)
+
+int compress_stream_lz4(int fdf, int fdt, off_t max_bytes) {
+
+#ifdef HAVE_LZ4
+
+        _cleanup_free_ char *buf1 = NULL, *buf2 = NULL, *out = NULL;
+        char *buf;
+        LZ4_stream_t lz4_data = {};
+        le32_t header;
+        size_t total_in = 0, total_out = sizeof(header);
+        ssize_t n;
+
+        assert(fdf >= 0);
+        assert(fdt >= 0);
+
+        buf1 = malloc(LZ4_BUFSIZE);
+        buf2 = malloc(LZ4_BUFSIZE);
+        out = malloc(LZ4_COMPRESSBOUND(LZ4_BUFSIZE));
+        if (!buf1 || !buf2 || !out)
+                return log_oom();
+
+        buf = buf1;
+        for (;;) {
+                size_t m;
+                int r;
+
+                m = LZ4_BUFSIZE;
+                if (max_bytes != -1 && m > (size_t) max_bytes - total_in)
+                        m = max_bytes - total_in;
+
+                n = read(fdf, buf, m);
+                if (n < 0)
+                        return -errno;
+                if (n == 0)
+                        break;
+
+                total_in += n;
+
+                r = LZ4_compress_limitedOutput_continue(&lz4_data, buf, out, n, n);
+                if (r == 0) {
+                        log_debug("Compressed size exceeds original, aborting compression.");
+                        return -ENOBUFS;
+                }
+
+                header = htole32(r);
+                errno = 0;
+
+                n = write(fdt, &header, sizeof(header));
+                if (n < 0)
+                        return -errno;
+                if (n != sizeof(header))
+                        return errno ? -errno : -EIO;
+
+                n = loop_write(fdt, out, r, false);
+                if (n < 0)
+                        return n;
+                if (n != r)
+                        return errno ? -errno : -EIO;
+
+                total_out += sizeof(header) + r;
+
+                buf = buf == buf1 ? buf2 : buf1;
+        }
+
+        header = htole32(0);
+        n = write(fdt, &header, sizeof(header));
+        if (n < 0)
+                return -errno;
+        if (n != sizeof(header))
+                return errno ? -errno : -EIO;
+
+        log_debug("LZ4 compression finished (%zu -> %zu bytes, %.1f%%)",
+                  total_in, total_out,
+                  (double) total_out / total_in * 100);
+
+        return 0;
+#else
+        return -EPROTONOSUPPORT;
+#endif
+}
+
+int decompress_stream_xz(int fdf, int fdt, off_t max_bytes) {
+
+#ifdef HAVE_XZ
         _cleanup_(lzma_end) lzma_stream s = LZMA_STREAM_INIT;
         lzma_ret ret;
 
@@ -253,7 +503,7 @@ int decompress_stream(int fdf, int fdt, off_t max_bytes) {
         ret = lzma_stream_decoder(&s, UINT64_MAX, 0);
         if (ret != LZMA_OK) {
                 log_error("Failed to initialize XZ decoder: code %d", ret);
-                return -EINVAL;
+                return -ENOMEM;
         }
 
         for (;;) {
@@ -289,7 +539,7 @@ int decompress_stream(int fdf, int fdt, off_t max_bytes) {
 
                         if (max_bytes != -1) {
                                 if (max_bytes < n)
-                                        return -E2BIG;
+                                        return -EFBIG;
 
                                 max_bytes -= n;
                         }
@@ -302,7 +552,7 @@ int decompress_stream(int fdf, int fdt, off_t max_bytes) {
                                 return errno ? -errno : -EIO;
 
                         if (ret == LZMA_STREAM_END) {
-                                log_debug("Decompression finished (%zu -> %zu bytes, %.1f%%)",
+                                log_debug("XZ decompression finished (%zu -> %zu bytes, %.1f%%)",
                                           s.total_in, s.total_out,
                                           (double) s.total_out / s.total_in * 100);
 
@@ -310,4 +560,101 @@ int decompress_stream(int fdf, int fdt, off_t max_bytes) {
                         }
                 }
         }
+#else
+        log_error("Cannot decompress file. Compiled without XZ support.");
+        return -EPROTONOSUPPORT;
+#endif
+}
+
+int decompress_stream_lz4(int fdf, int fdt, off_t max_bytes) {
+
+#ifdef HAVE_LZ4
+        _cleanup_free_ char *buf = NULL, *out = NULL;
+        size_t buf_size = 0;
+        LZ4_streamDecode_t lz4_data = {};
+        le32_t header;
+        size_t total_in = sizeof(header), total_out = 0;
+
+        assert(fdf >= 0);
+        assert(fdt >= 0);
+
+        out = malloc(4*LZ4_BUFSIZE);
+        if (!out)
+                return log_oom();
+
+        for (;;) {
+                ssize_t n, m;
+                int r;
+
+                n = read(fdf, &header, sizeof(header));
+                if (n < 0)
+                        return -errno;
+                if (n != sizeof(header))
+                        return errno ? -errno : -EIO;
+
+                m = le32toh(header);
+                if (m == 0)
+                        break;
+
+                /* We refuse to use a bigger decompression buffer than
+                 * the one used for compression by 4 times. This means
+                 * that compression buffer size can be enlarged 4
+                 * times. This can be changed, but old binaries might
+                 * not accept buffers compressed by newer binaries then.
+                 */
+                if (m > LZ4_COMPRESSBOUND(LZ4_BUFSIZE * 4)) {
+                        log_error("Compressed stream block too big: %zd bytes", m);
+                        return -EBADMSG;
+                }
+
+                total_in += sizeof(header) + m;
+
+                if (!GREEDY_REALLOC(buf, buf_size, m))
+                        return log_oom();
+
+                errno = 0;
+                n = loop_read(fdf, buf, m, false);
+                if (n < 0)
+                        return n;
+                if (n != m)
+                        return errno ? -errno : -EIO;
+
+                r = LZ4_decompress_safe_continue(&lz4_data, buf, out, m, 4*LZ4_BUFSIZE);
+                if (r <= 0)
+                        log_error("LZ4 decompression failed.");
+
+                total_out += r;
+
+                if (max_bytes != -1 && total_out > (size_t) max_bytes) {
+                        log_debug("Decompressed stream longer than %zd bytes", max_bytes);
+                        return -EFBIG;
+                }
+
+                errno = 0;
+                n = loop_write(fdt, out, r, false);
+                if (n < 0)
+                        return n;
+                if (n != r)
+                        return errno ? -errno : -EIO;
+        }
+
+        log_debug("LZ4 decompression finished (%zu -> %zu bytes, %.1f%%)",
+                  total_in, total_out,
+                  (double) total_out / total_in * 100);
+
+        return 0;
+#else
+        log_error("Cannot decompress file. Compiled without LZ4 support.");
+        return -EPROTONOSUPPORT;
+#endif
+}
+
+int decompress_stream(const char *filename, int fdf, int fdt, off_t max_bytes) {
+
+        if (endswith(filename, ".lz4"))
+                return decompress_stream_lz4(fdf, fdt, max_bytes);
+        else if (endswith(filename, ".xz"))
+                return decompress_stream_xz(fdf, fdt, max_bytes);
+        else
+                return -EPROTONOSUPPORT;
 }
index f25fe86..92621ba 100644 (file)
 #include <stdbool.h>
 #include <unistd.h>
 
-bool compress_blob(const void *src, uint64_t src_size, void *dst, uint64_t *dst_size);
+#include "journal-def.h"
 
-bool uncompress_blob(const void *src, uint64_t src_size,
-                     void **dst, uint64_t *dst_alloc_size, uint64_t* dst_size, uint64_t dst_max);
+const char* object_compressed_to_string(int compression);
+int object_compressed_from_string(const char *compression);
 
-bool uncompress_startswith(const void *src, uint64_t src_size,
-                           void **buffer, uint64_t *buffer_size,
-                           const void *prefix, uint64_t prefix_len,
-                           uint8_t extra);
+int compress_blob_xz(const void *src, uint64_t src_size, void *dst, uint64_t *dst_size);
+int compress_blob_lz4(const void *src, uint64_t src_size, void *dst, uint64_t *dst_size);
 
-int compress_stream(int fdf, int fdt, uint32_t preset, off_t max_size);
-int decompress_stream(int fdf, int fdt, off_t max_size);
+static inline int compress_blob(const void *src, uint64_t src_size, void *dst, uint64_t *dst_size) {
+        int r;
+#ifdef HAVE_LZ4
+        r = compress_blob_lz4(src, src_size, dst, dst_size);
+        if (r == 0)
+                return OBJECT_COMPRESSED_LZ4;
+#else
+        r = compress_blob_xz(src, src_size, dst, dst_size);
+        if (r == 0)
+                return OBJECT_COMPRESSED_XZ;
+#endif
+        return r;
+}
+
+int decompress_blob_xz(const void *src, uint64_t src_size,
+                       void **dst, uint64_t *dst_alloc_size, uint64_t* dst_size, uint64_t dst_max);
+int decompress_blob_lz4(const void *src, uint64_t src_size,
+                        void **dst, uint64_t *dst_alloc_size, uint64_t* dst_size, uint64_t dst_max);
+int decompress_blob(int compression,
+                    const void *src, uint64_t src_size,
+                    void **dst, uint64_t *dst_alloc_size, uint64_t* dst_size, uint64_t dst_max);
+
+int decompress_startswith_xz(const void *src, uint64_t src_size,
+                             void **buffer, uint64_t *buffer_size,
+                             const void *prefix, uint64_t prefix_len,
+                             uint8_t extra);
+int decompress_startswith_lz4(const void *src, uint64_t src_size,
+                              void **buffer, uint64_t *buffer_size,
+                              const void *prefix, uint64_t prefix_len,
+                              uint8_t extra);
+int decompress_startswith(int compression,
+                          const void *src, uint64_t src_size,
+                          void **buffer, uint64_t *buffer_size,
+                          const void *prefix, uint64_t prefix_len,
+                          uint8_t extra);
+
+int compress_stream_xz(int fdf, int fdt, off_t max_bytes);
+int compress_stream_lz4(int fdf, int fdt, off_t max_bytes);
+
+int decompress_stream_xz(int fdf, int fdt, off_t max_size);
+int decompress_stream_lz4(int fdf, int fdt, off_t max_size);
+
+#ifdef HAVE_LZ4
+#  define compress_stream compress_stream_lz4
+#  define COMPRESSED_EXT ".lz4"
+#else
+#  define compress_stream compress_stream_xz
+#  define COMPRESSED_EXT ".xz"
+#endif
+
+int decompress_stream(const char *filename, int fdf, int fdt, off_t max_bytes);
index 78e89d3..59c6d4b 100644 (file)
 #  include "acl-util.h"
 #endif
 
-#ifdef HAVE_XZ
-#  include <lzma.h>
-#else
-#  define LZMA_PRESET_DEFAULT 0
-#endif
-
 /* The maximum size up to which we process coredumps */
 #define PROCESS_SIZE_MAX ((off_t) (2LLU*1024LLU*1024LLU*1024LLU))
 
@@ -357,7 +351,7 @@ static int save_external_coredump(
                 goto fail;
         }
 
-#ifdef HAVE_XZ
+#if defined(HAVE_XZ) || defined(HAVE_LZ4)
         /* If we will remove the coredump anyway, do not compress. */
         if (maybe_remove_external_coredump(NULL, st.st_size) == 0
             && arg_compress) {
@@ -365,15 +359,15 @@ static int save_external_coredump(
                 _cleanup_free_ char *fn_compressed = NULL, *tmp_compressed = NULL;
                 _cleanup_close_ int fd_compressed = -1;
 
-                fn_compressed = strappend(fn, ".xz");
+                fn_compressed = strappend(fn, COMPRESSED_EXT);
                 if (!fn_compressed) {
-                        r = log_oom();
+                        log_oom();
                         goto uncompressed;
                 }
 
                 tmp_compressed = tempfn_random(fn_compressed);
                 if (!tmp_compressed) {
-                        r = log_oom();
+                        log_oom();
                         goto uncompressed;
                 }
 
@@ -383,7 +377,7 @@ static int save_external_coredump(
                         goto uncompressed;
                 }
 
-                r = compress_stream(fd, fd_compressed, LZMA_PRESET_DEFAULT, -1);
+                r = compress_stream(fd, fd_compressed, -1);
                 if (r < 0) {
                         log_error("Failed to compress %s: %s", tmp_compressed, strerror(-r));
                         goto fail_compressed;
index 2158d73..5d6b2c7 100644 (file)
@@ -600,7 +600,7 @@ static int save_core(sd_journal *j, int fd, char **path, bool *unlink_temp) {
                 filename = NULL;
         }
 
-        if (filename && !endswith(filename, ".xz")) {
+        if (filename && !endswith(filename, ".xz") && !endswith(filename, ".lz4")) {
                 if (path) {
                         *path = filename;
                         filename = NULL;
@@ -646,7 +646,7 @@ static int save_core(sd_journal *j, int fd, char **path, bool *unlink_temp) {
                                 goto error;
                         }
                 } else if (filename) {
-#ifdef HAVE_XZ
+#if defined(HAVE_XZ) || defined(HAVE_LZ4)
                         _cleanup_close_ int fdf;
 
                         fdf = open(filename, O_RDONLY | O_CLOEXEC);
@@ -656,13 +656,13 @@ static int save_core(sd_journal *j, int fd, char **path, bool *unlink_temp) {
                                 goto error;
                         }
 
-                        r = decompress_stream(fdf, fd, -1);
+                        r = decompress_stream(filename, fdf, fd, -1);
                         if (r < 0) {
                                 log_error("Failed to decompress %s: %s", filename, strerror(-r));
                                 goto error;
                         }
 #else
-                        log_error("Cannot decompress file. Compiled without XZ support.");
+                        log_error("Cannot decompress file. Compiled without compression support.");
                         r = -ENOTSUP;
                         goto error;
 #endif
index 7e407a4..ecfa9a2 100644 (file)
@@ -66,9 +66,13 @@ enum {
 
 /* Object flags */
 enum {
-        OBJECT_COMPRESSED = 1
+        OBJECT_COMPRESSED_XZ = 1 << 0,
+        OBJECT_COMPRESSED_LZ4 = 1 << 1,
+        _OBJECT_COMPRESSED_MAX
 };
 
+#define OBJECT_COMPRESSION_MASK (OBJECT_COMPRESSED_XZ | OBJECT_COMPRESSED_LZ4)
+
 struct ObjectHeader {
         uint8_t type;
         uint8_t flags;
@@ -155,13 +159,33 @@ enum {
 
 /* Header flags */
 enum {
-        HEADER_INCOMPATIBLE_COMPRESSED = 1
+        HEADER_INCOMPATIBLE_COMPRESSED_XZ = 1 << 0,
+        HEADER_INCOMPATIBLE_COMPRESSED_LZ4 = 1 << 1,
 };
 
+#define HEADER_INCOMPATIBLE_ANY (HEADER_INCOMPATIBLE_COMPRESSED_XZ|HEADER_INCOMPATIBLE_COMPRESSED_LZ4)
+
+#if defined(HAVE_XZ) && defined(HAVE_LZ4)
+#  define HEADER_INCOMPATIBLE_SUPPORTED HEADER_INCOMPATIBLE_ANY
+#elif defined(HAVE_XZ)
+#  define HEADER_INCOMPATIBLE_SUPPORTED HEADER_INCOMPATIBLE_COMPRESSED_XZ
+#elif defined(HAVE_LZ4)
+#  define HEADER_INCOMPATIBLE_SUPPORTED HEADER_INCOMPATIBLE_COMPRESSED_LZ4
+#else
+#  define HEADER_INCOMPATIBLE_SUPPORTED 0
+#endif
+
 enum {
         HEADER_COMPATIBLE_SEALED = 1
 };
 
+#define HEADER_COMPATIBLE_ANY HEADER_COMPATIBLE_SEALED
+#if HAVE_GCRYPT
+#  define HEADER_COMPATIBLE_SUPPORTED HEADER_COMPATIBLE_SEALED
+#else
+#  define HEADER_COMPATIBLE_SUPPORTED 0
+#endif
+
 #define HEADER_SIGNATURE ((char[]) { 'L', 'P', 'K', 'S', 'H', 'H', 'R', 'H' })
 
 struct Header {
index b3b1ffc..b6de502 100644 (file)
@@ -138,7 +138,7 @@ void journal_file_close(JournalFile *f) {
 
         hashmap_free_free(f->chain_cache);
 
-#ifdef HAVE_XZ
+#if defined(HAVE_XZ) || defined(HAVE_LZ4)
         free(f->compress_buffer);
 #endif
 
@@ -158,21 +158,21 @@ void journal_file_close(JournalFile *f) {
 }
 
 static int journal_file_init_header(JournalFile *f, JournalFile *template) {
-        Header h;
+        Header h = {};
         ssize_t k;
         int r;
 
         assert(f);
 
-        zero(h);
         memcpy(h.signature, HEADER_SIGNATURE, 8);
         h.header_size = htole64(ALIGN64(sizeof(h)));
 
-        h.incompatible_flags =
-                htole32(f->compress ? HEADER_INCOMPATIBLE_COMPRESSED : 0);
+        h.incompatible_flags |= htole32(
+                f->compress_xz * HEADER_INCOMPATIBLE_COMPRESSED_XZ |
+                f->compress_lz4 * HEADER_INCOMPATIBLE_COMPRESSED_LZ4);
 
-        h.compatible_flags =
-                htole32(f->seal ? HEADER_COMPATIBLE_SEALED : 0);
+        h.compatible_flags = htole32(
+                f->seal * HEADER_COMPATIBLE_SEALED);
 
         r = sd_id128_randomize(&h.file_id);
         if (r < 0)
@@ -222,6 +222,8 @@ static int journal_file_refresh_header(JournalFile *f) {
 }
 
 static int journal_file_verify_header(JournalFile *f) {
+        uint32_t flags;
+
         assert(f);
 
         if (memcmp(f->header->signature, HEADER_SIGNATURE, 8))
@@ -229,24 +231,30 @@ static int journal_file_verify_header(JournalFile *f) {
 
         /* In both read and write mode we refuse to open files with
          * incompatible flags we don't know */
-#ifdef HAVE_XZ
-        if ((le32toh(f->header->incompatible_flags) & ~HEADER_INCOMPATIBLE_COMPRESSED) != 0)
-                return -EPROTONOSUPPORT;
-#else
-        if (f->header->incompatible_flags != 0)
+        flags = le32toh(f->header->incompatible_flags);
+        if (flags & ~HEADER_INCOMPATIBLE_SUPPORTED) {
+                if (flags & ~HEADER_INCOMPATIBLE_ANY)
+                        log_debug("Journal file %s has unknown incompatible flags %"PRIx32,
+                                  f->path, flags & ~HEADER_INCOMPATIBLE_ANY);
+                flags = (flags & HEADER_INCOMPATIBLE_ANY) & ~HEADER_INCOMPATIBLE_SUPPORTED;
+                if (flags)
+                        log_debug("Journal file %s uses incompatible flags %"PRIx32
+                                  " disabled at compilation time.", f->path, flags);
                 return -EPROTONOSUPPORT;
-#endif
+        }
 
         /* When open for writing we refuse to open files with
          * compatible flags, too */
-        if (f->writable) {
-#ifdef HAVE_GCRYPT
-                if ((le32toh(f->header->compatible_flags) & ~HEADER_COMPATIBLE_SEALED) != 0)
-                        return -EPROTONOSUPPORT;
-#else
-                if (f->header->compatible_flags != 0)
-                        return -EPROTONOSUPPORT;
-#endif
+        flags = le32toh(f->header->compatible_flags);
+        if (f->writable && (flags & ~HEADER_COMPATIBLE_SUPPORTED)) {
+                if (flags & ~HEADER_COMPATIBLE_ANY)
+                        log_debug("Journal file %s has unknown compatible flags %"PRIx32,
+                                  f->path, flags & ~HEADER_COMPATIBLE_ANY);
+                flags = (flags & HEADER_COMPATIBLE_ANY) & ~HEADER_COMPATIBLE_SUPPORTED;
+                if (flags)
+                        log_debug("Journal file %s uses compatible flags %"PRIx32
+                                  " disabled at compilation time.", f->path, flags);
+                return -EPROTONOSUPPORT;
         }
 
         if (f->header->state >= _STATE_MAX)
@@ -302,7 +310,8 @@ static int journal_file_verify_header(JournalFile *f) {
                 }
         }
 
-        f->compress = JOURNAL_HEADER_COMPRESSED(f->header);
+        f->compress_xz = JOURNAL_HEADER_COMPRESSED_XZ(f->header);
+        f->compress_lz4 = JOURNAL_HEADER_COMPRESSED_LZ4(f->header);
 
         f->seal = JOURNAL_HEADER_SEALED(f->header);
 
@@ -809,8 +818,7 @@ int journal_file_find_data_object_with_hash(
                 if (le64toh(o->data.hash) != hash)
                         goto next;
 
-                if (o->object.flags & OBJECT_COMPRESSED) {
-#ifdef HAVE_XZ
+                if (o->object.flags & OBJECT_COMPRESSION_MASK) {
                         uint64_t l, rsize;
 
                         l = le64toh(o->object.size);
@@ -819,8 +827,10 @@ int journal_file_find_data_object_with_hash(
 
                         l -= offsetof(Object, data.payload);
 
-                        if (!uncompress_blob(o->data.payload, l, &f->compress_buffer, &f->compress_buffer_size, &rsize, 0))
-                                return -EBADMSG;
+                        r = decompress_blob(o->object.flags & OBJECT_COMPRESSION_MASK,
+                                            o->data.payload, l, &f->compress_buffer, &f->compress_buffer_size, &rsize, 0);
+                        if (r < 0)
+                                return r;
 
                         if (rsize == size &&
                             memcmp(f->compress_buffer, data, size) == 0) {
@@ -833,9 +843,6 @@ int journal_file_find_data_object_with_hash(
 
                                 return 1;
                         }
-#else
-                        return -EPROTONOSUPPORT;
-#endif
 
                 } else if (le64toh(o->object.size) == osize &&
                            memcmp(o->data.payload, data, size) == 0) {
@@ -943,8 +950,7 @@ static int journal_file_append_data(
         uint64_t hash, p;
         uint64_t osize;
         Object *o;
-        int r;
-        bool compressed = false;
+        int r, compression = 0;
         const void *eq;
 
         assert(f);
@@ -973,23 +979,24 @@ static int journal_file_append_data(
 
         o->data.hash = htole64(hash);
 
-#ifdef HAVE_XZ
-        if (f->compress &&
+#if defined(HAVE_XZ) || defined(HAVE_LZ4)
+        if (f->compress_xz &&
             size >= COMPRESSION_SIZE_THRESHOLD) {
                 uint64_t rsize;
 
-                compressed = compress_blob(data, size, o->data.payload, &rsize);
+                compression = compress_blob(data, size, o->data.payload, &rsize);
 
-                if (compressed) {
+                if (compression) {
                         o->object.size = htole64(offsetof(Object, data.payload) + rsize);
-                        o->object.flags |= OBJECT_COMPRESSED;
+                        o->object.flags |= compression;
 
-                        log_debug("Compressed data object %"PRIu64" -> %"PRIu64, size, rsize);
+                        log_debug("Compressed data object %"PRIu64" -> %"PRIu64" using %s",
+                                  size, rsize, object_compressed_to_string(compression));
                 }
         }
 #endif
 
-        if (!compressed && size > 0)
+        if (!compression && size > 0)
                 memcpy(o->data.payload, data, size);
 
         r = journal_file_link_data(f, o, p, hash);
@@ -2332,8 +2339,9 @@ void journal_file_dump(JournalFile *f) {
                         break;
                 }
 
-                if (o->object.flags & OBJECT_COMPRESSED)
-                        printf("Flags: COMPRESSED\n");
+                if (o->object.flags & OBJECT_COMPRESSION_MASK)
+                        printf("Flags: %s\n",
+                               object_compressed_to_string(o->object.flags & OBJECT_COMPRESSION_MASK));
 
                 if (p == le64toh(f->header->tail_object_offset))
                         p = 0;
@@ -2370,7 +2378,7 @@ void journal_file_print_header(JournalFile *f) {
                "Sequential Number ID: %s\n"
                "State: %s\n"
                "Compatible Flags:%s%s\n"
-               "Incompatible Flags:%s%s\n"
+               "Incompatible Flags:%s%s%s\n"
                "Header size: %"PRIu64"\n"
                "Arena size: %"PRIu64"\n"
                "Data Hash Table Size: %"PRIu64"\n"
@@ -2392,9 +2400,10 @@ void journal_file_print_header(JournalFile *f) {
                f->header->state == STATE_ONLINE ? "ONLINE" :
                f->header->state == STATE_ARCHIVED ? "ARCHIVED" : "UNKNOWN",
                JOURNAL_HEADER_SEALED(f->header) ? " SEALED" : "",
-               (le32toh(f->header->compatible_flags) & ~HEADER_COMPATIBLE_SEALED) ? " ???" : "",
-               JOURNAL_HEADER_COMPRESSED(f->header) ? " COMPRESSED" : "",
-               (le32toh(f->header->incompatible_flags) & ~HEADER_INCOMPATIBLE_COMPRESSED) ? " ???" : "",
+               (le32toh(f->header->compatible_flags) & ~HEADER_COMPATIBLE_ANY) ? " ???" : "",
+               JOURNAL_HEADER_COMPRESSED_XZ(f->header) ? " COMPRESSED-XZ" : "",
+               JOURNAL_HEADER_COMPRESSED_LZ4(f->header) ? " COMPRESSED-LZ4" : "",
+               (le32toh(f->header->incompatible_flags) & ~HEADER_INCOMPATIBLE_ANY) ? " ???" : "",
                le64toh(f->header->header_size),
                le64toh(f->header->arena_size),
                le64toh(f->header->data_hash_table_size) / sizeof(HashItem),
@@ -2467,8 +2476,10 @@ int journal_file_open(
         f->flags = flags;
         f->prot = prot_from_flags(flags);
         f->writable = (flags & O_ACCMODE) != O_RDONLY;
-#ifdef HAVE_XZ
-        f->compress = compress;
+#if defined(HAVE_LZ)
+        f->compress_lz4 = compress;
+#elif defined(HAVE_XZ)
+        f->compress_xz = compress;
 #endif
 #ifdef HAVE_GCRYPT
         f->seal = seal;
@@ -2760,18 +2771,16 @@ int journal_file_copy_entry(JournalFile *from, JournalFile *to, Object *o, uint6
                 if ((uint64_t) t != l)
                         return -E2BIG;
 
-                if (o->object.flags & OBJECT_COMPRESSED) {
-#ifdef HAVE_XZ
+                if (o->object.flags & OBJECT_COMPRESSION_MASK) {
                         uint64_t rsize;
 
-                        if (!uncompress_blob(o->data.payload, l, &from->compress_buffer, &from->compress_buffer_size, &rsize, 0))
-                                return -EBADMSG;
+                        r = decompress_blob(o->object.flags & OBJECT_COMPRESSION_MASK,
+                                            o->data.payload, l, &from->compress_buffer, &from->compress_buffer_size, &rsize, 0);
+                        if (r < 0)
+                                return r;
 
                         data = from->compress_buffer;
                         l = rsize;
-#else
-                        return -EPROTONOSUPPORT;
-#endif
                 } else
                         data = o->data.payload;
 
index 50dbe29..b0c28b5 100644 (file)
@@ -56,7 +56,8 @@ typedef struct JournalFile {
         int flags;
         int prot;
         bool writable:1;
-        bool compress:1;
+        bool compress_xz:1;
+        bool compress_lz4:1;
         bool seal:1;
 
         bool tail_entry_monotonic_valid:1;
@@ -153,8 +154,11 @@ static inline bool VALID_EPOCH(uint64_t u) {
 #define JOURNAL_HEADER_SEALED(h) \
         (!!(le32toh((h)->compatible_flags) & HEADER_COMPATIBLE_SEALED))
 
-#define JOURNAL_HEADER_COMPRESSED(h) \
-        (!!(le32toh((h)->incompatible_flags) & HEADER_INCOMPATIBLE_COMPRESSED))
+#define JOURNAL_HEADER_COMPRESSED_XZ(h) \
+        (!!(le32toh((h)->incompatible_flags) & HEADER_INCOMPATIBLE_COMPRESSED_XZ))
+
+#define JOURNAL_HEADER_COMPRESSED_LZ4(h) \
+        (!!(le32toh((h)->incompatible_flags) & HEADER_INCOMPATIBLE_COMPRESSED_LZ4))
 
 int journal_file_move_to_object(JournalFile *f, int type, uint64_t offset, Object **ret);
 
index 31bae5a..c063d12 100644 (file)
@@ -45,7 +45,7 @@ static int journal_file_object_verify(JournalFile *f, uint64_t offset, Object *o
          * possible field values. It does not follow any references to
          * other objects. */
 
-        if ((o->object.flags & OBJECT_COMPRESSED) &&
+        if ((o->object.flags & OBJECT_COMPRESSED_XZ) &&
             o->object.type != OBJECT_DATA)
                 return -EBADMSG;
 
@@ -72,22 +72,38 @@ static int journal_file_object_verify(JournalFile *f, uint64_t offset, Object *o
 
                 h1 = le64toh(o->data.hash);
 
-                if (o->object.flags & OBJECT_COMPRESSED) {
+                if (o->object.flags & OBJECT_COMPRESSED_XZ) {
 #ifdef HAVE_XZ
-                        void *b = NULL;
+                        _cleanup_free_ void *b = NULL;
                         uint64_t alloc = 0, b_size;
 
-                        if (!uncompress_blob(o->data.payload,
-                                             le64toh(o->object.size) - offsetof(Object, data.payload),
-                                             &b, &alloc, &b_size, 0)) {
-                                log_error(OFSfmt": uncompression failed", offset);
+                        if (!decompress_blob_xz(o->data.payload,
+                                                le64toh(o->object.size) - offsetof(Object, data.payload),
+                                                &b, &alloc, &b_size, 0)) {
+                                log_error(OFSfmt": XZ decompression failed", offset);
                                 return -EBADMSG;
                         }
 
                         h2 = hash64(b, b_size);
-                        free(b);
 #else
-                        log_error("Compression is not supported");
+                        log_error("XZ compression is not supported");
+                        return -EPROTONOSUPPORT;
+#endif
+                } else if (o->object.flags & OBJECT_COMPRESSED_LZ4) {
+#ifdef HAVE_XZ
+                        _cleanup_free_ void *b = NULL;
+                        uint64_t alloc = 0, b_size;
+
+                        if (!decompress_blob_xz(o->data.payload,
+                                                le64toh(o->object.size) - offsetof(Object, data.payload),
+                                                &b, &alloc, &b_size, 0)) {
+                                log_error(OFSfmt": LZ4 decompression failed", offset);
+                                return -EBADMSG;
+                        }
+
+                        h2 = hash64(b, b_size);
+#else
+                        log_error("XZ compression is not supported");
                         return -EPROTONOSUPPORT;
 #endif
                 } else
@@ -875,8 +891,21 @@ int journal_file_verify(
                         goto fail;
                 }
 
-                if ((o->object.flags & OBJECT_COMPRESSED) && !JOURNAL_HEADER_COMPRESSED(f->header)) {
-                        log_error("Compressed object in file without compression at "OFSfmt, p);
+                if ((o->object.flags & OBJECT_COMPRESSED_XZ) &&
+                    (o->object.flags & OBJECT_COMPRESSED_LZ4)) {
+                        log_error("Objected with double compression at "OFSfmt, p);
+                        r = -EBADMSG;
+                        goto fail;
+                }
+
+                if ((o->object.flags & OBJECT_COMPRESSED_XZ) && !JOURNAL_HEADER_COMPRESSED_XZ(f->header)) {
+                        log_error("XZ compressed object in file without XZ compression at "OFSfmt, p);
+                        r = -EBADMSG;
+                        goto fail;
+                }
+
+                if ((o->object.flags & OBJECT_COMPRESSED_LZ4) && !JOURNAL_HEADER_COMPRESSED_LZ4(f->header)) {
+                        log_error("LZ4 compressed object in file without LZ4 compression at "OFSfmt, p);
                         r = -EBADMSG;
                         goto fail;
                 }
index ca805f8..96ba2bd 100644 (file)
@@ -1995,28 +1995,24 @@ _public_ int sd_journal_get_data(sd_journal *j, const char *field, const void **
 
                 l = le64toh(o->object.size) - offsetof(Object, data.payload);
 
-                if (o->object.flags & OBJECT_COMPRESSED) {
+                if ((o->object.flags & OBJECT_COMPRESSION_MASK) &&
+                    decompress_startswith(o->object.flags & OBJECT_COMPRESSION_MASK,
+                                          o->data.payload, l,
+                                          &f->compress_buffer, &f->compress_buffer_size,
+                                          field, field_length, '=')) {
 
-#ifdef HAVE_XZ
-                        if (uncompress_startswith(o->data.payload, l,
-                                                  &f->compress_buffer, &f->compress_buffer_size,
-                                                  field, field_length, '=')) {
-
-                                uint64_t rsize;
+                        uint64_t rsize;
 
-                                if (!uncompress_blob(o->data.payload, l,
-                                                     &f->compress_buffer, &f->compress_buffer_size, &rsize,
-                                                     j->data_threshold))
-                                        return -EBADMSG;
+                        r = decompress_blob_xz(o->data.payload, l,
+                                               &f->compress_buffer, &f->compress_buffer_size, &rsize,
+                                               j->data_threshold);
+                        if (r < 0)
+                                return r;
 
-                                *data = f->compress_buffer;
-                                *size = (size_t) rsize;
+                        *data = f->compress_buffer;
+                        *size = (size_t) rsize;
 
-                                return 0;
-                        }
-#else
-                        return -EPROTONOSUPPORT;
-#endif
+                        return 0;
 
                 } else if (l >= field_length+1 &&
                            memcmp(o->data.payload, field, field_length) == 0 &&
@@ -2052,11 +2048,11 @@ static int return_data(sd_journal *j, JournalFile *f, Object *o, const void **da
         if ((uint64_t) t != l)
                 return -E2BIG;
 
-        if (o->object.flags & OBJECT_COMPRESSED) {
+        if (o->object.flags & OBJECT_COMPRESSED_XZ) {
 #ifdef HAVE_XZ
                 uint64_t rsize;
 
-                if (!uncompress_blob(o->data.payload, l, &f->compress_buffer, &f->compress_buffer_size, &rsize, j->data_threshold))
+                if (!decompress_blob_xz(o->data.payload, l, &f->compress_buffer, &f->compress_buffer_size, &rsize, j->data_threshold))
                         return -EBADMSG;
 
                 *data = f->compress_buffer;
index 0806145..6ac8373 100644 (file)
 #include "util.h"
 #include "macro.h"
 
-static void test_compress_uncompress(void) {
+#ifdef HAVE_XZ
+# define XZ_OK 0
+#else
+# define XZ_OK -EPROTONOSUPPORT
+#endif
+
+#ifdef HAVE_XZ
+# define LZ4_OK 0
+#else
+# define LZ4_OK -EPROTONOSUPPORT
+#endif
+
+typedef int (compress_blob_t)(const void *src, uint64_t src_size,
+                              void *dst, uint64_t *dst_size);
+typedef int (decompress_blob_t)(const void *src, uint64_t src_size,
+                                void **dst, uint64_t *dst_alloc_size,
+                                uint64_t* dst_size, uint64_t dst_max);
+
+typedef int (decompress_sw_t)(const void *src, uint64_t src_size,
+                              void **buffer, uint64_t *buffer_size,
+                              const void *prefix, uint64_t prefix_len,
+                              uint8_t extra);
+
+typedef int (compress_stream_t)(int fdf, int fdt, off_t max_bytes);
+typedef int (decompress_stream_t)(int fdf, int fdt, off_t max_size);
+
+static void test_compress_decompress(int compression,
+                                     compress_blob_t compress,
+                                     decompress_blob_t decompress) {
         char text[] = "foofoofoofoo AAAA aaaaaaaaa ghost busters barbarbar FFF"
                       "foofoofoofoo AAAA aaaaaaaaa ghost busters barbarbar FFF";
         char compressed[512];
         uint64_t csize = 512;
         uint64_t usize = 0;
-        _cleanup_free_ char *uncompressed = NULL;
-
-        assert_se(compress_blob(text, sizeof(text), compressed, &csize));
-        assert_se(uncompress_blob(compressed,
-                                  csize,
-                                  (void **) &uncompressed,
-                                  &usize, &csize, 0));
-        assert_se(uncompressed);
-        assert_se(streq(uncompressed, text));
-        assert_se(!uncompress_blob("garbage",
-                                   7,
-                                   (void **) &uncompressed,
-                                   &usize, &csize, 0));
+        _cleanup_free_ char *decompressed = NULL;
+        int r;
+
+        log_info("/* testing %s blob compression/decompression */",
+                 object_compressed_to_string(compression));
+
+        r = compress(text, sizeof(text), compressed, &csize);
+        assert(r == 0);
+        r = decompress(compressed, csize,
+                       (void **) &decompressed, &usize, &csize, 0);
+        assert(r == 0);
+        assert_se(decompressed);
+        assert_se(streq(decompressed, text));
+
+        r = decompress("garbage", 7,
+                       (void **) &decompressed, &usize, &csize, 0);
+        assert(r < 0);
+
+        /* make sure to have the minimal lz4 compressed size */
+        r = decompress("00000000\1g", 9,
+                       (void **) &decompressed, &usize, &csize, 0);
+        assert(r < 0);
+
+        r = decompress("\100000000g", 9,
+                       (void **) &decompressed, &usize, &csize, 0);
+        assert(r < 0);
+
+        memzero(decompressed, usize);
 }
 
-static void test_uncompress_startswith(void) {
+static void test_decompress_startswith(int compression,
+                                       compress_blob_t compress,
+                                       decompress_sw_t decompress_sw) {
+
         char text[] = "foofoofoofoo AAAA aaaaaaaaa ghost busters barbarbar FFF"
                       "foofoofoofoo AAAA aaaaaaaaa ghost busters barbarbar FFF";
         char compressed[512];
         uint64_t csize = 512;
         uint64_t usize = 0;
-        _cleanup_free_ char *uncompressed = NULL;
-
-        assert_se(compress_blob(text, sizeof(text), compressed, &csize));
-        assert_se(uncompress_startswith(compressed,
-                                        csize,
-                                        (void **) &uncompressed,
-                                        &usize,
-                                        "foofoofoofoo", 12, ' '));
-        assert_se(!uncompress_startswith(compressed,
-                                         csize,
-                                        (void **) &uncompressed,
-                                        &usize,
-                                        "foofoofoofoo", 12, 'w'));
-        assert_se(!uncompress_startswith(compressed,
-                                         csize,
-                                        (void **) &uncompressed,
-                                        &usize,
-                                        "barbarbar", 9, ' '));
+        _cleanup_free_ char *decompressed = NULL;
+
+        log_info("/* testing decompress_startswith with %s */",
+                 object_compressed_to_string(compression));
+
+        assert_se(compress(text, sizeof(text), compressed, &csize) == 0);
+        assert_se(decompress_sw(compressed,
+                                csize,
+                                (void **) &decompressed,
+                                &usize,
+                                "foofoofoofoo", 12, ' ') > 0);
+        assert_se(decompress_sw(compressed,
+                                csize,
+                                (void **) &decompressed,
+                                &usize,
+                                "foofoofoofoo", 12, 'w') == 0);
+        assert_se(decompress_sw(compressed,
+                                csize,
+                                (void **) &decompressed,
+                                &usize,
+                                "barbarbar", 9, ' ') == 0);
+        assert_se(decompress_sw(compressed,
+                                csize,
+                                (void **) &decompressed,
+                                &usize,
+                                "foofoofoofoo", 12, ' ') > 0);
 }
 
-static void test_compress_stream(const char *srcfile) {
+static void test_compress_stream(int compression,
+                                 const char* cat,
+                                 compress_stream_t compress,
+                                 decompress_stream_t decompress,
+                                 const char *srcfile) {
+
         _cleanup_close_ int src = -1, dst = -1, dst2 = -1;
         char pattern[] = "/tmp/systemd-test.xz.XXXXXX",
              pattern2[] = "/tmp/systemd-test.xz.XXXXXX";
         int r;
-        _cleanup_free_ char *cmd, *cmd2;
+        _cleanup_free_ char *cmd = NULL, *cmd2;
         struct stat st = {};
 
+        log_debug("/* testing %s compression */",
+                  object_compressed_to_string(compression));
+
         log_debug("/* create source from %s */", srcfile);
 
         assert_se((src = open(srcfile, O_RDONLY|O_CLOEXEC)) >= 0);
@@ -84,11 +146,12 @@ static void test_compress_stream(const char *srcfile) {
 
         assert_se((dst = mkostemp_safe(pattern, O_RDWR|O_CLOEXEC)) >= 0);
 
-        r = compress_stream(src, dst, 1, -1);
-        assert(r == 0);
+        assert(compress(src, dst, -1) == 0);
 
-        assert_se(asprintf(&cmd, "xzcat %s | diff %s -", pattern, srcfile) > 0);
-        assert_se(system(cmd) == 0);
+        if (cat) {
+                assert_se(asprintf(&cmd, "%s %s | diff %s -", cat, pattern, srcfile) > 0);
+                assert(system(cmd) == 0);
+        }
 
         log_debug("/* test decompression */");
 
@@ -97,7 +160,7 @@ static void test_compress_stream(const char *srcfile) {
         assert_se(stat(srcfile, &st) == 0);
 
         assert_se(lseek(dst, 0, SEEK_SET) == 0);
-        r = decompress_stream(dst, dst2, st.st_size);
+        r = decompress(dst, dst2, st.st_size);
         assert(r == 0);
 
         assert_se(asprintf(&cmd2, "diff %s %s", srcfile, pattern2) > 0);
@@ -106,13 +169,13 @@ static void test_compress_stream(const char *srcfile) {
         log_debug("/* test faulty decompression */");
 
         assert_se(lseek(dst, 1, SEEK_SET) == 1);
-        r = decompress_stream(dst, dst2, st.st_size);
+        r = decompress(dst, dst2, st.st_size);
         assert(r == -EBADMSG);
 
         assert_se(lseek(dst, 0, SEEK_SET) == 0);
         assert_se(lseek(dst2, 0, SEEK_SET) == 0);
-        r = decompress_stream(dst, dst2, st.st_size - 1);
-        assert(r == -E2BIG);
+        r = decompress(dst, dst2, st.st_size - 1);
+        assert(r == -EFBIG);
 
         assert_se(unlink(pattern) == 0);
         assert_se(unlink(pattern2) == 0);
@@ -122,9 +185,24 @@ int main(int argc, char *argv[]) {
 
         log_set_max_level(LOG_DEBUG);
 
-        test_compress_uncompress();
-        test_uncompress_startswith();
-        test_compress_stream(argv[0]);
+#ifdef HAVE_XZ
+        test_compress_decompress(OBJECT_COMPRESSED_XZ, compress_blob_xz, decompress_blob_xz);
+        test_decompress_startswith(OBJECT_COMPRESSED_XZ, compress_blob_xz, decompress_startswith_xz);
+        test_compress_stream(OBJECT_COMPRESSED_XZ, "xzcat",
+                             compress_stream_xz, decompress_stream_xz, argv[0]);
+#else
+        log_info("/* XZ test skipped */");
+#endif
+#ifdef HAVE_LZ4
+        test_compress_decompress(OBJECT_COMPRESSED_LZ4, compress_blob_lz4, decompress_blob_lz4);
+        test_decompress_startswith(OBJECT_COMPRESSED_LZ4, compress_blob_lz4, decompress_startswith_lz4);
+
+        /* Produced stream is not compatible with lz4 binary, skip lz4cat check. */
+        test_compress_stream(OBJECT_COMPRESSED_LZ4, NULL,
+                             compress_stream_lz4, decompress_stream_lz4, argv[0]);
+#else
+        log_info("/* LZ4 test skipped */");
+#endif
 
         return 0;
 }