journal/compress: add stream compression/decompression functions

diff --git a/src/journal/compress.c b/src/journal/compress.c
index cafe8f4f54e4758993a9531d746d04613d25e44c..f36c4308017283540aeecc0a87416efa198ca9b3 100644 (file)
--- a/src/journal/compress.c
+++ b/src/journal/compress.c
@@ -22,10 +22,12 @@
 #include <assert.h>
 #include <stdlib.h>
 #include <string.h>
 #include <assert.h>
 #include <stdlib.h>
 #include <string.h>

+#include <unistd.h>

 #include <lzma.h>
 
 #include <lzma.h>
 

-#include "macro.h"

 #include "compress.h"
 #include "compress.h"

+#include "macro.h"
+#include "util.h"

 
 bool compress_blob(const void *src, uint64_t src_size, void *dst, uint64_t *dst_size) {
         lzma_ret ret;
 
 bool compress_blob(const void *src, uint64_t src_size, void *dst, uint64_t *dst_size) {
         lzma_ret ret;


@@ -40,12 +42,12 @@ bool compress_blob(const void *src, uint64_t src_size, void *dst, uint64_t *dst_
          * compressed result is longer than the original */
 
         ret = lzma_easy_buffer_encode(LZMA_PRESET_DEFAULT, LZMA_CHECK_NONE, NULL,
          * compressed result is longer than the original */
 
         ret = lzma_easy_buffer_encode(LZMA_PRESET_DEFAULT, LZMA_CHECK_NONE, NULL,

-                                      src, src_size, dst, &out_pos, *dst_size);
+                                      src, src_size, dst, &out_pos, src_size);

         if (ret != LZMA_OK)
                 return false;
 
         /* Is it actually shorter? */
         if (ret != LZMA_OK)
                 return false;
 
         /* Is it actually shorter? */

-        if (out_pos == *dst_size)
+        if (out_pos == src_size)

                 return false;
 
         *dst_size = out_pos;
                 return false;
 
         *dst_size = out_pos;


@@ -200,3 +202,149 @@ fail:
 
         return b;
 }
 
         return b;
 }

+
+int compress_stream(int fdf, int fdt, off_t max_bytes) {
+        _cleanup_(lzma_end) lzma_stream s = LZMA_STREAM_INIT;
+        lzma_ret ret;
+
+        uint8_t buf[BUFSIZ], out[BUFSIZ];
+        lzma_action action = LZMA_RUN;
+
+        assert(fdf >= 0);
+        assert(fdt >= 0);
+
+        ret = lzma_easy_encoder(&s, LZMA_PRESET_DEFAULT, LZMA_CHECK_CRC64);
+        if (ret != LZMA_OK) {
+                log_error("Failed to initialize XZ encoder: code %d", ret);
+                return -EINVAL;
+        }
+
+        for (;;) {
+                if (s.avail_in == 0 && action == LZMA_RUN) {
+                        size_t m = sizeof(buf);
+                        ssize_t n;
+
+                        if (max_bytes != -1 && m > (size_t) max_bytes)
+                                m = max_bytes;
+
+                        n = read(fdf, buf, m);
+                        if (n < 0)
+                                return -errno;
+                        if (n == 0)
+                                action = LZMA_FINISH;
+                        else {
+                                s.next_in = buf;
+                                s.avail_in = n;
+
+                                if (max_bytes != -1) {
+                                        assert(max_bytes >= n);
+                                        max_bytes -= n;
+                                }
+                        }
+                }
+
+                if (s.avail_out == 0) {
+                        s.next_out = out;
+                        s.avail_out = sizeof(out);
+                }
+
+                ret = lzma_code(&s, action);
+                if (ret != LZMA_OK && ret != LZMA_STREAM_END) {
+                        log_error("Compression failed: code %d", ret);
+                        return -EBADMSG;
+                }
+
+                if (s.avail_out == 0 || ret == LZMA_STREAM_END) {
+                        ssize_t n, k;
+
+                        n = sizeof(out) - s.avail_out;
+
+                        errno = 0;
+                        k = loop_write(fdt, out, n, false);
+                        if (k < 0)
+                                return k;
+                        if (k != n)
+                                return errno ? -errno : -EIO;
+
+                        if (ret == LZMA_STREAM_END) {
+                                log_debug("Compression finished (%zu -> %zu bytes, %.1f%%)",
+                                          s.total_in, s.total_out,
+                                          (double) s.total_out / s.total_in * 100);
+
+                                return 0;
+                        }
+                }
+        }
+}
+
+int decompress_stream(int fdf, int fdt, off_t max_bytes) {
+        _cleanup_(lzma_end) lzma_stream s = LZMA_STREAM_INIT;
+        lzma_ret ret;
+
+        uint8_t buf[BUFSIZ], out[BUFSIZ];
+        lzma_action action = LZMA_RUN;
+
+        assert(fdf >= 0);
+        assert(fdt >= 0);
+
+        ret = lzma_stream_decoder(&s, UINT64_MAX, 0);
+        if (ret != LZMA_OK) {
+                log_error("Failed to initialize XZ decoder: code %d", ret);
+                return -EINVAL;
+        }
+
+        for (;;) {
+                if (s.avail_in == 0 && action == LZMA_RUN) {
+                        ssize_t n;
+
+                        n = read(fdf, buf, sizeof(buf));
+                        if (n < 0)
+                                return -errno;
+                        if (n == 0)
+                                action = LZMA_FINISH;
+                        else {
+                                s.next_in = buf;
+                                s.avail_in = n;
+                        }
+                }
+
+                if (s.avail_out == 0) {
+                        s.next_out = out;
+                        s.avail_out = sizeof(out);
+                }
+
+                ret = lzma_code(&s, action);
+                if (ret != LZMA_OK && ret != LZMA_STREAM_END) {
+                        log_error("Decompression failed: code %d", ret);
+                        return -EBADMSG;
+                }
+
+                if (s.avail_out == 0 || ret == LZMA_STREAM_END) {
+                        ssize_t n, k;
+
+                        n = sizeof(out) - s.avail_out;
+
+                        if (max_bytes != -1) {
+                                if (max_bytes < n)
+                                        return -E2BIG;
+
+                                max_bytes -= n;
+                        }
+
+                        errno = 0;
+                        k = loop_write(fdt, out, n, false);
+                        if (k < 0)
+                                return k;
+                        if (k != n)
+                                return errno ? -errno : -EIO;
+
+                        if (ret == LZMA_STREAM_END) {
+                                log_debug("Decompression finished (%zu -> %zu bytes, %.1f%%)",
+                                          s.total_in, s.total_out,
+                                          (double) s.total_out / s.total_in * 100);
+
+                                return 0;
+                        }
+                }
+        }
+}

diff --git a/src/journal/compress.h b/src/journal/compress.h
index 2b87e73a267af9898bc9ecc1aaa6453813687e57..f37a6b3d148e59efae4da213f3b06e883223fec4 100644 (file)
--- a/src/journal/compress.h
+++ b/src/journal/compress.h
@@ -23,6 +23,7 @@
 
 #include <inttypes.h>
 #include <stdbool.h>
 
 #include <inttypes.h>
 #include <stdbool.h>

+#include <unistd.h>

 
 bool compress_blob(const void *src, uint64_t src_size, void *dst, uint64_t *dst_size);
 
 
 bool compress_blob(const void *src, uint64_t src_size, void *dst, uint64_t *dst_size);
 


@@ -33,3 +34,6 @@ bool uncompress_startswith(const void *src, uint64_t src_size,
                            void **buffer, uint64_t *buffer_size,
                            const void *prefix, uint64_t prefix_len,
                            uint8_t extra);
                            void **buffer, uint64_t *buffer_size,
                            const void *prefix, uint64_t prefix_len,
                            uint8_t extra);

+
+int compress_stream(int fdf, int fdt, off_t max_size);
+int decompress_stream(int fdf, int fdt, off_t max_size);

diff --git a/src/journal/test-compress.c b/src/journal/test-compress.c
index 15b3f9aa00b2f4b76be210e408f21272fff0890f..b098ef93a850c02bb5c3a5a7c8e42a40874c82c9 100644 (file)
--- a/src/journal/test-compress.c
+++ b/src/journal/test-compress.c
@@ -68,9 +68,63 @@ static void test_uncompress_startswith(void) {
                                         "barbarbar", 9, ' '));
 }
 
                                         "barbarbar", 9, ' '));
 }
 

+static void test_compress_stream(const char *srcfile) {
+        _cleanup_close_ int src = -1, dst = -1, dst2 = -1;
+        char pattern[] = "/tmp/systemd-test.xz.XXXXXX",
+             pattern2[] = "/tmp/systemd-test.xz.XXXXXX";
+        int r;
+        _cleanup_free_ char *cmd, *cmd2;
+        struct stat st = {};
+
+        log_debug("/* create source from %s */", srcfile);
+
+        assert_se((src = open(srcfile, O_RDONLY|O_CLOEXEC)) >= 0);
+
+        log_debug("/* test compression */");
+
+        assert_se((dst = mkostemp_safe(pattern, O_RDWR|O_CLOEXEC)) >= 0);
+
+        r = compress_stream(src, dst, -1);
+        assert(r == 0);
+
+        assert_se(asprintf(&cmd, "xzcat %s | diff %s -", pattern, srcfile) > 0);
+        assert_se(system(cmd) == 0);
+
+        log_debug("/* test decompression */");
+
+        assert_se((dst2 = mkostemp_safe(pattern2, O_RDWR|O_CLOEXEC)) >= 0);
+
+        assert_se(stat(srcfile, &st) == 0);
+
+        assert_se(lseek(dst, 0, SEEK_SET) == 0);
+        r = decompress_stream(dst, dst2, st.st_size);
+        assert(r == 0);
+
+        assert_se(asprintf(&cmd2, "diff %s %s", srcfile, pattern2) > 0);
+        assert_se(system(cmd2) == 0);
+
+        log_debug("/* test faulty decompression */");
+
+        assert_se(lseek(dst, 1, SEEK_SET) == 1);
+        r = decompress_stream(dst, dst2, st.st_size);
+        assert(r == -EBADMSG);
+
+        assert_se(lseek(dst, 0, SEEK_SET) == 0);
+        assert_se(lseek(dst2, 0, SEEK_SET) == 0);
+        r = decompress_stream(dst, dst2, st.st_size - 1);
+        assert(r == -E2BIG);
+
+        assert_se(unlink(pattern) == 0);
+        assert_se(unlink(pattern2) == 0);
+}
+

 int main(int argc, char *argv[]) {
 int main(int argc, char *argv[]) {

+
+        log_set_max_level(LOG_DEBUG);
+

         test_compress_uncompress();
         test_uncompress_startswith();
         test_compress_uncompress();
         test_uncompress_startswith();

+        test_compress_stream(argv[0]);

 
         return 0;
 }
 
         return 0;
 }

diff --git a/src/shared/copy.c b/src/shared/copy.c
index ebd66998384db7b33610ecbbbe83cdb577dc2724..3744797b9502788b8d16335085ec3ca89a7492f5 100644 (file)
--- a/src/shared/copy.c
+++ b/src/shared/copy.c
@@ -29,9 +29,7 @@ int copy_bytes(int fdf, int fdt, off_t max_bytes) {
         for (;;) {
                 char buf[PIPE_BUF];
                 ssize_t n, k;
         for (;;) {
                 char buf[PIPE_BUF];
                 ssize_t n, k;

-                size_t m;
-
-                m = sizeof(buf);
+                size_t m = sizeof(buf);

 
                 if (max_bytes != (off_t) -1) {
 
 
                 if (max_bytes != (off_t) -1) {