1 /*-*- Mode: C; c-basic-offset: 8; indent-tabs-mode: nil -*-*/
4 This file is part of systemd.
6 Copyright 2011 Lennart Poettering
8 systemd is free software; you can redistribute it and/or modify it
9 under the terms of the GNU Lesser General Public License as published by
10 the Free Software Foundation; either version 2.1 of the License, or
11 (at your option) any later version.
13 systemd is distributed in the hope that it will be useful, but
14 WITHOUT ANY WARRANTY; without even the implied warranty of
15 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
16 Lesser General Public License for more details.
18 You should have received a copy of the GNU Lesser General Public License
19 along with systemd; If not, see <http://www.gnu.org/licenses/>.
38 #include "sparse-endian.h"
39 #include "journal-def.h"
41 #define ALIGN_8(l) ALIGN_TO(l, sizeof(size_t))
43 static const char* const object_compressed_table[_OBJECT_COMPRESSED_MAX] = {
44 [OBJECT_COMPRESSED_XZ] = "XZ",
45 [OBJECT_COMPRESSED_LZ4] = "LZ4",
48 DEFINE_STRING_TABLE_LOOKUP(object_compressed, int);
50 int compress_blob_xz(const void *src, uint64_t src_size, void *dst, uint64_t *dst_size) {
52 static const lzma_options_lzma opt = {
53 1u << 20u, NULL, 0, LZMA_LC_DEFAULT, LZMA_LP_DEFAULT,
54 LZMA_PB_DEFAULT, LZMA_MODE_FAST, 128, LZMA_MF_HC3, 4};
55 static const lzma_filter filters[2] = {
56 {LZMA_FILTER_LZMA2, (lzma_options_lzma*) &opt},
57 {LZMA_VLI_UNKNOWN, NULL}
67 /* Returns < 0 if we couldn't compress the data or the
68 * compressed result is longer than the original */
73 ret = lzma_stream_buffer_encode((lzma_filter*) filters, LZMA_CHECK_NONE, NULL,
74 src, src_size, dst, &out_pos, src_size - 1);
81 return -EPROTONOSUPPORT;
85 int compress_blob_lz4(const void *src, uint64_t src_size, void *dst, uint64_t *dst_size) {
94 /* Returns < 0 if we couldn't compress the data or the
95 * compressed result is longer than the original */
100 r = LZ4_compress_limitedOutput(src, dst + 8, src_size, src_size - 8 - 1);
104 *(le64_t*) dst = htole64(src_size);
109 return -EPROTONOSUPPORT;
114 int decompress_blob_xz(const void *src, uint64_t src_size,
115 void **dst, uint64_t *dst_alloc_size, uint64_t* dst_size, uint64_t dst_max) {
118 _cleanup_(lzma_end) lzma_stream s = LZMA_STREAM_INIT;
123 assert(src_size > 0);
125 assert(dst_alloc_size);
127 assert(*dst_alloc_size == 0 || *dst);
129 ret = lzma_stream_decoder(&s, UINT64_MAX, 0);
133 space = MIN(src_size * 2, dst_max ?: (uint64_t) -1);
134 if (!greedy_realloc(dst, dst_alloc_size, space, 1))
138 s.avail_in = src_size;
146 ret = lzma_code(&s, LZMA_FINISH);
148 if (ret == LZMA_STREAM_END)
150 else if (ret != LZMA_OK)
153 if (dst_max > 0 && (space - s.avail_out) >= dst_max)
155 else if (dst_max > 0 && space == dst_max)
158 used = space - s.avail_out;
159 space = MIN(2 * space, dst_max ?: (uint64_t) -1);
160 if (!greedy_realloc(dst, dst_alloc_size, space, 1))
163 s.avail_out = space - used;
164 s.next_out = *dst + used;
167 *dst_size = space - s.avail_out;
170 return -EPROTONOSUPPORT;
174 int decompress_blob_lz4(const void *src, uint64_t src_size,
175 void **dst, uint64_t *dst_alloc_size, uint64_t* dst_size, uint64_t dst_max) {
183 assert(src_size > 0);
185 assert(dst_alloc_size);
187 assert(*dst_alloc_size == 0 || *dst);
192 size = le64toh( *(le64_t*)src );
193 if (size > *dst_alloc_size) {
194 out = realloc(*dst, size);
198 *dst_alloc_size = size;
202 r = LZ4_decompress_safe(src + 8, out, src_size - 8, size);
203 if (r < 0 || (uint64_t) r != size)
209 return -EPROTONOSUPPORT;
213 int decompress_blob(int compression,
214 const void *src, uint64_t src_size,
215 void **dst, uint64_t *dst_alloc_size, uint64_t* dst_size, uint64_t dst_max) {
216 if (compression == OBJECT_COMPRESSED_XZ)
217 return decompress_blob_xz(src, src_size,
218 dst, dst_alloc_size, dst_size, dst_max);
219 else if (compression == OBJECT_COMPRESSED_LZ4)
220 return decompress_blob_lz4(src, src_size,
221 dst, dst_alloc_size, dst_size, dst_max);
227 int decompress_startswith_xz(const void *src, uint64_t src_size,
228 void **buffer, uint64_t *buffer_size,
229 const void *prefix, uint64_t prefix_len,
233 _cleanup_(lzma_end) lzma_stream s = LZMA_STREAM_INIT;
236 /* Checks whether the decompressed blob starts with the
237 * mentioned prefix. The byte extra needs to follow the
241 assert(src_size > 0);
245 assert(*buffer_size == 0 || *buffer);
247 ret = lzma_stream_decoder(&s, UINT64_MAX, 0);
251 if (!(greedy_realloc(buffer, buffer_size, ALIGN_8(prefix_len + 1), 1)))
255 s.avail_in = src_size;
257 s.next_out = *buffer;
258 s.avail_out = *buffer_size;
261 ret = lzma_code(&s, LZMA_FINISH);
263 if (ret != LZMA_STREAM_END && ret != LZMA_OK)
266 if (*buffer_size - s.avail_out >= prefix_len + 1)
267 return memcmp(*buffer, prefix, prefix_len) == 0 &&
268 ((const uint8_t*) *buffer)[prefix_len] == extra;
270 if (ret == LZMA_STREAM_END)
273 s.avail_out += *buffer_size;
275 if (!(greedy_realloc(buffer, buffer_size, *buffer_size * 2, 1)))
278 s.next_out = *buffer + *buffer_size - s.avail_out;
282 return -EPROTONOSUPPORT;
286 int decompress_startswith_lz4(const void *src, uint64_t src_size,
287 void **buffer, uint64_t *buffer_size,
288 const void *prefix, uint64_t prefix_len,
291 /* Checks whether the decompressed blob starts with the
292 * mentioned prefix. The byte extra needs to follow the
298 assert(src_size > 0);
302 assert(*buffer_size == 0 || *buffer);
307 if (!(greedy_realloc(buffer, buffer_size, ALIGN_8(prefix_len + 1), 1)))
310 r = LZ4_decompress_safe_partial(src + 8, *buffer, src_size - 8,
311 prefix_len + 1, *buffer_size);
315 if ((unsigned) r >= prefix_len + 1)
316 return memcmp(*buffer, prefix, prefix_len) == 0 &&
317 ((const uint8_t*) *buffer)[prefix_len] == extra;
322 return -EPROTONOSUPPORT;
326 int decompress_startswith(int compression,
327 const void *src, uint64_t src_size,
328 void **buffer, uint64_t *buffer_size,
329 const void *prefix, uint64_t prefix_len,
331 if (compression == OBJECT_COMPRESSED_XZ)
332 return decompress_startswith_xz(src, src_size,
336 else if (compression == OBJECT_COMPRESSED_LZ4)
337 return decompress_startswith_lz4(src, src_size,
345 int compress_stream_xz(int fdf, int fdt, off_t max_bytes) {
346 _cleanup_(lzma_end) lzma_stream s = LZMA_STREAM_INIT;
349 uint8_t buf[BUFSIZ], out[BUFSIZ];
350 lzma_action action = LZMA_RUN;
355 ret = lzma_easy_encoder(&s, LZMA_PRESET_DEFAULT, LZMA_CHECK_CRC64);
356 if (ret != LZMA_OK) {
357 log_error("Failed to initialize XZ encoder: code %d", ret);
362 if (s.avail_in == 0 && action == LZMA_RUN) {
363 size_t m = sizeof(buf);
366 if (max_bytes != -1 && m > (size_t) max_bytes)
369 n = read(fdf, buf, m);
373 action = LZMA_FINISH;
378 if (max_bytes != -1) {
379 assert(max_bytes >= n);
385 if (s.avail_out == 0) {
387 s.avail_out = sizeof(out);
390 ret = lzma_code(&s, action);
391 if (ret != LZMA_OK && ret != LZMA_STREAM_END) {
392 log_error("Compression failed: code %d", ret);
396 if (s.avail_out == 0 || ret == LZMA_STREAM_END) {
399 n = sizeof(out) - s.avail_out;
402 k = loop_write(fdt, out, n, false);
406 return errno ? -errno : -EIO;
408 if (ret == LZMA_STREAM_END) {
409 log_debug("XZ compression finished (%zu -> %zu bytes, %.1f%%)",
410 s.total_in, s.total_out,
411 (double) s.total_out / s.total_in * 100);
419 #define LZ4_BUFSIZE (512*1024)
421 int compress_stream_lz4(int fdf, int fdt, off_t max_bytes) {
425 _cleanup_free_ char *buf1 = NULL, *buf2 = NULL, *out = NULL;
427 LZ4_stream_t lz4_data = {};
429 size_t total_in = 0, total_out = sizeof(header);
435 buf1 = malloc(LZ4_BUFSIZE);
436 buf2 = malloc(LZ4_BUFSIZE);
437 out = malloc(LZ4_COMPRESSBOUND(LZ4_BUFSIZE));
438 if (!buf1 || !buf2 || !out)
447 if (max_bytes != -1 && m > (size_t) max_bytes - total_in)
448 m = max_bytes - total_in;
450 n = read(fdf, buf, m);
458 r = LZ4_compress_limitedOutput_continue(&lz4_data, buf, out, n, n);
460 log_debug("Compressed size exceeds original, aborting compression.");
467 n = write(fdt, &header, sizeof(header));
470 if (n != sizeof(header))
471 return errno ? -errno : -EIO;
473 n = loop_write(fdt, out, r, false);
477 return errno ? -errno : -EIO;
479 total_out += sizeof(header) + r;
481 buf = buf == buf1 ? buf2 : buf1;
485 n = write(fdt, &header, sizeof(header));
488 if (n != sizeof(header))
489 return errno ? -errno : -EIO;
491 log_debug("LZ4 compression finished (%zu -> %zu bytes, %.1f%%)",
493 (double) total_out / total_in * 100);
497 return -EPROTONOSUPPORT;
501 int decompress_stream_xz(int fdf, int fdt, off_t max_bytes) {
504 _cleanup_(lzma_end) lzma_stream s = LZMA_STREAM_INIT;
507 uint8_t buf[BUFSIZ], out[BUFSIZ];
508 lzma_action action = LZMA_RUN;
513 ret = lzma_stream_decoder(&s, UINT64_MAX, 0);
514 if (ret != LZMA_OK) {
515 log_error("Failed to initialize XZ decoder: code %d", ret);
520 if (s.avail_in == 0 && action == LZMA_RUN) {
523 n = read(fdf, buf, sizeof(buf));
527 action = LZMA_FINISH;
534 if (s.avail_out == 0) {
536 s.avail_out = sizeof(out);
539 ret = lzma_code(&s, action);
540 if (ret != LZMA_OK && ret != LZMA_STREAM_END) {
541 log_error("Decompression failed: code %d", ret);
545 if (s.avail_out == 0 || ret == LZMA_STREAM_END) {
548 n = sizeof(out) - s.avail_out;
550 if (max_bytes != -1) {
558 k = loop_write(fdt, out, n, false);
562 return errno ? -errno : -EIO;
564 if (ret == LZMA_STREAM_END) {
565 log_debug("XZ decompression finished (%zu -> %zu bytes, %.1f%%)",
566 s.total_in, s.total_out,
567 (double) s.total_out / s.total_in * 100);
574 log_error("Cannot decompress file. Compiled without XZ support.");
575 return -EPROTONOSUPPORT;
579 int decompress_stream_lz4(int fdf, int fdt, off_t max_bytes) {
582 _cleanup_free_ char *buf = NULL, *out = NULL;
584 LZ4_streamDecode_t lz4_data = {};
586 size_t total_in = sizeof(header), total_out = 0;
591 out = malloc(4*LZ4_BUFSIZE);
599 n = read(fdf, &header, sizeof(header));
602 if (n != sizeof(header))
603 return errno ? -errno : -EIO;
609 /* We refuse to use a bigger decompression buffer than
610 * the one used for compression by 4 times. This means
611 * that compression buffer size can be enlarged 4
612 * times. This can be changed, but old binaries might
613 * not accept buffers compressed by newer binaries then.
615 if (m > LZ4_COMPRESSBOUND(LZ4_BUFSIZE * 4)) {
616 log_error("Compressed stream block too big: %zd bytes", m);
620 total_in += sizeof(header) + m;
622 if (!GREEDY_REALLOC(buf, buf_size, m))
626 n = loop_read(fdf, buf, m, false);
630 return errno ? -errno : -EIO;
632 r = LZ4_decompress_safe_continue(&lz4_data, buf, out, m, 4*LZ4_BUFSIZE);
634 log_error("LZ4 decompression failed.");
638 if (max_bytes != -1 && total_out > (size_t) max_bytes) {
639 log_debug("Decompressed stream longer than %zd bytes", max_bytes);
644 n = loop_write(fdt, out, r, false);
648 return errno ? -errno : -EIO;
651 log_debug("LZ4 decompression finished (%zu -> %zu bytes, %.1f%%)",
653 (double) total_out / total_in * 100);
657 log_error("Cannot decompress file. Compiled without LZ4 support.");
658 return -EPROTONOSUPPORT;
662 int decompress_stream(const char *filename, int fdf, int fdt, off_t max_bytes) {
664 if (endswith(filename, ".lz4"))
665 return decompress_stream_lz4(fdf, fdt, max_bytes);
666 else if (endswith(filename, ".xz"))
667 return decompress_stream_xz(fdf, fdt, max_bytes);
669 return -EPROTONOSUPPORT;