1 /*-*- Mode: C; c-basic-offset: 8; indent-tabs-mode: nil -*-*/
4 This file is part of systemd.
6 Copyright 2011 Lennart Poettering
8 systemd is free software; you can redistribute it and/or modify it
9 under the terms of the GNU Lesser General Public License as published by
10 the Free Software Foundation; either version 2.1 of the License, or
11 (at your option) any later version.
13 systemd is distributed in the hope that it will be useful, but
14 WITHOUT ANY WARRANTY; without even the implied warranty of
15 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
16 Lesser General Public License for more details.
18 You should have received a copy of the GNU Lesser General Public License
19 along with systemd; If not, see <http://www.gnu.org/licenses/>.
38 #include "sparse-endian.h"
39 #include "journal-def.h"
41 #define ALIGN_8(l) ALIGN_TO(l, sizeof(size_t))
43 static const char* const object_compressed_table[_OBJECT_COMPRESSED_MAX] = {
44 [OBJECT_COMPRESSED_XZ] = "XZ",
45 [OBJECT_COMPRESSED_LZ4] = "LZ4",
48 DEFINE_STRING_TABLE_LOOKUP(object_compressed, int);
50 int compress_blob_xz(const void *src, uint64_t src_size, void *dst, size_t *dst_size) {
52 static const lzma_options_lzma opt = {
53 1u << 20u, NULL, 0, LZMA_LC_DEFAULT, LZMA_LP_DEFAULT,
54 LZMA_PB_DEFAULT, LZMA_MODE_FAST, 128, LZMA_MF_HC3, 4};
55 static const lzma_filter filters[2] = {
56 {LZMA_FILTER_LZMA2, (lzma_options_lzma*) &opt},
57 {LZMA_VLI_UNKNOWN, NULL}
67 /* Returns < 0 if we couldn't compress the data or the
68 * compressed result is longer than the original */
73 ret = lzma_stream_buffer_encode((lzma_filter*) filters, LZMA_CHECK_NONE, NULL,
74 src, src_size, dst, &out_pos, src_size - 1);
81 return -EPROTONOSUPPORT;
85 int compress_blob_lz4(const void *src, uint64_t src_size, void *dst, size_t *dst_size) {
94 /* Returns < 0 if we couldn't compress the data or the
95 * compressed result is longer than the original */
100 r = LZ4_compress_limitedOutput(src, dst + 8, src_size, src_size - 8 - 1);
104 *(le64_t*) dst = htole64(src_size);
109 return -EPROTONOSUPPORT;
114 int decompress_blob_xz(const void *src, uint64_t src_size,
115 void **dst, size_t *dst_alloc_size, size_t* dst_size, size_t dst_max) {
118 _cleanup_(lzma_end) lzma_stream s = LZMA_STREAM_INIT;
123 assert(src_size > 0);
125 assert(dst_alloc_size);
127 assert(*dst_alloc_size == 0 || *dst);
129 ret = lzma_stream_decoder(&s, UINT64_MAX, 0);
133 space = MIN(src_size * 2, dst_max ?: (size_t) -1);
134 if (!greedy_realloc(dst, dst_alloc_size, space, 1))
138 s.avail_in = src_size;
146 ret = lzma_code(&s, LZMA_FINISH);
148 if (ret == LZMA_STREAM_END)
150 else if (ret != LZMA_OK)
153 if (dst_max > 0 && (space - s.avail_out) >= dst_max)
155 else if (dst_max > 0 && space == dst_max)
158 used = space - s.avail_out;
159 space = MIN(2 * space, dst_max ?: (size_t) -1);
160 if (!greedy_realloc(dst, dst_alloc_size, space, 1))
163 s.avail_out = space - used;
164 s.next_out = *dst + used;
167 *dst_size = space - s.avail_out;
170 return -EPROTONOSUPPORT;
174 int decompress_blob_lz4(const void *src, uint64_t src_size,
175 void **dst, size_t *dst_alloc_size, size_t* dst_size, size_t dst_max) {
179 int r, size; /* LZ4 uses int for size */
182 assert(src_size > 0);
184 assert(dst_alloc_size);
186 assert(*dst_alloc_size == 0 || *dst);
191 size = le64toh( *(le64_t*)src );
192 if (size < 0 || (le64_t) size != *(le64_t*)src)
194 if ((size_t) size > *dst_alloc_size) {
195 out = realloc(*dst, size);
199 *dst_alloc_size = size;
203 r = LZ4_decompress_safe(src + 8, out, src_size - 8, size);
204 if (r < 0 || r != size)
210 return -EPROTONOSUPPORT;
214 int decompress_blob(int compression,
215 const void *src, uint64_t src_size,
216 void **dst, size_t *dst_alloc_size, size_t* dst_size, size_t dst_max) {
217 if (compression == OBJECT_COMPRESSED_XZ)
218 return decompress_blob_xz(src, src_size,
219 dst, dst_alloc_size, dst_size, dst_max);
220 else if (compression == OBJECT_COMPRESSED_LZ4)
221 return decompress_blob_lz4(src, src_size,
222 dst, dst_alloc_size, dst_size, dst_max);
228 int decompress_startswith_xz(const void *src, uint64_t src_size,
229 void **buffer, size_t *buffer_size,
230 const void *prefix, size_t prefix_len,
234 _cleanup_(lzma_end) lzma_stream s = LZMA_STREAM_INIT;
237 /* Checks whether the decompressed blob starts with the
238 * mentioned prefix. The byte extra needs to follow the
242 assert(src_size > 0);
246 assert(*buffer_size == 0 || *buffer);
248 ret = lzma_stream_decoder(&s, UINT64_MAX, 0);
252 if (!(greedy_realloc(buffer, buffer_size, ALIGN_8(prefix_len + 1), 1)))
256 s.avail_in = src_size;
258 s.next_out = *buffer;
259 s.avail_out = *buffer_size;
262 ret = lzma_code(&s, LZMA_FINISH);
264 if (ret != LZMA_STREAM_END && ret != LZMA_OK)
267 if (*buffer_size - s.avail_out >= prefix_len + 1)
268 return memcmp(*buffer, prefix, prefix_len) == 0 &&
269 ((const uint8_t*) *buffer)[prefix_len] == extra;
271 if (ret == LZMA_STREAM_END)
274 s.avail_out += *buffer_size;
276 if (!(greedy_realloc(buffer, buffer_size, *buffer_size * 2, 1)))
279 s.next_out = *buffer + *buffer_size - s.avail_out;
283 return -EPROTONOSUPPORT;
287 int decompress_startswith_lz4(const void *src, uint64_t src_size,
288 void **buffer, size_t *buffer_size,
289 const void *prefix, size_t prefix_len,
292 /* Checks whether the decompressed blob starts with the
293 * mentioned prefix. The byte extra needs to follow the
299 assert(src_size > 0);
303 assert(*buffer_size == 0 || *buffer);
308 if (!(greedy_realloc(buffer, buffer_size, ALIGN_8(prefix_len + 1), 1)))
311 r = LZ4_decompress_safe_partial(src + 8, *buffer, src_size - 8,
312 prefix_len + 1, *buffer_size);
316 if ((unsigned) r >= prefix_len + 1)
317 return memcmp(*buffer, prefix, prefix_len) == 0 &&
318 ((const uint8_t*) *buffer)[prefix_len] == extra;
323 return -EPROTONOSUPPORT;
327 int decompress_startswith(int compression,
328 const void *src, uint64_t src_size,
329 void **buffer, size_t *buffer_size,
330 const void *prefix, size_t prefix_len,
332 if (compression == OBJECT_COMPRESSED_XZ)
333 return decompress_startswith_xz(src, src_size,
337 else if (compression == OBJECT_COMPRESSED_LZ4)
338 return decompress_startswith_lz4(src, src_size,
346 int compress_stream_xz(int fdf, int fdt, off_t max_bytes) {
348 _cleanup_(lzma_end) lzma_stream s = LZMA_STREAM_INIT;
351 uint8_t buf[BUFSIZ], out[BUFSIZ];
352 lzma_action action = LZMA_RUN;
357 ret = lzma_easy_encoder(&s, LZMA_PRESET_DEFAULT, LZMA_CHECK_CRC64);
358 if (ret != LZMA_OK) {
359 log_error("Failed to initialize XZ encoder: code %d", ret);
364 if (s.avail_in == 0 && action == LZMA_RUN) {
365 size_t m = sizeof(buf);
368 if (max_bytes != -1 && m > (size_t) max_bytes)
371 n = read(fdf, buf, m);
375 action = LZMA_FINISH;
380 if (max_bytes != -1) {
381 assert(max_bytes >= n);
387 if (s.avail_out == 0) {
389 s.avail_out = sizeof(out);
392 ret = lzma_code(&s, action);
393 if (ret != LZMA_OK && ret != LZMA_STREAM_END) {
394 log_error("Compression failed: code %d", ret);
398 if (s.avail_out == 0 || ret == LZMA_STREAM_END) {
401 n = sizeof(out) - s.avail_out;
403 k = loop_write(fdt, out, n, false);
407 if (ret == LZMA_STREAM_END) {
408 log_debug("XZ compression finished (%"PRIu64" -> %"PRIu64" bytes, %.1f%%)",
409 s.total_in, s.total_out,
410 (double) s.total_out / s.total_in * 100);
417 return -EPROTONOSUPPORT;
421 #define LZ4_BUFSIZE (512*1024)
423 int compress_stream_lz4(int fdf, int fdt, off_t max_bytes) {
427 _cleanup_free_ char *buf1 = NULL, *buf2 = NULL, *out = NULL;
429 LZ4_stream_t lz4_data = {};
431 size_t total_in = 0, total_out = sizeof(header);
437 buf1 = malloc(LZ4_BUFSIZE);
438 buf2 = malloc(LZ4_BUFSIZE);
439 out = malloc(LZ4_COMPRESSBOUND(LZ4_BUFSIZE));
440 if (!buf1 || !buf2 || !out)
449 if (max_bytes != -1 && m > (size_t) max_bytes - total_in)
450 m = max_bytes - total_in;
452 n = read(fdf, buf, m);
460 r = LZ4_compress_continue(&lz4_data, buf, out, n);
462 log_error("LZ4 compression failed.");
469 n = write(fdt, &header, sizeof(header));
472 if (n != sizeof(header))
473 return errno ? -errno : -EIO;
475 n = loop_write(fdt, out, r, false);
479 total_out += sizeof(header) + r;
481 buf = buf == buf1 ? buf2 : buf1;
485 n = write(fdt, &header, sizeof(header));
488 if (n != sizeof(header))
489 return errno ? -errno : -EIO;
491 log_debug("LZ4 compression finished (%zu -> %zu bytes, %.1f%%)",
493 (double) total_out / total_in * 100);
497 return -EPROTONOSUPPORT;
501 int decompress_stream_xz(int fdf, int fdt, off_t max_bytes) {
504 _cleanup_(lzma_end) lzma_stream s = LZMA_STREAM_INIT;
507 uint8_t buf[BUFSIZ], out[BUFSIZ];
508 lzma_action action = LZMA_RUN;
513 ret = lzma_stream_decoder(&s, UINT64_MAX, 0);
514 if (ret != LZMA_OK) {
515 log_error("Failed to initialize XZ decoder: code %d", ret);
520 if (s.avail_in == 0 && action == LZMA_RUN) {
523 n = read(fdf, buf, sizeof(buf));
527 action = LZMA_FINISH;
534 if (s.avail_out == 0) {
536 s.avail_out = sizeof(out);
539 ret = lzma_code(&s, action);
540 if (ret != LZMA_OK && ret != LZMA_STREAM_END) {
541 log_error("Decompression failed: code %d", ret);
545 if (s.avail_out == 0 || ret == LZMA_STREAM_END) {
548 n = sizeof(out) - s.avail_out;
550 if (max_bytes != -1) {
557 k = loop_write(fdt, out, n, false);
561 if (ret == LZMA_STREAM_END) {
562 log_debug("XZ decompression finished (%"PRIu64" -> %"PRIu64" bytes, %.1f%%)",
563 s.total_in, s.total_out,
564 (double) s.total_out / s.total_in * 100);
571 log_error("Cannot decompress file. Compiled without XZ support.");
572 return -EPROTONOSUPPORT;
576 int decompress_stream_lz4(int fdf, int fdt, off_t max_bytes) {
579 _cleanup_free_ char *buf = NULL, *out = NULL;
581 LZ4_streamDecode_t lz4_data = {};
583 size_t total_in = sizeof(header), total_out = 0;
588 out = malloc(4*LZ4_BUFSIZE);
596 n = read(fdf, &header, sizeof(header));
599 if (n != sizeof(header))
600 return errno ? -errno : -EIO;
606 /* We refuse to use a bigger decompression buffer than
607 * the one used for compression by 4 times. This means
608 * that compression buffer size can be enlarged 4
609 * times. This can be changed, but old binaries might
610 * not accept buffers compressed by newer binaries then.
612 if (m > LZ4_COMPRESSBOUND(LZ4_BUFSIZE * 4)) {
613 log_error("Compressed stream block too big: %zd bytes", m);
617 total_in += sizeof(header) + m;
619 if (!GREEDY_REALLOC(buf, buf_size, m))
623 n = loop_read(fdf, buf, m, false);
627 return errno ? -errno : -EIO;
629 r = LZ4_decompress_safe_continue(&lz4_data, buf, out, m, 4*LZ4_BUFSIZE);
631 log_error("LZ4 decompression failed.");
635 if (max_bytes != -1 && total_out > (size_t) max_bytes) {
636 log_debug("Decompressed stream longer than %zd bytes", max_bytes);
640 n = loop_write(fdt, out, r, false);
645 log_debug("LZ4 decompression finished (%zu -> %zu bytes, %.1f%%)",
647 (double) total_out / total_in * 100);
651 log_error("Cannot decompress file. Compiled without LZ4 support.");
652 return -EPROTONOSUPPORT;
656 int decompress_stream(const char *filename, int fdf, int fdt, off_t max_bytes) {
658 if (endswith(filename, ".lz4"))
659 return decompress_stream_lz4(fdf, fdt, max_bytes);
660 else if (endswith(filename, ".xz"))
661 return decompress_stream_xz(fdf, fdt, max_bytes);
663 return -EPROTONOSUPPORT;