1 /*-*- Mode: C; c-basic-offset: 8; indent-tabs-mode: nil -*-*/
4 This file is part of systemd.
6 Copyright 2011 Lennart Poettering
8 systemd is free software; you can redistribute it and/or modify it
9 under the terms of the GNU Lesser General Public License as published by
10 the Free Software Foundation; either version 2.1 of the License, or
11 (at your option) any later version.
13 systemd is distributed in the hope that it will be useful, but
14 WITHOUT ANY WARRANTY; without even the implied warranty of
15 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
16 Lesser General Public License for more details.
18 You should have received a copy of the GNU Lesser General Public License
19 along with systemd; If not, see <http://www.gnu.org/licenses/>.
38 #include "sparse-endian.h"
39 #include "journal-def.h"
41 #define ALIGN_8(l) ALIGN_TO(l, sizeof(size_t))
43 static const char* const object_compressed_table[_OBJECT_COMPRESSED_MAX] = {
44 [OBJECT_COMPRESSED_XZ] = "XZ",
45 [OBJECT_COMPRESSED_LZ4] = "LZ4",
48 DEFINE_STRING_TABLE_LOOKUP(object_compressed, int);
50 int compress_blob_xz(const void *src, uint64_t src_size, void *dst, uint64_t *dst_size) {
60 /* Returns < 0 if we couldn't compress the data or the
61 * compressed result is longer than the original */
63 ret = lzma_easy_buffer_encode(LZMA_PRESET_DEFAULT, LZMA_CHECK_NONE, NULL,
64 src, src_size, dst, &out_pos, src_size - 1);
71 return -EPROTONOSUPPORT;
75 int compress_blob_lz4(const void *src, uint64_t src_size, void *dst, uint64_t *dst_size) {
84 /* Returns < 0 if we couldn't compress the data or the
85 * compressed result is longer than the original */
90 r = LZ4_compress_limitedOutput(src, dst + 8, src_size, src_size - 8 - 1);
94 *(le64_t*) dst = htole64(src_size);
99 return -EPROTONOSUPPORT;
104 int decompress_blob_xz(const void *src, uint64_t src_size,
105 void **dst, uint64_t *dst_alloc_size, uint64_t* dst_size, uint64_t dst_max) {
108 _cleanup_(lzma_end) lzma_stream s = LZMA_STREAM_INIT;
113 assert(src_size > 0);
115 assert(dst_alloc_size);
117 assert(*dst_alloc_size == 0 || *dst);
119 ret = lzma_stream_decoder(&s, UINT64_MAX, 0);
123 space = MIN(src_size * 2, dst_max ?: (uint64_t) -1);
124 if (!greedy_realloc(dst, dst_alloc_size, space, 1))
128 s.avail_in = src_size;
136 ret = lzma_code(&s, LZMA_FINISH);
138 if (ret == LZMA_STREAM_END)
140 else if (ret != LZMA_OK)
143 if (dst_max > 0 && (space - s.avail_out) >= dst_max)
145 else if (dst_max > 0 && space == dst_max)
148 used = space - s.avail_out;
149 space = MIN(2 * space, dst_max ?: (uint64_t) -1);
150 if (!greedy_realloc(dst, dst_alloc_size, space, 1))
153 s.avail_out = space - used;
154 s.next_out = *dst + used;
157 *dst_size = space - s.avail_out;
160 return -EPROTONOSUPPORT;
164 int decompress_blob_lz4(const void *src, uint64_t src_size,
165 void **dst, uint64_t *dst_alloc_size, uint64_t* dst_size, uint64_t dst_max) {
173 assert(src_size > 0);
175 assert(dst_alloc_size);
177 assert(*dst_alloc_size == 0 || *dst);
182 size = le64toh( *(le64_t*)src );
183 if (size > *dst_alloc_size) {
184 out = realloc(*dst, size);
188 *dst_alloc_size = size;
192 r = LZ4_decompress_safe(src + 8, out, src_size - 8, size);
193 if (r < 0 || (uint64_t) r != size)
199 return -EPROTONOSUPPORT;
203 int decompress_blob(int compression,
204 const void *src, uint64_t src_size,
205 void **dst, uint64_t *dst_alloc_size, uint64_t* dst_size, uint64_t dst_max) {
206 if (compression == OBJECT_COMPRESSED_XZ)
207 return decompress_blob_xz(src, src_size,
208 dst, dst_alloc_size, dst_size, dst_max);
209 else if (compression == OBJECT_COMPRESSED_LZ4)
210 return decompress_blob_lz4(src, src_size,
211 dst, dst_alloc_size, dst_size, dst_max);
217 int decompress_startswith_xz(const void *src, uint64_t src_size,
218 void **buffer, uint64_t *buffer_size,
219 const void *prefix, uint64_t prefix_len,
223 _cleanup_(lzma_end) lzma_stream s = LZMA_STREAM_INIT;
226 /* Checks whether the decompressed blob starts with the
227 * mentioned prefix. The byte extra needs to follow the
231 assert(src_size > 0);
235 assert(*buffer_size == 0 || *buffer);
237 ret = lzma_stream_decoder(&s, UINT64_MAX, 0);
241 if (!(greedy_realloc(buffer, buffer_size, ALIGN_8(prefix_len + 1), 1)))
245 s.avail_in = src_size;
247 s.next_out = *buffer;
248 s.avail_out = *buffer_size;
251 ret = lzma_code(&s, LZMA_FINISH);
253 if (ret != LZMA_STREAM_END && ret != LZMA_OK)
256 if (*buffer_size - s.avail_out >= prefix_len + 1)
257 return memcmp(*buffer, prefix, prefix_len) == 0 &&
258 ((const uint8_t*) *buffer)[prefix_len] == extra;
260 if (ret == LZMA_STREAM_END)
263 s.avail_out += *buffer_size;
265 if (!(greedy_realloc(buffer, buffer_size, *buffer_size * 2, 1)))
268 s.next_out = *buffer + *buffer_size - s.avail_out;
272 return -EPROTONOSUPPORT;
276 int decompress_startswith_lz4(const void *src, uint64_t src_size,
277 void **buffer, uint64_t *buffer_size,
278 const void *prefix, uint64_t prefix_len,
281 /* Checks whether the decompressed blob starts with the
282 * mentioned prefix. The byte extra needs to follow the
288 assert(src_size > 0);
292 assert(*buffer_size == 0 || *buffer);
297 if (!(greedy_realloc(buffer, buffer_size, ALIGN_8(prefix_len + 1), 1)))
300 r = LZ4_decompress_safe_partial(src + 8, *buffer, src_size - 8,
301 prefix_len + 1, *buffer_size);
305 if ((unsigned) r >= prefix_len + 1)
306 return memcmp(*buffer, prefix, prefix_len) == 0 &&
307 ((const uint8_t*) *buffer)[prefix_len] == extra;
312 return -EPROTONOSUPPORT;
316 int decompress_startswith(int compression,
317 const void *src, uint64_t src_size,
318 void **buffer, uint64_t *buffer_size,
319 const void *prefix, uint64_t prefix_len,
321 if (compression == OBJECT_COMPRESSED_XZ)
322 return decompress_startswith_xz(src, src_size,
326 else if (compression == OBJECT_COMPRESSED_LZ4)
327 return decompress_startswith_lz4(src, src_size,
335 int compress_stream_xz(int fdf, int fdt, off_t max_bytes) {
336 _cleanup_(lzma_end) lzma_stream s = LZMA_STREAM_INIT;
339 uint8_t buf[BUFSIZ], out[BUFSIZ];
340 lzma_action action = LZMA_RUN;
345 ret = lzma_easy_encoder(&s, LZMA_PRESET_DEFAULT, LZMA_CHECK_CRC64);
346 if (ret != LZMA_OK) {
347 log_error("Failed to initialize XZ encoder: code %d", ret);
352 if (s.avail_in == 0 && action == LZMA_RUN) {
353 size_t m = sizeof(buf);
356 if (max_bytes != -1 && m > (size_t) max_bytes)
359 n = read(fdf, buf, m);
363 action = LZMA_FINISH;
368 if (max_bytes != -1) {
369 assert(max_bytes >= n);
375 if (s.avail_out == 0) {
377 s.avail_out = sizeof(out);
380 ret = lzma_code(&s, action);
381 if (ret != LZMA_OK && ret != LZMA_STREAM_END) {
382 log_error("Compression failed: code %d", ret);
386 if (s.avail_out == 0 || ret == LZMA_STREAM_END) {
389 n = sizeof(out) - s.avail_out;
392 k = loop_write(fdt, out, n, false);
396 return errno ? -errno : -EIO;
398 if (ret == LZMA_STREAM_END) {
399 log_debug("XZ compression finished (%zu -> %zu bytes, %.1f%%)",
400 s.total_in, s.total_out,
401 (double) s.total_out / s.total_in * 100);
409 #define LZ4_BUFSIZE (512*1024)
411 int compress_stream_lz4(int fdf, int fdt, off_t max_bytes) {
415 _cleanup_free_ char *buf1 = NULL, *buf2 = NULL, *out = NULL;
417 LZ4_stream_t lz4_data = {};
419 size_t total_in = 0, total_out = sizeof(header);
425 buf1 = malloc(LZ4_BUFSIZE);
426 buf2 = malloc(LZ4_BUFSIZE);
427 out = malloc(LZ4_COMPRESSBOUND(LZ4_BUFSIZE));
428 if (!buf1 || !buf2 || !out)
437 if (max_bytes != -1 && m > (size_t) max_bytes - total_in)
438 m = max_bytes - total_in;
440 n = read(fdf, buf, m);
448 r = LZ4_compress_limitedOutput_continue(&lz4_data, buf, out, n, n);
450 log_debug("Compressed size exceeds original, aborting compression.");
457 n = write(fdt, &header, sizeof(header));
460 if (n != sizeof(header))
461 return errno ? -errno : -EIO;
463 n = loop_write(fdt, out, r, false);
467 return errno ? -errno : -EIO;
469 total_out += sizeof(header) + r;
471 buf = buf == buf1 ? buf2 : buf1;
475 n = write(fdt, &header, sizeof(header));
478 if (n != sizeof(header))
479 return errno ? -errno : -EIO;
481 log_debug("LZ4 compression finished (%zu -> %zu bytes, %.1f%%)",
483 (double) total_out / total_in * 100);
487 return -EPROTONOSUPPORT;
491 int decompress_stream_xz(int fdf, int fdt, off_t max_bytes) {
494 _cleanup_(lzma_end) lzma_stream s = LZMA_STREAM_INIT;
497 uint8_t buf[BUFSIZ], out[BUFSIZ];
498 lzma_action action = LZMA_RUN;
503 ret = lzma_stream_decoder(&s, UINT64_MAX, 0);
504 if (ret != LZMA_OK) {
505 log_error("Failed to initialize XZ decoder: code %d", ret);
510 if (s.avail_in == 0 && action == LZMA_RUN) {
513 n = read(fdf, buf, sizeof(buf));
517 action = LZMA_FINISH;
524 if (s.avail_out == 0) {
526 s.avail_out = sizeof(out);
529 ret = lzma_code(&s, action);
530 if (ret != LZMA_OK && ret != LZMA_STREAM_END) {
531 log_error("Decompression failed: code %d", ret);
535 if (s.avail_out == 0 || ret == LZMA_STREAM_END) {
538 n = sizeof(out) - s.avail_out;
540 if (max_bytes != -1) {
548 k = loop_write(fdt, out, n, false);
552 return errno ? -errno : -EIO;
554 if (ret == LZMA_STREAM_END) {
555 log_debug("XZ decompression finished (%zu -> %zu bytes, %.1f%%)",
556 s.total_in, s.total_out,
557 (double) s.total_out / s.total_in * 100);
564 log_error("Cannot decompress file. Compiled without XZ support.");
565 return -EPROTONOSUPPORT;
569 int decompress_stream_lz4(int fdf, int fdt, off_t max_bytes) {
572 _cleanup_free_ char *buf = NULL, *out = NULL;
574 LZ4_streamDecode_t lz4_data = {};
576 size_t total_in = sizeof(header), total_out = 0;
581 out = malloc(4*LZ4_BUFSIZE);
589 n = read(fdf, &header, sizeof(header));
592 if (n != sizeof(header))
593 return errno ? -errno : -EIO;
599 /* We refuse to use a bigger decompression buffer than
600 * the one used for compression by 4 times. This means
601 * that compression buffer size can be enlarged 4
602 * times. This can be changed, but old binaries might
603 * not accept buffers compressed by newer binaries then.
605 if (m > LZ4_COMPRESSBOUND(LZ4_BUFSIZE * 4)) {
606 log_error("Compressed stream block too big: %zd bytes", m);
610 total_in += sizeof(header) + m;
612 if (!GREEDY_REALLOC(buf, buf_size, m))
616 n = loop_read(fdf, buf, m, false);
620 return errno ? -errno : -EIO;
622 r = LZ4_decompress_safe_continue(&lz4_data, buf, out, m, 4*LZ4_BUFSIZE);
624 log_error("LZ4 decompression failed.");
628 if (max_bytes != -1 && total_out > (size_t) max_bytes) {
629 log_debug("Decompressed stream longer than %zd bytes", max_bytes);
634 n = loop_write(fdt, out, r, false);
638 return errno ? -errno : -EIO;
641 log_debug("LZ4 decompression finished (%zu -> %zu bytes, %.1f%%)",
643 (double) total_out / total_in * 100);
647 log_error("Cannot decompress file. Compiled without LZ4 support.");
648 return -EPROTONOSUPPORT;
652 int decompress_stream(const char *filename, int fdf, int fdt, off_t max_bytes) {
654 if (endswith(filename, ".lz4"))
655 return decompress_stream_lz4(fdf, fdt, max_bytes);
656 else if (endswith(filename, ".xz"))
657 return decompress_stream_xz(fdf, fdt, max_bytes);
659 return -EPROTONOSUPPORT;