1 /*-*- Mode: C; c-basic-offset: 8; indent-tabs-mode: nil -*-*/
4 This file is part of systemd.
6 Copyright 2011 Lennart Poettering
8 systemd is free software; you can redistribute it and/or modify it
9 under the terms of the GNU Lesser General Public License as published by
10 the Free Software Foundation; either version 2.1 of the License, or
11 (at your option) any later version.
13 systemd is distributed in the hope that it will be useful, but
14 WITHOUT ANY WARRANTY; without even the implied warranty of
15 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
16 Lesser General Public License for more details.
18 You should have received a copy of the GNU Lesser General Public License
19 along with systemd; If not, see <http://www.gnu.org/licenses/>.
37 #include "sparse-endian.h"
38 #include "journal-def.h"
40 #define ALIGN_8(l) ALIGN_TO(l, sizeof(size_t))
42 static const char* const object_compressed_table[_OBJECT_COMPRESSED_MAX] = {
43 [OBJECT_COMPRESSED_XZ] = "XZ",
44 [OBJECT_COMPRESSED_LZ4] = "LZ4",
47 DEFINE_STRING_TABLE_LOOKUP(object_compressed, int);
49 int compress_blob_xz(const void *src, uint64_t src_size, void *dst, size_t *dst_size) {
51 static const lzma_options_lzma opt = {
52 1u << 20u, NULL, 0, LZMA_LC_DEFAULT, LZMA_LP_DEFAULT,
53 LZMA_PB_DEFAULT, LZMA_MODE_FAST, 128, LZMA_MF_HC3, 4};
54 static const lzma_filter filters[2] = {
55 {LZMA_FILTER_LZMA2, (lzma_options_lzma*) &opt},
56 {LZMA_VLI_UNKNOWN, NULL}
66 /* Returns < 0 if we couldn't compress the data or the
67 * compressed result is longer than the original */
72 ret = lzma_stream_buffer_encode((lzma_filter*) filters, LZMA_CHECK_NONE, NULL,
73 src, src_size, dst, &out_pos, src_size - 1);
80 return -EPROTONOSUPPORT;
84 int compress_blob_lz4(const void *src, uint64_t src_size, void *dst, size_t *dst_size) {
93 /* Returns < 0 if we couldn't compress the data or the
94 * compressed result is longer than the original */
99 r = LZ4_compress_limitedOutput(src, dst + 8, src_size, src_size - 8 - 1);
103 *(le64_t*) dst = htole64(src_size);
108 return -EPROTONOSUPPORT;
113 int decompress_blob_xz(const void *src, uint64_t src_size,
114 void **dst, size_t *dst_alloc_size, size_t* dst_size, size_t dst_max) {
117 _cleanup_(lzma_end) lzma_stream s = LZMA_STREAM_INIT;
122 assert(src_size > 0);
124 assert(dst_alloc_size);
126 assert(*dst_alloc_size == 0 || *dst);
128 ret = lzma_stream_decoder(&s, UINT64_MAX, 0);
132 space = MIN(src_size * 2, dst_max ?: (size_t) -1);
133 if (!greedy_realloc(dst, dst_alloc_size, space, 1))
137 s.avail_in = src_size;
145 ret = lzma_code(&s, LZMA_FINISH);
147 if (ret == LZMA_STREAM_END)
149 else if (ret != LZMA_OK)
152 if (dst_max > 0 && (space - s.avail_out) >= dst_max)
154 else if (dst_max > 0 && space == dst_max)
157 used = space - s.avail_out;
158 space = MIN(2 * space, dst_max ?: (size_t) -1);
159 if (!greedy_realloc(dst, dst_alloc_size, space, 1))
162 s.avail_out = space - used;
163 s.next_out = *dst + used;
166 *dst_size = space - s.avail_out;
169 return -EPROTONOSUPPORT;
173 int decompress_blob_lz4(const void *src, uint64_t src_size,
174 void **dst, size_t *dst_alloc_size, size_t* dst_size, size_t dst_max) {
178 int r, size; /* LZ4 uses int for size */
181 assert(src_size > 0);
183 assert(dst_alloc_size);
185 assert(*dst_alloc_size == 0 || *dst);
190 size = le64toh( *(le64_t*)src );
191 if (size < 0 || (le64_t) size != *(le64_t*)src)
193 if ((size_t) size > *dst_alloc_size) {
194 out = realloc(*dst, size);
198 *dst_alloc_size = size;
202 r = LZ4_decompress_safe(src + 8, out, src_size - 8, size);
203 if (r < 0 || r != size)
209 return -EPROTONOSUPPORT;
213 int decompress_blob(int compression,
214 const void *src, uint64_t src_size,
215 void **dst, size_t *dst_alloc_size, size_t* dst_size, size_t dst_max) {
216 if (compression == OBJECT_COMPRESSED_XZ)
217 return decompress_blob_xz(src, src_size,
218 dst, dst_alloc_size, dst_size, dst_max);
219 else if (compression == OBJECT_COMPRESSED_LZ4)
220 return decompress_blob_lz4(src, src_size,
221 dst, dst_alloc_size, dst_size, dst_max);
227 int decompress_startswith_xz(const void *src, uint64_t src_size,
228 void **buffer, size_t *buffer_size,
229 const void *prefix, size_t prefix_len,
233 _cleanup_(lzma_end) lzma_stream s = LZMA_STREAM_INIT;
236 /* Checks whether the decompressed blob starts with the
237 * mentioned prefix. The byte extra needs to follow the
241 assert(src_size > 0);
245 assert(*buffer_size == 0 || *buffer);
247 ret = lzma_stream_decoder(&s, UINT64_MAX, 0);
251 if (!(greedy_realloc(buffer, buffer_size, ALIGN_8(prefix_len + 1), 1)))
255 s.avail_in = src_size;
257 s.next_out = *buffer;
258 s.avail_out = *buffer_size;
261 ret = lzma_code(&s, LZMA_FINISH);
263 if (ret != LZMA_STREAM_END && ret != LZMA_OK)
266 if (*buffer_size - s.avail_out >= prefix_len + 1)
267 return memcmp(*buffer, prefix, prefix_len) == 0 &&
268 ((const uint8_t*) *buffer)[prefix_len] == extra;
270 if (ret == LZMA_STREAM_END)
273 s.avail_out += *buffer_size;
275 if (!(greedy_realloc(buffer, buffer_size, *buffer_size * 2, 1)))
278 s.next_out = *buffer + *buffer_size - s.avail_out;
282 return -EPROTONOSUPPORT;
286 int decompress_startswith_lz4(const void *src, uint64_t src_size,
287 void **buffer, size_t *buffer_size,
288 const void *prefix, size_t prefix_len,
291 /* Checks whether the decompressed blob starts with the
292 * mentioned prefix. The byte extra needs to follow the
298 assert(src_size > 0);
302 assert(*buffer_size == 0 || *buffer);
307 if (!(greedy_realloc(buffer, buffer_size, ALIGN_8(prefix_len + 1), 1)))
310 r = LZ4_decompress_safe_partial(src + 8, *buffer, src_size - 8,
311 prefix_len + 1, *buffer_size);
315 if ((unsigned) r >= prefix_len + 1)
316 return memcmp(*buffer, prefix, prefix_len) == 0 &&
317 ((const uint8_t*) *buffer)[prefix_len] == extra;
322 return -EPROTONOSUPPORT;
326 int decompress_startswith(int compression,
327 const void *src, uint64_t src_size,
328 void **buffer, size_t *buffer_size,
329 const void *prefix, size_t prefix_len,
331 if (compression == OBJECT_COMPRESSED_XZ)
332 return decompress_startswith_xz(src, src_size,
336 else if (compression == OBJECT_COMPRESSED_LZ4)
337 return decompress_startswith_lz4(src, src_size,
345 int compress_stream_xz(int fdf, int fdt, off_t max_bytes) {
347 _cleanup_(lzma_end) lzma_stream s = LZMA_STREAM_INIT;
350 uint8_t buf[BUFSIZ], out[BUFSIZ];
351 lzma_action action = LZMA_RUN;
356 ret = lzma_easy_encoder(&s, LZMA_PRESET_DEFAULT, LZMA_CHECK_CRC64);
357 if (ret != LZMA_OK) {
358 log_error("Failed to initialize XZ encoder: code %u", ret);
363 if (s.avail_in == 0 && action == LZMA_RUN) {
364 size_t m = sizeof(buf);
367 if (max_bytes != -1 && m > (size_t) max_bytes)
370 n = read(fdf, buf, m);
374 action = LZMA_FINISH;
379 if (max_bytes != -1) {
380 assert(max_bytes >= n);
386 if (s.avail_out == 0) {
388 s.avail_out = sizeof(out);
391 ret = lzma_code(&s, action);
392 if (ret != LZMA_OK && ret != LZMA_STREAM_END) {
393 log_error("Compression failed: code %u", ret);
397 if (s.avail_out == 0 || ret == LZMA_STREAM_END) {
400 n = sizeof(out) - s.avail_out;
402 k = loop_write(fdt, out, n, false);
406 if (ret == LZMA_STREAM_END) {
407 log_debug("XZ compression finished (%"PRIu64" -> %"PRIu64" bytes, %.1f%%)",
408 s.total_in, s.total_out,
409 (double) s.total_out / s.total_in * 100);
416 return -EPROTONOSUPPORT;
420 #define LZ4_BUFSIZE (512*1024)
422 int compress_stream_lz4(int fdf, int fdt, off_t max_bytes) {
426 _cleanup_free_ char *buf1 = NULL, *buf2 = NULL, *out = NULL;
428 LZ4_stream_t lz4_data = {};
430 size_t total_in = 0, total_out = sizeof(header);
436 buf1 = malloc(LZ4_BUFSIZE);
437 buf2 = malloc(LZ4_BUFSIZE);
438 out = malloc(LZ4_COMPRESSBOUND(LZ4_BUFSIZE));
439 if (!buf1 || !buf2 || !out)
448 if (max_bytes != -1 && m > (size_t) max_bytes - total_in)
449 m = max_bytes - total_in;
451 n = read(fdf, buf, m);
459 r = LZ4_compress_continue(&lz4_data, buf, out, n);
461 log_error("LZ4 compression failed.");
468 n = write(fdt, &header, sizeof(header));
471 if (n != sizeof(header))
472 return errno ? -errno : -EIO;
474 n = loop_write(fdt, out, r, false);
478 total_out += sizeof(header) + r;
480 buf = buf == buf1 ? buf2 : buf1;
484 n = write(fdt, &header, sizeof(header));
487 if (n != sizeof(header))
488 return errno ? -errno : -EIO;
490 log_debug("LZ4 compression finished (%zu -> %zu bytes, %.1f%%)",
492 (double) total_out / total_in * 100);
496 return -EPROTONOSUPPORT;
500 int decompress_stream_xz(int fdf, int fdt, off_t max_bytes) {
503 _cleanup_(lzma_end) lzma_stream s = LZMA_STREAM_INIT;
506 uint8_t buf[BUFSIZ], out[BUFSIZ];
507 lzma_action action = LZMA_RUN;
512 ret = lzma_stream_decoder(&s, UINT64_MAX, 0);
513 if (ret != LZMA_OK) {
514 log_error("Failed to initialize XZ decoder: code %u", ret);
519 if (s.avail_in == 0 && action == LZMA_RUN) {
522 n = read(fdf, buf, sizeof(buf));
526 action = LZMA_FINISH;
533 if (s.avail_out == 0) {
535 s.avail_out = sizeof(out);
538 ret = lzma_code(&s, action);
539 if (ret != LZMA_OK && ret != LZMA_STREAM_END) {
540 log_error("Decompression failed: code %u", ret);
544 if (s.avail_out == 0 || ret == LZMA_STREAM_END) {
547 n = sizeof(out) - s.avail_out;
549 if (max_bytes != -1) {
556 k = loop_write(fdt, out, n, false);
560 if (ret == LZMA_STREAM_END) {
561 log_debug("XZ decompression finished (%"PRIu64" -> %"PRIu64" bytes, %.1f%%)",
562 s.total_in, s.total_out,
563 (double) s.total_out / s.total_in * 100);
570 log_error("Cannot decompress file. Compiled without XZ support.");
571 return -EPROTONOSUPPORT;
575 int decompress_stream_lz4(int fdf, int fdt, off_t max_bytes) {
578 _cleanup_free_ char *buf = NULL, *out = NULL;
580 LZ4_streamDecode_t lz4_data = {};
582 size_t total_in = sizeof(header), total_out = 0;
587 out = malloc(4*LZ4_BUFSIZE);
595 r = loop_read_exact(fdf, &header, sizeof(header), false);
603 /* We refuse to use a bigger decompression buffer than
604 * the one used for compression by 4 times. This means
605 * that compression buffer size can be enlarged 4
606 * times. This can be changed, but old binaries might
607 * not accept buffers compressed by newer binaries then.
609 if (m > LZ4_COMPRESSBOUND(LZ4_BUFSIZE * 4)) {
610 log_error("Compressed stream block too big: %zd bytes", m);
614 total_in += sizeof(header) + m;
616 if (!GREEDY_REALLOC(buf, buf_size, m))
619 r = loop_read_exact(fdf, buf, m, false);
623 r = LZ4_decompress_safe_continue(&lz4_data, buf, out, m, 4*LZ4_BUFSIZE);
625 log_error("LZ4 decompression failed.");
629 if (max_bytes != -1 && total_out > (size_t) max_bytes) {
630 log_debug("Decompressed stream longer than %zd bytes", max_bytes);
634 r = loop_write(fdt, out, r, false);
639 log_debug("LZ4 decompression finished (%zu -> %zu bytes, %.1f%%)",
641 (double) total_out / total_in * 100);
645 log_error("Cannot decompress file. Compiled without LZ4 support.");
646 return -EPROTONOSUPPORT;
650 int decompress_stream(const char *filename, int fdf, int fdt, off_t max_bytes) {
652 if (endswith(filename, ".lz4"))
653 return decompress_stream_lz4(fdf, fdt, max_bytes);
654 else if (endswith(filename, ".xz"))
655 return decompress_stream_xz(fdf, fdt, max_bytes);
657 return -EPROTONOSUPPORT;