// // Copyright (C) 2021 The Android Open Source Project // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. // #include "lz4diff_compress.h" #include "update_engine/common/utils.h" #include "update_engine/common/hash_calculator.h" #include "update_engine/payload_generator/delta_diff_generator.h" #include "update_engine/payload_generator/payload_generation_config.h" #include #include #include namespace chromeos_update_engine { bool TryCompressBlob(std::string_view blob, const std::vector& block_info, const bool zero_padding_enabled, const CompressionAlgorithm compression_algo, const SinkFunc& sink) { size_t uncompressed_size = 0; for (const auto& block : block_info) { CHECK_EQ(uncompressed_size, block.uncompressed_offset) << "Compressed block info is expected to be sorted."; uncompressed_size += block.uncompressed_length; } auto hc = LZ4_createStreamHC(); DEFER { if (hc) { LZ4_freeStreamHC(hc); hc = nullptr; } }; size_t compressed_offset = 0; Blob block_buffer; for (const auto& block : block_info) { const auto uncompressed_block = blob.substr(block.uncompressed_offset, block.uncompressed_length); if (!block.IsCompressed()) { TEST_EQ(sink(reinterpret_cast(uncompressed_block.data()), uncompressed_block.size()), uncompressed_block.size()); continue; } block_buffer.resize(block.compressed_length); // Execute the increment at end of each loop DEFER { compressed_offset += block.compressed_length; block_buffer.clear(); }; int ret = 0; // LZ4 spec enforces that last op of a compressed block must be an insert op // of at least 5 bytes. Compressors will try to conform to that requirement // if the input size is just right. We don't want that. So always give a // little bit more data. switch (int src_size = uncompressed_size - block.uncompressed_offset; compression_algo.type()) { case CompressionAlgorithm::LZ4HC: ret = LZ4_compress_HC_destSize( hc, uncompressed_block.data(), reinterpret_cast(block_buffer.data()), &src_size, block.compressed_length, compression_algo.level()); break; case CompressionAlgorithm::LZ4: ret = LZ4_compress_destSize(uncompressed_block.data(), reinterpret_cast(block_buffer.data()), &src_size, block.compressed_length); break; default: LOG(ERROR) << "Unrecognized compression algorithm: " << compression_algo.type(); return {}; } TEST_GT(ret, 0); const uint64_t bytes_written = ret; // Last block may have trailing zeros TEST_LE(bytes_written, block.compressed_length); if (bytes_written < block.compressed_length) { if (zero_padding_enabled) { const auto padding = block.compressed_length - bytes_written; std::memmove( block_buffer.data() + padding, block_buffer.data(), bytes_written); std::fill(block_buffer.data(), block_buffer.data() + padding, 0); } else { std::fill(block_buffer.data() + bytes_written, block_buffer.data() + block.compressed_length, 0); } } TEST_EQ(sink(block_buffer.data(), block_buffer.size()), block_buffer.size()); } // Any trailing data will be copied to the output buffer. TEST_EQ( sink(reinterpret_cast(blob.data()) + uncompressed_size, blob.size() - uncompressed_size), blob.size() - uncompressed_size); return true; } Blob TryCompressBlob(std::string_view blob, const std::vector& block_info, const bool zero_padding_enabled, const CompressionAlgorithm compression_algo) { size_t uncompressed_size = 0; size_t compressed_size = 0; for (const auto& block : block_info) { CHECK_EQ(uncompressed_size, block.uncompressed_offset) << "Compressed block info is expected to be sorted."; uncompressed_size += block.uncompressed_length; compressed_size += block.compressed_length; } TEST_EQ(uncompressed_size, blob.size()); Blob output; output.reserve(utils::RoundUp(compressed_size, kBlockSize)); if (!TryCompressBlob(blob, block_info, zero_padding_enabled, compression_algo, [&output](const uint8_t* data, size_t size) { output.insert(output.end(), data, data + size); return size; })) { return {}; } return output; } Blob TryDecompressBlob(std::string_view blob, const std::vector& block_info, const bool zero_padding_enabled) { if (block_info.empty()) { return {}; } size_t uncompressed_size = 0; size_t compressed_size = 0; for (const auto& block : block_info) { CHECK_EQ(uncompressed_size, block.uncompressed_offset) << " Compressed block info is expected to be sorted, expected offset " << uncompressed_size << ", actual block " << block; uncompressed_size += block.uncompressed_length; compressed_size += block.compressed_length; } if (blob.size() < compressed_size) { LOG(INFO) << "File is chunked. Skip lz4 decompress. Expected size: " << compressed_size << ", actual size: " << blob.size(); return {}; } Blob output; output.reserve(uncompressed_size); size_t compressed_offset = 0; for (const auto& block : block_info) { std::string_view cluster = blob.substr(compressed_offset, block.compressed_length); if (!block.IsCompressed()) { CHECK_NE(cluster.size(), 0UL); output.insert(output.end(), cluster.begin(), cluster.end()); compressed_offset += cluster.size(); continue; } size_t inputmargin = 0; if (zero_padding_enabled) { while (inputmargin < std::min(kBlockSize, cluster.size()) && cluster[inputmargin] == 0) { inputmargin++; } } output.resize(output.size() + block.uncompressed_length); const auto bytes_decompressed = LZ4_decompress_safe_partial( cluster.data() + inputmargin, reinterpret_cast(output.data()) + output.size() - block.uncompressed_length, cluster.size() - inputmargin, block.uncompressed_length, block.uncompressed_length); if (bytes_decompressed < 0) { LOG(FATAL) << "Failed to decompress, " << bytes_decompressed << ", output_cursor = " << output.size() - block.uncompressed_length << ", input_cursor = " << compressed_offset << ", blob.size() = " << blob.size() << ", cluster_size = " << block.compressed_length << ", dest capacity = " << block.uncompressed_length << ", input margin = " << inputmargin << " " << HashCalculator::SHA256Digest(cluster) << " " << HashCalculator::SHA256Digest(blob); return {}; } compressed_offset += block.compressed_length; CHECK_EQ(static_cast(bytes_decompressed), block.uncompressed_length); } CHECK_EQ(output.size(), uncompressed_size); // Trailing data not recorded by compressed block info will be treated as // uncompressed, most of the time these are xattrs or trailing zeros. CHECK_EQ(blob.size(), compressed_offset) << " Unexpected data the end of compressed data "; if (compressed_offset < blob.size()) { output.insert(output.end(), blob.begin() + compressed_offset, blob.end()); } return output; } Blob TryDecompressBlob(const Blob& blob, const std::vector& block_info, const bool zero_padding_enabled) { return TryDecompressBlob( ToStringView(blob), block_info, zero_padding_enabled); } std::ostream& operator<<(std::ostream& out, const CompressedBlock& block) { out << "CompressedBlock{.uncompressed_offset = " << block.uncompressed_offset << ", .compressed_length = " << block.compressed_length << ", .uncompressed_length = " << block.uncompressed_length << "}"; return out; } std::ostream& operator<<(std::ostream& out, const CompressedBlockInfo& info) { out << "BlockInfo { compressed_length: " << info.compressed_length() << ", uncompressed_length: " << info.uncompressed_length() << ", uncompressed_offset: " << info.uncompressed_offset(); if (!info.sha256_hash().empty()) { out << ", sha256_hash: " << HexEncode(info.sha256_hash()); } if (!info.postfix_bspatch().empty()) { out << ", postfix_bspatch: " << info.postfix_bspatch().size(); } out << "}"; return out; } } // namespace chromeos_update_engine