371 lines
12 KiB
C++
371 lines
12 KiB
C++
//
|
|
// Copyright (C) 2021 The Android Open Source Project
|
|
//
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
// you may not use this file except in compliance with the License.
|
|
// You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
// See the License for the specific language governing permissions and
|
|
// limitations under the License.
|
|
//
|
|
|
|
#include "lz4patch.h"
|
|
|
|
#include <endian.h>
|
|
#include <unistd.h>
|
|
#include <fcntl.h>
|
|
|
|
#include <algorithm>
|
|
#include <string_view>
|
|
|
|
#include <bsdiff/bspatch.h>
|
|
#include <bsdiff/memory_file.h>
|
|
#include <bsdiff/file.h>
|
|
#include <puffin/memory_stream.h>
|
|
|
|
#include "android-base/strings.h"
|
|
#include "lz4diff/lz4diff.h"
|
|
#include "lz4diff/lz4diff.pb.h"
|
|
#include "lz4diff_compress.h"
|
|
#include "lz4diff_format.h"
|
|
#include "puffin/puffpatch.h"
|
|
#include "update_engine/common/hash_calculator.h"
|
|
#include "update_engine/common/utils.h"
|
|
|
|
namespace chromeos_update_engine {
|
|
|
|
namespace {
|
|
|
|
template <typename T>
|
|
constexpr void BigEndianToHost(T& t) {
|
|
static_assert(std::is_integral_v<T>);
|
|
static_assert(sizeof(t) == 4 || sizeof(t) == 8 || sizeof(t) == 2);
|
|
if constexpr (sizeof(t) == 4) {
|
|
t = be32toh(t);
|
|
} else if constexpr (sizeof(t) == 8) {
|
|
t = be64toh(t);
|
|
} else if constexpr (sizeof(t) == 2) {
|
|
t = be16toh(t);
|
|
}
|
|
}
|
|
|
|
// In memory representation of an LZ4Diff patch, it's not marked as packed
|
|
// because parsing isn't as simple as reinterpret_cast<> any way.
|
|
struct Lz4diffPatch {
|
|
char magic[kLz4diffMagic.size()];
|
|
uint32_t version;
|
|
uint32_t pb_header_size; // size of protobuf message
|
|
Lz4diffHeader pb_header;
|
|
std::string_view inner_patch;
|
|
};
|
|
|
|
// Utility class to interact with puffin API. C++ does not have standard
|
|
// Read/Write trait. So everybody invent their own file descriptor wrapper.
|
|
class StringViewStream : public puffin::StreamInterface {
|
|
public:
|
|
~StringViewStream() override = default;
|
|
|
|
bool GetSize(uint64_t* size) const override {
|
|
*size = read_memory_.size();
|
|
return true;
|
|
}
|
|
|
|
bool GetOffset(uint64_t* offset) const override {
|
|
*offset = offset_;
|
|
return true;
|
|
}
|
|
|
|
bool Seek(uint64_t offset) override {
|
|
TEST_AND_RETURN_FALSE(open_);
|
|
uint64_t size;
|
|
GetSize(&size);
|
|
TEST_AND_RETURN_FALSE(offset <= size);
|
|
offset_ = offset;
|
|
return true;
|
|
}
|
|
|
|
bool Read(void* buffer, size_t length) override {
|
|
TEST_AND_RETURN_FALSE(open_);
|
|
TEST_AND_RETURN_FALSE(offset_ + length <= read_memory_.size());
|
|
memcpy(buffer, read_memory_.data() + offset_, length);
|
|
offset_ += length;
|
|
return true;
|
|
}
|
|
|
|
bool Write(const void* buffer, size_t length) override {
|
|
LOG(ERROR) << "Unsupported operation " << __FUNCTION__;
|
|
return false;
|
|
}
|
|
|
|
bool Close() override {
|
|
open_ = false;
|
|
return true;
|
|
}
|
|
|
|
constexpr StringViewStream(std::string_view read_memory)
|
|
: read_memory_(read_memory) {
|
|
CHECK(!read_memory.empty());
|
|
}
|
|
|
|
private:
|
|
// The memory buffer for reading.
|
|
std::string_view read_memory_;
|
|
|
|
// The current offset.
|
|
uint64_t offset_{};
|
|
bool open_{true};
|
|
};
|
|
|
|
bool ParseLz4DifffPatch(std::string_view patch_data, Lz4diffPatch* output) {
|
|
CHECK_NE(output, nullptr);
|
|
if (!android::base::StartsWith(patch_data, kLz4diffMagic)) {
|
|
LOG(ERROR) << "Invalid lz4diff magic: "
|
|
<< HexEncode(patch_data.substr(0, kLz4diffMagic.size()))
|
|
<< ", expected: " << HexEncode(kLz4diffMagic);
|
|
return false;
|
|
}
|
|
Lz4diffPatch& patch = *output;
|
|
std::memcpy(patch.magic, patch_data.data(), kLz4diffMagic.size());
|
|
std::memcpy(&patch.version,
|
|
patch_data.data() + kLz4diffMagic.size(),
|
|
sizeof(patch.version));
|
|
BigEndianToHost(patch.version);
|
|
if (patch.version != kLz4diffVersion) {
|
|
LOG(ERROR) << "Unsupported lz4diff version: " << patch.version
|
|
<< ", supported version: " << kLz4diffVersion;
|
|
return false;
|
|
}
|
|
std::memcpy(&patch.pb_header_size,
|
|
patch_data.data() + kLz4diffMagic.size() + sizeof(patch.version),
|
|
sizeof(patch.pb_header_size));
|
|
BigEndianToHost(patch.pb_header_size);
|
|
TEST_AND_RETURN_FALSE(patch.pb_header.ParseFromArray(
|
|
patch_data.data() + kLz4diffHeaderSize, patch.pb_header_size));
|
|
patch.inner_patch =
|
|
patch_data.substr(kLz4diffHeaderSize + patch.pb_header_size);
|
|
return true;
|
|
}
|
|
|
|
bool bspatch(std::string_view input_data,
|
|
std::string_view patch_data,
|
|
Blob* output) {
|
|
CHECK_NE(output, nullptr);
|
|
output->clear();
|
|
CHECK_GT(patch_data.size(), 0UL);
|
|
int err =
|
|
bsdiff::bspatch(reinterpret_cast<const uint8_t*>(input_data.data()),
|
|
input_data.size(),
|
|
reinterpret_cast<const uint8_t*>(patch_data.data()),
|
|
patch_data.size(),
|
|
[output](const uint8_t* data, size_t size) -> size_t {
|
|
output->insert(output->end(), data, data + size);
|
|
return size;
|
|
});
|
|
return err == 0;
|
|
}
|
|
|
|
bool puffpatch(std::string_view input_data,
|
|
std::string_view patch_data,
|
|
Blob* output) {
|
|
return puffin::PuffPatch(std::make_unique<StringViewStream>(input_data),
|
|
puffin::MemoryStream::CreateForWrite(output),
|
|
reinterpret_cast<const uint8_t*>(patch_data.data()),
|
|
patch_data.size());
|
|
}
|
|
|
|
std::vector<CompressedBlock> ToCompressedBlockVec(
|
|
const google::protobuf::RepeatedPtrField<CompressedBlockInfo>& rpf) {
|
|
std::vector<CompressedBlock> ret;
|
|
ret.reserve(rpf.size());
|
|
for (const auto& block : rpf) {
|
|
auto& info = ret.emplace_back();
|
|
info.compressed_length = block.compressed_length();
|
|
info.uncompressed_length = block.uncompressed_length();
|
|
info.uncompressed_offset = block.uncompressed_offset();
|
|
}
|
|
return ret;
|
|
}
|
|
|
|
bool HasPosfixPatches(const Lz4diffPatch& patch) {
|
|
for (const auto& info : patch.pb_header.dst_info().block_info()) {
|
|
if (!info.postfix_bspatch().empty()) {
|
|
return true;
|
|
}
|
|
}
|
|
return false;
|
|
}
|
|
|
|
size_t GetCompressedSize(
|
|
const google::protobuf::RepeatedPtrField<CompressedBlockInfo>& info) {
|
|
size_t compressed_size = 0;
|
|
for (const auto& block : info) {
|
|
compressed_size += block.compressed_length();
|
|
}
|
|
return compressed_size;
|
|
}
|
|
|
|
size_t GetDecompressedSize(
|
|
const google::protobuf::RepeatedPtrField<CompressedBlockInfo>& info) {
|
|
size_t decompressed_size = 0;
|
|
for (const auto& block : info) {
|
|
decompressed_size += block.uncompressed_length();
|
|
}
|
|
return decompressed_size;
|
|
}
|
|
|
|
bool ApplyInnerPatch(Blob decompressed_src,
|
|
const Lz4diffPatch& patch,
|
|
Blob* decompressed_dst) {
|
|
switch (patch.pb_header.inner_type()) {
|
|
case InnerPatchType::BSDIFF:
|
|
TEST_AND_RETURN_FALSE(bspatch(
|
|
ToStringView(decompressed_src), patch.inner_patch, decompressed_dst));
|
|
break;
|
|
case InnerPatchType::PUFFDIFF:
|
|
TEST_AND_RETURN_FALSE(puffpatch(
|
|
ToStringView(decompressed_src), patch.inner_patch, decompressed_dst));
|
|
break;
|
|
default:
|
|
LOG(ERROR) << "Unsupported patch type: " << patch.pb_header.inner_type();
|
|
return false;
|
|
}
|
|
return true;
|
|
}
|
|
|
|
// TODO(zhangkelvin) Rewrite this in C++ 20 coroutine once that's available.
|
|
// Hand coding CPS is not fun.
|
|
bool Lz4Patch(std::string_view src_data,
|
|
const Lz4diffPatch& patch,
|
|
const SinkFunc& sink) {
|
|
auto decompressed_src = TryDecompressBlob(
|
|
src_data,
|
|
ToCompressedBlockVec(patch.pb_header.src_info().block_info()),
|
|
patch.pb_header.src_info().zero_padding_enabled());
|
|
TEST_AND_RETURN_FALSE(!decompressed_src.empty());
|
|
Blob decompressed_dst;
|
|
const auto decompressed_dst_size =
|
|
GetDecompressedSize(patch.pb_header.dst_info().block_info());
|
|
decompressed_dst.reserve(decompressed_dst_size);
|
|
|
|
ApplyInnerPatch(std::move(decompressed_src), patch, &decompressed_dst);
|
|
|
|
if (!HasPosfixPatches(patch)) {
|
|
return TryCompressBlob(
|
|
ToStringView(decompressed_dst),
|
|
ToCompressedBlockVec(patch.pb_header.dst_info().block_info()),
|
|
patch.pb_header.dst_info().zero_padding_enabled(),
|
|
patch.pb_header.dst_info().algo(),
|
|
sink);
|
|
}
|
|
auto postfix_patcher =
|
|
[&sink,
|
|
block_idx = 0,
|
|
&dst_block_info = patch.pb_header.dst_info().block_info()](
|
|
const uint8_t* data, size_t size) mutable -> size_t {
|
|
if (block_idx >= dst_block_info.size()) {
|
|
return sink(data, size);
|
|
}
|
|
const auto& block_info = dst_block_info[block_idx];
|
|
TEST_EQ(size, block_info.compressed_length());
|
|
DEFER { block_idx++; };
|
|
if (block_info.postfix_bspatch().empty()) {
|
|
return sink(data, size);
|
|
}
|
|
if (!block_info.sha256_hash().empty()) {
|
|
Blob actual_hash;
|
|
TEST_AND_RETURN_FALSE(
|
|
HashCalculator::RawHashOfBytes(data, size, &actual_hash));
|
|
if (ToStringView(actual_hash) != block_info.sha256_hash()) {
|
|
LOG(ERROR) << "Block " << block_info
|
|
<< " is corrupted. This usually means the patch generator "
|
|
"used a different version of LZ4, or an incompatible LZ4 "
|
|
"patch generator was used, or LZ4 produces different "
|
|
"output on different platforms. Expected hash: "
|
|
<< HexEncode(block_info.sha256_hash())
|
|
<< ", actual hash: " << HexEncode(actual_hash);
|
|
return 0;
|
|
}
|
|
}
|
|
Blob fixed_block;
|
|
TEST_AND_RETURN_FALSE(
|
|
bspatch(std::string_view(reinterpret_cast<const char*>(data), size),
|
|
block_info.postfix_bspatch(),
|
|
&fixed_block));
|
|
return sink(fixed_block.data(), fixed_block.size());
|
|
};
|
|
|
|
return TryCompressBlob(
|
|
ToStringView(decompressed_dst),
|
|
ToCompressedBlockVec(patch.pb_header.dst_info().block_info()),
|
|
patch.pb_header.dst_info().zero_padding_enabled(),
|
|
patch.pb_header.dst_info().algo(),
|
|
postfix_patcher);
|
|
}
|
|
|
|
bool Lz4Patch(std::string_view src_data,
|
|
const Lz4diffPatch& patch,
|
|
Blob* output) {
|
|
Blob blob;
|
|
const auto output_size =
|
|
GetCompressedSize(patch.pb_header.dst_info().block_info());
|
|
blob.reserve(output_size);
|
|
TEST_AND_RETURN_FALSE(Lz4Patch(
|
|
src_data, patch, [&blob](const uint8_t* data, size_t size) -> size_t {
|
|
blob.insert(blob.end(), data, data + size);
|
|
return size;
|
|
}));
|
|
*output = std::move(blob);
|
|
return true;
|
|
}
|
|
|
|
} // namespace
|
|
|
|
bool Lz4Patch(std::string_view src_data,
|
|
std::string_view patch_data,
|
|
Blob* output) {
|
|
Lz4diffPatch patch;
|
|
TEST_AND_RETURN_FALSE(ParseLz4DifffPatch(patch_data, &patch));
|
|
return Lz4Patch(src_data, patch, output);
|
|
}
|
|
|
|
bool Lz4Patch(std::string_view src_data,
|
|
std::string_view patch_data,
|
|
const SinkFunc& sink) {
|
|
Lz4diffPatch patch;
|
|
TEST_AND_RETURN_FALSE(ParseLz4DifffPatch(patch_data, &patch));
|
|
return Lz4Patch(src_data, patch, sink);
|
|
}
|
|
|
|
bool Lz4Patch(const Blob& src_data, const Blob& patch_data, Blob* output) {
|
|
return Lz4Patch(ToStringView(src_data), ToStringView(patch_data), output);
|
|
}
|
|
|
|
std::ostream& operator<<(std::ostream& out, const CompressionAlgorithm& info) {
|
|
out << "Algo {type: " << info.Type_Name(info.type());
|
|
if (info.level() != 0) {
|
|
out << ", level: " << info.level();
|
|
}
|
|
out << "}";
|
|
|
|
return out;
|
|
}
|
|
|
|
std::ostream& operator<<(std::ostream& out, const CompressionInfo& info) {
|
|
out << "CompressionInfo {block_info: " << info.block_info()
|
|
<< ", algo: " << info.algo() << "}";
|
|
return out;
|
|
}
|
|
|
|
std::ostream& operator<<(std::ostream& out, const Lz4diffHeader& header) {
|
|
out << "Lz4diffHeader {src_info: " << header.src_info()
|
|
<< ", dst_info: " << header.dst_info() << "}";
|
|
return out;
|
|
}
|
|
|
|
} // namespace chromeos_update_engine
|