Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 7 additions & 3 deletions base/cvd/cuttlefish/host/libs/zip/BUILD.bazel
Original file line number Diff line number Diff line change
Expand Up @@ -11,11 +11,13 @@ cf_cc_library(
deps = [
"//cuttlefish/host/libs/zip/libzip_cc:archive",
"//cuttlefish/host/libs/zip/libzip_cc:seekable_source",
"//cuttlefish/host/libs/zip/libzip_cc:source_callback",
"//cuttlefish/io",
"//cuttlefish/io:fake_seek",
"//cuttlefish/io:length",
"//cuttlefish/io:read_exact",
"//cuttlefish/io:reader_seeker",
"//cuttlefish/io:visitable",
"//cuttlefish/result",
"@abseil-cpp//absl/log",
],
)

Expand All @@ -40,7 +42,9 @@ cf_cc_library(
"//cuttlefish/host/libs/web/http_client",
"//cuttlefish/host/libs/web/http_client:http_probe",
"//cuttlefish/host/libs/zip/libzip_cc:seekable_source",
"//cuttlefish/host/libs/zip/libzip_cc:source_callback",
"//cuttlefish/io",
"//cuttlefish/io:fake_seek",
"//cuttlefish/io:visitable",
"//cuttlefish/result",
"@abseil-cpp//absl/log",
"@fmt",
Expand Down
142 changes: 65 additions & 77 deletions base/cvd/cuttlefish/host/libs/zip/buffered_zip_source.cc
Original file line number Diff line number Diff line change
Expand Up @@ -21,123 +21,111 @@

#include <algorithm>
#include <memory>
#include <mutex>
#include <shared_mutex>
#include <utility>
#include <vector>

#include "absl/log/log.h"

#include "cuttlefish/host/libs/zip/libzip_cc/archive.h"
#include "cuttlefish/host/libs/zip/libzip_cc/seekable_source.h"
#include "cuttlefish/host/libs/zip/libzip_cc/source_callback.h"
#include "cuttlefish/io/fake_seek.h"
#include "cuttlefish/io/io.h"
#include "cuttlefish/io/length.h"
#include "cuttlefish/io/read_exact.h"
#include "cuttlefish/io/reader_seeker.h"
#include "cuttlefish/io/visitable.h"
#include "cuttlefish/result/result.h"

namespace cuttlefish {
namespace {

class BufferedZipSourceCallbacks : public SeekableZipSourceCallback {
class BufferedZipSourceCallbacks : public ReaderFakeSeeker {
public:
static Result<std::unique_ptr<BufferedZipSourceCallbacks>> Create(
std::unique_ptr<ReaderSeeker> data_provider, size_t buffer_size) {
std::unique_ptr<ReaderSeeker> data_provider, uint64_t buffer_size) {
CF_EXPECT(data_provider.get());

uint64_t backing_size = CF_EXPECT(Length(*data_provider));
buffer_size = std::min(buffer_size, backing_size);

std::unique_ptr<BufferedZipSourceCallbacks> callbacks(
new BufferedZipSourceCallbacks(buffer_size));
new BufferedZipSourceCallbacks(buffer_size, backing_size));

callbacks->size_ = CF_EXPECT(Length(*data_provider));
callbacks->data_provider_ = std::move(data_provider);

return callbacks;
}

bool Close() override {
offset_ = 0;
buffer_remaining_ = 0;
offset_in_buffer_ = 0;
buffer_remaining_ = 0;

return true;
Result<void> Visit(IoVisitor& visitor) override {
CF_EXPECT(visitor.Accept(*this));
return {};
}
bool Open() override {
offset_ = 0;
buffer_remaining_ = 0;
offset_in_buffer_ = 0;
buffer_remaining_ = 0;

return data_provider_->SeekSet(0).has_value();
Result<uint64_t, int> PRead(void* data, uint64_t len,
uint64_t offset) const override {
BufferedZipSourceCallbacks* mut =
const_cast<BufferedZipSourceCallbacks*>(this);
return CF_EXPECT(mut->PReadMut(data, len, offset));
}
int64_t Read(char* data, uint64_t len) override {
if (len > buffer_.size()) {
buffer_remaining_ = 0;
if (!data_provider_->SeekSet(offset_).has_value()) {
return false;
}
VLOG(1) << "Bypassing buffer, reading " << len;
if (Result<uint64_t> read_len = data_provider_->Read(data, len);
read_len.has_value()) {
offset_ += *read_len;
return *read_len;
} else {
return -1;
}
}
if (buffer_remaining_ > 0) {
size_t to_read = std::min<size_t>(len, buffer_remaining_);
memcpy(data, &buffer_[offset_in_buffer_], to_read);
buffer_remaining_ -= to_read;
offset_in_buffer_ += to_read;
offset_ += to_read;
return to_read;
}
uint64_t buffer_fill = buffer_.size();
if (offset_ + buffer_fill > size_) {
buffer_fill = size_ - offset_;
}
if (buffer_fill == 0) {

private:
explicit BufferedZipSourceCallbacks(uint64_t buffer_size,
uint64_t backing_size)
: ReaderFakeSeeker(backing_size),
buffer_(buffer_size),
backing_size_(backing_size) {}

Result<uint64_t, int> PReadMut(void* data, uint64_t len, uint64_t offset) {
if (offset >= backing_size_ || len == 0) {
return 0;
}
if (!data_provider_->SeekSet(offset_).has_value()) {
return -1;
len = std::min(len, backing_size_ - offset);
if (len > buffer_.size()) {
return CF_EXPECT(data_provider_->PRead(data, len, offset));
}
VLOG(1) << "Filling buffer with " << buffer_fill;
Result<size_t> data_provider_read =
data_provider_->Read(buffer_.data(), buffer_fill);
if (!data_provider_read.has_value()) {
return -1;
{ // If data can be read from the buffer, can share read access
std::shared_lock lock(mutex_);
if (uint64_t buf_read = ReadBufLocked(data, len, offset); buf_read > 0) {
return buf_read;
}
}
buffer_remaining_ = *data_provider_read;
offset_in_buffer_ = 0;
return Read(data, len);
}
uint64_t Size() override { return size_; }
bool SetOffset(int64_t new_offset) override {
if (new_offset >= offset_ && new_offset < offset_ + buffer_remaining_) {
offset_in_buffer_ += new_offset - offset_;
buffer_remaining_ -= new_offset - offset_;
} else {
buffer_remaining_ = 0;
{ // If we need to load new data, get exclusive access to fill the buffer
std::lock_guard lock(mutex_);
// Another thread may have loaded data we want between the unlock / lock
if (uint64_t buf_rd = ReadBufLocked(data, len, offset); buf_rd > 0) {
return buf_rd;
} // We need new data.
buf_begin_ = std::min(backing_size_ - buffer_.size(), offset);
has_data_ = false; // If PReadExact fails, assume buffer_ is corrupted.
CF_EXPECT(PReadExact(*data_provider_, buffer_.data(), buffer_.size(),
buf_begin_));
has_data_ = true;
return ReadBufLocked(data, len, offset);
}
offset_ = new_offset;
return true;
}
int64_t Offset() override { return offset_; }

private:
explicit BufferedZipSourceCallbacks(size_t buffer_size)
: buffer_(buffer_size) {}
uint64_t ReadBufLocked(void* data, uint64_t len, uint64_t off) {
if (!has_data_ || off < buf_begin_ || off >= buf_begin_ + buffer_.size()) {
return 0;
}
uint64_t buf_offset = off - buf_begin_;
uint64_t buf_read_len = std::min(len, buffer_.size() - buf_offset);
memcpy(data, &buffer_[buf_offset], buf_read_len);
return buf_read_len;
}

std::unique_ptr<ReaderSeeker> data_provider_;
std::shared_mutex mutex_;
bool has_data_ = false;
std::vector<char> buffer_;
uint64_t size_ = 0;
int64_t offset_ = 0;
size_t offset_in_buffer_ = 0;
size_t buffer_remaining_ = 0;
uint64_t backing_size_ = 0;
uint64_t buf_begin_ = 0;
};

} // namespace

Result<SeekableZipSource> BufferZipSource(
std::unique_ptr<ReaderSeeker> data_provider, size_t buffer_size) {
std::unique_ptr<ReaderSeeker> data_provider, uint64_t buffer_size) {
std::unique_ptr<BufferedZipSourceCallbacks> callbacks =
CF_EXPECT(BufferedZipSourceCallbacks::Create(std::move(data_provider),
buffer_size));
Expand Down
4 changes: 2 additions & 2 deletions base/cvd/cuttlefish/host/libs/zip/buffered_zip_source.h
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@

#pragma once

#include <stddef.h>
#include <stdint.h>

#include <memory>

Expand All @@ -27,7 +27,7 @@
namespace cuttlefish {

Result<SeekableZipSource> BufferZipSource(std::unique_ptr<ReaderSeeker>,
size_t buffer_size);
uint64_t buffer_size);

Result<ReadableZip> BufferAndOpenZip(std::unique_ptr<ReaderSeeker>);

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
#include "cuttlefish/host/libs/zip/buffered_zip_source.h"

#include <stddef.h>
#include <stdint.h>

#include <memory>
#include <vector>
Expand Down Expand Up @@ -57,7 +58,7 @@ TEST(BufferedZipSourceTest, ManySmallReads) {
size_t read_offset = 0;
while (true) {
char* ptr = &data_out[read_offset];
size_t amount_read = CF_ASSERT(reader.Read(ptr, kPrimeReadSize));
uint64_t amount_read = CF_ASSERT(reader.Read(ptr, kPrimeReadSize));
if (amount_read == 0) {
break;
}
Expand Down
9 changes: 2 additions & 7 deletions base/cvd/cuttlefish/host/libs/zip/libzip_cc/BUILD.bazel
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,6 @@ cf_cc_library(
deps = [
"//cuttlefish/host/libs/zip/libzip_cc:error",
"//cuttlefish/host/libs/zip/libzip_cc:managed",
"//cuttlefish/host/libs/zip/libzip_cc:source_callback",
"//cuttlefish/host/libs/zip/libzip_cc:stat",
"//cuttlefish/io",
"//cuttlefish/result",
Expand All @@ -74,12 +73,13 @@ cf_cc_library(
"//cuttlefish/host/libs/zip/libzip_cc:error",
"//cuttlefish/host/libs/zip/libzip_cc:managed",
"//cuttlefish/host/libs/zip/libzip_cc:readable_source",
"//cuttlefish/host/libs/zip/libzip_cc:source_callback",
"//cuttlefish/io",
"//cuttlefish/io:fake_pread_pwrite",
"//cuttlefish/io:length",
"//cuttlefish/io:reader_seeker",
"//cuttlefish/io:visitable",
"//cuttlefish/result",
"@abseil-cpp//absl/base:no_destructor",
"@libzip",
],
)
Expand All @@ -89,11 +89,6 @@ cf_cc_library(
hdrs = ["stat.h"],
)

cf_cc_library(
name = "source_callback",
hdrs = ["source_callback.h"],
)

cf_cc_library(
name = "writable_source",
srcs = ["writable_source.cc"],
Expand Down
82 changes: 0 additions & 82 deletions base/cvd/cuttlefish/host/libs/zip/libzip_cc/readable_source.cc
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,6 @@
#include <stddef.h>
#include <stdint.h>

#include <memory>
#include <mutex>
#include <optional>
#include <string>
Expand All @@ -29,35 +28,13 @@

#include "cuttlefish/host/libs/zip/libzip_cc/error.h"
#include "cuttlefish/host/libs/zip/libzip_cc/managed.h"
#include "cuttlefish/host/libs/zip/libzip_cc/source_callback.h"
#include "cuttlefish/host/libs/zip/libzip_cc/stat.h"
#include "cuttlefish/io/io.h"
#include "cuttlefish/result/result.h"

namespace cuttlefish {
namespace {

struct ReadableCallbackSource {
ReadableCallbackSource(std::unique_ptr<ReadableZipSourceCallback> callbacks)
: callbacks_(std::move(callbacks)), error_(NewZipError()) {}

std::unique_ptr<ReadableZipSourceCallback> callbacks_;
ManagedZipError error_;
};

// https://libzip.org/documentation/zip_source_function.html
int64_t ReadableZipSourceCallbackFn(void* userdata, void* data, uint64_t len,
zip_source_cmd_t cmd) {
ReadableCallbackSource* source =
reinterpret_cast<ReadableCallbackSource*>(userdata);
if (cmd == ZIP_SOURCE_FREE) {
delete source;
return 0;
}
return HandleCallback(*source->callbacks_, source->error_.get(), data, len,
cmd);
}

std::optional<ZipCompression> CompressionFromRaw(uint16_t method) {
switch (method) {
case ZIP_CM_DEFAULT:
Expand All @@ -79,65 +56,6 @@ std::optional<ZipCompression> CompressionFromRaw(uint16_t method) {

} // namespace

int64_t HandleCallback(ReadableZipSourceCallback& callbacks, zip_error_t* error,
void* data, uint64_t len, zip_source_cmd_t cmd) {
switch (cmd) {
case ZIP_SOURCE_CLOSE: {
bool callback_res = callbacks.Close();
if (!callback_res) {
zip_error_set(error, ZIP_ER_CLOSE, errno);
}
return callback_res ? 0 : -1;
}
case ZIP_SOURCE_ERROR:
return zip_error_to_data(error, data, len);
case ZIP_SOURCE_OPEN: {
bool callback_res = callbacks.Open();
if (!callback_res) {
zip_error_set(error, ZIP_ER_OPEN, errno);
}
return callback_res ? 0 : -1;
}
case ZIP_SOURCE_READ: {
int64_t callback_res = callbacks.Read(reinterpret_cast<char*>(data), len);
if (callback_res < 0) {
zip_error_set(error, ZIP_ER_READ, errno);
}
return callback_res;
}
case ZIP_SOURCE_STAT: {
zip_stat_t* stat_out = ZIP_SOURCE_GET_ARGS(zip_stat_t, data, len, error);
zip_stat_init(stat_out);
stat_out->valid = ZIP_STAT_SIZE;
stat_out->size = callbacks.Size();
return 0;
}
case ZIP_SOURCE_SUPPORTS:
return zip_source_make_command_bitmap(
ZIP_SOURCE_CLOSE, ZIP_SOURCE_ERROR, ZIP_SOURCE_FREE, ZIP_SOURCE_OPEN,
ZIP_SOURCE_READ, ZIP_SOURCE_STAT, ZIP_SOURCE_SUPPORTS, -1);
default:
zip_error_set(error, ZIP_ER_OPNOTSUPP, EINVAL);
return -1;
}
}

Result<ReadableZipSource> ReadableZipSource::FromCallbacks(
std::unique_ptr<ReadableZipSourceCallback> callbacks) {
CF_EXPECT(callbacks.get());

std::unique_ptr<ReadableCallbackSource> wrapped_source =
std::make_unique<ReadableCallbackSource>(std::move(callbacks));

ManagedZipError error = NewZipError();
ManagedZipSource source(zip_source_function_create(
ReadableZipSourceCallbackFn, wrapped_source.release(), error.get()));

CF_EXPECT(source.get(), ZipErrorString(error.get()));

return ReadableZipSource(std::move(source));
}

// These have to be defined in the `.cc` file to avoid linker errors because of
// bazel weirdness around cmake files.
ReadableZipSource::ReadableZipSource(ReadableZipSource&&) = default;
Expand Down
Loading
Loading