Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions riegeli/bytes/istream_reader.h
Original file line number Diff line number Diff line change
Expand Up @@ -112,6 +112,10 @@ class IStreamReaderBase : public BufferedReader {
void Reset(Closed);
void Reset(BufferOptions buffer_options, bool growing_source);
void Initialize(std::istream* src, std::optional<Position> assumed_pos);
void set_growing_source(bool growing_source) {
growing_source_ = growing_source;
}
bool growing_source() const { return growing_source_; }
ABSL_ATTRIBUTE_COLD bool FailOperation(absl::string_view operation);

void Done() override;
Expand Down
5 changes: 5 additions & 0 deletions riegeli/bytes/reader.cc
Original file line number Diff line number Diff line change
Expand Up @@ -837,6 +837,11 @@ std::unique_ptr<Reader> Reader::NewReaderImpl(Position initial_pos) {
return nullptr;
}

std::unique_ptr<Reader> Reader::NewReaderWithLengthHintImpl(
Position initial_pos, Position /*length_hint*/) {
return NewReaderImpl(initial_pos);
}

std::unique_ptr<Reader> Reader::NewReaderCurrentPosImpl() {
return NewReader(pos());
}
Expand Down
27 changes: 27 additions & 0 deletions riegeli/bytes/reader.h
Original file line number Diff line number Diff line change
Expand Up @@ -459,6 +459,20 @@ class Reader : public Object {
return NewReaderImpl(initial_pos);
}

// Like `NewReader(initial_pos)`, but hints that about `length_hint` bytes
// will be read from `initial_pos`.
//
// This lets a source for which reading more than needed is costly, e.g. a
// remote object, request only this range. Reading beyond it is supported,
// possibly less efficiently. By default the hint is ignored.
//
// The same thread safety guarantees and return value as for
// `NewReader(initial_pos)` apply.
std::unique_ptr<Reader> NewReader(Position initial_pos,
Position length_hint) {
return NewReaderWithLengthHintImpl(initial_pos, length_hint);
}

// Like `NewReader(pos())`. It can be more efficient because for some classes
// the current buffer can be shared. It offers fewer thread safety guarantees.
//
Expand Down Expand Up @@ -629,7 +643,20 @@ class Reader : public Object {
// By default fails.
virtual std::optional<Position> SizeImpl();

// Implementation of `NewReader(initial_pos)`.
//
// By default fails.
virtual std::unique_ptr<Reader> NewReaderImpl(Position initial_pos);

// Implementation of `NewReader(initial_pos, length_hint)`.
//
// By default calls `NewReaderImpl(initial_pos)`.
virtual std::unique_ptr<Reader> NewReaderWithLengthHintImpl(
Position initial_pos, Position length_hint);

// Implementation of `NewReaderCurrentPos()`.
//
// By default calls `NewReader(pos())`.
virtual std::unique_ptr<Reader> NewReaderCurrentPosImpl();

private:
Expand Down
63 changes: 53 additions & 10 deletions riegeli/gcs/gcs_reader.cc
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
#include <stdint.h>

#include <functional>
#include <limits>
#include <memory>
#include <optional>
#include <utility>
Expand Down Expand Up @@ -45,16 +46,22 @@ GcsReader::GcsReader(
NewReaderTag, const google::cloud::storage::Client& client,
const GcsObject& object,
const std::function<google::cloud::storage::ObjectReadStream(
GcsReader&, int64_t)>& read_object,
BufferOptions buffer_options, Position read_from_offset)
GcsReader&, int64_t, std::optional<int64_t>)>& read_object,
BufferOptions buffer_options, Position read_from_offset,
std::optional<Position> read_to_offset)
: IStreamReader(kClosed),
client_(client),
object_(object),
read_object_(read_object) {
IStreamReader::Reset(read_object(*this, IntCast<int64_t>(read_from_offset)),
IStreamReaderBase::Options()
.set_assumed_pos(read_from_offset)
.set_buffer_options(buffer_options));
IStreamReader::Reset(
read_object(*this, IntCast<int64_t>(read_from_offset),
read_to_offset == std::nullopt
? std::nullopt
: std::make_optional(IntCast<int64_t>(*read_to_offset))),
IStreamReaderBase::Options()
.set_assumed_pos(read_from_offset)
.set_growing_source(read_to_offset != std::nullopt)
.set_buffer_options(buffer_options));
PropagateStatus();
set_limit_pos(read_from_offset);
}
Expand Down Expand Up @@ -136,7 +143,28 @@ bool GcsReader::ReadInternal(size_t min_length, size_t max_length, char* dest) {
"max_length < min_length";
RIEGELI_ASSERT_OK(*this)
<< "Failed precondition of BufferedReader::ReadInternal()";
const bool result = IStreamReader::ReadInternal(min_length, max_length, dest);
const Position pos_before = limit_pos();
bool result = IStreamReader::ReadInternal(min_length, max_length, dest);
PropagateStatus();
if (ABSL_PREDICT_TRUE(result)) return true;
if (!growing_source() || ABSL_PREDICT_FALSE(!ok())) return false;
// The requested range could not satisfy this read. Continue with a request
// for the rest of the object.
const size_t length_read = IntCast<size_t>(limit_pos() - pos_before);
RIEGELI_ASSERT_LT(length_read, min_length)
<< "IStreamReader::ReadInternal() returned false but read enough";
dest += length_read;
min_length -= length_read;
max_length -= length_read;
// As in `SeekBehindBuffer()`, this request is not pinned to `generation()`
// of the previous one. Unless a generation was requested explicitly, the
// data come from the current generation, which is a different one if the
// object was overwritten in the meantime.
src() = read_object_(*this, IntCast<int64_t>(limit_pos()), std::nullopt);
set_growing_source(false);
PropagateStatus();
if (ABSL_PREDICT_FALSE(!ok())) return false;
result = IStreamReader::ReadInternal(min_length, max_length, dest);
PropagateStatus();
return result;
}
Expand All @@ -157,7 +185,8 @@ bool GcsReader::SeekBehindBuffer(Position new_pos) {
// `ObjectReadStream` does not support seeking to the very end.
const Position read_from_offset =
UnsignedMin(new_pos, SaturatingSub(*exact_size(), Position{1}));
src() = read_object_(*this, IntCast<int64_t>(read_from_offset));
src() = read_object_(*this, IntCast<int64_t>(read_from_offset), std::nullopt);
set_growing_source(false);
PropagateStatus();
set_limit_pos(read_from_offset);
if (new_pos > read_from_offset) {
Expand All @@ -172,19 +201,33 @@ bool GcsReader::SeekBehindBuffer(Position new_pos) {
bool GcsReader::SupportsNewReader() { return exact_size() != std::nullopt; }

std::unique_ptr<Reader> GcsReader::NewReaderImpl(Position initial_pos) {
// A length hint reaching the end of the object requests the rest of it.
return GcsReader::NewReaderWithLengthHintImpl(
initial_pos, std::numeric_limits<Position>::max());
}

std::unique_ptr<Reader> GcsReader::NewReaderWithLengthHintImpl(
Position initial_pos, Position length_hint) {
if (ABSL_PREDICT_FALSE(!GcsReader::SupportsNewReader())) {
// Delegate to the base class to avoid repeating the error message.
return IStreamReader::NewReaderImpl(initial_pos);
}
if (ABSL_PREDICT_FALSE(!ok())) return nullptr;
// `NewReaderImpl()` is thread-safe from this point.
// `NewReaderWithLengthHintImpl()` is thread-safe from this point.

// `ObjectReadStream` does not support seeking to the very end.
const Position read_from_offset =
UnsignedMin(initial_pos, SaturatingSub(*exact_size(), Position{1}));
// Request only the hinted range if it ends before the object does. Otherwise
// request the rest of the object: in particular a read of the whole object
// then sends no `Range` header and keeps checksum validation.
std::optional<Position> read_to_offset;
if (length_hint > 0 && length_hint < *exact_size() - read_from_offset) {
read_to_offset = read_from_offset + length_hint;
}
std::unique_ptr<GcsReader> reader(
new GcsReader(NewReaderTag(), client(), object(), read_object_,
buffer_options(), read_from_offset));
buffer_options(), read_from_offset, read_to_offset));
reader->set_exact_size(exact_size());
if (initial_pos > read_from_offset) {
if (ABSL_PREDICT_TRUE(reader->Pull())) reader->move_cursor(1);
Expand Down
31 changes: 24 additions & 7 deletions riegeli/gcs/gcs_reader.h
Original file line number Diff line number Diff line change
Expand Up @@ -124,6 +124,8 @@ class GcsReader
bool ReadInternal(size_t min_length, size_t max_length, char* dest) override;
bool SeekBehindBuffer(Position new_pos) override;
std::unique_ptr<Reader> NewReaderImpl(Position initial_pos) override;
std::unique_ptr<Reader> NewReaderWithLengthHintImpl(
Position initial_pos, Position length_hint) override;

private:
static constexpr uint64_t kMaxPosition = std::numeric_limits<int64_t>::max();
Expand All @@ -145,8 +147,9 @@ class GcsReader
NewReaderTag, const google::cloud::storage::Client& client,
const GcsObject& object,
const std::function<google::cloud::storage::ObjectReadStream(
GcsReader&, int64_t)>& read_object,
BufferOptions buffer_options, Position read_from_offset);
GcsReader&, int64_t, std::optional<int64_t>)>& read_object,
BufferOptions buffer_options, Position read_from_offset,
std::optional<Position> read_to_offset);

template <typename... ReadObjectOptions>
static RangeOptions GetRangeOptions(
Expand Down Expand Up @@ -192,7 +195,10 @@ class GcsReader

std::optional<google::cloud::storage::Client> client_;
GcsObject object_;
std::function<google::cloud::storage::ObjectReadStream(GcsReader&, int64_t)>
// Opens a stream from the first argument to the second argument (exclusive)
// if given, otherwise to the end of the range specified in the constructor.
std::function<google::cloud::storage::ObjectReadStream(
GcsReader&, int64_t, std::optional<int64_t>)>
read_object_;
};

Expand Down Expand Up @@ -300,7 +306,16 @@ inline void GcsReader::SetReadObject(
read_object_ =
[new_read_object_options = DecayTuple(
Filter<NewReadObjectOptionPredicate>(read_object_options...))](
GcsReader& self, int64_t read_from_offset) mutable {
GcsReader& self, int64_t read_from_offset,
std::optional<int64_t> read_to_offset) mutable {
if (read_to_offset != std::nullopt) {
return self.ApplyReadObject(
new_read_object_options,
std::make_index_sequence<
std::tuple_size_v<decltype(new_read_object_options)>>(),
google::cloud::storage::ReadRange(read_from_offset,
*read_to_offset));
}
return self.ApplyReadObject(
new_read_object_options,
std::make_index_sequence<
Expand All @@ -311,13 +326,15 @@ inline void GcsReader::SetReadObject(
read_object_ =
[new_read_object_options = DecayTuple(
Filter<NewReadObjectOptionPredicate>(read_object_options...)),
max_size = *range_options.max_size](GcsReader& self,
int64_t read_from_offset) mutable {
max_size = *range_options.max_size](
GcsReader& self, int64_t read_from_offset,
std::optional<int64_t> read_to_offset) mutable {
return self.ApplyReadObject(
new_read_object_options,
std::make_index_sequence<
std::tuple_size_v<decltype(new_read_object_options)>>(),
google::cloud::storage::ReadRange(read_from_offset, max_size));
google::cloud::storage::ReadRange(
read_from_offset, read_to_offset.value_or(max_size)));
};
}
}
Expand Down
Loading