From 64d553df9f878b176b018096601df812af471df6 Mon Sep 17 00:00:00 2001 From: Compression Team Date: Tue, 17 Nov 2026 08:10:27 -0800 Subject: [PATCH] Add `Reader::NewReader(initial_pos, length_hint)`, and let GcsReader bound the request to the hinted range. Random single-record reads of ArrayRecord files on GCS create a child reader with `NewReader(offset)` and read one chunk. `GcsReader` sent an open-ended request (`Range: bytes=offset-`), and the request was cancelled when the reader was destroyed after the chunk was read. The caller knows how much it will read, but `NewReader(Position)` cannot express it. `NewReader(initial_pos, length_hint)` is like `NewReader(initial_pos)`, but hints that about `length_hint` bytes will be read from `initial_pos`. Reading beyond the hinted range is supported. By default the hint is ignored, by the new virtual `NewReaderWithLengthHintImpl()` calling `NewReaderImpl()`. `GcsReader` overrides it: if the hinted range ends before the object (or the range given to the constructor) does, the request is `ReadRange(pos, pos + length_hint)`, so it completes normally and the connection is reused. The request is sent when the reader is created, as for `NewReader(initial_pos)`, so request failures and metadata are reported the same way. A read which needs more than what is left of the range continues with an open-ended request from the current position. A hint reaching the end of the object sends the same request as `NewReader(initial_pos)`; in particular a read of the whole object from offset 0 still sends no `Range` header and keeps checksum validation. All other requests are unchanged. ArrayRecord passes the chunk length in a follow-up change. PiperOrigin-RevId: 1023749871 --- riegeli/bytes/istream_reader.h | 4 +++ riegeli/bytes/reader.cc | 5 +++ riegeli/bytes/reader.h | 27 +++++++++++++++ riegeli/gcs/gcs_reader.cc | 63 ++++++++++++++++++++++++++++------ riegeli/gcs/gcs_reader.h | 31 +++++++++++++---- 5 files changed, 113 insertions(+), 17 deletions(-) diff --git a/riegeli/bytes/istream_reader.h b/riegeli/bytes/istream_reader.h index d853a777..ffae3dcd 100644 --- a/riegeli/bytes/istream_reader.h +++ b/riegeli/bytes/istream_reader.h @@ -112,6 +112,10 @@ class IStreamReaderBase : public BufferedReader { void Reset(Closed); void Reset(BufferOptions buffer_options, bool growing_source); void Initialize(std::istream* src, std::optional assumed_pos); + void set_growing_source(bool growing_source) { + growing_source_ = growing_source; + } + bool growing_source() const { return growing_source_; } ABSL_ATTRIBUTE_COLD bool FailOperation(absl::string_view operation); void Done() override; diff --git a/riegeli/bytes/reader.cc b/riegeli/bytes/reader.cc index 1e0bcd70..5f4e024d 100644 --- a/riegeli/bytes/reader.cc +++ b/riegeli/bytes/reader.cc @@ -837,6 +837,11 @@ std::unique_ptr Reader::NewReaderImpl(Position initial_pos) { return nullptr; } +std::unique_ptr Reader::NewReaderWithLengthHintImpl( + Position initial_pos, Position /*length_hint*/) { + return NewReaderImpl(initial_pos); +} + std::unique_ptr Reader::NewReaderCurrentPosImpl() { return NewReader(pos()); } diff --git a/riegeli/bytes/reader.h b/riegeli/bytes/reader.h index e5561df6..db38c902 100644 --- a/riegeli/bytes/reader.h +++ b/riegeli/bytes/reader.h @@ -459,6 +459,20 @@ class Reader : public Object { return NewReaderImpl(initial_pos); } + // Like `NewReader(initial_pos)`, but hints that about `length_hint` bytes + // will be read from `initial_pos`. + // + // This lets a source for which reading more than needed is costly, e.g. a + // remote object, request only this range. Reading beyond it is supported, + // possibly less efficiently. By default the hint is ignored. + // + // The same thread safety guarantees and return value as for + // `NewReader(initial_pos)` apply. + std::unique_ptr NewReader(Position initial_pos, + Position length_hint) { + return NewReaderWithLengthHintImpl(initial_pos, length_hint); + } + // Like `NewReader(pos())`. It can be more efficient because for some classes // the current buffer can be shared. It offers fewer thread safety guarantees. // @@ -629,7 +643,20 @@ class Reader : public Object { // By default fails. virtual std::optional SizeImpl(); + // Implementation of `NewReader(initial_pos)`. + // + // By default fails. virtual std::unique_ptr NewReaderImpl(Position initial_pos); + + // Implementation of `NewReader(initial_pos, length_hint)`. + // + // By default calls `NewReaderImpl(initial_pos)`. + virtual std::unique_ptr NewReaderWithLengthHintImpl( + Position initial_pos, Position length_hint); + + // Implementation of `NewReaderCurrentPos()`. + // + // By default calls `NewReader(pos())`. virtual std::unique_ptr NewReaderCurrentPosImpl(); private: diff --git a/riegeli/gcs/gcs_reader.cc b/riegeli/gcs/gcs_reader.cc index 81b18663..c2c407a4 100644 --- a/riegeli/gcs/gcs_reader.cc +++ b/riegeli/gcs/gcs_reader.cc @@ -18,6 +18,7 @@ #include #include +#include #include #include #include @@ -45,16 +46,22 @@ GcsReader::GcsReader( NewReaderTag, const google::cloud::storage::Client& client, const GcsObject& object, const std::function& read_object, - BufferOptions buffer_options, Position read_from_offset) + GcsReader&, int64_t, std::optional)>& read_object, + BufferOptions buffer_options, Position read_from_offset, + std::optional read_to_offset) : IStreamReader(kClosed), client_(client), object_(object), read_object_(read_object) { - IStreamReader::Reset(read_object(*this, IntCast(read_from_offset)), - IStreamReaderBase::Options() - .set_assumed_pos(read_from_offset) - .set_buffer_options(buffer_options)); + IStreamReader::Reset( + read_object(*this, IntCast(read_from_offset), + read_to_offset == std::nullopt + ? std::nullopt + : std::make_optional(IntCast(*read_to_offset))), + IStreamReaderBase::Options() + .set_assumed_pos(read_from_offset) + .set_growing_source(read_to_offset != std::nullopt) + .set_buffer_options(buffer_options)); PropagateStatus(); set_limit_pos(read_from_offset); } @@ -136,7 +143,28 @@ bool GcsReader::ReadInternal(size_t min_length, size_t max_length, char* dest) { "max_length < min_length"; RIEGELI_ASSERT_OK(*this) << "Failed precondition of BufferedReader::ReadInternal()"; - const bool result = IStreamReader::ReadInternal(min_length, max_length, dest); + const Position pos_before = limit_pos(); + bool result = IStreamReader::ReadInternal(min_length, max_length, dest); + PropagateStatus(); + if (ABSL_PREDICT_TRUE(result)) return true; + if (!growing_source() || ABSL_PREDICT_FALSE(!ok())) return false; + // The requested range could not satisfy this read. Continue with a request + // for the rest of the object. + const size_t length_read = IntCast(limit_pos() - pos_before); + RIEGELI_ASSERT_LT(length_read, min_length) + << "IStreamReader::ReadInternal() returned false but read enough"; + dest += length_read; + min_length -= length_read; + max_length -= length_read; + // As in `SeekBehindBuffer()`, this request is not pinned to `generation()` + // of the previous one. Unless a generation was requested explicitly, the + // data come from the current generation, which is a different one if the + // object was overwritten in the meantime. + src() = read_object_(*this, IntCast(limit_pos()), std::nullopt); + set_growing_source(false); + PropagateStatus(); + if (ABSL_PREDICT_FALSE(!ok())) return false; + result = IStreamReader::ReadInternal(min_length, max_length, dest); PropagateStatus(); return result; } @@ -157,7 +185,8 @@ bool GcsReader::SeekBehindBuffer(Position new_pos) { // `ObjectReadStream` does not support seeking to the very end. const Position read_from_offset = UnsignedMin(new_pos, SaturatingSub(*exact_size(), Position{1})); - src() = read_object_(*this, IntCast(read_from_offset)); + src() = read_object_(*this, IntCast(read_from_offset), std::nullopt); + set_growing_source(false); PropagateStatus(); set_limit_pos(read_from_offset); if (new_pos > read_from_offset) { @@ -172,19 +201,33 @@ bool GcsReader::SeekBehindBuffer(Position new_pos) { bool GcsReader::SupportsNewReader() { return exact_size() != std::nullopt; } std::unique_ptr GcsReader::NewReaderImpl(Position initial_pos) { + // A length hint reaching the end of the object requests the rest of it. + return GcsReader::NewReaderWithLengthHintImpl( + initial_pos, std::numeric_limits::max()); +} + +std::unique_ptr GcsReader::NewReaderWithLengthHintImpl( + Position initial_pos, Position length_hint) { if (ABSL_PREDICT_FALSE(!GcsReader::SupportsNewReader())) { // Delegate to the base class to avoid repeating the error message. return IStreamReader::NewReaderImpl(initial_pos); } if (ABSL_PREDICT_FALSE(!ok())) return nullptr; - // `NewReaderImpl()` is thread-safe from this point. + // `NewReaderWithLengthHintImpl()` is thread-safe from this point. // `ObjectReadStream` does not support seeking to the very end. const Position read_from_offset = UnsignedMin(initial_pos, SaturatingSub(*exact_size(), Position{1})); + // Request only the hinted range if it ends before the object does. Otherwise + // request the rest of the object: in particular a read of the whole object + // then sends no `Range` header and keeps checksum validation. + std::optional read_to_offset; + if (length_hint > 0 && length_hint < *exact_size() - read_from_offset) { + read_to_offset = read_from_offset + length_hint; + } std::unique_ptr reader( new GcsReader(NewReaderTag(), client(), object(), read_object_, - buffer_options(), read_from_offset)); + buffer_options(), read_from_offset, read_to_offset)); reader->set_exact_size(exact_size()); if (initial_pos > read_from_offset) { if (ABSL_PREDICT_TRUE(reader->Pull())) reader->move_cursor(1); diff --git a/riegeli/gcs/gcs_reader.h b/riegeli/gcs/gcs_reader.h index f4811b81..8eb3dc89 100644 --- a/riegeli/gcs/gcs_reader.h +++ b/riegeli/gcs/gcs_reader.h @@ -124,6 +124,8 @@ class GcsReader bool ReadInternal(size_t min_length, size_t max_length, char* dest) override; bool SeekBehindBuffer(Position new_pos) override; std::unique_ptr NewReaderImpl(Position initial_pos) override; + std::unique_ptr NewReaderWithLengthHintImpl( + Position initial_pos, Position length_hint) override; private: static constexpr uint64_t kMaxPosition = std::numeric_limits::max(); @@ -145,8 +147,9 @@ class GcsReader NewReaderTag, const google::cloud::storage::Client& client, const GcsObject& object, const std::function& read_object, - BufferOptions buffer_options, Position read_from_offset); + GcsReader&, int64_t, std::optional)>& read_object, + BufferOptions buffer_options, Position read_from_offset, + std::optional read_to_offset); template static RangeOptions GetRangeOptions( @@ -192,7 +195,10 @@ class GcsReader std::optional client_; GcsObject object_; - std::function + // Opens a stream from the first argument to the second argument (exclusive) + // if given, otherwise to the end of the range specified in the constructor. + std::function)> read_object_; }; @@ -300,7 +306,16 @@ inline void GcsReader::SetReadObject( read_object_ = [new_read_object_options = DecayTuple( Filter(read_object_options...))]( - GcsReader& self, int64_t read_from_offset) mutable { + GcsReader& self, int64_t read_from_offset, + std::optional read_to_offset) mutable { + if (read_to_offset != std::nullopt) { + return self.ApplyReadObject( + new_read_object_options, + std::make_index_sequence< + std::tuple_size_v>(), + google::cloud::storage::ReadRange(read_from_offset, + *read_to_offset)); + } return self.ApplyReadObject( new_read_object_options, std::make_index_sequence< @@ -311,13 +326,15 @@ inline void GcsReader::SetReadObject( read_object_ = [new_read_object_options = DecayTuple( Filter(read_object_options...)), - max_size = *range_options.max_size](GcsReader& self, - int64_t read_from_offset) mutable { + max_size = *range_options.max_size]( + GcsReader& self, int64_t read_from_offset, + std::optional read_to_offset) mutable { return self.ApplyReadObject( new_read_object_options, std::make_index_sequence< std::tuple_size_v>(), - google::cloud::storage::ReadRange(read_from_offset, max_size)); + google::cloud::storage::ReadRange( + read_from_offset, read_to_offset.value_or(max_size))); }; } }