From e0abae3af2b63a19689d93e71199f7fbe13c3cfe Mon Sep 17 00:00:00 2001 From: Matthew Date: Fri, 13 Nov 2026 09:57:12 -0500 Subject: [PATCH] localstore: find files when the base path is `.` With a base path of `.`, walk filtered on the `./` prefix, but filepath.Walk(".") reports `file` and not `./file`, so every file was skipped and Walk, WalkFrom and the listings returned nothing. Resolve the base path to an absolute path when the store is created. Fixes streamingfast/firehose-core#56 (`firecore tools check merged-blocks .`). Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 2 ++ localstore.go | 7 ++++++- localstore_test.go | 34 ++++++++++++++++++++++++++++++++++ 3 files changed, 42 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cead260..be19d24 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -36,6 +36,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ### Fixed +* A local store whose base path is `.` now finds its files. `Walk`, `WalkFrom` and the listings returned nothing, because `filepath.Walk(".")` reports `file` and not `./file`, which never matched the `./` prefix the walk filtered on. The base path is now made absolute when the store is created. + * `S3Store.WriteObject` no longer commits a partial object when its reader fails partway. The reader's error was turned into a clean end of stream before the upload was cancelled, so an upload that got there first completed with the content read so far and `WriteObject` returned `nil`. The upload now fails with the reader's error whatever the timing. * zstd stores now close the object body, a file or an HTTP response, when the reader returned by `OpenObject` is closed, as gzip stores already did. Closing only stopped the decoder, so a reader closed before the end of the object kept its HTTP stream or connection open until its context was canceled, never when opened with `context.Background()`, and local stores kept the file descriptor until garbage collection. The body is also closed when the zstd reader cannot be created. diff --git a/localstore.go b/localstore.go index 6c32014..7a99716 100644 --- a/localstore.go +++ b/localstore.go @@ -38,7 +38,12 @@ func newLocalStoreContext(_ context.Context, baseURL *url.URL, extension, compre } rand.Seed(time.Now().UnixNano()) - basePath := filepath.Clean(baseURL.Path) + // Absolute so that walk can match the paths filepath.Walk returns against it: for a + // base path of `.`, filepath.Walk returns `file` and not `./file`. + basePath, err := filepath.Abs(baseURL.Path) + if err != nil { + return nil, fmt.Errorf("unable to resolve base path %q: %w", baseURL.Path, err) + } zlog.Debug("sanitized base path", zap.String("original_base_path", baseURL.Path), zap.String("sanitized_base_path", basePath)) myBaseURL := *baseURL diff --git a/localstore_test.go b/localstore_test.go index 419b7ca..cee8185 100644 --- a/localstore_test.go +++ b/localstore_test.go @@ -2,8 +2,11 @@ package dstore import ( "context" + "io" "math" "net/url" + "os" + "path/filepath" "strings" "testing" @@ -38,6 +41,37 @@ func TestNewLocalStore_SubStoreRelative(t *testing.T) { require.True(t, strings.HasSuffix(sub.BaseURL().Path, "sub-folder")) } +func TestNewLocalStore_CurrentDirectory(t *testing.T) { + dir := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(dir, "0000000100"), []byte("a"), 0644)) + require.NoError(t, os.WriteFile(filepath.Join(dir, "0000000200"), []byte("b"), 0644)) + t.Chdir(dir) + + store, err := NewLocalStore(&url.URL{Scheme: "", Path: "."}, "", "", false) + require.NoError(t, err) + + var seen []string + require.NoError(t, store.Walk(context.Background(), "", func(filename string) error { + seen = append(seen, filename) + return nil + })) + assert.Equal(t, []string{"0000000100", "0000000200"}, seen) + + seen = nil + require.NoError(t, store.WalkFrom(context.Background(), "", "0000000200", func(filename string) error { + seen = append(seen, filename) + return nil + })) + assert.Equal(t, []string{"0000000200"}, seen) + + reader, err := store.OpenObject(context.Background(), "0000000100") + require.NoError(t, err) + content, err := io.ReadAll(reader) + require.NoError(t, err) + reader.Close() + assert.Equal(t, "a", string(content)) +} + func TestNewLocalStore_SubStore(t *testing.T) { store, err := NewLocalStore(&url.URL{Scheme: "", Path: "/tmp/storage/"}, "", "", false) require.NoError(t, err)