Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),

### Fixed

* A local store whose base path is `.` now finds its files. `Walk`, `WalkFrom` and the listings returned nothing, because `filepath.Walk(".")` reports `file` and not `./file`, which never matched the `./` prefix the walk filtered on. The base path is now made absolute when the store is created.

* `S3Store.WriteObject` no longer commits a partial object when its reader fails partway. The reader's error was turned into a clean end of stream before the upload was cancelled, so an upload that got there first completed with the content read so far and `WriteObject` returned `nil`. The upload now fails with the reader's error whatever the timing.

* zstd stores now close the object body, a file or an HTTP response, when the reader returned by `OpenObject` is closed, as gzip stores already did. Closing only stopped the decoder, so a reader closed before the end of the object kept its HTTP stream or connection open until its context was canceled, never when opened with `context.Background()`, and local stores kept the file descriptor until garbage collection. The body is also closed when the zstd reader cannot be created.
Expand Down
7 changes: 6 additions & 1 deletion localstore.go
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,12 @@ func newLocalStoreContext(_ context.Context, baseURL *url.URL, extension, compre
}

rand.Seed(time.Now().UnixNano())
basePath := filepath.Clean(baseURL.Path)
// Absolute so that walk can match the paths filepath.Walk returns against it: for a
// base path of `.`, filepath.Walk returns `file` and not `./file`.
basePath, err := filepath.Abs(baseURL.Path)
if err != nil {
return nil, fmt.Errorf("unable to resolve base path %q: %w", baseURL.Path, err)
}
zlog.Debug("sanitized base path", zap.String("original_base_path", baseURL.Path), zap.String("sanitized_base_path", basePath))

myBaseURL := *baseURL
Expand Down
34 changes: 34 additions & 0 deletions localstore_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,11 @@ package dstore

import (
"context"
"io"
"math"
"net/url"
"os"
"path/filepath"
"strings"
"testing"

Expand Down Expand Up @@ -38,6 +41,37 @@ func TestNewLocalStore_SubStoreRelative(t *testing.T) {
require.True(t, strings.HasSuffix(sub.BaseURL().Path, "sub-folder"))

}
func TestNewLocalStore_CurrentDirectory(t *testing.T) {
dir := t.TempDir()
require.NoError(t, os.WriteFile(filepath.Join(dir, "0000000100"), []byte("a"), 0644))
require.NoError(t, os.WriteFile(filepath.Join(dir, "0000000200"), []byte("b"), 0644))
t.Chdir(dir)

store, err := NewLocalStore(&url.URL{Scheme: "", Path: "."}, "", "", false)
require.NoError(t, err)

var seen []string
require.NoError(t, store.Walk(context.Background(), "", func(filename string) error {
seen = append(seen, filename)
return nil
}))
assert.Equal(t, []string{"0000000100", "0000000200"}, seen)

seen = nil
require.NoError(t, store.WalkFrom(context.Background(), "", "0000000200", func(filename string) error {
seen = append(seen, filename)
return nil
}))
assert.Equal(t, []string{"0000000200"}, seen)

reader, err := store.OpenObject(context.Background(), "0000000100")
require.NoError(t, err)
content, err := io.ReadAll(reader)
require.NoError(t, err)
reader.Close()
assert.Equal(t, "a", string(content))
}

func TestNewLocalStore_SubStore(t *testing.T) {
store, err := NewLocalStore(&url.URL{Scheme: "", Path: "/tmp/storage/"}, "", "", false)
require.NoError(t, err)
Expand Down
Loading