Eric Bower
·
2026-10-12
1package storage_test
2
3import (
4 "bytes"
5 "crypto/rand"
6 "errors"
7 "fmt"
8 "io"
9 "io/fs"
10 "log/slog"
11 "slices"
12 "strings"
13 "testing"
14 "time"
15
16 "github.com/picosh/pico/pkg/storage"
17 "github.com/picosh/pico/pkg/storage/storagetest"
18)
19
20func backends(t *testing.T) map[string]func(t *testing.T) storage.StorageServe {
21 return map[string]func(t *testing.T) storage.StorageServe{
22 "fs": func(t *testing.T) storage.StorageServe {
23 st, err := storage.NewStorageFS(slog.New(slog.DiscardHandler), t.TempDir())
24 if err != nil {
25 t.Fatal(err)
26 }
27 return st
28 },
29 "memory": func(t *testing.T) storage.StorageServe {
30 st, err := storage.NewStorageMemory(map[string]map[string]string{})
31 if err != nil {
32 t.Fatal(err)
33 }
34 return st
35 },
36 }
37}
38
39func put(t *testing.T, st storage.StorageServe, b storage.Bucket, name string, data []byte, mtime time.Time) {
40 t.Helper()
41 _, size, err := st.PutObject(b, name, bytes.NewReader(data), &storage.ObjectInfo{LastModified: mtime})
42 if err != nil {
43 t.Fatalf("put %s: %v", name, err)
44 }
45 if size != int64(len(data)) {
46 t.Fatalf("put %s: size %d, want %d", name, size, len(data))
47 }
48}
49
50// entry renders a listing entry so listings compare as sorted strings.
51func entry(name string, isDir bool, size int64, mtime time.Time) string {
52 if isDir {
53 return name + "/"
54 }
55 return fmt.Sprintf("%s %d %d", name, size, mtime.Unix())
56}
57
58func list(t *testing.T, st storage.StorageServe, b storage.Bucket, dir string, recursive bool) []string {
59 t.Helper()
60 infos, err := st.ListObjects(b, dir, recursive)
61 if err != nil {
62 t.Fatalf("list %s: %v", dir, err)
63 }
64 out := []string{}
65 for _, info := range infos {
66 out = append(out, entry(info.Name(), info.IsDir(), info.Size(), info.ModTime()))
67 }
68 slices.Sort(out)
69 return out
70}
71
72func TestStorageConformance(t *testing.T) {
73 for name, newStorage := range backends(t) {
74 t.Run(name, func(t *testing.T) {
75 st := newStorage(t)
76 b := storagetest.Bucket(t, st, "conformance")
77
78 t1 := time.Unix(1_700_000_000, 0)
79 t2 := time.Unix(1_700_100_000, 0)
80 put(t, st, b, "/proj/index.html", []byte("<html></html>"), t1)
81 put(t, st, b, "/proj/css/site.css", []byte("body{}"), t2)
82 put(t, st, b, "/proj/css/deep/x.css", []byte("a{}"), t1)
83 put(t, st, b, "/other.txt", []byte("other"), t2)
84 before := time.Now().Add(-time.Second)
85 put(t, st, b, "/proj/now.txt", []byte("now"), time.Time{})
86
87 t.Run("get", func(t *testing.T) {
88 r, info, err := st.GetObject(b, "/proj/index.html")
89 if err != nil {
90 t.Fatal(err)
91 }
92 defer func() { _ = r.Close() }()
93 data, _ := io.ReadAll(r)
94 if string(data) != "<html></html>" || info.Size != 13 || !info.LastModified.Equal(t1) {
95 t.Fatalf("got %q size=%d mtime=%v", data, info.Size, info.LastModified)
96 }
97
98 _, info, err = st.GetObject(b, "/proj/now.txt")
99 if err != nil {
100 t.Fatal(err)
101 }
102 if info.LastModified.Before(before) || info.LastModified.After(time.Now().Add(time.Second)) {
103 t.Fatalf("object stored without an mtime has mtime %v", info.LastModified)
104 }
105
106 if _, _, err := st.GetObject(b, "/proj/missing.txt"); err == nil {
107 t.Fatal("missing object returned no error")
108 }
109
110 for _, dir := range []string{"/proj/css", "/proj/css/"} {
111 if _, _, err := st.GetObject(b, dir); !errors.Is(err, fs.ErrNotExist) {
112 t.Errorf("GetObject(%q) returned %v, want fs.ErrNotExist", dir, err)
113 }
114 }
115 })
116
117 t.Run("stat", func(t *testing.T) {
118 info, err := st.StatObject(b, "/proj/css/site.css")
119 if err != nil {
120 t.Fatal(err)
121 }
122 if info.Size != 6 || !info.LastModified.Equal(t2) {
123 t.Fatalf("size=%d mtime=%v", info.Size, info.LastModified)
124 }
125 for _, missing := range []string{"/proj/missing.txt", "/proj/css", "/proj/css/"} {
126 if _, err := st.StatObject(b, missing); !errors.Is(err, fs.ErrNotExist) {
127 t.Errorf("StatObject(%q) returned %v, want fs.ErrNotExist", missing, err)
128 }
129 }
130 })
131
132 t.Run("list", func(t *testing.T) {
133 now := list(t, st, b, "/proj/now.txt", false)[0]
134 tests := []struct {
135 dir string
136 recursive bool
137 want []string
138 }{
139 {"/proj/index.html", false, []string{entry("index.html", false, 13, t1)}},
140 {"/proj", false, []string{"/"}},
141 {"/proj/", false, []string{"css/", entry("index.html", false, 13, t1), now}},
142 {"/proj/", true, []string{
143 "css/",
144 entry("css/deep/x.css", false, 3, t1),
145 "css/deep/",
146 entry("css/site.css", false, 6, t2),
147 entry("index.html", false, 13, t1),
148 now,
149 }},
150 {"/proj/css/", true, []string{entry("deep/x.css", false, 3, t1), "deep/", entry("site.css", false, 6, t2)}},
151 {"/missing", false, []string{}},
152 {"/missing/", true, []string{}},
153 }
154 for _, tt := range tests {
155 want := slices.Clone(tt.want)
156 slices.Sort(want)
157 if got := list(t, st, b, tt.dir, tt.recursive); !slices.Equal(got, want) {
158 t.Errorf("list(%q, recursive=%v)\n got %q\nwant %q", tt.dir, tt.recursive, got, want)
159 }
160 }
161 })
162
163 t.Run("overwrite", func(t *testing.T) {
164 put(t, st, b, "/proj/css/site.css", []byte("body{color:red}"), t1)
165 r, info, err := st.GetObject(b, "/proj/css/site.css")
166 if err != nil {
167 t.Fatal(err)
168 }
169 data, _ := io.ReadAll(r)
170 _ = r.Close()
171 if string(data) != "body{color:red}" || !info.LastModified.Equal(t1) {
172 t.Fatalf("got %q mtime=%v", data, info.LastModified)
173 }
174 })
175
176 t.Run("random access", func(t *testing.T) {
177 data := make([]byte, 4<<20+12345)
178 _, _ = rand.Read(data)
179 put(t, st, b, "/proj/big.bin", data, t2)
180
181 r, info, err := st.GetObject(b, "/proj/big.bin")
182 if err != nil {
183 t.Fatal(err)
184 }
185 defer func() { _ = r.Close() }()
186 if info.Size != int64(len(data)) || !info.LastModified.Equal(t2) {
187 t.Fatalf("size=%d mtime=%v", info.Size, info.LastModified)
188 }
189 for _, off := range []int64{0, 1<<20 - 10, 5, int64(len(data)) - 100, 3 << 20} {
190 buf := make([]byte, 64<<10)
191 n, err := r.ReadAt(buf, off)
192 want := data[off:min(off+int64(len(buf)), int64(len(data)))]
193 if !bytes.Equal(buf[:n], want) {
194 t.Fatalf("ReadAt(%d) returned wrong data", off)
195 }
196 if n < len(buf) && err != io.EOF {
197 t.Fatalf("short ReadAt(%d) returned %v", off, err)
198 }
199 }
200 if _, err := r.Seek(int64(len(data))-1000, io.SeekStart); err != nil {
201 t.Fatal(err)
202 }
203 tail, err := io.ReadAll(r)
204 if err != nil || !bytes.Equal(tail, data[len(data)-1000:]) {
205 t.Fatalf("read after seek: %d bytes, %v", len(tail), err)
206 }
207 if _, err := r.Seek(0, io.SeekStart); err != nil {
208 t.Fatal(err)
209 }
210 all, err := io.ReadAll(r)
211 if err != nil || !bytes.Equal(all, data) {
212 t.Fatalf("full read: %d bytes, %v", len(all), err)
213 }
214 })
215
216 t.Run("failed upload", func(t *testing.T) {
217 body := io.MultiReader(strings.NewReader(strings.Repeat("x", 1000)), errReader{})
218 _, _, err := st.PutObject(b, "/proj/broken.txt", body, &storage.ObjectInfo{})
219 if err == nil {
220 t.Fatal("upload from a failing reader succeeded")
221 }
222 r, _, err := st.GetObject(b, "/proj/broken.txt")
223 if err == nil {
224 data, _ := io.ReadAll(r)
225 _ = r.Close()
226 t.Fatalf("failed upload left an object of %d bytes", len(data))
227 }
228 })
229
230 t.Run("delete", func(t *testing.T) {
231 if err := st.DeleteObject(b, "/proj/css/deep/x.css"); err != nil {
232 t.Fatal(err)
233 }
234 if err := st.DeleteObject(b, "/proj/css/deep/x.css"); err != nil {
235 t.Fatalf("deleting a missing object: %v", err)
236 }
237 if _, _, err := st.GetObject(b, "/proj/css/deep/x.css"); err == nil {
238 t.Fatal("deleted object still readable")
239 }
240 if _, err := st.StatObject(b, "/proj/css/deep/x.css"); !errors.Is(err, fs.ErrNotExist) {
241 t.Fatalf("StatObject after delete returned %v", err)
242 }
243 got := list(t, st, b, "/proj/css/", true)
244 want := []string{"deep/", entry("site.css", false, 15, t1)}
245 if !slices.Equal(got, want) {
246 t.Fatalf("after delete got %q, want %q", got, want)
247 }
248 })
249
250 t.Run("directories", func(t *testing.T) {
251 if err := st.PutDir(b, "/proj/empty/nested"); err != nil {
252 t.Fatal(err)
253 }
254 if got, want := list(t, st, b, "/proj/empty/", true), []string{"nested/"}; !slices.Equal(got, want) {
255 t.Fatalf("list after PutDir got %q, want %q", got, want)
256 }
257 if got := list(t, st, b, "/proj/empty/nested", false); !slices.Equal(got, []string{"/"}) {
258 t.Fatalf("empty directory not found: %q", got)
259 }
260 if _, err := st.StatObject(b, "/proj/empty/nested"); !errors.Is(err, fs.ErrNotExist) {
261 t.Fatalf("StatObject on a directory returned %v", err)
262 }
263
264 if err := st.DeleteObject(b, "/proj/empty"); err == nil {
265 t.Fatal("deleting a non-empty directory succeeded")
266 }
267 for _, dir := range []string{"/proj/empty/nested", "/proj/empty", "/proj/css/deep"} {
268 if err := st.DeleteObject(b, dir); err != nil {
269 t.Fatalf("delete %s: %v", dir, err)
270 }
271 if got := list(t, st, b, dir, false); len(got) != 0 {
272 t.Fatalf("%s still listed as %q", dir, got)
273 }
274 }
275
276 put(t, st, b, "/solo/only.txt", []byte("x"), t1)
277 if err := st.DeleteObject(b, "/solo/only.txt"); err != nil {
278 t.Fatal(err)
279 }
280 if got := list(t, st, b, "/", false); !slices.Contains(got, "solo/") {
281 t.Fatalf("directory went with its last file: %q", got)
282 }
283 if err := st.DeleteObject(b, "/solo"); err != nil {
284 t.Fatal(err)
285 }
286 if got := list(t, st, b, "/", false); slices.Contains(got, "solo/") {
287 t.Fatalf("deleted directory still listed: %q", got)
288 }
289 })
290
291 t.Run("quota", func(t *testing.T) {
292 size, err := st.GetBucketQuota(b)
293 if err != nil {
294 t.Fatal(err)
295 }
296 want := uint64(13 + 15 + 5 + 3 + 4<<20 + 12345)
297 if size != want {
298 t.Fatalf("quota %d, want %d", size, want)
299 }
300 })
301 })
302 }
303}
304
305type errReader struct{}
306
307func (errReader) Read([]byte) (int, error) { return 0, fmt.Errorf("connection reset") }