Eric Bower
·
2026-04-20
fs.go
1package storage
2
3import (
4 "crypto/md5"
5 "encoding/hex"
6 "fmt"
7 "io"
8 "io/fs"
9 "log/slog"
10 "os"
11 "path"
12 "path/filepath"
13 "strings"
14 "time"
15
16 "github.com/google/renameio/v2"
17 "github.com/picosh/pico/pkg/send/utils"
18 "github.com/picosh/pico/pkg/shared/mime"
19)
20
21var KB = 1000
22var MB = KB * 1000
23
24// https://stackoverflow.com/a/32482941
25func dirSize(path string) (int64, error) {
26 var size int64
27 err := filepath.Walk(path, func(_ string, info os.FileInfo, err error) error {
28 if err != nil {
29 return err
30 }
31 if !info.IsDir() {
32 size += info.Size()
33 }
34 return err
35 })
36
37 return size, err
38}
39
40type StorageFS struct {
41 Dir string
42 Logger *slog.Logger
43}
44
45var _ StorageServe = &StorageFS{}
46var _ StorageServe = (*StorageFS)(nil)
47
48func NewStorageFS(logger *slog.Logger, dir string) (*StorageFS, error) {
49 return &StorageFS{Logger: logger, Dir: dir}, nil
50}
51
52func (s *StorageFS) GetBucket(name string) (Bucket, error) {
53 dirPath := filepath.Join(s.Dir, name)
54 bucket := Bucket{
55 Name: name,
56 Path: dirPath,
57 }
58 // s.Logger.Info("get bucket", "dir", dirPath)
59
60 info, err := os.Stat(dirPath)
61 if os.IsNotExist(err) {
62 return bucket, fmt.Errorf("directory does not exist: %v %w", dirPath, err)
63 }
64
65 if err != nil {
66 return bucket, fmt.Errorf("directory error: %v %w", dirPath, err)
67
68 }
69
70 if !info.IsDir() {
71 return bucket, fmt.Errorf("directory is a file, not a directory: %#v", dirPath)
72 }
73
74 return bucket, nil
75}
76
77func (s *StorageFS) UpsertBucket(name string) (Bucket, error) {
78 s.Logger.Info("upsert bucket", "name", name)
79 bucket, err := s.GetBucket(name)
80 if err == nil {
81 return bucket, nil
82 }
83
84 dir := filepath.Join(s.Dir, name)
85 s.Logger.Info("bucket not found, creating", "dir", dir, "err", err)
86 err = os.MkdirAll(dir, os.ModePerm)
87 if err != nil {
88 return bucket, err
89 }
90
91 return bucket, nil
92}
93
94func (s *StorageFS) GetBucketQuota(bucket Bucket) (uint64, error) {
95 dsize, err := dirSize(bucket.Path)
96 return uint64(dsize), err
97}
98
99// DeleteBucket will delete all contents regardless if files exist inside of it.
100func (s *StorageFS) DeleteBucket(bucket Bucket) error {
101 return os.RemoveAll(bucket.Path)
102}
103
104func (s *StorageFS) GetObject(bucket Bucket, fpath string) (utils.ReadAndReaderAtCloser, *ObjectInfo, error) {
105 objInfo := &ObjectInfo{
106 Size: 0,
107 LastModified: time.Time{},
108 ETag: "",
109 ContentType: "",
110 }
111
112 dat, err := os.Open(filepath.Join(bucket.Path, fpath))
113 if err != nil {
114 return nil, objInfo, err
115 }
116
117 info, err := dat.Stat()
118 if err != nil {
119 _ = dat.Close()
120 return nil, objInfo, err
121 }
122
123 etag := ""
124 // only generate etag if file is less than 10MB
125 if info.Size() <= int64(10*MB) {
126 // calculate etag
127 h := md5.New()
128 if _, err := io.Copy(h, dat); err != nil {
129 _ = dat.Close()
130 return nil, objInfo, err
131 }
132 md5Sum := h.Sum(nil)
133 etag = hex.EncodeToString(md5Sum)
134
135 // reset os.File reader
136 _, err = dat.Seek(0, io.SeekStart)
137 if err != nil {
138 _ = dat.Close()
139 return nil, objInfo, err
140 }
141 }
142
143 objInfo.ETag = etag
144 objInfo.Size = info.Size()
145 objInfo.LastModified = info.ModTime()
146 objInfo.ContentType = mime.GetMimeType(fpath)
147 return dat, objInfo, nil
148}
149
150func (s *StorageFS) PutObject(bucket Bucket, fpath string, contents io.Reader, info *ObjectInfo) (string, int64, error) {
151 loc := filepath.Join(bucket.Path, fpath)
152 err := os.MkdirAll(filepath.Dir(loc), os.ModePerm)
153 if err != nil {
154 return "", 0, err
155 }
156 out, err := renameio.NewPendingFile(loc, renameio.WithPermissions(os.ModePerm))
157 if err != nil {
158 return "", 0, err
159 }
160
161 size, err := io.Copy(out, contents)
162 if err != nil {
163 return "", 0, err
164 }
165
166 if err := out.CloseAtomicallyReplace(); err != nil {
167 return "", 0, err
168 }
169
170 if !info.LastModified.IsZero() {
171 uTime := info.LastModified
172 _ = os.Chtimes(loc, uTime, uTime)
173 }
174
175 return loc, size, nil
176}
177
178func (s *StorageFS) DeleteObject(bucket Bucket, fpath string) error {
179 loc := filepath.Join(bucket.Path, fpath)
180 err := os.Remove(loc)
181 if err != nil {
182 if os.IsNotExist(err) {
183 return nil
184 }
185 return err
186 }
187
188 // traverse up the folder tree and remove all empty folders
189 dir := filepath.Dir(loc)
190 for dir != "" {
191 f, err := os.Open(dir)
192 if err != nil {
193 s.Logger.Info("open dir", "dir", dir, "err", err)
194 break
195 }
196 defer func() {
197 _ = f.Close()
198 }()
199
200 // https://stackoverflow.com/a/30708914
201 contents, err := f.Readdirnames(-1)
202 if err != nil {
203 s.Logger.Info("read dir", "dir", dir, "err", err)
204 break
205 }
206 if len(contents) > 0 {
207 break
208 }
209
210 err = os.Remove(dir)
211 if err != nil {
212 s.Logger.Info("remove dir", "dir", dir, "err", err)
213 break
214 }
215 fp := strings.Split(dir, "/")
216 prefix := ""
217 if strings.HasPrefix(loc, "/") {
218 prefix = "/"
219 }
220 dir = prefix + filepath.Join(fp[:len(fp)-1]...)
221 }
222
223 return nil
224}
225
226func (s *StorageFS) ListBuckets() ([]string, error) {
227 entries, err := os.ReadDir(s.Dir)
228 if err != nil {
229 return []string{}, err
230 }
231
232 buckets := []string{}
233 for _, e := range entries {
234 if !e.IsDir() {
235 continue
236 }
237 buckets = append(buckets, e.Name())
238 }
239 return buckets, nil
240}
241
242func (s *StorageFS) ListObjects(bucket Bucket, dir string, recursive bool) ([]os.FileInfo, error) {
243 fileList := []os.FileInfo{}
244
245 fpath := path.Join(bucket.Path, dir)
246
247 info, err := os.Stat(fpath)
248 if err != nil {
249 if os.IsNotExist(err) {
250 return fileList, nil
251 }
252 return fileList, err
253 }
254
255 if info.IsDir() && !strings.HasSuffix(dir, "/") {
256 fileList = append(fileList, &utils.VirtualFile{
257 FName: "",
258 FIsDir: info.IsDir(),
259 FSize: info.Size(),
260 FModTime: info.ModTime(),
261 })
262
263 return fileList, err
264 }
265
266 var files []utils.VirtualFile
267
268 if recursive {
269 err = filepath.WalkDir(fpath, func(s string, d fs.DirEntry, err error) error {
270 if err != nil {
271 return err
272 }
273 info, err := d.Info()
274 if err != nil {
275 return nil
276 }
277 fname := strings.TrimPrefix(s, fpath)
278 if fname == "" {
279 return nil
280 }
281 // rsync does not expect prefixed `/` so without this `rsync --delete` is borked
282 fname = strings.TrimPrefix(fname, "/")
283 files = append(files, utils.VirtualFile{
284 FName: fname,
285 FIsDir: info.IsDir(),
286 FSize: info.Size(),
287 FModTime: info.ModTime(),
288 })
289 return nil
290 })
291 if err != nil {
292 fileList = append(fileList, info)
293 return fileList, nil
294 }
295 } else {
296 fls, err := os.ReadDir(fpath)
297 if err != nil {
298 fileList = append(fileList, info)
299 return fileList, nil
300 }
301 for _, d := range fls {
302 info, err := d.Info()
303 if err != nil {
304 continue
305 }
306 fp := info.Name()
307 files = append(files, utils.VirtualFile{
308 FName: fp,
309 FIsDir: info.IsDir(),
310 FSize: info.Size(),
311 FModTime: info.ModTime(),
312 })
313 }
314 }
315
316 for _, f := range files {
317 fileList = append(fileList, &f)
318 }
319
320 return fileList, err
321}