Commit 4af7648
Eric Bower
·
2026-06-17 22:50:21 -0400 EDT
parent 96bba54
fix(rsync): properly delete `._pico_keep_dir` first so os.Remove suceeds
2 files changed,
+429,
-3
+48,
-3
1@@ -490,12 +490,57 @@ func (h *UploadAssetHandler) Delete(s *pssh.SSHServerConnSession, entry *senduti
2 projectName := shared.GetProjectName(entry)
3 logger = logger.With("project", projectName)
4
5- if assetFilepath == filepath.Join("/", projectName, "._pico_keep_dir") {
6- return os.ErrPermission
7+ logger.Info("deleting file")
8+
9+ // Check if this path represents a directory (has a . _pico_keep_dir marker)
10+ keepDirPath := filepath.Join(assetFilepath, "._pico_keep_dir")
11+ keepDirReader, _, keepDirErr := h.Cfg.Storage.GetObject(bucket, keepDirPath)
12+ if keepDirReader != nil {
13+ defer func() {
14+ _ = keepDirReader.Close()
15+ }()
16 }
17
18- logger.Info("deleting file")
19+ if keepDirErr == nil {
20+ // This is a directory being deleted. We must delete all nested
21+ // . _pico_keep_dir files first, otherwise os.Remove() on the
22+ // directory will fail with "directory not empty".
23+ nested, err := h.Cfg.Storage.ListObjects(bucket, assetFilepath+"/", true)
24+ if err != nil {
25+ return err
26+ }
27+
28+ baseDepth := strings.Count(assetFilepath, string(os.PathSeparator))
29+ for _, nestedEntry := range nested {
30+ if filepath.Base(nestedEntry.Name()) != "._pico_keep_dir" {
31+ continue
32+ }
33+ // Only delete keep_dir files that are direct or nested children
34+ nestedDepth := strings.Count(nestedEntry.Name(), string(os.PathSeparator))
35+ if nestedDepth <= baseDepth {
36+ continue
37+ }
38+ nestedKeepDirPath := filepath.Join(assetFilepath, nestedEntry.Name())
39+ if delErr := h.Cfg.Storage.DeleteObject(bucket, nestedKeepDirPath); delErr != nil {
40+ return delErr
41+ }
42+ }
43+
44+ // Delete this directory's own . _pico_keep_dir
45+ err = h.Cfg.Storage.DeleteObject(bucket, keepDirPath)
46+ if err != nil {
47+ return err
48+ }
49+
50+ // Delete the directory itself (no-op for S3-style storage, removes the dir for fs storage)
51+ _ = h.Cfg.Storage.DeleteObject(bucket, assetFilepath)
52+
53+ surrogate := getSurrogateKey(user.Name, projectName)
54+ h.Cfg.CacheClearingQueue <- surrogate
55+ return nil
56+ }
57
58+ // Regular file deletion: create . _pico_keep_dir if the directory becomes empty
59 pathDir := filepath.Dir(assetFilepath)
60 fileName := filepath.Base(assetFilepath)
61
+381,
-0
1@@ -0,0 +1,381 @@
2+package pgs
3+
4+import (
5+ "encoding/pem"
6+ "fmt"
7+ "io"
8+ "log/slog"
9+ "net"
10+ "os"
11+ "os/exec"
12+ "path/filepath"
13+ "strings"
14+ "testing"
15+ "time"
16+
17+ pgsdb "github.com/picosh/pico/pkg/apps/pgs/db"
18+ "github.com/picosh/pico/pkg/db"
19+ "github.com/picosh/pico/pkg/pssh"
20+ "github.com/picosh/pico/pkg/shared"
21+ "github.com/picosh/pico/pkg/storage"
22+ "github.com/pkg/sftp"
23+ "github.com/prometheus/client_golang/prometheus"
24+)
25+
26+// TestRsyncDeleteDirectoryWithKeepDir verifies that rsync --delete can
27+// successfully delete a directory that contains . _pico_keep_dir markers.
28+//
29+// Regression test for: "remove /storage/.../project/... directory not empty"
30+// The bug occurred because . _pico_keep_dir files were not cleaned up when
31+// their parent directory was explicitly deleted, causing os.Remove() to fail.
32+func TestRsyncDeleteDirectoryWithKeepDir(t *testing.T) {
33+ opts := &slog.HandlerOptions{
34+ AddSource: true,
35+ Level: slog.LevelDebug,
36+ }
37+ logger := slog.New(
38+ slog.NewTextHandler(os.Stdout, opts),
39+ )
40+ slog.SetDefault(logger)
41+ dbpool := pgsdb.NewDBMemory(logger)
42+ dbpool.SetupTestData()
43+
44+ // Use filesystem storage so the "directory not empty" bug manifests
45+ tmpDir, err := os.MkdirTemp("", "pgs-test-storage-*")
46+ if err != nil {
47+ t.Fatalf("failed to create temp dir: %v", err)
48+ }
49+ defer func() {
50+ _ = os.RemoveAll(tmpDir)
51+ }()
52+
53+ st, err := storage.NewStorageFS(logger, tmpDir)
54+ if err != nil {
55+ t.Fatalf("failed to create storage: %v", err)
56+ }
57+
58+ pubsub := NewPubsubChan()
59+ defer func() {
60+ _ = pubsub.Close()
61+ }()
62+
63+ _ = os.Setenv("PGS_SSH_PORT", "0")
64+ _ = os.Setenv("PGS_PROM_PORT", "0")
65+
66+ cfg := NewPgsConfig(logger, dbpool, st, pubsub)
67+ done := make(chan error)
68+ readyCh := make(chan *pssh.SSHServer)
69+ prometheus.DefaultRegisterer = prometheus.NewRegistry()
70+
71+ go StartSshServerForTesting(cfg, done, readyCh)
72+
73+ server := <-readyCh
74+ if server == nil {
75+ t.Fatal("failed to create ssh server")
76+ }
77+
78+ var actualAddr string
79+ for i := 0; i < 100; i++ {
80+ server.Mu.Lock()
81+ listener := server.Listener
82+ server.Mu.Unlock()
83+ if listener != nil {
84+ actualAddr = listener.Addr().String()
85+ break
86+ }
87+ time.Sleep(10 * time.Millisecond)
88+ }
89+ if actualAddr == "" {
90+ t.Fatal("server listener not ready")
91+ }
92+
93+ user := GenerateUser()
94+ dbpool.Pubkeys = append(dbpool.Pubkeys, &db.PublicKey{
95+ ID: "test-pubkey-keepdir",
96+ UserID: dbpool.Users[0].ID,
97+ Key: shared.KeyForKeyText(user.signer.PublicKey()),
98+ })
99+
100+ conn, err := user.NewClientAddr(actualAddr)
101+ if err != nil {
102+ t.Fatalf("failed to connect: %v", err)
103+ }
104+ defer func() {
105+ _ = conn.Close()
106+ }()
107+
108+ client, err := sftp.NewClient(conn)
109+ if err != nil {
110+ t.Fatalf("failed to create sftp client: %v", err)
111+ }
112+ defer func() {
113+ _ = client.Close()
114+ }()
115+
116+ // Create temp directory with nested structure
117+ name, err := os.MkdirTemp("", "rsync-dir-test-*")
118+ if err != nil {
119+ t.Fatalf("failed to create temp dir: %v", err)
120+ }
121+ defer func() {
122+ _ = os.RemoveAll(name)
123+ }()
124+
125+ // Create: project/subdir/nested/file.txt
126+ nestedDir := filepath.Join(name, "subdir", "nested")
127+ err = os.MkdirAll(nestedDir, 0755)
128+ if err != nil {
129+ t.Fatalf("failed to create nested dir: %v", err)
130+ }
131+ err = os.WriteFile(filepath.Join(nestedDir, "deep.txt"), []byte("deep content"), 0644)
132+ if err != nil {
133+ t.Fatalf("failed to write deep.txt: %v", err)
134+ }
135+
136+ // Create: project/subdir/file.txt
137+ err = os.WriteFile(filepath.Join(name, "subdir", "file.txt"), []byte("subdir content"), 0644)
138+ if err != nil {
139+ t.Fatalf("failed to write file.txt: %v", err)
140+ }
141+
142+ // Create: project/index.html (stays after delete)
143+ err = os.WriteFile(filepath.Join(name, "index.html"), []byte("index content"), 0644)
144+ if err != nil {
145+ t.Fatalf("failed to write index.html: %v", err)
146+ }
147+
148+ block := &pem.Block{
149+ Type: "OPENSSH PRIVATE KEY",
150+ Bytes: user.privateKey,
151+ }
152+ keyFile := filepath.Join(name, "id_ed25519")
153+ err = os.WriteFile(keyFile, pem.EncodeToMemory(block), 0600)
154+ if err != nil {
155+ t.Fatalf("failed to write key file: %v", err)
156+ }
157+
158+ _, port, err := net.SplitHostPort(actualAddr)
159+ if err != nil {
160+ t.Fatalf("failed to parse server address: %v", err)
161+ }
162+
163+ eCmd := fmt.Sprintf(
164+ "ssh -p %s -o IdentitiesOnly=yes -i %s -o StrictHostKeyChecking=no",
165+ port, keyFile,
166+ )
167+
168+ // Upload files including the subdir/ directory
169+ cmd := exec.Command("rsync", "-rv", "-e", eCmd, name+"/", "localhost:/testdir")
170+ result, err := cmd.CombinedOutput()
171+ if err != nil {
172+ t.Fatalf("rsync upload failed: %v\noutput: %s", err, result)
173+ }
174+
175+ // Verify files exist on the server
176+ _, err = client.Lstat("/testdir/subdir/file.txt")
177+ if err != nil {
178+ t.Fatalf("subdir/file.txt not found after upload: %v", err)
179+ }
180+ _, err = client.Lstat("/testdir/subdir/nested/deep.txt")
181+ if err != nil {
182+ t.Fatalf("subdir/nested/deep.txt not found after upload: %v", err)
183+ }
184+
185+ // Now remove the entire subdir/ from the local directory
186+ err = os.RemoveAll(filepath.Join(name, "subdir"))
187+ if err != nil {
188+ t.Fatalf("failed to remove local subdir: %v", err)
189+ }
190+
191+ // Run rsync --delete - this should delete the subdir/ directory
192+ // WITHOUT failing with "directory not empty"
193+ delCmd := exec.Command("rsync", "-rv", "--delete", "-e", eCmd, name+"/", "localhost:/testdir")
194+ result, err = delCmd.CombinedOutput()
195+ if err != nil {
196+ t.Fatalf("rsync --delete failed (this was the bug - 'directory not empty'): %v\noutput: %s", err, result)
197+ }
198+
199+ // Verify the subdir and its contents are gone
200+ _, err = client.Lstat("/testdir/subdir")
201+ if err == nil {
202+ t.Fatal("subdir/ should have been deleted but still exists")
203+ }
204+ // SFTP can return "no such file" or "file does not exist" depending on version
205+ if !strings.Contains(err.Error(), "no such file") && !strings.Contains(err.Error(), "does not exist") {
206+ t.Fatalf("expected 'not found' error, got: %v", err)
207+ }
208+
209+ // Verify the . _pico_keep_dir markers are also cleaned up
210+ _, err = client.Lstat("/testdir/subdir/._pico_keep_dir")
211+ if err == nil {
212+ t.Fatal("subdir/._pico_keep_dir should have been deleted but still exists")
213+ }
214+
215+ // Verify index.html still exists (it wasn't deleted)
216+ fi, err := client.Lstat("/testdir/index.html")
217+ if err != nil {
218+ t.Fatalf("index.html should still exist: %v", err)
219+ }
220+ if fi.Size() != 13 {
221+ t.Errorf("index.html has wrong size: got %d, want 13", fi.Size())
222+ }
223+
224+ close(done)
225+ time.Sleep(100 * time.Millisecond)
226+}
227+
228+// TestRsyncDeleteNestedEmptyDirectories verifies that rsync --delete handles
229+// deeply nested directory structures where intermediate directories become empty
230+// and need . _pico_keep_dir markers managed correctly.
231+func TestRsyncDeleteNestedEmptyDirectories(t *testing.T) {
232+ opts := &slog.HandlerOptions{
233+ AddSource: true,
234+ Level: slog.LevelDebug,
235+ }
236+ logger := slog.New(
237+ slog.NewTextHandler(io.Discard, opts),
238+ )
239+ slog.SetDefault(logger)
240+ dbpool := pgsdb.NewDBMemory(logger)
241+ dbpool.SetupTestData()
242+
243+ tmpDir, err := os.MkdirTemp("", "pgs-test-nested-*")
244+ if err != nil {
245+ t.Fatalf("failed to create temp dir: %v", err)
246+ }
247+ defer func() {
248+ _ = os.RemoveAll(tmpDir)
249+ }()
250+
251+ st, err := storage.NewStorageFS(logger, tmpDir)
252+ if err != nil {
253+ t.Fatalf("failed to create storage: %v", err)
254+ }
255+
256+ pubsub := NewPubsubChan()
257+ defer func() {
258+ _ = pubsub.Close()
259+ }()
260+
261+ _ = os.Setenv("PGS_SSH_PORT", "0")
262+ _ = os.Setenv("PGS_PROM_PORT", "0")
263+
264+ cfg := NewPgsConfig(logger, dbpool, st, pubsub)
265+ done := make(chan error)
266+ readyCh := make(chan *pssh.SSHServer)
267+ prometheus.DefaultRegisterer = prometheus.NewRegistry()
268+
269+ go StartSshServerForTesting(cfg, done, readyCh)
270+
271+ server := <-readyCh
272+ if server == nil {
273+ t.Fatal("failed to create ssh server")
274+ }
275+
276+ var actualAddr string
277+ for i := 0; i < 100; i++ {
278+ server.Mu.Lock()
279+ listener := server.Listener
280+ server.Mu.Unlock()
281+ if listener != nil {
282+ actualAddr = listener.Addr().String()
283+ break
284+ }
285+ time.Sleep(10 * time.Millisecond)
286+ }
287+ if actualAddr == "" {
288+ t.Fatal("server listener not ready")
289+ }
290+
291+ user := GenerateUser()
292+ dbpool.Pubkeys = append(dbpool.Pubkeys, &db.PublicKey{
293+ ID: "test-pubkey-nested",
294+ UserID: dbpool.Users[0].ID,
295+ Key: shared.KeyForKeyText(user.signer.PublicKey()),
296+ })
297+
298+ conn, err := user.NewClientAddr(actualAddr)
299+ if err != nil {
300+ t.Fatalf("failed to connect: %v", err)
301+ }
302+ defer func() {
303+ _ = conn.Close()
304+ }()
305+
306+ client, err := sftp.NewClient(conn)
307+ if err != nil {
308+ t.Fatalf("failed to create sftp client: %v", err)
309+ }
310+ defer func() {
311+ _ = client.Close()
312+ }()
313+
314+ // Create temp directory with deeply nested structure
315+ name, err := os.MkdirTemp("", "rsync-nested-test-*")
316+ if err != nil {
317+ t.Fatalf("failed to create temp dir: %v", err)
318+ }
319+ defer func() {
320+ _ = os.RemoveAll(name)
321+ }()
322+
323+ // Create: a/b/c/d/file.txt
324+ deepDir := filepath.Join(name, "a", "b", "c", "d")
325+ err = os.MkdirAll(deepDir, 0755)
326+ if err != nil {
327+ t.Fatalf("failed to create deep dir: %v", err)
328+ }
329+ err = os.WriteFile(filepath.Join(deepDir, "file.txt"), []byte("deep"), 0644)
330+ if err != nil {
331+ t.Fatalf("failed to write file: %v", err)
332+ }
333+
334+ block := &pem.Block{
335+ Type: "OPENSSH PRIVATE KEY",
336+ Bytes: user.privateKey,
337+ }
338+ keyFile := filepath.Join(name, "id_ed25519")
339+ err = os.WriteFile(keyFile, pem.EncodeToMemory(block), 0600)
340+ if err != nil {
341+ t.Fatalf("failed to write key file: %v", err)
342+ }
343+
344+ _, port, err := net.SplitHostPort(actualAddr)
345+ if err != nil {
346+ t.Fatalf("failed to parse server address: %v", err)
347+ }
348+
349+ eCmd := fmt.Sprintf(
350+ "ssh -p %s -o IdentitiesOnly=yes -i %s -o StrictHostKeyChecking=no",
351+ port, keyFile,
352+ )
353+
354+ // Upload
355+ cmd := exec.Command("rsync", "-rv", "-e", eCmd, name+"/", "localhost:/deep")
356+ result, err := cmd.CombinedOutput()
357+ if err != nil {
358+ t.Fatalf("rsync upload failed: %v\noutput: %s", err, result)
359+ }
360+
361+ // Remove the entire a/ directory locally
362+ err = os.RemoveAll(filepath.Join(name, "a"))
363+ if err != nil {
364+ t.Fatalf("failed to remove local a/: %v", err)
365+ }
366+
367+ // rsync --delete should handle the deeply nested structure
368+ delCmd := exec.Command("rsync", "-rv", "--delete", "-e", eCmd, name+"/", "localhost:/deep")
369+ result, err = delCmd.CombinedOutput()
370+ if err != nil {
371+ t.Fatalf("rsync --delete failed on nested dirs: %v\noutput: %s", err, result)
372+ }
373+
374+ // Verify the entire tree is gone
375+ _, err = client.Lstat("/deep/a")
376+ if err == nil {
377+ t.Fatal("/deep/a should have been deleted but still exists")
378+ }
379+
380+ close(done)
381+ time.Sleep(100 * time.Millisecond)
382+}