Antonio Mika
·
2026-10-11
1package rsync
2
3import (
4 "bytes"
5 "crypto/rand"
6 "errors"
7 "fmt"
8 "io"
9 "io/fs"
10 "os"
11 "os/exec"
12 "path/filepath"
13 "slices"
14 "strings"
15 "testing"
16 "time"
17)
18
19// dirFS serves a local directory.
20type dirFS struct{ root string }
21
22func (d dirFS) full(name string) string { return filepath.Join(d.root, filepath.FromSlash(name)) }
23
24func (d dirFS) Stat(name string) (FileInfo, error) {
25 st, err := os.Stat(d.full(name))
26 if err != nil {
27 return FileInfo{}, err
28 }
29 return FileInfo{Name: st.Name(), Size: st.Size(), ModTime: st.ModTime(), IsDir: st.IsDir()}, nil
30}
31
32func (d dirFS) ReadDir(dir string, recursive bool) ([]FileInfo, error) {
33 base := d.full(dir)
34 var out []FileInfo
35 err := filepath.WalkDir(base, func(p string, e fs.DirEntry, err error) error {
36 if err != nil {
37 return err
38 }
39 if p == base {
40 return nil
41 }
42 st, err := e.Info()
43 if err != nil {
44 return err
45 }
46 rel, _ := filepath.Rel(base, p)
47 out = append(out, FileInfo{Name: filepath.ToSlash(rel), Size: st.Size(), ModTime: st.ModTime(), IsDir: st.IsDir()})
48 if e.IsDir() && !recursive {
49 return filepath.SkipDir
50 }
51 return nil
52 })
53 return out, err
54}
55
56func (d dirFS) Open(name string) (File, FileInfo, error) {
57 f, err := os.Open(d.full(name))
58 if err != nil {
59 return nil, FileInfo{}, err
60 }
61 st, err := f.Stat()
62 if err != nil {
63 _ = f.Close()
64 return nil, FileInfo{}, err
65 }
66 return f, FileInfo{Name: st.Name(), Size: st.Size(), ModTime: st.ModTime()}, nil
67}
68
69func (d dirFS) Put(name string, info FileInfo, r io.Reader) (string, error) {
70 p := d.full(name)
71 if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil {
72 return "", err
73 }
74 tmp, err := os.CreateTemp(filepath.Dir(p), ".tmp-")
75 if err != nil {
76 return "", err
77 }
78 if _, err := io.Copy(tmp, r); err != nil {
79 _ = tmp.Close()
80 _ = os.Remove(tmp.Name())
81 return "", err
82 }
83 if err := tmp.Close(); err != nil {
84 return "", err
85 }
86 if err := os.Rename(tmp.Name(), p); err != nil {
87 return "", err
88 }
89 return "", os.Chtimes(p, info.ModTime, info.ModTime)
90}
91
92func (d dirFS) Remove(name string, isDir bool) error { return os.Remove(d.full(name)) }
93
94type rejectingFS struct {
95 dirFS
96 err error
97}
98
99func (d rejectingFS) Put(name string, info FileInfo, r io.Reader) (string, error) {
100 if filepath.Base(name) == "skip.txt" {
101 return "", d.err
102 }
103 return d.dirFS.Put(name, info, r)
104}
105
106// TestHelperProcess is the remote shell rsync runs in these tests. It serves
107// the directory in RSYNC_TEST_ROOT on stdin and stdout.
108func TestHelperProcess(t *testing.T) {
109 root := os.Getenv("RSYNC_TEST_ROOT")
110 if root == "" {
111 t.Skip("helper process")
112 }
113 args := os.Args
114 for len(args) > 0 && args[0] != "--" {
115 args = args[1:]
116 }
117 // Skip "--", the host name and "rsync".
118 if len(args) < 3 {
119 os.Exit(2)
120 }
121 args = args[3:]
122 rw := struct {
123 io.Reader
124 io.Writer
125 }{os.Stdin, os.Stdout}
126 var fsys FS = dirFS{root: root}
127 switch os.Getenv("RSYNC_TEST_PUT_RESULT") {
128 case "ignored":
129 fsys = rejectingFS{dirFS{root: root}, fmt.Errorf("policy: %w", ErrFileIgnored)}
130 case "failed":
131 fsys = rejectingFS{dirFS{root: root}, errors.New("storage backend failed")}
132 }
133 err := Serve(Config{FS: fsys, Stderr: os.Stderr}, rw, args)
134 code := 0
135 var exit *ExitError
136 if errors.As(err, &exit) {
137 code = exit.Code
138 } else if err != nil {
139 code = 1
140 }
141 os.Exit(code)
142}
143
144type rsyncEnv struct {
145 t *testing.T
146 remote string // directory our server serves
147 local string
148}
149
150func newRsyncEnv(t *testing.T) *rsyncEnv {
151 t.Helper()
152 if _, err := exec.LookPath("rsync"); err != nil {
153 t.Skip("rsync binary not found")
154 }
155 return &rsyncEnv{t: t, remote: t.TempDir(), local: t.TempDir()}
156}
157
158// run executes the rsync client with our server on the other end.
159func (e *rsyncEnv) run(args ...string) (string, error) {
160 e.t.Helper()
161 rsh := fmt.Sprintf("%s -test.run=^TestHelperProcess$ --", os.Args[0])
162 cmd := exec.Command("rsync", append([]string{"-e", rsh}, args...)...)
163 // A race build sleeps a second before exiting, once per helper.
164 gorace := strings.TrimSpace(os.Getenv("GORACE") + " atexit_sleep_ms=0")
165 cmd.Env = append(os.Environ(), "RSYNC_TEST_ROOT="+e.remote, "GORACE="+gorace)
166 out, err := cmd.CombinedOutput()
167 return string(out), err
168}
169
170func (e *rsyncEnv) mustRun(args ...string) string {
171 e.t.Helper()
172 out, err := e.run(args...)
173 if err != nil {
174 e.t.Fatalf("rsync %s: %v\n%s", strings.Join(args, " "), err, out)
175 }
176 return out
177}
178
179func writeFile(t *testing.T, path string, data []byte) {
180 t.Helper()
181 if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
182 t.Fatal(err)
183 }
184 if err := os.WriteFile(path, data, 0o644); err != nil {
185 t.Fatal(err)
186 }
187}
188
189func randomBytes(t *testing.T, n int) []byte {
190 t.Helper()
191 b := make([]byte, n)
192 if _, err := rand.Read(b); err != nil {
193 t.Fatal(err)
194 }
195 return b
196}
197
198// textBytes returns compressible data.
199func textBytes(n int, seed string) []byte {
200 var b bytes.Buffer
201 for i := 0; b.Len() < n; i++ {
202 fmt.Fprintf(&b, "%s line %d: the quick brown fox jumps over the lazy dog\n", seed, i)
203 }
204 return b.Bytes()[:n]
205}
206
207// tree returns the regular files below root with their contents.
208func tree(t *testing.T, root string) map[string]string {
209 t.Helper()
210 out := map[string]string{}
211 err := filepath.WalkDir(root, func(p string, e fs.DirEntry, err error) error {
212 if err != nil {
213 return err
214 }
215 if e.Type().IsRegular() {
216 b, err := os.ReadFile(p)
217 if err != nil {
218 return err
219 }
220 rel, _ := filepath.Rel(root, p)
221 out[filepath.ToSlash(rel)] = string(b)
222 }
223 return nil
224 })
225 if err != nil {
226 t.Fatal(err)
227 }
228 return out
229}
230
231func assertSameTree(t *testing.T, want, got string) {
232 t.Helper()
233 a, b := tree(t, want), tree(t, got)
234 var names []string
235 for k := range a {
236 names = append(names, k)
237 }
238 for k := range b {
239 if _, ok := a[k]; !ok {
240 names = append(names, k)
241 }
242 }
243 slices.Sort(names)
244 for _, n := range slices.Compact(names) {
245 av, aok := a[n]
246 bv, bok := b[n]
247 switch {
248 case !aok:
249 t.Errorf("unexpected file %s", n)
250 case !bok:
251 t.Errorf("missing file %s", n)
252 case av != bv:
253 t.Errorf("file %s differs (%d bytes, want %d)", n, len(bv), len(av))
254 }
255 }
256}
257
258func populate(t *testing.T, root string) {
259 t.Helper()
260 writeFile(t, filepath.Join(root, "index.html"), textBytes(3000, "index"))
261 writeFile(t, filepath.Join(root, "empty.txt"), nil)
262 writeFile(t, filepath.Join(root, "css/site.css"), textBytes(800, "css"))
263 writeFile(t, filepath.Join(root, "img/logo.bin"), randomBytes(t, 200_000))
264 writeFile(t, filepath.Join(root, "a/b/c/deep.txt"), textBytes(50, "deep"))
265 writeFile(t, filepath.Join(root, "a.b"), []byte("dot file next to dir a"))
266 writeFile(t, filepath.Join(root, "a-b"), []byte("dash file next to dir a"))
267 writeFile(t, filepath.Join(root, "big.dat"), randomBytes(t, 3_000_000))
268}
269
270var protocols = []string{"27", "28", "29", "30", "31", ""}
271
272func protoArgs(p string) []string {
273 if p == "" {
274 return nil
275 }
276 return []string{"--protocol=" + p}
277}
278
279func TestRsyncUpload(t *testing.T) {
280 for _, proto := range protocols {
281 for _, compress := range [][]string{nil, {"-z"}, {"--compress-choice=zlib"}, {"--compress-choice=zlibx"}} {
282 name := fmt.Sprintf("proto=%s/%s", proto, strings.Join(compress, ""))
283 t.Run(name, func(t *testing.T) {
284 if proto != "" && proto < "30" && len(compress) > 0 && compress[0] != "-z" {
285 t.Skip("compression choice needs protocol 30")
286 }
287 e := newRsyncEnv(t)
288 populate(t, e.local)
289 args := append(protoArgs(proto), compress...)
290 e.mustRun(append(args, "-rt", e.local+"/", "host:site")...)
291 assertSameTree(t, e.local, filepath.Join(e.remote, "site"))
292
293 // Change part of a big file and upload again: only the
294 // changed region should cross the wire.
295 big := filepath.Join(e.local, "big.dat")
296 data, _ := os.ReadFile(big)
297 copy(data[1_000_000:], randomBytes(t, 5000))
298 data = append(data[:2_000_000], append(randomBytes(t, 1234), data[2_000_000:]...)...)
299 writeFile(t, big, data)
300 writeFile(t, filepath.Join(e.local, "css/site.css"), textBytes(900, "css2"))
301 out := e.mustRun(append(args, "-rt", "--stats", e.local+"/", "host:site")...)
302 assertSameTree(t, e.local, filepath.Join(e.remote, "site"))
303 literal := statValue(t, out, "Literal data")
304 if literal > 100_000 {
305 t.Errorf("literal data %d, want a delta transfer\n%s", literal, out)
306 }
307 if matched := statValue(t, out, "Matched data"); matched < 2_000_000 {
308 t.Errorf("matched data %d, want most of big.dat\n%s", matched, out)
309 }
310 })
311 }
312 }
313}
314
315func TestRsyncDownload(t *testing.T) {
316 for _, proto := range protocols {
317 for _, compress := range [][]string{nil, {"-z"}, {"--compress-choice=zlib"}} {
318 name := fmt.Sprintf("proto=%s/%s", proto, strings.Join(compress, ""))
319 t.Run(name, func(t *testing.T) {
320 if proto != "" && proto < "30" && len(compress) > 0 && compress[0] != "-z" {
321 t.Skip("compression choice needs protocol 30")
322 }
323 e := newRsyncEnv(t)
324 site := filepath.Join(e.remote, "site")
325 populate(t, site)
326 args := append(protoArgs(proto), compress...)
327 e.mustRun(append(args, "-rt", "host:site/", e.local+"/")...)
328 assertSameTree(t, site, e.local)
329
330 // Change the server's copy and download again.
331 big := filepath.Join(site, "big.dat")
332 data, _ := os.ReadFile(big)
333 copy(data[500_000:], randomBytes(t, 3000))
334 writeFile(t, big, data)
335 // rsync compares whole seconds, so make the change visible.
336 later := time.Now().Add(time.Minute)
337 if err := os.Chtimes(big, later, later); err != nil {
338 t.Fatal(err)
339 }
340 out := e.mustRun(append(args, "-rt", "--stats", "host:site/", e.local+"/")...)
341 assertSameTree(t, site, e.local)
342 if literal := statValue(t, out, "Literal data"); literal > 100_000 {
343 t.Errorf("literal data %d, want a delta transfer\n%s", literal, out)
344 }
345 })
346 }
347 }
348}
349
350func TestRsyncChecksumChoices(t *testing.T) {
351 for _, choice := range []string{"md4", "md5", "xxh64", "md5,md4"} {
352 t.Run(choice, func(t *testing.T) {
353 e := newRsyncEnv(t)
354 populate(t, e.local)
355 e.mustRun("-r", "--checksum-choice="+choice, e.local+"/", "host:site")
356 assertSameTree(t, e.local, filepath.Join(e.remote, "site"))
357 e.mustRun("-r", "-c", "--checksum-choice="+choice, "host:site/", t.TempDir())
358 })
359 }
360}
361
362func TestRsyncAlwaysChecksum(t *testing.T) {
363 for _, proto := range protocols {
364 t.Run("proto="+proto, func(t *testing.T) {
365 e := newRsyncEnv(t)
366 populate(t, e.local)
367 args := protoArgs(proto)
368 e.mustRun(append(args, "-r", e.local+"/", "host:site")...)
369
370 // Same size and time, different content: only -c notices.
371 p := filepath.Join(e.local, "css/site.css")
372 st, _ := os.Stat(filepath.Join(e.remote, "site/css/site.css"))
373 data, _ := os.ReadFile(p)
374 data[10] ^= 0xff
375 writeFile(t, p, data)
376 _ = os.Chtimes(p, st.ModTime(), st.ModTime())
377
378 out := e.mustRun(append(args, "-rv", e.local+"/", "host:site")...)
379 if strings.Contains(out, "css/site.css") {
380 t.Fatalf("quick check should have skipped the file\n%s", out)
381 }
382 out = e.mustRun(append(args, "-rvc", e.local+"/", "host:site")...)
383 if !strings.Contains(out, "css/site.css") {
384 t.Fatalf("checksum run did not transfer the file\n%s", out)
385 }
386 assertSameTree(t, e.local, filepath.Join(e.remote, "site"))
387
388 // And the other direction.
389 dl := t.TempDir()
390 e.mustRun(append(args, "-rt", "host:site/", dl+"/")...)
391 data, _ = os.ReadFile(filepath.Join(dl, "index.html"))
392 st, _ = os.Stat(filepath.Join(dl, "index.html"))
393 data[0] ^= 0xff
394 writeFile(t, filepath.Join(dl, "index.html"), data)
395 _ = os.Chtimes(filepath.Join(dl, "index.html"), st.ModTime(), st.ModTime())
396 e.mustRun(append(args, "-rtc", "host:site/", dl+"/")...)
397 assertSameTree(t, filepath.Join(e.remote, "site"), dl)
398 })
399 }
400}
401
402func TestRsyncDelete(t *testing.T) {
403 for _, proto := range protocols {
404 for _, when := range []string{"--delete", "--delete-before", "--delete-during", "--delete-after", "--delete-delay"} {
405 t.Run(fmt.Sprintf("proto=%s/%s", proto, when), func(t *testing.T) {
406 if proto != "" && proto < "30" && when == "--delete-delay" {
407 t.Skip("needs protocol 30")
408 }
409 e := newRsyncEnv(t)
410 populate(t, e.local)
411 args := protoArgs(proto)
412 e.mustRun(append(args, "-r", e.local+"/", "host:site")...)
413
414 writeFile(t, filepath.Join(e.remote, "site/stale.txt"), []byte("old"))
415 writeFile(t, filepath.Join(e.remote, "site/old/dir/file.txt"), []byte("old"))
416 writeFile(t, filepath.Join(e.remote, "site/keep.log"), []byte("protected"))
417 if err := os.RemoveAll(filepath.Join(e.local, "a")); err != nil {
418 t.Fatal(err)
419 }
420
421 out := e.mustRun(append(args, "-rv", when, "--exclude=*.log", e.local+"/", "host:site")...)
422 if !strings.Contains(out, "deleting stale.txt") {
423 t.Errorf("missing deletion message\n%s", out)
424 }
425 remote := tree(t, filepath.Join(e.remote, "site"))
426 if _, ok := remote["keep.log"]; !ok {
427 t.Error("excluded file was deleted")
428 }
429 delete(remote, "keep.log")
430 if err := os.Remove(filepath.Join(e.remote, "site/keep.log")); err != nil {
431 t.Fatal(err)
432 }
433 assertSameTree(t, e.local, filepath.Join(e.remote, "site"))
434 if _, err := os.Stat(filepath.Join(e.remote, "site/old")); !os.IsNotExist(err) {
435 t.Errorf("stale directory survived: %v", err)
436 }
437 })
438 }
439 }
440}
441
442func TestRsyncDownloadFilters(t *testing.T) {
443 e := newRsyncEnv(t)
444 site := filepath.Join(e.remote, "site")
445 populate(t, site)
446 e.mustRun("-r", "--exclude=*.bin", "--exclude=/a/", "host:site/", e.local+"/")
447 got := tree(t, e.local)
448 if _, ok := got["img/logo.bin"]; ok {
449 t.Error("excluded file was sent")
450 }
451 if _, ok := got["a/b/c/deep.txt"]; ok {
452 t.Error("excluded directory was sent")
453 }
454 if _, ok := got["index.html"]; !ok {
455 t.Error("index.html missing")
456 }
457}
458
459func TestRsyncSingleFile(t *testing.T) {
460 e := newRsyncEnv(t)
461 writeFile(t, filepath.Join(e.local, "one.txt"), []byte("hello"))
462
463 e.mustRun(filepath.Join(e.local, "one.txt"), "host:site/renamed.txt")
464 if b, err := os.ReadFile(filepath.Join(e.remote, "site/renamed.txt")); err != nil || string(b) != "hello" {
465 t.Fatalf("single file upload: %q %v", b, err)
466 }
467 e.mustRun(filepath.Join(e.local, "one.txt"), "host:site/")
468 if b, err := os.ReadFile(filepath.Join(e.remote, "site/one.txt")); err != nil || string(b) != "hello" {
469 t.Fatalf("upload into dir: %q %v", b, err)
470 }
471
472 dl := t.TempDir()
473 e.mustRun("host:site/renamed.txt", dl+"/")
474 if b, err := os.ReadFile(filepath.Join(dl, "renamed.txt")); err != nil || string(b) != "hello" {
475 t.Fatalf("single file download: %q %v", b, err)
476 }
477}
478
479func TestRsyncListOnly(t *testing.T) {
480 e := newRsyncEnv(t)
481 populate(t, filepath.Join(e.remote, "site"))
482 out := e.mustRun("host:site/")
483 for _, want := range []string{"index.html", "css", "big.dat"} {
484 if !strings.Contains(out, want) {
485 t.Errorf("listing lacks %s\n%s", want, out)
486 }
487 }
488 if strings.Contains(out, "site.css") {
489 t.Errorf("non-recursive listing descended\n%s", out)
490 }
491 out = e.mustRun("-r", "host:site/")
492 if !strings.Contains(out, "css/site.css") {
493 t.Errorf("recursive listing lacks css/site.css\n%s", out)
494 }
495}
496
497func TestRsyncDryRun(t *testing.T) {
498 e := newRsyncEnv(t)
499 populate(t, e.local)
500 out := e.mustRun("-rvn", e.local+"/", "host:site")
501 if !strings.Contains(out, "big.dat") {
502 t.Errorf("dry run did not list files\n%s", out)
503 }
504 if _, err := os.Stat(filepath.Join(e.remote, "site")); !os.IsNotExist(err) {
505 t.Errorf("dry run wrote files: %v", err)
506 }
507}
508
509func TestRsyncMissingSource(t *testing.T) {
510 e := newRsyncEnv(t)
511 out, err := e.run("-r", "host:nope/", e.local+"/")
512 var exit *exec.ExitError
513 if !errors.As(err, &exit) || exit.ExitCode() != exitPartial {
514 t.Fatalf("want exit code %d, got %v\n%s", exitPartial, err, out)
515 }
516 if !strings.Contains(out, "No such file or directory") {
517 t.Errorf("missing error message\n%s", out)
518 }
519}
520
521func TestRsyncUnchangedSkipped(t *testing.T) {
522 e := newRsyncEnv(t)
523 populate(t, e.local)
524 e.mustRun("-rt", e.local+"/", "host:site")
525 out := e.mustRun("-rt", "--stats", e.local+"/", "host:site")
526 if n := statValue(t, out, "Number of regular files transferred"); n != 0 {
527 t.Errorf("transferred %d files, want 0\n%s", n, out)
528 }
529}
530
531func statValue(t *testing.T, out, label string) int64 {
532 t.Helper()
533 for _, line := range strings.Split(out, "\n") {
534 rest, ok := strings.CutPrefix(line, label+": ")
535 if !ok {
536 continue
537 }
538 rest = strings.TrimSuffix(rest, " bytes")
539 rest = strings.ReplaceAll(rest, ",", "")
540 var v int64
541 if _, err := fmt.Sscan(rest, &v); err == nil {
542 return v
543 }
544 }
545 t.Fatalf("no %q in output\n%s", label, out)
546 return 0
547}
548
549func TestRsyncCompression(t *testing.T) {
550 type tc struct {
551 proto string
552 args []string
553 }
554 cases := []tc{
555 {"27", []string{"-z"}},
556 {"29", []string{"-z"}},
557 {"30", []string{"-z"}},
558 {"30", []string{"--compress-choice=zlib"}},
559 {"31", []string{"--compress-choice=zlib"}},
560 {"", []string{"-z"}},
561 {"", []string{"--compress-choice=zlibx"}},
562 {"", []string{"-z", "--compress-level=1"}},
563 {"", []string{"-z", "--compress-level=9"}},
564 }
565 for _, c := range cases {
566 for _, block := range []string{"", "--block-size=100000"} {
567 t.Run(fmt.Sprintf("proto=%s/%s/%s", c.proto, strings.Join(c.args, ""), block), func(t *testing.T) {
568 e := newRsyncEnv(t)
569 text := textBytes(2_000_000, "compressible")
570 writeFile(t, filepath.Join(e.local, "text.txt"), text)
571 args := append(protoArgs(c.proto), c.args...)
572 if block != "" {
573 args = append(args, block)
574 }
575
576 out := e.mustRun(append(args, "-rt", "--stats", e.local+"/", "host:site")...)
577 assertSameTree(t, e.local, filepath.Join(e.remote, "site"))
578 if sent := statValue(t, out, "Total bytes sent"); sent > 500_000 {
579 t.Errorf("upload sent %d bytes, compression is not in effect\n%s", sent, out)
580 }
581
582 // A delta with matched blocks exercises the deflate history.
583 text = append(textBytes(150_000, "prefix"), text...)
584 text = append(text, textBytes(70_000, "suffix")...)
585 writeFile(t, filepath.Join(e.local, "text.txt"), text)
586 out = e.mustRun(append(args, "-rt", "--stats", e.local+"/", "host:site")...)
587 assertSameTree(t, e.local, filepath.Join(e.remote, "site"))
588 if matched := statValue(t, out, "Matched data"); matched < 1_000_000 {
589 t.Errorf("matched %d bytes, want a delta\n%s", matched, out)
590 }
591
592 dl := t.TempDir()
593 writeFile(t, filepath.Join(dl, "text.txt"), textBytes(2_000_000, "compressible"))
594 old := time.Now().Add(-time.Hour)
595 _ = os.Chtimes(filepath.Join(dl, "text.txt"), old, old)
596 out = e.mustRun(append(args, "-rt", "--stats", "host:site/", dl+"/")...)
597 assertSameTree(t, filepath.Join(e.remote, "site"), dl)
598 if recv := statValue(t, out, "Total bytes received"); recv > 300_000 {
599 t.Errorf("download received %d bytes, want a compressed delta\n%s", recv, out)
600 }
601 if matched := statValue(t, out, "Matched data"); matched < 1_000_000 {
602 t.Errorf("download matched %d bytes, want a delta\n%s", matched, out)
603 }
604 })
605 }
606 }
607}
608
609func TestRsyncArchiveMode(t *testing.T) {
610 for _, proto := range protocols {
611 t.Run("proto="+proto, func(t *testing.T) {
612 e := newRsyncEnv(t)
613 populate(t, e.local)
614 if err := os.Symlink("index.html", filepath.Join(e.local, "link.html")); err != nil {
615 t.Fatal(err)
616 }
617 args := protoArgs(proto)
618 out := e.mustRun(append(args, "-av", e.local+"/", "host:site")...)
619 if !strings.Contains(out, `skipping non-regular file "link.html"`) {
620 t.Errorf("symlink not reported as skipped\n%s", out)
621 }
622 _ = os.Remove(filepath.Join(e.local, "link.html"))
623 assertSameTree(t, e.local, filepath.Join(e.remote, "site"))
624
625 dl := t.TempDir()
626 e.mustRun(append(args, "-a", "host:site/", dl+"/")...)
627 assertSameTree(t, e.local, dl)
628 })
629 }
630}
631
632func TestRsyncTransferOptions(t *testing.T) {
633 cases := [][]string{
634 {"-s"},
635 {"-W"},
636 {"--inplace"},
637 {"-u"},
638 {"--size-only"},
639 {"-I"},
640 {"--block-size=2048"},
641 {"--checksum-seed=12345"},
642 {"--numeric-ids", "-og"},
643 {"-U"},
644 }
645 for _, opts := range cases {
646 t.Run(strings.Join(opts, " "), func(t *testing.T) {
647 e := newRsyncEnv(t)
648 populate(t, e.local)
649 e.mustRun(append(slices.Clone(opts), "-rt", e.local+"/", "host:site")...)
650 assertSameTree(t, e.local, filepath.Join(e.remote, "site"))
651
652 big := filepath.Join(e.remote, "site/big.dat")
653 data, _ := os.ReadFile(big)
654 copy(data[100_000:], randomBytes(t, 4000))
655 copy(data[2_500_000:], data[10_000:20_000])
656 writeFile(t, big, data)
657 later := time.Now().Add(time.Minute)
658 _ = os.Chtimes(big, later, later)
659
660 dl := t.TempDir()
661 e.mustRun(append(slices.Clone(opts), "-rt", e.local+"/", dl+"/")...)
662 e.mustRun(append(slices.Clone(opts), "-rt", "host:site/", dl+"/")...)
663 if opts[0] != "--size-only" {
664 assertSameTree(t, filepath.Join(e.remote, "site"), dl)
665 }
666 })
667 }
668}
669
670func TestRsyncSkipOptions(t *testing.T) {
671 e := newRsyncEnv(t)
672 writeFile(t, filepath.Join(e.local, "new.txt"), []byte("new"))
673 writeFile(t, filepath.Join(e.local, "old.txt"), []byte("changed locally"))
674 writeFile(t, filepath.Join(e.local, "huge.txt"), textBytes(50_000, "huge"))
675 writeFile(t, filepath.Join(e.remote, "site/old.txt"), []byte("remote"))
676
677 e.mustRun("-r", "--existing", e.local+"/", "host:site")
678 got := tree(t, filepath.Join(e.remote, "site"))
679 if _, ok := got["new.txt"]; ok || got["old.txt"] != "changed locally" {
680 t.Errorf("--existing: %v", got)
681 }
682
683 writeFile(t, filepath.Join(e.remote, "site/old.txt"), []byte("remote"))
684 e.mustRun("-r", "--ignore-existing", "--max-size=10k", e.local+"/", "host:site")
685 got = tree(t, filepath.Join(e.remote, "site"))
686 if got["old.txt"] != "remote" || got["new.txt"] != "new" {
687 t.Errorf("--ignore-existing: %v", got)
688 }
689 if _, ok := got["huge.txt"]; ok {
690 t.Error("--max-size let a large file through")
691 }
692}
693
694func TestRsyncPutResults(t *testing.T) {
695 for _, proto := range protocols {
696 for _, result := range []string{"ignored", "failed"} {
697 t.Run("protocol_"+proto+"/"+result, func(t *testing.T) {
698 t.Setenv("RSYNC_TEST_PUT_RESULT", result)
699 e := newRsyncEnv(t)
700 for _, name := range []string{"a.txt", "skip.txt", "z.txt"} {
701 writeFile(t, filepath.Join(e.local, name), textBytes(100_000, name))
702 }
703 writeFile(t, filepath.Join(e.remote, "site/skip.txt"), []byte("original"))
704 out, err := e.run(append(protoArgs(proto), "-r", "--remove-source-files", e.local+"/", "host:site")...)
705 if result == "ignored" {
706 if err != nil {
707 t.Fatalf("ignored file failed the batch: %v\n%s", err, out)
708 }
709 } else {
710 var exit *exec.ExitError
711 if !errors.As(err, &exit) || exit.ExitCode() != 23 {
712 t.Fatalf("want exit 23, got %v\n%s", err, out)
713 }
714 }
715 got := tree(t, filepath.Join(e.remote, "site"))
716 if len(got) != 3 || got["skip.txt"] != "original" {
717 t.Fatalf("rejected file changed: %v", got)
718 }
719 for _, name := range []string{"a.txt", "z.txt"} {
720 if got[name] != string(textBytes(100_000, name)) {
721 t.Errorf("accepted file %s not stored", name)
722 }
723 if _, err := os.Stat(filepath.Join(e.local, name)); !errors.Is(err, os.ErrNotExist) {
724 t.Errorf("accepted file %s not removed from source: %v", name, err)
725 }
726 }
727 if data, err := os.ReadFile(filepath.Join(e.local, "skip.txt")); err != nil || !bytes.Equal(data, textBytes(100_000, "skip.txt")) {
728 t.Fatalf("rejected source file changed: %v", err)
729 }
730 })
731 }
732 }
733}
734
735// writeTree writes files, named relative to root.
736func writeTree(t *testing.T, root string, files map[string]string) {
737 t.Helper()
738 for name, contents := range files {
739 writeFile(t, filepath.Join(root, name), []byte(contents))
740 }
741}
742
743// mergeFilterTree has per-directory merge files covering inheritance,
744// overrides, anchoring, clearing and protect rules.
745var mergeFilterTree = map[string]string{
746 ".rsync-filter": "- keep.log\nP precious/\n- *.tmp\n-p *.perish\n",
747 "keep.log": "keep",
748 "stale.txt": "stale",
749 "x.tmp": "tmp",
750 "precious/a.txt": "precious",
751 "sub/.rsync-filter": "+ sub-only.tmp\n- /anchored.txt\n",
752 "sub/a.tmp": "tmp",
753 "sub/sub-only.tmp": "kept by include",
754 "sub/anchored.txt": "anchored",
755 "sub/deeper/anchored.txt": "not anchored here",
756 "sub/deeper/keep.log": "inherited",
757 "cleared/.rsync-filter": "!\n- only.txt\n",
758 "cleared/keep.log": "no longer excluded",
759 "cleared/only.txt": "only",
760 "gone/.rsync-filter": "- inside.log\n",
761 "gone/inside.log": "protects its directory",
762 "gone/other.txt": "other",
763 ".cvsignore": "precious.dat\n",
764 "precious.dat": "cvs",
765 "obj.o": "default cvs pattern",
766 "nested/.cvsignore": "nested.dat",
767 "nested/nested.dat": "nested",
768 "nested/deeper/nested.dat": "cvsignore does not inherit",
769 "nested/deeper/precious.dat": "nor does the parent's",
770 "top.perish": "perishable rules protect a kept directory's entries",
771 "perish/keep.perish": "but not a deleted directory's",
772 "perish/deeper/keep.perish": "at any depth",
773 "objdir/x.o": "default CVS patterns are perishable from protocol 30",
774}
775
776// TestRsyncMergeFiltersMatchRsync runs the same transfer against real rsync
777// and against our server, each from a copy of mergeFilterTree, and expects
778// the same result.
779func TestRsyncMergeFiltersMatchRsync(t *testing.T) {
780 local := map[string]string{"index.html": "new", "sub/new.txt": "new"}
781 pushes := [][]string{
782 {"-F", "--delete"},
783 {"-F", "--delete-before"},
784 {"-F", "--delete-during"},
785 {"-F", "--delete-after"},
786 {"-F", "--delete-delay"},
787 {"-FF", "--delete"},
788 {"-F", "--delete", "--delete-excluded"},
789 {"-C", "--delete"},
790 {"-CF", "--delete"},
791 {"--filter=:e .rsync-filter", "--delete"},
792 {"--filter=:n- .rsync-filter", "--delete"},
793 {"--filter=dir-merge,- .rsync-filter", "--delete"},
794 }
795 for _, proto := range []string{"29", "30", "31", ""} {
796 for _, flags := range pushes {
797 t.Run(fmt.Sprintf("push/proto=%s/%s", proto, strings.Join(flags, " ")), func(t *testing.T) {
798 if proto == "29" && slices.Contains(flags, "--delete-delay") {
799 t.Skip("needs protocol 30")
800 }
801 e := newRsyncEnv(t)
802 writeTree(t, e.local, local)
803 ref := filepath.Join(t.TempDir(), "site")
804 writeTree(t, ref, mergeFilterTree)
805 writeTree(t, filepath.Join(e.remote, "site"), mergeFilterTree)
806 args := append(protoArgs(proto), "-r")
807 args = append(args, flags...)
808
809 refCmd := exec.Command("rsync", append(slices.Clone(args), e.local+"/", ref)...)
810 if out, err := refCmd.CombinedOutput(); err != nil {
811 t.Fatalf("reference rsync: %v\n%s", err, out)
812 }
813 e.mustRun(append(args, e.local+"/", "host:site")...)
814 if slices.Contains(flags, "--delete-delay") {
815 // rsync drops inherited per-directory rules before its
816 // delayed deletions, so it removes this file. We keep it.
817 inherited := filepath.Join(e.remote, "site/sub/deeper/keep.log")
818 if err := os.Remove(inherited); err != nil {
819 t.Fatalf("file protected by an inherited rule was deleted: %v", err)
820 }
821 }
822 assertSameTree(t, ref, filepath.Join(e.remote, "site"))
823 })
824 }
825 }
826
827 pulls := [][]string{{"-F"}, {"-FF"}, {"-C"}, {"--filter=:n .rsync-filter"}}
828 for _, proto := range []string{"29", "30", "31", ""} {
829 for _, flags := range pulls {
830 for _, src := range []string{"site/", "site"} {
831 t.Run(fmt.Sprintf("pull/proto=%s/%s/%s", proto, strings.Join(flags, " "), src), func(t *testing.T) {
832 e := newRsyncEnv(t)
833 writeTree(t, filepath.Join(e.remote, "site"), mergeFilterTree)
834 refSrc := t.TempDir()
835 writeTree(t, filepath.Join(refSrc, "site"), mergeFilterTree)
836 ref := t.TempDir()
837 args := append(protoArgs(proto), "-r")
838 args = append(args, flags...)
839
840 refCmd := exec.Command("rsync", append(slices.Clone(args), filepath.Join(refSrc, src), ref+"/")...)
841 if strings.HasSuffix(src, "/") {
842 refCmd.Args[len(refCmd.Args)-2] += "/"
843 }
844 if out, err := refCmd.CombinedOutput(); err != nil {
845 t.Fatalf("reference rsync: %v\n%s", err, out)
846 }
847 e.mustRun(append(args, "host:"+src, e.local+"/")...)
848 assertSameTree(t, ref, e.local)
849 })
850 }
851 }
852 }
853}
854
855// TestRsyncMergeFilterParentScan checks that "-F" reads .rsync-filter in
856// the directories above the transfer, as rsync does up to "/".
857func TestRsyncMergeFilterParentScan(t *testing.T) {
858 e := newRsyncEnv(t)
859 writeFile(t, filepath.Join(e.local, "index.html"), []byte("new"))
860 writeTree(t, e.remote, map[string]string{
861 ".rsync-filter": "- /site/scanned.txt\n",
862 "site/scanned.txt": "protected from above",
863 "site/stale.txt": "stale",
864 "other/scanned.txt": "outside the transfer",
865 })
866 e.mustRun("-r", "-F", "--delete", e.local+"/", "host:site")
867 got := tree(t, filepath.Join(e.remote, "site"))
868 if _, ok := got["scanned.txt"]; !ok {
869 t.Error("rule from the parent directory's merge file was ignored")
870 }
871 if _, ok := got["stale.txt"]; ok {
872 t.Error("unprotected file survived --delete")
873 }
874}
875
876// TestRsyncMergeFilterErrors checks that a merge file we can't use stops
877// deletions on upload and the whole transfer on download.
878func TestRsyncMergeFilterErrors(t *testing.T) {
879 for _, contents := range []string{"merge other\n", "bogus rule\n"} {
880 t.Run("push/"+contents, func(t *testing.T) {
881 e := newRsyncEnv(t)
882 writeFile(t, filepath.Join(e.local, "index.html"), []byte("new"))
883 writeTree(t, filepath.Join(e.remote, "site"), map[string]string{
884 "sub/.rsync-filter": contents,
885 "sub/stale.txt": "stale",
886 "stale.txt": "stale",
887 })
888 out, err := e.run("-r", "-F", "--delete-after", e.local+"/", "host:site")
889 var exit *exec.ExitError
890 if !errors.As(err, &exit) || exit.ExitCode() != exitPartial {
891 t.Fatalf("want exit %d, got %v\n%s", exitPartial, err, out)
892 }
893 if !strings.Contains(out, "skipping file deletion") {
894 t.Errorf("no explanation for skipped deletion\n%s", out)
895 }
896 got := tree(t, filepath.Join(e.remote, "site"))
897 if got["index.html"] != "new" {
898 t.Error("file was not transferred")
899 }
900 for _, name := range []string{"stale.txt", "sub/stale.txt"} {
901 if _, ok := got[name]; !ok {
902 t.Errorf("%s was deleted despite the unusable merge file", name)
903 }
904 }
905 })
906 t.Run("pull/"+contents, func(t *testing.T) {
907 e := newRsyncEnv(t)
908 writeTree(t, filepath.Join(e.remote, "site"), map[string]string{
909 "sub/.rsync-filter": contents,
910 "index.html": "index",
911 })
912 out, err := e.run("-r", "-F", "host:site/", e.local+"/")
913 if err == nil || !strings.Contains(out, "per-directory filter file") {
914 t.Fatalf("want a merge-file error, got %v\n%s", err, out)
915 }
916 if got := tree(t, e.local); len(got) != 0 {
917 t.Errorf("files transferred despite the error: %v", got)
918 }
919 })
920 }
921}
922
923func TestRsyncUnsupportedOptions(t *testing.T) {
924 e := newRsyncEnv(t)
925 populate(t, e.local)
926 for _, opt := range []string{"-H", "-A", "-X", "-R"} {
927 out, err := e.run("-r", opt, e.local+"/", "host:site")
928 if err == nil {
929 t.Errorf("%s: want an error\n%s", opt, out)
930 }
931 if !strings.Contains(out, "not supported") {
932 t.Errorf("%s: missing explanation\n%s", opt, out)
933 }
934 }
935}