Commit 9eb8e76

Eric Bower  ·  2026-05-31 09:31:45 -0400 EDT
parent 8936652 
refactor: move go-rsync-receiver into pico monorepo
36 files changed,  +3873, -12
M go.mod
M go.sum
A pkg/rsync-receiver/nofollow/nofollow.go
A pkg/rsync-receiver/rsync/consts.go
A pkg/rsync-receiver/rsync/doc.go
A pkg/rsync-receiver/rsync/logger.go
A pkg/rsync-receiver/rsync/types.go
A pkg/rsync-receiver/rsyncchecksum/checksum_test.go
A pkg/rsync-receiver/rsyncchecksum/rsyncchecksum.go
A pkg/rsync-receiver/rsynccommon/rsynccommon.go
A pkg/rsync-receiver/rsyncopts/popt.go
A pkg/rsync-receiver/rsyncopts/rsyncopts.go
A pkg/rsync-receiver/rsyncreceiver/client.go
A pkg/rsync-receiver/rsyncreceiver/do.go
A pkg/rsync-receiver/rsyncreceiver/flist.go
A pkg/rsync-receiver/rsyncreceiver/generator.go
A pkg/rsync-receiver/rsyncreceiver/receiver.go
A pkg/rsync-receiver/rsyncreceiver/token.go
A pkg/rsync-receiver/rsyncreceiver/transfer.go
A pkg/rsync-receiver/rsyncreceiver/uidlist.go
A pkg/rsync-receiver/rsyncsender/client.go
A pkg/rsync-receiver/rsyncsender/do.go
A pkg/rsync-receiver/rsyncsender/exclude.go
A pkg/rsync-receiver/rsyncsender/fileio.go
A pkg/rsync-receiver/rsyncsender/flist.go
A pkg/rsync-receiver/rsyncsender/match.go
A pkg/rsync-receiver/rsyncsender/sender.go
A pkg/rsync-receiver/rsyncsender/stat.go
A pkg/rsync-receiver/rsyncsender/token.go
A pkg/rsync-receiver/rsyncsender/transfer.go
A pkg/rsync-receiver/rsyncstats/rsyncstats.go
A pkg/rsync-receiver/rsyncwire/wire.go
A pkg/rsync-receiver/utils/file.go
A pkg/rsync-receiver/utils/fs.go
M pkg/send/protocols/rsync/rsync.go
M pkg/send/protocols/rsync/rsync_test.go
M go.mod
+2, -5
......@@ -6,8 +6,6 @@ go 1.25.0
66
77 // replace github.com/picosh/send => ../send
88
9-// replace github.com/picosh/go-rsync-receiver => ../go-rsync-receiver
10-
119 // replace github.com/picosh/pobj => ../pobj
1210
1311 // replace github.com/picosh/pubsub => ../pubsub
......@@ -41,8 +39,8 @@ require (
4139 github.com/matryer/is v1.4.1
4240 github.com/microcosm-cc/bluemonday v1.0.27
4341 github.com/mmcdole/gofeed v1.3.0
42+ github.com/mmcloughlin/md4 v0.1.2
4443 github.com/neurosnap/go-exif-remove v0.0.0-20221010134343-50d1e3c35577
45- github.com/picosh/go-rsync-receiver v0.0.0-20250304201040-fcc11dd22d79
4644 github.com/picosh/utils v0.0.0-20260125160622-5c3a9e231ec6
4745 github.com/pkg/sftp v1.13.10
4846 github.com/prometheus/client_golang v1.23.2
......@@ -58,6 +56,7 @@ require (
5856 go.abhg.dev/goldmark/hashtag v0.4.0
5957 go.abhg.dev/goldmark/toc v0.12.0
6058 golang.org/x/crypto v0.50.0
59+ golang.org/x/sync v0.20.0
6160 gopkg.in/yaml.v2 v2.4.0
6261 modernc.org/sqlite v1.49.1
6362 )
......@@ -126,7 +125,6 @@ require (
126125 github.com/mattn/go-runewidth v0.0.23 // indirect
127126 github.com/mattn/go-sixel v0.0.9 // indirect
128127 github.com/mmcdole/goxpp v1.1.1 // indirect
129- github.com/mmcloughlin/md4 v0.1.2 // indirect
130128 github.com/moby/docker-image-spec v1.3.1 // indirect
131129 github.com/moby/go-archive v0.1.0 // indirect
132130 github.com/moby/patternmatcher v0.6.0 // indirect
......@@ -173,7 +171,6 @@ require (
173171 go.yaml.in/yaml/v2 v2.4.4 // indirect
174172 golang.org/x/image v0.39.0 // indirect
175173 golang.org/x/net v0.53.0 // indirect
176- golang.org/x/sync v0.20.0 // indirect
177174 golang.org/x/sys v0.43.0 // indirect
178175 golang.org/x/text v0.36.0 // indirect
179176 golang.org/x/time v0.15.0 // indirect
M go.sum
+0, -2
......@@ -260,8 +260,6 @@ github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8
260260 github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
261261 github.com/opencontainers/image-spec v1.1.1 h1:y0fUlFfIZhPF1W537XOLg0/fcx6zcHCJwooC2xJA040=
262262 github.com/opencontainers/image-spec v1.1.1/go.mod h1:qpqAh3Dmcf36wStyyWU+kCeDgrGnAve2nCC8+7h8Q0M=
263-github.com/picosh/go-rsync-receiver v0.0.0-20250304201040-fcc11dd22d79 h1:MyB9P43hlQ6A2FoP9LGeiTBL3WKToW4gcWd6lQPg/Zg=
264-github.com/picosh/go-rsync-receiver v0.0.0-20250304201040-fcc11dd22d79/go.mod h1:4ZICsr6bESoHP8He9DqROlZiMw4hHHjcbDzhtTTDQzA=
265263 github.com/picosh/utils v0.0.0-20260125160622-5c3a9e231ec6 h1:9KfCtfcx7vrSyGU1K9whdE1crll9Aq+nAZ6c0FzuzvE=
266264 github.com/picosh/utils v0.0.0-20260125160622-5c3a9e231ec6/go.mod h1:HogYEyJ43IGXrOa3D/kjM1pkzNAyh+pejRyv8Eo//pk=
267265 github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=
A pkg/rsync-receiver/nofollow/nofollow.go
+5, -0
......@@ -0,0 +1,5 @@
1+package nofollow
2+
3+// Maybe resolves to unix.O_NOFOLLOW on unix systems,
4+// 0 on other platforms. TODO(go1.24): use os.Root.
5+const Maybe = 0
A pkg/rsync-receiver/rsync/consts.go
+36, -0
......@@ -0,0 +1,36 @@
1+package rsync
2+
3+// rsync.h.
4+const (
5+ XMIT_TOP_DIR = (1 << 0)
6+ XMIT_SAME_MODE = (1 << 1)
7+ XMIT_EXTENDED_FLAGS = (1 << 2)
8+ XMIT_SAME_RDEV_pre28 = XMIT_EXTENDED_FLAGS /* Only in protocols < 28 */
9+ XMIT_SAME_UID = (1 << 3)
10+ XMIT_SAME_GID = (1 << 4)
11+ XMIT_SAME_NAME = (1 << 5)
12+ XMIT_LONG_NAME = (1 << 6)
13+ XMIT_SAME_TIME = (1 << 7)
14+ XMIT_SAME_RDEV_MAJOR = (1 << 8)
15+ XMIT_HAS_IDEV_DATA = (1 << 9)
16+ XMIT_SAME_DEV = (1 << 10)
17+ XMIT_RDEV_MINOR_IS_SMALL = (1 << 11)
18+)
19+
20+// as per /usr/include/bits/stat.h.
21+const (
22+ S_IFMT = 0o0170000 // bits determining the file type
23+ S_IFDIR = 0o0040000 // Directory
24+ S_IFCHR = 0o0020000 // Character device
25+ S_IFBLK = 0o0060000 // Block device
26+ S_IFREG = 0o0100000 // Regular file
27+ S_IFIFO = 0o0010000 // FIFO
28+ S_IFLNK = 0o0120000 // Symbolic link
29+ S_IFSOCK = 0o0140000 // Socket
30+)
31+
32+// ProtocolVersion defines the currently implemented rsync protocol
33+// version. Protocol version 27 seems to be the safest bet for wide
34+// compatibility: version 27 was introduced by rsync 2.6.0 (released 2004), and
35+// is supported by openrsync and rsyn.
36+const ProtocolVersion = 27
A pkg/rsync-receiver/rsync/doc.go
+6, -0
......@@ -0,0 +1,6 @@
1+// Package rsync contains a native Go rsync implementation.
2+//
3+// The only component currently is gokr-rsyncd, a read-only rsync daemon
4+// sender-only Go implementation of rsyncd. rsync daemon is a custom
5+// (un-standardized) network protocol, running on port 873 by default.
6+package rsync
A pkg/rsync-receiver/rsync/logger.go
+7, -0
......@@ -0,0 +1,7 @@
1+package rsync
2+
3+import "log/slog"
4+
5+// Logger is an interface that allows specifying your own logger.
6+// By default, the Go log package is used, which prints to stderr.
7+type Logger = slog.Logger
A pkg/rsync-receiver/rsync/types.go
+86, -0
......@@ -0,0 +1,86 @@
1+package rsync
2+
3+import (
4+ "fmt"
5+
6+ "github.com/picosh/pico/pkg/rsync-receiver/rsyncwire"
7+)
8+
9+// rsync/rsync.h:struct sum_buf.
10+type SumBuf struct {
11+ Offset int64
12+ Len int64
13+ Index int32
14+ Sum1 uint32
15+ Sum2 [16]byte
16+}
17+
18+// TODO: remove connection.go:sumHead in favor of this type.
19+type SumHead struct {
20+ // “number of blocks” (openrsync)
21+ // “how many chunks” (rsync)
22+ ChecksumCount int32
23+
24+ // “block length in the file” (openrsync)
25+ // maximum (1 << 29) for older rsync, (1 << 17) for newer
26+ BlockLength int32
27+
28+ // “long checksum length” (openrsync)
29+ ChecksumLength int32
30+
31+ // “terminal (remainder) block length” (openrsync)
32+ // RemainderLength is flength % BlockLength
33+ RemainderLength int32
34+
35+ Sums []SumBuf
36+}
37+
38+func (sh *SumHead) ReadFrom(c *rsyncwire.Conn) error {
39+ // TODO(protocol>=30): update maxBlockLen
40+ const maxBlockLen = 1 << 29 // see rsync.h:OLD_MAX_BLOCK_SIZE
41+
42+ var err error
43+ sh.ChecksumCount, err = c.ReadInt32()
44+ if err != nil {
45+ return err
46+ }
47+ if sh.ChecksumCount < 0 {
48+ return fmt.Errorf("invalid checksum count %d", sh.ChecksumCount)
49+ }
50+
51+ sh.BlockLength, err = c.ReadInt32()
52+ if err != nil {
53+ return err
54+ }
55+ if sh.BlockLength < 0 || sh.BlockLength > maxBlockLen {
56+ return fmt.Errorf("invalid block length %d", sh.BlockLength)
57+ }
58+
59+ sh.ChecksumLength, err = c.ReadInt32()
60+ if err != nil {
61+ return err
62+ }
63+ // TODO(protocol>=27): update max sh.ChecksumLength check
64+ if sh.ChecksumLength < 0 || sh.ChecksumLength > 16 {
65+ return fmt.Errorf("invalid checksum length %d", sh.ChecksumLength)
66+ }
67+
68+ sh.RemainderLength, err = c.ReadInt32()
69+ if err != nil {
70+ return err
71+ }
72+ if sh.RemainderLength < 0 || sh.RemainderLength > sh.BlockLength {
73+ return fmt.Errorf("invalid remainder length %d", sh.RemainderLength)
74+ }
75+
76+ return nil
77+}
78+
79+func (sh *SumHead) WriteTo(c *rsyncwire.Conn) error {
80+ var buf rsyncwire.Buffer
81+ buf.WriteInt32(sh.ChecksumCount)
82+ buf.WriteInt32(sh.BlockLength)
83+ buf.WriteInt32(sh.ChecksumLength)
84+ buf.WriteInt32(sh.RemainderLength)
85+ return c.WriteString(buf.String())
86+}
A pkg/rsync-receiver/rsyncchecksum/checksum_test.go
+73, -0
......@@ -0,0 +1,73 @@
1+package rsyncchecksum_test
2+
3+import (
4+ "bytes"
5+ "os"
6+ "path/filepath"
7+ "testing"
8+
9+ "github.com/picosh/pico/pkg/rsync-receiver/rsyncchecksum"
10+)
11+
12+func constructLargeDataFile(headPattern, bodyPattern, endPattern []byte) []byte {
13+ // create large data file in source directory to be copied
14+ head := bytes.Repeat(headPattern, 1*1024*1024)
15+ body := bytes.Repeat(bodyPattern, 1*1024*1024)
16+ end := bytes.Repeat(endPattern, 1*1024*1024)
17+ return append(append(head, body...), end...)
18+}
19+
20+func writeLargeDataFile(t *testing.T, source string, headPattern, bodyPattern, endPattern []byte) {
21+ // create large data file in source directory to be copied
22+ content := constructLargeDataFile(headPattern, bodyPattern, endPattern)
23+ large := filepath.Join(source, "large-data-file")
24+ if err := os.MkdirAll(filepath.Dir(large), 0755); err != nil {
25+ t.Fatal(err)
26+ }
27+ if err := os.WriteFile(large, content, 0644); err != nil {
28+ t.Fatal(err)
29+ }
30+}
31+
32+func TestSyncExtended(t *testing.T) {
33+ tmp := t.TempDir()
34+ source := filepath.Join(tmp, "source")
35+
36+ writeLargeDataFile(t, source, []byte{0x11}, []byte{0xbb}, []byte{0xee})
37+
38+ // These values are taken from the rsync debug output:
39+ const k = 1768
40+ want := make([]uint32, 1780)
41+ for i := 0; i <= 592; i++ {
42+ want[i] = 0xa5d47568
43+ }
44+ want[593] = 0x23645688
45+ for i := 594; i <= 1185; i++ {
46+ want[i] = 0x8c1c2378
47+ }
48+ want[1186] = 0x12504720
49+ for i := 1187; i <= 1778; i++ {
50+ want[i] = 0x7d9883b0
51+ }
52+ want[1779] = 0x61b8dff0
53+
54+ sourceLarge := filepath.Join(source, "large-data-file")
55+ f, err := os.Open(sourceLarge)
56+ if err != nil {
57+ t.Fatal(err)
58+ }
59+ defer func() { _ = f.Close() }()
60+ buf := make([]byte, k)
61+ for idx, wantChecksum := range want {
62+ n, err := f.Read(buf)
63+ if err != nil {
64+ t.Fatal(err)
65+ }
66+
67+ chunk := buf[:n]
68+ sum := rsyncchecksum.Checksum1(chunk)
69+ if sum != wantChecksum {
70+ t.Fatalf("checksum calculation error: got %08x, want %08x (idx %d), chunk: %#v", sum, wantChecksum, idx, chunk)
71+ }
72+ }
73+}
A pkg/rsync-receiver/rsyncchecksum/rsyncchecksum.go
+73, -0
......@@ -0,0 +1,73 @@
1+package rsyncchecksum
2+
3+import (
4+ "encoding/binary"
5+ "io"
6+ "os"
7+
8+ "github.com/mmcloughlin/md4"
9+)
10+
11+func Tag2(s1, s2 uint16) uint16 {
12+ return (((s1) + (s2)) & 0xFFFF)
13+}
14+
15+func Tag(sum uint32) uint16 {
16+ return Tag2(uint16(sum&0xFFFF), uint16(sum>>16))
17+}
18+
19+// SignExtend mirrors how C converts from (signed char) to uint32, i.e. using
20+// sign extension. get_checksum1 treats the buffer as (signed char*) instead of
21+// (unsigned char*), which likely was not a conscious choice, but here we are.
22+//
23+// This function is exported for use in the rolling checksum in match.go.
24+func SignExtend(b byte) uint32 {
25+ val := uint32(b)
26+ return uint32(int32(val<<24) >> 24)
27+}
28+
29+func Checksum1(buf []byte) uint32 {
30+ bufLen := len(buf)
31+ var s1, s2 uint32
32+ var i int
33+
34+ if bufLen > 4 {
35+ for i = 0; i < (bufLen - 4); i += 4 {
36+ s2 += 4*(s1+SignExtend(buf[i])) +
37+ 3*SignExtend(buf[i+1]) +
38+ 2*SignExtend(buf[i+2]) +
39+ SignExtend(buf[i+3])
40+ s1 += SignExtend(buf[i+0]) +
41+ SignExtend(buf[i+1]) +
42+ SignExtend(buf[i+2]) +
43+ SignExtend(buf[i+3])
44+ }
45+ }
46+ for ; i < bufLen; i++ {
47+ s1 += SignExtend(buf[i])
48+ s2 += s1
49+ }
50+ return (s1 & 0xffff) + (s2 << 16)
51+}
52+
53+func Checksum2(seed int32, buf []byte) []byte {
54+ h := md4.New()
55+ h.Write(buf)
56+ _ = binary.Write(h, binary.LittleEndian, seed) // hash.Hash.Write never fails
57+ return h.Sum(nil)
58+}
59+
60+func FileChecksum(fn string) ([]byte, error) {
61+ f, err := os.Open(fn)
62+ if err != nil {
63+ return nil, err
64+ }
65+ defer func() { _ = f.Close() }()
66+ h := md4.New()
67+ if _, err := io.Copy(h, f); err != nil {
68+ return nil, err
69+ }
70+ return h.Sum(nil), nil
71+}
72+
73+const Size = md4.Size
A pkg/rsync-receiver/rsynccommon/rsynccommon.go
+37, -0
......@@ -0,0 +1,37 @@
1+// Package rsynccommon contains functionality that both the sender and the
2+// receiver implementation need.
3+package rsynccommon
4+
5+import (
6+ "math"
7+
8+ "github.com/picosh/pico/pkg/rsync-receiver/rsync"
9+)
10+
11+const blockSize = 700 // rsync/rsync.h
12+
13+// Corresponds to rsync/generator.c:sum_sizes_sqroot.
14+func SumSizesSqroot(contentLen int64) rsync.SumHead {
15+ // * The block size is a rounded square root of file length.
16+
17+ // The block size algorithm plays a crucial role in the protocol efficiency. In general, the block size is the rounded square root of the total file size. The minimum block size, however, is 700 B. Otherwise, the square root computation is simply sqrt(3) followed by ceil(3)
18+
19+ // For reasons unknown, the square root result is rounded up to the nearest multiple of eight.
20+
21+ // TODO: round this
22+ blockLength := max(int32(math.Sqrt(float64(contentLen))), blockSize)
23+
24+ // * The checksum size is determined according to:
25+ // * blocksum_bits = BLOCKSUM_EXP + 2*log2(file_len) - log2(block_len)
26+ // * provided by Donovan Baarda which gives a probability of rsync
27+ // * algorithm corrupting data and falling back using the whole md4
28+ // * checksums.
29+ const checksumLength = 16 // TODO?
30+
31+ return rsync.SumHead{
32+ ChecksumCount: int32((contentLen + (int64(blockLength) - 1)) / int64(blockLength)),
33+ RemainderLength: int32(contentLen % int64(blockLength)),
34+ BlockLength: blockLength,
35+ ChecksumLength: checksumLength,
36+ }
37+}
A pkg/rsync-receiver/rsyncopts/popt.go
+255, -0
......@@ -0,0 +1,255 @@
1+package rsyncopts
2+
3+import (
4+ "fmt"
5+ "math"
6+ "strconv"
7+ "strings"
8+)
9+
10+type poptOption struct {
11+ longName string
12+ shortName string
13+ argInfo int
14+ arg any // depends on argInfo
15+ val int // 0 means don't return, just update arg
16+ // descrip string
17+ // argDescrip string
18+}
19+
20+func (o *poptOption) name() string {
21+ if o.longName == "" {
22+ return "-" + o.shortName
23+ }
24+ return "--" + o.longName
25+}
26+
27+// see popt(3).
28+const (
29+ POPT_ARG_NONE = iota // int; No argument expected
30+ POPT_ARG_STRING // char*; No type checking to be performed
31+ POPT_ARG_INT // int; An integer argument is expected
32+ POPT_ARG_LONG // long; A long integer is expected
33+ POPT_ARG_INCLUDE_TABLE // nest another option table
34+ POPT_ARG_CALLBACK // call a function
35+ POPT_ARG_INTL_DOMAIN // <not documented in popt(3)>
36+ POPT_ARG_VAL // int; Integer value taken from val
37+ POPT_ARG_FLOAT // float; A float argument is expected
38+ POPT_ARG_DOUBLE // double; A double argument is expected
39+ POPT_ARG_LONGLONG // long long; A long long integer is expected
40+ POPT_ARG_MAINCALL = 16 + 11
41+ POPT_ARG_ARGV = 12
42+ POPT_ARG_SHORT = 13
43+ POPT_ARG_BITSET = 16 + 14
44+)
45+
46+const POPT_ARG_MASK = 0x000000FF
47+
48+const (
49+ POPT_ARGFLAG_OR = 0x08000000
50+)
51+
52+const (
53+ POPT_BIT_SET = POPT_ARG_VAL | POPT_ARGFLAG_OR
54+)
55+
56+type PoptError struct {
57+ Errno int32
58+ Err error
59+}
60+
61+func (pe *PoptError) Unwrap() error { return pe.Err }
62+
63+func (pe *PoptError) Error() string { return pe.Err.Error() }
64+
65+// TODO(later): turn these into sentinel error values.
66+// which stringify like poptStrerror().
67+const (
68+ POPT_ERROR_NOARG = -10 // missing argument
69+ POPT_ERROR_BADOPT = -11 // unknown option
70+ POPT_ERROR_UNWANTEDARG = -12 // option does not take an argument
71+ POPT_ERROR_OPTSTOODEEP = -13 // aliases nested too deeply
72+ POPT_ERROR_BADQUOTE = -15 // error in parameter quoting
73+ POPT_ERROR_ERRNO = -16 // errno set, use strerror(errno)
74+ POPT_ERROR_BADNUMBER = -17 // invalid numeric value
75+ POPT_ERROR_OVERFLOW = -18 // number too large or too small
76+ POPT_ERROR_BADOPERATION = -19 // mutually exclusive logical operations requested
77+ POPT_ERROR_NULLARG = -20 // opt->arg should not be NULL
78+ POPT_ERROR_MALLOC = -21 // memory allocation failed
79+ POPT_ERROR_BADCONFIG = -22 // config file failed sanity test
80+)
81+
82+type Context struct {
83+ // state
84+ table []poptOption
85+ args []string
86+ nextCharArg string
87+ nextArg string
88+
89+ // output
90+ Options *Options
91+ RemainingArgs []string
92+}
93+
94+func (pc *Context) findOption(longName, shortName string) *poptOption {
95+ for idx, opt := range pc.table {
96+ if longName != "" && opt.longName == longName {
97+ return &pc.table[idx]
98+ }
99+ if shortName != "" && opt.shortName == shortName {
100+ return &pc.table[idx]
101+ }
102+ }
103+ return nil
104+}
105+
106+func (pc *Context) poptSaveInt(opt *poptOption, val int) bool {
107+ intPtr := opt.arg.(*int)
108+ if intPtr == nil {
109+ return false
110+ }
111+ if opt.argInfo&POPT_ARGFLAG_OR != 0 {
112+ *intPtr |= val
113+ } else {
114+ *intPtr = val
115+ }
116+ return true
117+}
118+
119+func (pc *Context) poptSaveArg(opt *poptOption, nextArg string) int32 {
120+ argType := opt.argInfo & POPT_ARG_MASK
121+ switch argType {
122+ case POPT_ARG_INT:
123+ i, err := strconv.ParseInt(nextArg, 0, 64)
124+ if err != nil {
125+ return POPT_ERROR_BADNUMBER
126+ }
127+ if i < math.MinInt32 || i > math.MaxInt32 {
128+ return POPT_ERROR_OVERFLOW
129+ }
130+ pc.poptSaveInt(opt, int(i))
131+ return 0
132+
133+ case POPT_ARG_STRING:
134+ stringPtr := opt.arg.(*string)
135+ if stringPtr == nil {
136+ return 0
137+ }
138+ *stringPtr = nextArg
139+ return 0
140+ }
141+
142+ return POPT_ERROR_BADOPERATION
143+}
144+
145+func (pc *Context) poptGetNextOpt() (int32, error) {
146+ var opt *poptOption
147+ for {
148+ var longArg string
149+ if pc.nextCharArg == "" && len(pc.args) == 0 {
150+ return -1, nil // done
151+ }
152+ if pc.nextCharArg == "" {
153+ // process next long option
154+ origOptString := pc.args[0]
155+ pc.args = pc.args[1:]
156+ if origOptString == "" {
157+ return -1, &PoptError{
158+ Errno: POPT_ERROR_BADOPT,
159+ Err: fmt.Errorf("unknown option: origOptString empty"),
160+ }
161+ }
162+ if origOptString[0] != '-' || origOptString == "-" {
163+ pc.RemainingArgs = append(pc.RemainingArgs, origOptString)
164+ continue
165+ }
166+ before, after, found := strings.Cut(origOptString, "=")
167+ if found {
168+ longArg = after
169+ }
170+ // remove the one dash we ensured is present
171+ before = strings.TrimPrefix(before, "-")
172+ // a second dash is permitted
173+ before = strings.TrimPrefix(before, "-")
174+ opt = pc.findOption(before, "")
175+ if opt == nil {
176+ // try and parse it as a short option
177+ pc.nextCharArg = origOptString[1:]
178+ longArg = ""
179+ }
180+ }
181+ if pc.nextCharArg != "" {
182+ // process next short option
183+ opt = pc.findOption("", pc.nextCharArg[:1])
184+ if opt == nil {
185+ return -1, &PoptError{
186+ Errno: POPT_ERROR_BADOPT,
187+ Err: fmt.Errorf("option %q not found", pc.nextCharArg[:1]),
188+ }
189+ }
190+ pc.nextCharArg = pc.nextCharArg[1:]
191+ }
192+ if opt == nil {
193+ // neither long nor short? how can we end up here?
194+ return -1, &PoptError{
195+ Errno: POPT_ERROR_BADOPT,
196+ Err: fmt.Errorf("neither long nor short option found"),
197+ }
198+ }
199+ argType := opt.argInfo & POPT_ARG_MASK
200+ if argType == POPT_ARG_NONE || argType == POPT_ARG_VAL {
201+ if longArg != "" || strings.HasPrefix(pc.nextCharArg, "=") {
202+ return -1, &PoptError{
203+ Errno: POPT_ERROR_UNWANTEDARG,
204+ Err: fmt.Errorf("option %s does not take an argument", opt.name()),
205+ }
206+ }
207+ if opt.arg != nil {
208+ val := 1
209+ if argType == POPT_ARG_VAL {
210+ val = opt.val
211+ }
212+ if !pc.poptSaveInt(opt, val) {
213+ return -1, &PoptError{
214+ Errno: POPT_ERROR_BADOPERATION,
215+ Err: fmt.Errorf("poptSaveInt"),
216+ }
217+ }
218+ }
219+ } else {
220+ nextArg := longArg
221+ if longArg != "" {
222+ } else if pc.nextCharArg != "" {
223+ nextArg = strings.TrimPrefix(pc.nextCharArg, "=")
224+ pc.nextCharArg = ""
225+ } else {
226+ if len(pc.args) == 0 {
227+ return -1, &PoptError{
228+ Errno: POPT_ERROR_NOARG,
229+ Err: fmt.Errorf("missing argument for option %s", opt.name()),
230+ }
231+ }
232+ nextArg = pc.args[0]
233+ pc.args = pc.args[1:]
234+ }
235+ pc.nextArg = nextArg
236+ if opt.arg != nil {
237+ if errno := pc.poptSaveArg(opt, nextArg); errno != 0 {
238+ return -1, &PoptError{
239+ Errno: errno,
240+ Err: fmt.Errorf("poptSaveArg"),
241+ }
242+ }
243+ }
244+ }
245+ if opt.val != 0 && argType != POPT_ARG_VAL {
246+ return int32(opt.val), nil
247+ }
248+ }
249+}
250+
251+func (pc *Context) poptGetOptArg() string {
252+ ret := pc.nextArg
253+ pc.nextArg = ""
254+ return ret
255+}
A pkg/rsync-receiver/rsyncopts/rsyncopts.go
+943, -0
......@@ -0,0 +1,943 @@
1+// Package rsyncopts implements a parser for command-line options that
2+// implements a subset of popt(3) semantics; just enough to parse typical
3+// rsync(1) invocations without the advanced popt features like aliases
4+// or option prefix matching (not --del, only --delete).
5+//
6+// If we encounter arguments that rsync(1) parses differently compared to this
7+// package, then this package should be adjusted to match rsync(1).
8+package rsyncopts
9+
10+import (
11+ "errors"
12+ "fmt"
13+ "log/slog"
14+ "math"
15+ "slices"
16+ "strconv"
17+ "strings"
18+ "syscall"
19+ "unicode"
20+)
21+
22+const (
23+ OPT_SERVER = 1000 + iota
24+ OPT_DAEMON
25+ OPT_SENDER
26+ OPT_EXCLUDE
27+ OPT_EXCLUDE_FROM
28+ OPT_FILTER
29+ OPT_COMPARE_DEST
30+ OPT_COPY_DEST
31+ OPT_LINK_DEST
32+ OPT_HELP
33+ OPT_INCLUDE
34+ OPT_INCLUDE_FROM
35+ OPT_MODIFY_WINDOW
36+ OPT_MIN_SIZE
37+ OPT_CHMOD
38+ OPT_READ_BATCH
39+ OPT_WRITE_BATCH
40+ OPT_ONLY_WRITE_BATCH
41+ OPT_MAX_SIZE
42+ OPT_NO_D
43+ OPT_APPEND
44+ OPT_NO_ICONV
45+ OPT_INFO
46+ OPT_DEBUG
47+ OPT_BLOCK_SIZE
48+ OPT_USERMAP
49+ OPT_GROUPMAP
50+ OPT_CHOWN
51+ OPT_BWLIMIT
52+ OPT_STDERR
53+ OPT_OLD_COMPRESS
54+ OPT_NEW_COMPRESS
55+ OPT_NO_COMPRESS
56+ OPT_OLD_ARGS
57+ OPT_STOP_AFTER
58+ OPT_STOP_AT
59+ OPT_REFUSED_BASE = 9000
60+)
61+
62+type infoLevel int
63+
64+const (
65+ INFO_BACKUP infoLevel = iota
66+ INFO_COPY
67+ INFO_DEL
68+ INFO_FLIST
69+ INFO_MISC
70+ INFO_MOUNT
71+ INFO_NAME
72+ INFO_NONREG
73+ INFO_PROGRESS
74+ INFO_REMOVE
75+ INFO_SKIP
76+ INFO_STATS
77+ INFO_SYMSAFE
78+ COUNT_INFO
79+)
80+
81+// NewOptions returns an Options struct with all options initialized to their
82+// default values. Note that ParseArguments will set some options (that default
83+// to -1) based on the encountered command-line flags and built-in rules.
84+func NewOptions() *Options {
85+ return &Options{
86+ msgs2stderr: 2, // Default: send errors to stderr for local & remote-shell transfers
87+ output_motd: 1,
88+ human_readable: 1,
89+ allow_inc_recurse: 1,
90+ xfer_dirs: -1,
91+ relative_paths: -1,
92+ implied_dirs: 1,
93+ max_delete: math.MinInt32,
94+ whole_file: -1,
95+ do_compression_level: math.MinInt32,
96+ rsync_path: "rsync",
97+ default_af_hint: syscall.AF_INET6,
98+ blocking_io: -1,
99+ protocol_version: 27,
100+ }
101+}
102+
103+// GokrazyOptions contains additional command-line flags, prefixed with
104+// gokr. (like --gokr.modulemap) to not clash with rsync flag names.
105+type GokrazyOptions struct {
106+ Config string
107+ Listen string
108+ MonitoringListen string
109+ AnonSSHListen string
110+ ModuleMap string
111+}
112+
113+func (o *GokrazyOptions) table() []poptOption {
114+ return []poptOption{
115+ /* longName, shortName, argInfo, arg, val */
116+ {"gokr.config", "", POPT_ARG_STRING, &o.Config, 0},
117+ {"gokr.listen", "", POPT_ARG_STRING, &o.Listen, 0},
118+ {"gokr.monitoring_listen", "", POPT_ARG_STRING, &o.MonitoringListen, 0},
119+ {"gokr.anonssh_listen", "", POPT_ARG_STRING, &o.AnonSSHListen, 0},
120+ {"gokr.modulemap", "", POPT_ARG_STRING, &o.ModuleMap, 0},
121+ }
122+}
123+
124+type Options struct {
125+ Gokrazy GokrazyOptions
126+
127+ // not directly referenced in the table, but used in the special case code.
128+ do_compression int
129+ info [COUNT_INFO]uint16
130+ local_server int
131+
132+ // order matches long_options order
133+ verbose int
134+ msgs2stderr int
135+ quiet int
136+ output_motd int
137+ do_stats int
138+ human_readable int
139+ dry_run int
140+ recurse int
141+ allow_inc_recurse int
142+ xfer_dirs int
143+ preserve_perms int
144+ preserve_executability int
145+ preserve_acls int
146+ preserve_xattrs int
147+ preserve_mtimes int
148+ preserve_atimes int
149+ open_noatime int
150+ preserve_crtimes int
151+ omit_dir_times int
152+ omit_link_times int
153+ modify_window int
154+ am_root int // 0 = normal, 1 = root, 2 = --super, -1 = --fake-super
155+ preserve_uid int
156+ preserve_gid int
157+ preserve_devices int
158+ copy_devices int
159+ write_devices int
160+ preserve_specials int
161+ preserve_links int
162+ copy_links int
163+ copy_unsafe_links int
164+ safe_symlinks int
165+ munge_symlinks int
166+ copy_dirlinks int
167+ keep_dirlinks int
168+ preserve_hard_links int
169+ relative_paths int
170+ implied_dirs int
171+ ignore_times int
172+ size_only int
173+ one_file_system int
174+ update_only int
175+ ignore_non_existing int
176+ ignore_existing int
177+ max_size_arg string
178+ min_size_arg string
179+ max_alloc_arg string
180+ sparse_files int
181+ preallocate_files int
182+ inplace int
183+ append_mode int
184+ delete_during int
185+ delete_mode int
186+ delete_before int
187+ delete_after int
188+ delete_excluded int
189+ missing_args int // 0 = FERROR_XFER, 1 = ignore, 2 = delete
190+ remove_source_files int
191+ force_delete int
192+ ignore_errors int
193+ max_delete int
194+ cvs_exclude int
195+ // If 1, send the whole file as literal data rather than trying to create an
196+ // incremental diff.
197+ // If -1, then look at whether we're local or remote and go by that.
198+ // See also disable_deltas_p()
199+ whole_file int
200+ always_checksum int
201+ checksum_choice string
202+ fuzzy_basis int
203+ compress_choice string
204+ skip_compress string
205+ do_compression_level int
206+ do_progress int
207+ keep_partial int
208+ partial_dir string
209+ delay_updates int
210+ prune_empty_dirs int
211+ logfile_name string
212+ logfile_format string
213+ stdout_format string
214+ itemize_changes int
215+ bwlimit_arg string
216+ bwlimit int
217+ make_backups int
218+ backup_dir string
219+ backup_suffix string
220+ list_only int
221+ batch_name string
222+ files_from string
223+ eol_nulls int
224+ old_style_args int // intentionally set to 0; unsupported
225+ protect_args int // intentionally set to 0; currently unsupported
226+ trust_sender int
227+ numeric_ids int
228+ io_timeout int
229+ connect_timeout int
230+ do_fsync int
231+ shell_cmd string
232+ rsync_path string
233+ tmpdir string
234+ iconv_opt string
235+ default_af_hint int
236+ allow_8bit_chars int
237+ mkpath_dest_arg int
238+ use_qsort int
239+ copy_as string
240+ bind_address string // numeric IPv4 or IPv6, or a hostname
241+ rsync_port int
242+ sockopts string
243+ password_file string
244+ early_input_file string
245+ blocking_io int
246+ outbuf_mode string
247+ protocol_version int
248+ checksum_seed int
249+ am_server int
250+ am_sender int
251+ am_daemon int
252+
253+ daemon_bwlimit int
254+ config_file string
255+ daemon_opt int
256+ no_detach int
257+}
258+
259+type priority int
260+
261+const (
262+ DEFAULT_PRIORITY priority = iota
263+ HELP_PRIORITY
264+ USER_PRIORITY
265+ LIMIT_PRIORITY
266+)
267+
268+const (
269+ W_CLI = 1 << iota
270+ W_SRV
271+ W_SND
272+ W_REC
273+)
274+
275+type output struct {
276+ name string
277+ where int
278+ help string
279+}
280+
281+var infoWords = [...]output{
282+ {"BACKUP", W_REC, "Mention files backed up"},
283+ {"COPY", W_REC, "Mention files copied locally on the receiving side"},
284+ {"DEL", W_REC, "Mention deletions on the receiving side"},
285+ {"FLIST", W_CLI, "Mention file-list receiving/sending (levels 1-2)"},
286+ {"MISC", W_SND | W_REC, "Mention miscellaneous information (levels 1-2)"},
287+ {"MOUNT", W_SND | W_REC, "Mention mounts that were found or skipped"},
288+ {"NAME", W_SND | W_REC, "Mention 1) updated file/dir names, 2) unchanged names"},
289+ {"NONREG", W_REC, "Mention skipped non-regular files (default 1, 0 disables)"},
290+ {"PROGRESS", W_CLI, "Mention 1) per-file progress or 2) total transfer progress"},
291+ {"REMOVE", W_SND, "Mention files removed on the sending side"},
292+ {"SKIP", W_REC, "Mention files skipped due to transfer overrides (levels 1-2)"},
293+ {"STATS", W_CLI | W_SRV, "Mention statistics at end of run (levels 1-3)"},
294+ {"SYMSAFE", W_SND | W_REC, "Mention symlinks that are unsafe"},
295+}
296+
297+func parseOutputWords(words []output, levels []uint16, str string, prio priority) {
298+Level:
299+ for s := range strings.SplitSeq(str, ",") {
300+ if strings.TrimSpace(s) == "" {
301+ continue
302+ }
303+ trimmed := strings.TrimRightFunc(s, unicode.IsNumber)
304+ lev := 1
305+ if len(trimmed) < len(s) {
306+ var err error
307+ lev, err = strconv.Atoi(s[len(trimmed):])
308+ if err != nil {
309+ continue
310+ }
311+ }
312+ trimmed = strings.ToLower(trimmed)
313+ all := false
314+ switch trimmed {
315+ case "none":
316+ lev = 0
317+ case "all":
318+ all = true
319+ }
320+ for j := range words {
321+ word := words[j]
322+ if strings.ToLower(word.name) == trimmed || all {
323+ levels[j] = uint16(lev)
324+ if !all {
325+ continue Level
326+ }
327+ }
328+ }
329+ }
330+}
331+
332+func (o *Options) setOutputVerbosity(prio priority) {
333+ // debugVerbosity is reserved for future use with debug logging levels.
334+ // debugVerbosity := [...]string{
335+ // "",
336+ // "",
337+ // "BIND,CMD,CONNECT,DEL,DELTASUM,DUP,FILTER,FLIST,ICONV",
338+ // "ACL,BACKUP,CONNECT2,DELTASUM2,DEL2,EXIT,FILTER2,FLIST2,FUZZY,GENR,OWN,RECV,SEND,TIME",
339+ // "CMD2,DELTASUM3,DEL3,EXIT2,FLIST3,ICONV2,OWN2,PROTO,TIME2",
340+ // "CHDIR,DELTASUM4,FLIST4,FUZZY2,HASH,HLINK",
341+ // }
342+ infoVerbosity := [...]string{
343+ "NONREG",
344+ "COPY,DEL,FLIST,MISC,NAME,STATS,SYMSAFE",
345+ "BACKUP,MISC2,MOUNT,NAME2,REMOVE,SKIP",
346+ }
347+ for j := 0; j <= o.verbose; j++ {
348+ if j < len(infoVerbosity) {
349+ parseOutputWords(infoWords[:], o.info[:], infoVerbosity[j], prio)
350+ }
351+ // TODO: enable debug verbosity when debug logging is implemented
352+ // if j < len(debugVerbosity) {
353+ // parseOutputWords(debugWords[:], o.debug[:], debugVerbosity[j], prio)
354+ // }
355+ }
356+}
357+
358+func (o *Options) Help() string {
359+ return ""
360+}
361+
362+func (o *Options) ShellCommand() string { return o.shell_cmd }
363+func (o *Options) UpdateOnly() bool { return o.update_only != 0 }
364+func (o *Options) DryRun() bool { return o.dry_run != 0 }
365+func (o *Options) PreserveLinks() bool { return o.preserve_links != 0 }
366+func (o *Options) PreserveUid() bool { return o.preserve_uid != 0 }
367+func (o *Options) PreserveGid() bool { return o.preserve_gid != 0 }
368+func (o *Options) PreserveDevices() bool { return o.preserve_devices != 0 }
369+func (o *Options) PreserveMTimes() bool { return o.preserve_mtimes != 0 }
370+func (o *Options) PreservePerms() bool { return o.preserve_perms != 0 }
371+func (o *Options) PreserveSpecials() bool { return o.preserve_specials != 0 }
372+func (o *Options) PreserveHardLinks() bool { return o.preserve_hard_links != 0 }
373+func (o *Options) Recurse() bool { return o.recurse != 0 }
374+func (o *Options) Verbose() bool { return o.verbose != 0 }
375+func (o *Options) DeleteMode() bool { return o.delete_mode != 0 }
376+func (o *Options) Sender() bool { return o.am_sender != 0 }
377+func (o *Options) SetSender() { o.am_sender = 1 }
378+func (o *Options) LocalServer() bool { return o.local_server != 0 }
379+func (o *Options) SetLocalServer() { o.local_server = 1 }
380+func (o *Options) Server() bool { return o.am_server != 0 }
381+func (o *Options) Daemon() bool { return o.am_daemon != 0 }
382+func (o *Options) ConnectTimeoutSeconds() int { return o.connect_timeout }
383+func (o *Options) AlwaysChecksum() bool { return o.always_checksum != 0 }
384+func (o *Options) Compress() bool { return o.do_compression != 0 }
385+func (o *Options) CompressChoice() string { return o.compress_choice }
386+func (o *Options) CompressLevel() int { return o.do_compression_level }
387+func (o *Options) IgnoreTimes() bool { return o.ignore_times == 1 }
388+func (o *Options) SizeOnly() bool { return o.size_only == 1 }
389+
390+func (o *Options) daemonTable() []poptOption {
391+ return []poptOption{
392+ /* longName, shortName, argInfo, arg, val */
393+ {"help", "", POPT_ARG_NONE, nil, OPT_HELP},
394+ {"address", "", POPT_ARG_STRING, &o.bind_address, 0},
395+ {"bwlimit", "", POPT_ARG_INT, &o.daemon_bwlimit, 0},
396+ {"config", "", POPT_ARG_STRING, &o.config_file, 0},
397+ {"daemon", "", POPT_ARG_NONE, &o.daemon_opt, 0},
398+ {"dparam", "M", POPT_ARG_STRING, nil, 'M'},
399+ {"ipv4", "4", POPT_ARG_VAL, &o.default_af_hint, syscall.AF_INET},
400+ {"ipv6", "6", POPT_ARG_VAL, &o.default_af_hint, syscall.AF_INET6},
401+ {"detach", "", POPT_ARG_VAL, &o.no_detach, 0},
402+ {"no-detach", "", POPT_ARG_VAL, &o.no_detach, 1},
403+ {"log-file", "", POPT_ARG_STRING, &o.logfile_name, 0},
404+ {"log-file-format", "", POPT_ARG_STRING, &o.logfile_format, 0},
405+ {"port", "", POPT_ARG_INT, &o.rsync_port, 0},
406+ {"sockopts", "", POPT_ARG_STRING, &o.sockopts, 0},
407+ {"protocol", "", POPT_ARG_INT, &o.protocol_version, 0},
408+ {"server", "", POPT_ARG_NONE, &o.am_server, 0},
409+ {"temp-dir", "T", POPT_ARG_STRING, &o.tmpdir, 0},
410+ {"verbose", "v", POPT_ARG_NONE, 0, 'v'},
411+ {"no-verbose", "", POPT_ARG_VAL, &o.verbose, 0},
412+ {"no-v", "", POPT_ARG_VAL, &o.verbose, 0},
413+ {"help", "h", POPT_ARG_NONE, 0, 'h'},
414+ }
415+}
416+
417+func (o *Options) table() []poptOption {
418+ return []poptOption{
419+ /* longName, shortName, argInfo, arg, val */
420+ {"help", "", POPT_ARG_NONE, nil, OPT_HELP},
421+ {"version", "V", POPT_ARG_NONE, nil, 'V'},
422+ {"verbose", "v", POPT_ARG_NONE, nil, 'v'},
423+ {"no-verbose", "", POPT_ARG_VAL, &o.verbose, 0},
424+ {"no-v", "", POPT_ARG_VAL, &o.verbose, 0},
425+ {"info", "", POPT_ARG_STRING, nil, OPT_INFO},
426+ {"debug", "", POPT_ARG_STRING, nil, OPT_DEBUG},
427+ {"stderr", "", POPT_ARG_STRING, nil, OPT_STDERR},
428+ {"msgs2stderr", "", POPT_ARG_VAL, &o.msgs2stderr, 1},
429+ {"no-msgs2stderr", "", POPT_ARG_VAL, &o.msgs2stderr, 0},
430+ {"quiet", "q", POPT_ARG_NONE, nil, 'q'},
431+ {"motd", "", POPT_ARG_VAL, &o.output_motd, 1},
432+ {"no-motd", "", POPT_ARG_VAL, &o.output_motd, 0},
433+ {"stats", "", POPT_ARG_NONE, &o.do_stats, 0},
434+ {"human-readable", "h", POPT_ARG_NONE, nil, 'h'},
435+ {"no-human-readable", "", POPT_ARG_VAL, &o.human_readable, 0},
436+ {"no-h", "", POPT_ARG_VAL, &o.human_readable, 0},
437+ {"dry-run", "n", POPT_ARG_NONE, &o.dry_run, 0},
438+ {"archive", "a", POPT_ARG_NONE, nil, 'a'},
439+ {"recursive", "r", POPT_ARG_VAL, &o.recurse, 2},
440+ {"no-recursive", "", POPT_ARG_VAL, &o.recurse, 0},
441+ {"no-r", "", POPT_ARG_VAL, &o.recurse, 0},
442+ {"inc-recursive", "", POPT_ARG_VAL, &o.allow_inc_recurse, 1},
443+ {"no-inc-recursive", "", POPT_ARG_VAL, &o.allow_inc_recurse, 0},
444+ {"i-r", "", POPT_ARG_VAL, &o.allow_inc_recurse, 1},
445+ {"no-i-r", "", POPT_ARG_VAL, &o.allow_inc_recurse, 0},
446+ {"dirs", "d", POPT_ARG_VAL, &o.xfer_dirs, 2},
447+ {"no-dirs", "", POPT_ARG_VAL, &o.xfer_dirs, 0},
448+ {"no-d", "", POPT_ARG_VAL, &o.xfer_dirs, 0},
449+ {"old-dirs", "", POPT_ARG_VAL, &o.xfer_dirs, 4},
450+ {"old-d", "", POPT_ARG_VAL, &o.xfer_dirs, 4},
451+ {"perms", "p", POPT_ARG_VAL, &o.preserve_perms, 1},
452+ {"no-perms", "", POPT_ARG_VAL, &o.preserve_perms, 0},
453+ {"no-p", "", POPT_ARG_VAL, &o.preserve_perms, 0},
454+ {"executability", "E", POPT_ARG_NONE, &o.preserve_executability, 0},
455+ {"acls", "A", POPT_ARG_NONE, nil, 'A'},
456+ {"no-acls", "", POPT_ARG_VAL, &o.preserve_acls, 0},
457+ {"no-A", "", POPT_ARG_VAL, &o.preserve_acls, 0},
458+ {"xattrs", "X", POPT_ARG_NONE, nil, 'X'},
459+ {"no-xattrs", "", POPT_ARG_VAL, &o.preserve_xattrs, 0},
460+ {"no-X", "", POPT_ARG_VAL, &o.preserve_xattrs, 0},
461+ {"times", "t", POPT_ARG_VAL, &o.preserve_mtimes, 1},
462+ {"no-times", "", POPT_ARG_VAL, &o.preserve_mtimes, 0},
463+ {"no-t", "", POPT_ARG_VAL, &o.preserve_mtimes, 0},
464+ {"atimes", "U", POPT_ARG_NONE, nil, 'U'},
465+ {"no-atimes", "", POPT_ARG_VAL, &o.preserve_atimes, 0},
466+ {"no-U", "", POPT_ARG_VAL, &o.preserve_atimes, 0},
467+ {"open-noatime", "", POPT_ARG_VAL, &o.open_noatime, 1},
468+ {"no-open-noatime", "", POPT_ARG_VAL, &o.open_noatime, 0},
469+ {"crtimes", "N", POPT_ARG_NONE, &o.preserve_crtimes, 1}, // refused
470+ {"no-crtimes", "", POPT_ARG_VAL, &o.preserve_crtimes, 0},
471+ {"no-N", "", POPT_ARG_VAL, &o.preserve_crtimes, 0},
472+ {"omit-dir-times", "O", POPT_ARG_VAL, &o.omit_dir_times, 1},
473+ {"no-omit-dir-times", "", POPT_ARG_VAL, &o.omit_dir_times, 0},
474+ {"no-O", "", POPT_ARG_VAL, &o.omit_dir_times, 0},
475+ {"omit-link-times", "J", POPT_ARG_VAL, &o.omit_link_times, 1},
476+ {"no-omit-link-times", "", POPT_ARG_VAL, &o.omit_link_times, 0},
477+ {"no-J", "", POPT_ARG_VAL, &o.omit_link_times, 0},
478+ {"modify-window", "@", POPT_ARG_INT, &o.modify_window, OPT_MODIFY_WINDOW},
479+ {"super", "", POPT_ARG_VAL, &o.am_root, 2},
480+ {"no-super", "", POPT_ARG_VAL, &o.am_root, 0},
481+ {"fake-super", "", POPT_ARG_VAL, &o.am_root, -1},
482+ {"owner", "o", POPT_ARG_VAL, &o.preserve_uid, 1},
483+ {"no-owner", "", POPT_ARG_VAL, &o.preserve_uid, 0},
484+ {"no-o", "", POPT_ARG_VAL, &o.preserve_uid, 0},
485+ {"group", "g", POPT_ARG_VAL, &o.preserve_gid, 1},
486+ {"no-group", "", POPT_ARG_VAL, &o.preserve_gid, 0},
487+ {"no-g", "", POPT_ARG_VAL, &o.preserve_gid, 0},
488+ {"", "D", POPT_ARG_NONE, nil, 'D'},
489+ {"no-D", "", POPT_ARG_NONE, nil, OPT_NO_D},
490+ {"devices", "", POPT_ARG_VAL, &o.preserve_devices, 1},
491+ {"no-devices", "", POPT_ARG_VAL, &o.preserve_devices, 0},
492+ {"copy-devices", "", POPT_ARG_NONE, &o.copy_devices, 0},
493+ {"write-devices", "", POPT_ARG_VAL, &o.write_devices, 1},
494+ {"no-write-devices", "", POPT_ARG_VAL, &o.write_devices, 0},
495+ {"specials", "", POPT_ARG_VAL, &o.preserve_specials, 1},
496+ {"no-specials", "", POPT_ARG_VAL, &o.preserve_specials, 0},
497+ {"links", "l", POPT_ARG_VAL, &o.preserve_links, 1},
498+ {"no-links", "", POPT_ARG_VAL, &o.preserve_links, 0},
499+ {"no-l", "", POPT_ARG_VAL, &o.preserve_links, 0},
500+ {"copy-links", "L", POPT_ARG_NONE, &o.copy_links, 0},
501+ {"copy-unsafe-links", "", POPT_ARG_NONE, &o.copy_unsafe_links, 0},
502+ {"safe-links", "", POPT_ARG_NONE, &o.safe_symlinks, 0},
503+ {"munge-links", "", POPT_ARG_VAL, &o.munge_symlinks, 1},
504+ {"no-munge-links", "", POPT_ARG_VAL, &o.munge_symlinks, 0},
505+ {"copy-dirlinks", "k", POPT_ARG_NONE, &o.copy_dirlinks, 0},
506+ {"keep-dirlinks", "K", POPT_ARG_NONE, &o.keep_dirlinks, 0},
507+ {"hard-links", "H", POPT_ARG_NONE, nil, 'H'},
508+ {"no-hard-links", "", POPT_ARG_VAL, &o.preserve_hard_links, 0},
509+ {"no-H", "", POPT_ARG_VAL, &o.preserve_hard_links, 0},
510+ {"relative", "R", POPT_ARG_VAL, &o.relative_paths, 1},
511+ {"no-relative", "", POPT_ARG_VAL, &o.relative_paths, 0},
512+ {"no-R", "", POPT_ARG_VAL, &o.relative_paths, 0},
513+ {"implied-dirs", "", POPT_ARG_VAL, &o.implied_dirs, 1},
514+ {"no-implied-dirs", "", POPT_ARG_VAL, &o.implied_dirs, 0},
515+ {"i-d", "", POPT_ARG_VAL, &o.implied_dirs, 1},
516+ {"no-i-d", "", POPT_ARG_VAL, &o.implied_dirs, 0},
517+ {"chmod", "", POPT_ARG_STRING, nil, OPT_CHMOD},
518+ {"ignore-times", "I", POPT_ARG_NONE, &o.ignore_times, 0},
519+ {"size-only", "", POPT_ARG_NONE, &o.size_only, 0},
520+ {"one-file-system", "x", POPT_ARG_NONE, nil, 'x'},
521+ {"no-one-file-system", "", POPT_ARG_VAL, &o.one_file_system, 0},
522+ {"no-x", "", POPT_ARG_VAL, &o.one_file_system, 0},
523+ {"update", "u", POPT_ARG_NONE, &o.update_only, 0},
524+ {"existing", "", POPT_ARG_NONE, &o.ignore_non_existing, 0},
525+ {"ignore-non-existing", "", POPT_ARG_NONE, &o.ignore_non_existing, 0},
526+ {"ignore-existing", "", POPT_ARG_NONE, &o.ignore_existing, 0},
527+ {"max-size", "", POPT_ARG_STRING, &o.max_size_arg, OPT_MAX_SIZE},
528+ {"min-size", "", POPT_ARG_STRING, &o.min_size_arg, OPT_MIN_SIZE},
529+ {"max-alloc", "", POPT_ARG_STRING, &o.max_alloc_arg, 0},
530+ {"sparse", "S", POPT_ARG_VAL, &o.sparse_files, 1},
531+ {"no-sparse", "", POPT_ARG_VAL, &o.sparse_files, 0},
532+ {"no-S", "", POPT_ARG_VAL, &o.sparse_files, 0},
533+ {"preallocate", "", POPT_ARG_NONE, &o.preallocate_files, 0},
534+ {"inplace", "", POPT_ARG_VAL, &o.inplace, 1},
535+ {"no-inplace", "", POPT_ARG_VAL, &o.inplace, 0},
536+ {"append", "", POPT_ARG_NONE, nil, OPT_APPEND},
537+ {"append-verify", "", POPT_ARG_VAL, &o.append_mode, 2},
538+ {"no-append", "", POPT_ARG_VAL, &o.append_mode, 0},
539+ {"del", "", POPT_ARG_NONE, &o.delete_during, 0},
540+ {"delete", "", POPT_ARG_NONE, &o.delete_mode, 0},
541+ {"delete-before", "", POPT_ARG_NONE, &o.delete_before, 0},
542+ {"delete-during", "", POPT_ARG_VAL, &o.delete_during, 1},
543+ {"delete-delay", "", POPT_ARG_VAL, &o.delete_during, 2},
544+ {"delete-after", "", POPT_ARG_NONE, &o.delete_after, 0},
545+ {"delete-excluded", "", POPT_ARG_NONE, &o.delete_excluded, 0},
546+ {"delete-missing-args", "", POPT_BIT_SET, &o.missing_args, 2},
547+ {"ignore-missing-args", "", POPT_BIT_SET, &o.missing_args, 1},
548+ {"remove-sent-files", "", POPT_ARG_VAL, &o.remove_source_files, 2}, /* deprecated */
549+ {"remove-source-files", "", POPT_ARG_VAL, &o.remove_source_files, 1},
550+ {"force", "", POPT_ARG_VAL, &o.force_delete, 1},
551+ {"no-force", "", POPT_ARG_VAL, &o.force_delete, 0},
552+ {"ignore-errors", "", POPT_ARG_VAL, &o.ignore_errors, 1},
553+ {"no-ignore-errors", "", POPT_ARG_VAL, &o.ignore_errors, 0},
554+ {"max-delete", "", POPT_ARG_INT, &o.max_delete, 0},
555+ {"", "F", POPT_ARG_NONE, nil, 'F'},
556+ {"filter", "f", POPT_ARG_STRING, nil, OPT_FILTER},
557+ {"exclude", "", POPT_ARG_STRING, nil, OPT_EXCLUDE},
558+ {"include", "", POPT_ARG_STRING, nil, OPT_INCLUDE},
559+ {"exclude-from", "", POPT_ARG_STRING, nil, OPT_EXCLUDE_FROM},
560+ {"include-from", "", POPT_ARG_STRING, nil, OPT_INCLUDE_FROM},
561+ {"cvs-exclude", "C", POPT_ARG_NONE, &o.cvs_exclude, 0},
562+ {"whole-file", "W", POPT_ARG_VAL, &o.whole_file, 1},
563+ {"no-whole-file", "", POPT_ARG_VAL, &o.whole_file, 0},
564+ {"no-W", "", POPT_ARG_VAL, &o.whole_file, 0},
565+ {"checksum", "c", POPT_ARG_VAL, &o.always_checksum, 1},
566+ {"no-checksum", "", POPT_ARG_VAL, &o.always_checksum, 0},
567+ {"no-c", "", POPT_ARG_VAL, &o.always_checksum, 0},
568+ {"checksum-choice", "", POPT_ARG_STRING, &o.checksum_choice, 0},
569+ {"cc", "", POPT_ARG_STRING, &o.checksum_choice, 0},
570+ {"block-size", "B", POPT_ARG_STRING, nil, OPT_BLOCK_SIZE},
571+ {"compare-dest", "", POPT_ARG_STRING, nil, OPT_COMPARE_DEST},
572+ {"copy-dest", "", POPT_ARG_STRING, nil, OPT_COPY_DEST},
573+ {"link-dest", "", POPT_ARG_STRING, nil, OPT_LINK_DEST},
574+ {"fuzzy", "y", POPT_ARG_NONE, nil, 'y'},
575+ {"no-fuzzy", "", POPT_ARG_VAL, &o.fuzzy_basis, 0},
576+ {"no-y", "", POPT_ARG_VAL, &o.fuzzy_basis, 0},
577+ {"compress", "z", POPT_ARG_NONE, nil, 'z'},
578+ {"old-compress", "", POPT_ARG_NONE, nil, OPT_OLD_COMPRESS},
579+ {"new-compress", "", POPT_ARG_NONE, nil, OPT_NEW_COMPRESS},
580+ {"no-compress", "", POPT_ARG_NONE, nil, OPT_NO_COMPRESS},
581+ {"no-z", "", POPT_ARG_NONE, nil, OPT_NO_COMPRESS},
582+ {"compress-choice", "", POPT_ARG_STRING, &o.compress_choice, 0},
583+ {"zc", "", POPT_ARG_STRING, &o.compress_choice, 0},
584+ {"skip-compress", "", POPT_ARG_STRING, &o.skip_compress, 0},
585+ {"compress-level", "", POPT_ARG_INT, &o.do_compression_level, 0},
586+ {"zl", "", POPT_ARG_INT, &o.do_compression_level, 0},
587+ {"", "P", POPT_ARG_NONE, nil, 'P'},
588+ {"progress", "", POPT_ARG_VAL, &o.do_progress, 1},
589+ {"no-progress", "", POPT_ARG_VAL, &o.do_progress, 0},
590+ {"partial", "", POPT_ARG_VAL, &o.keep_partial, 1},
591+ {"no-partial", "", POPT_ARG_VAL, &o.keep_partial, 0},
592+ {"partial-dir", "", POPT_ARG_STRING, &o.partial_dir, 0},
593+ {"delay-updates", "", POPT_ARG_VAL, &o.delay_updates, 1},
594+ {"no-delay-updates", "", POPT_ARG_VAL, &o.delay_updates, 0},
595+ {"prune-empty-dirs", "m", POPT_ARG_VAL, &o.prune_empty_dirs, 1},
596+ {"no-prune-empty-dirs", "", POPT_ARG_VAL, &o.prune_empty_dirs, 0},
597+ {"no-m", "", POPT_ARG_VAL, &o.prune_empty_dirs, 0},
598+ {"log-file", "", POPT_ARG_STRING, &o.logfile_name, 0},
599+ {"log-file-format", "", POPT_ARG_STRING, &o.logfile_format, 0},
600+ {"out-format", "", POPT_ARG_STRING, &o.stdout_format, 0},
601+ {"log-format", "", POPT_ARG_STRING, &o.stdout_format, 0}, /* DEPRECATED */
602+ {"itemize-changes", "i", POPT_ARG_NONE, nil, 'i'},
603+ {"no-itemize-changes", "", POPT_ARG_VAL, &o.itemize_changes, 0},
604+ {"no-i", "", POPT_ARG_VAL, &o.itemize_changes, 0},
605+ {"bwlimit", "", POPT_ARG_STRING, &o.bwlimit_arg, OPT_BWLIMIT},
606+ {"no-bwlimit", "", POPT_ARG_VAL, &o.bwlimit, 0},
607+ {"backup", "b", POPT_ARG_VAL, &o.make_backups, 1},
608+ {"no-backup", "", POPT_ARG_VAL, &o.make_backups, 0},
609+ {"backup-dir", "", POPT_ARG_STRING, &o.backup_dir, 0},
610+ {"suffix", "", POPT_ARG_STRING, &o.backup_suffix, 0},
611+ {"list-only", "", POPT_ARG_VAL, &o.list_only, 2},
612+ {"read-batch", "", POPT_ARG_STRING, &o.batch_name, OPT_READ_BATCH},
613+ {"write-batch", "", POPT_ARG_STRING, &o.batch_name, OPT_WRITE_BATCH},
614+ {"only-write-batch", "", POPT_ARG_STRING, &o.batch_name, OPT_ONLY_WRITE_BATCH},
615+ {"files-from", "", POPT_ARG_STRING, &o.files_from, 0},
616+ {"from0", "0", POPT_ARG_VAL, &o.eol_nulls, 1},
617+ {"no-from0", "", POPT_ARG_VAL, &o.eol_nulls, 0},
618+ {"old-args", "", POPT_ARG_NONE, nil, OPT_OLD_ARGS},
619+ {"no-old-args", "", POPT_ARG_VAL, &o.old_style_args, 0},
620+ {"secluded-args", "s", POPT_ARG_VAL, &o.protect_args, 1},
621+ {"no-secluded-args", "", POPT_ARG_VAL, &o.protect_args, 0},
622+ {"protect-args", "", POPT_ARG_VAL, &o.protect_args, 1},
623+ {"no-protect-args", "", POPT_ARG_VAL, &o.protect_args, 0},
624+ {"no-s", "", POPT_ARG_VAL, &o.protect_args, 0},
625+ {"trust-sender", "", POPT_ARG_VAL, &o.trust_sender, 1},
626+ {"numeric-ids", "", POPT_ARG_VAL, &o.numeric_ids, 1},
627+ {"no-numeric-ids", "", POPT_ARG_VAL, &o.numeric_ids, 0},
628+ {"usermap", "", POPT_ARG_STRING, nil, OPT_USERMAP},
629+ {"groupmap", "", POPT_ARG_STRING, nil, OPT_GROUPMAP},
630+ {"chown", "", POPT_ARG_STRING, nil, OPT_CHOWN},
631+ {"timeout", "", POPT_ARG_INT, &o.io_timeout, 0},
632+ {"no-timeout", "", POPT_ARG_VAL, &o.io_timeout, 0},
633+ {"contimeout", "", POPT_ARG_INT, &o.connect_timeout, 0},
634+ {"no-contimeout", "", POPT_ARG_VAL, &o.connect_timeout, 0},
635+ {"fsync", "", POPT_ARG_NONE, &o.do_fsync, 0},
636+ {"stop-after", "", POPT_ARG_STRING, nil, OPT_STOP_AFTER},
637+ {"time-limit", "", POPT_ARG_STRING, nil, OPT_STOP_AFTER}, /* earlier stop-after name */
638+ {"stop-at", "", POPT_ARG_STRING, nil, OPT_STOP_AT},
639+ {"rsh", "e", POPT_ARG_STRING, &o.shell_cmd, 0},
640+ {"rsync-path", "", POPT_ARG_STRING, &o.rsync_path, 0},
641+ {"temp-dir", "T", POPT_ARG_STRING, &o.tmpdir, 0},
642+ {"iconv", "", POPT_ARG_STRING, &o.iconv_opt, 0},
643+ {"no-iconv", "", POPT_ARG_NONE, nil, OPT_NO_ICONV},
644+ {"ipv4", "4", POPT_ARG_VAL, &o.default_af_hint, syscall.AF_INET},
645+ {"ipv6", "6", POPT_ARG_VAL, &o.default_af_hint, syscall.AF_INET6},
646+ {"8-bit-output", "8", POPT_ARG_VAL, &o.allow_8bit_chars, 1},
647+ {"no-8-bit-output", "", POPT_ARG_VAL, &o.allow_8bit_chars, 0},
648+ {"no-8", "", POPT_ARG_VAL, &o.allow_8bit_chars, 0},
649+ {"mkpath", "", POPT_ARG_VAL, &o.mkpath_dest_arg, 1},
650+ {"no-mkpath", "", POPT_ARG_VAL, &o.mkpath_dest_arg, 0},
651+ {"qsort", "", POPT_ARG_NONE, &o.use_qsort, 0},
652+ {"copy-as", "", POPT_ARG_STRING, &o.copy_as, 0},
653+ {"address", "", POPT_ARG_STRING, &o.bind_address, 0},
654+ {"port", "", POPT_ARG_INT, &o.rsync_port, 0},
655+ {"sockopts", "", POPT_ARG_STRING, &o.sockopts, 0},
656+ {"password-file", "", POPT_ARG_STRING, &o.password_file, 0},
657+ {"early-input", "", POPT_ARG_STRING, &o.early_input_file, 0},
658+ {"blocking-io", "", POPT_ARG_VAL, &o.blocking_io, 1},
659+ {"no-blocking-io", "", POPT_ARG_VAL, &o.blocking_io, 0},
660+ {"outbuf", "", POPT_ARG_STRING, &o.outbuf_mode, 0},
661+ {"remote-option", "M", POPT_ARG_STRING, nil, 'M'},
662+ {"protocol", "", POPT_ARG_INT, &o.protocol_version, 0},
663+ {"checksum-seed", "", POPT_ARG_INT, &o.checksum_seed, 0},
664+ {"server", "", POPT_ARG_NONE, nil, OPT_SERVER},
665+ {"sender", "", POPT_ARG_NONE, nil, OPT_SENDER},
666+ /* All the following options switch us into daemon-mode option-parsing. */
667+ {"config", "", POPT_ARG_STRING, nil, OPT_DAEMON},
668+ {"daemon", "", POPT_ARG_NONE, nil, OPT_DAEMON},
669+ {"dparam", "", POPT_ARG_STRING, nil, OPT_DAEMON},
670+ {"detach", "", POPT_ARG_NONE, nil, OPT_DAEMON},
671+ {"no-detach", "", POPT_ARG_NONE, nil, OPT_DAEMON},
672+ }
673+}
674+
675+var errNotYetImplemented = errors.New("option not yet implemented in gokrazy/rsync")
676+
677+// rsync/options.c:parse_arguments.
678+func ParseArguments(args []string, gokrazyTable bool) (*Context, error) {
679+ // NOTE: We do not implement support for refusing options per rsyncd.conf
680+ // here, as we have our own configuration file.
681+
682+ version_opt_cnt := 0
683+
684+ opts := NewOptions()
685+ table := opts.table()
686+ if gokrazyTable {
687+ // We need to make the --gokr.* flags known, otherwise the first parsing
688+ // attempt fails and the daemon mode parsing is never run.
689+ table = slices.Concat(opts.Gokrazy.table(), table)
690+ }
691+ pc := Context{
692+ Options: opts,
693+ table: table,
694+ args: args,
695+ }
696+
697+ for {
698+ opt, err := pc.poptGetNextOpt()
699+ if err != nil {
700+ return nil, err
701+ }
702+ if opt == -1 {
703+ break // done
704+ }
705+ // Most options are handled by poptGetNextOpt, only special cases
706+ // are returned and handled here.
707+ switch opt {
708+ case 'V':
709+ version_opt_cnt++
710+
711+ case OPT_SERVER:
712+ opts.am_server = 1
713+
714+ case OPT_SENDER:
715+ if opts.am_server == 0 {
716+ return nil, fmt.Errorf("--sender only allowed with --server")
717+ }
718+ opts.am_sender = 1
719+
720+ case OPT_DAEMON:
721+ // Parse the whole command-line using the daemon options table.
722+ table := opts.daemonTable()
723+ if gokrazyTable {
724+ table = slices.Concat(opts.Gokrazy.table(), table)
725+ }
726+ pc := Context{
727+ Options: opts,
728+ table: table,
729+ args: args,
730+ }
731+
732+ for {
733+ opt, err := pc.poptGetNextOpt()
734+ if err != nil {
735+ return nil, err
736+ }
737+ if opt == -1 {
738+ break // done
739+ }
740+ // Most options are handled by poptGetNextOpt, only special cases
741+ // are returned and handled here.
742+ switch opt {
743+ case 'M':
744+ return nil, errNotYetImplemented
745+ case 'v':
746+ opts.verbose++
747+ default:
748+ return nil, fmt.Errorf("unhandled special case opt: %v", opt)
749+ }
750+ }
751+
752+ opts.am_daemon = 1
753+
754+ return &pc, nil
755+
756+ case OPT_FILTER,
757+ OPT_EXCLUDE,
758+ OPT_INCLUDE,
759+ OPT_INCLUDE_FROM,
760+ OPT_EXCLUDE_FROM:
761+ return nil, errNotYetImplemented
762+
763+ case 'a':
764+ if opts.recurse == 0 {
765+ opts.recurse = 1
766+ }
767+ opts.preserve_links = 1
768+ opts.preserve_perms = 1
769+ opts.preserve_mtimes = 1
770+ opts.preserve_gid = 1
771+ opts.preserve_uid = 1
772+ opts.preserve_devices = 1
773+ opts.preserve_specials = 1
774+
775+ case 'D':
776+ opts.preserve_devices = 1
777+ opts.preserve_specials = 1
778+
779+ case OPT_NO_D:
780+ opts.preserve_devices = 0
781+ opts.preserve_specials = 0
782+
783+ case 'h':
784+ opts.human_readable++
785+
786+ case 'H':
787+ opts.preserve_hard_links = 1
788+
789+ case 'i':
790+ opts.itemize_changes++
791+
792+ case 'U':
793+ opts.preserve_atimes++
794+ if opts.preserve_atimes > 1 {
795+ opts.open_noatime = 1
796+ }
797+
798+ case 'v':
799+ opts.verbose++
800+
801+ case 'y':
802+ return nil, errNotYetImplemented
803+
804+ case 'q':
805+ opts.quiet++
806+
807+ case 'x':
808+ opts.one_file_system++
809+
810+ case 'F':
811+ return nil, errNotYetImplemented
812+
813+ case 'P':
814+ opts.do_progress = 1
815+ opts.keep_partial = 1
816+
817+ case 'z':
818+ opts.do_compression++
819+
820+ case OPT_OLD_COMPRESS:
821+ opts.compress_choice = "zlib"
822+
823+ case OPT_NEW_COMPRESS:
824+ opts.compress_choice = "zlibx"
825+
826+ case OPT_NO_COMPRESS:
827+ opts.do_compression = 0
828+ opts.compress_choice = ""
829+
830+ case OPT_OLD_ARGS:
831+ return nil, errNotYetImplemented
832+
833+ case 'M': // --remote-option
834+ return nil, errNotYetImplemented
835+
836+ case OPT_WRITE_BATCH,
837+ OPT_ONLY_WRITE_BATCH,
838+ OPT_READ_BATCH:
839+ return nil, errNotYetImplemented
840+
841+ case OPT_BLOCK_SIZE:
842+ return nil, errNotYetImplemented
843+
844+ case OPT_MAX_SIZE, // (needs parse_size_arg)
845+ OPT_MIN_SIZE,
846+ OPT_BWLIMIT:
847+ return nil, errNotYetImplemented
848+
849+ case OPT_APPEND:
850+ return nil, errNotYetImplemented
851+
852+ case OPT_LINK_DEST,
853+ OPT_COPY_DEST,
854+ OPT_COMPARE_DEST:
855+ return nil, errNotYetImplemented
856+
857+ case OPT_CHMOD: // (needs parse_chmod):
858+ return nil, errNotYetImplemented
859+
860+ case OPT_INFO:
861+ parseOutputWords(infoWords[:], opts.info[:], pc.poptGetOptArg(), USER_PRIORITY)
862+
863+ case OPT_DEBUG:
864+ // TODO: plumb the debug level that make sense for our implementation
865+ slog.Info("TODO: set debug level", "to", pc.poptGetOptArg())
866+
867+ case OPT_USERMAP,
868+ OPT_GROUPMAP,
869+ OPT_CHOWN:
870+ return nil, errNotYetImplemented
871+
872+ case 'A':
873+ return nil, fmt.Errorf("ACLs are not supported by gokrazy/rsync")
874+
875+ case 'X':
876+ opts.preserve_xattrs++
877+
878+ case OPT_STOP_AFTER,
879+ OPT_STOP_AT,
880+ OPT_STDERR:
881+ return nil, errNotYetImplemented
882+
883+ default:
884+ return nil, fmt.Errorf("unhandled special case opt: %v", opt)
885+ }
886+ }
887+
888+ // rsync/options.c line 1973 and following set option defaults based on
889+ // other options
890+
891+ opts.setOutputVerbosity(DEFAULT_PRIORITY)
892+
893+ if opts.recurse != 0 {
894+ opts.xfer_dirs = 1
895+ }
896+ if opts.xfer_dirs < 0 {
897+ if opts.list_only != 0 {
898+ opts.xfer_dirs = 1
899+ } else {
900+ opts.xfer_dirs = 0
901+ }
902+ }
903+
904+ if opts.relative_paths < 0 {
905+ if opts.files_from != "" {
906+ opts.relative_paths = 1
907+ } else {
908+ opts.relative_paths = 0
909+ }
910+ }
911+
912+ if opts.relative_paths == 0 {
913+ opts.implied_dirs = 0
914+ }
915+
916+ // NOTE: This simplification means that even if we ignore POPT_ARGFLAG_OR
917+ // and store ints without regards for bit sets, we get the same result.
918+ // Nevertheless, we support bit to be future-proof as new options are added.
919+ if opts.missing_args == 3 {
920+ // simplify if both options were specified
921+ opts.missing_args = 2
922+ }
923+
924+ if opts.backup_suffix == "" && opts.backup_dir == "" {
925+ opts.backup_suffix = "~"
926+ }
927+
928+ if opts.backup_dir != "" {
929+ opts.make_backups = 1 // --backup-dir implies --backup
930+ }
931+
932+ if opts.do_progress != 0 /* && !opts.am_server */ {
933+ if opts.info[INFO_NAME] == 0 {
934+ opts.info[INFO_NAME] = 1
935+ }
936+ }
937+
938+ if opts.info[INFO_NAME] >= 1 && opts.stdout_format == "" {
939+ opts.stdout_format = "%n%L"
940+ }
941+
942+ return &pc, nil
943+}
A pkg/rsync-receiver/rsyncreceiver/client.go
+109, -0
......@@ -0,0 +1,109 @@
1+package rsyncreceiver
2+
3+import (
4+ "fmt"
5+ "io"
6+ "log/slog"
7+ "os"
8+
9+ "github.com/picosh/pico/pkg/rsync-receiver/rsync"
10+ "github.com/picosh/pico/pkg/rsync-receiver/rsyncopts"
11+ "github.com/picosh/pico/pkg/rsync-receiver/rsyncsender"
12+ "github.com/picosh/pico/pkg/rsync-receiver/rsyncwire"
13+ "github.com/picosh/pico/pkg/rsync-receiver/utils"
14+)
15+
16+func ClientRun(logger *slog.Logger, opts *rsyncopts.Options, conn io.ReadWriter, filesystem utils.FS, paths []string, negotiate bool) error {
17+ var err error
18+
19+ crd, cwr := rsyncwire.CounterPair(conn, conn)
20+
21+ const sessionChecksumSeed = 666
22+
23+ c := &rsyncwire.Conn{
24+ Reader: crd,
25+ Writer: cwr,
26+ }
27+
28+ if negotiate {
29+ remoteProtocol, err := c.ReadInt32()
30+ if err != nil {
31+ return err
32+ }
33+ logger.Debug("remote protocol", "protocol", remoteProtocol)
34+ if err := c.WriteInt32(rsync.ProtocolVersion); err != nil {
35+ return err
36+ }
37+ }
38+
39+ if err := c.WriteInt32(sessionChecksumSeed); err != nil {
40+ return err
41+ }
42+
43+ // Switch to multiplexing protocol, but only for server-side transmissions.
44+ // Transmissions received from the client are not multiplexed.
45+ mpx := &rsyncwire.MultiplexWriter{Writer: c.Writer}
46+ c.Writer = mpx
47+
48+ defer func() {
49+ if err != nil {
50+ _, _ = mpx.WriteMsg(rsyncwire.MsgError, fmt.Appendf(nil, "gokr-rsync [receiver]: %v\n", err))
51+ }
52+ }()
53+
54+ rt := &Transfer{
55+ Opts: &TransferOpts{
56+ DryRun: opts.DryRun(),
57+
58+ DeleteMode: opts.DeleteMode(),
59+ PreserveGid: opts.PreserveGid(),
60+ PreserveUid: opts.PreserveUid(),
61+ PreserveLinks: opts.PreserveLinks(),
62+ PreservePerms: opts.PreservePerms(),
63+ PreserveDevices: opts.PreserveDevices(),
64+ PreserveSpecials: opts.PreserveSpecials(),
65+ PreserveTimes: opts.PreserveMTimes(),
66+ IgnoreTimes: opts.IgnoreTimes(),
67+ SizeOnly: opts.SizeOnly(),
68+ AlwaysChecksum: opts.AlwaysChecksum(),
69+ // TODO: PreserveHardlinks: opts.PreserveHardlinks,
70+ },
71+ Dest: "/",
72+ // TODO: what is Env used for and can we get rid of it?
73+ Env: Osenv{
74+ Stdout: os.Stdout,
75+ Stderr: os.Stderr,
76+ Stdin: os.Stdin,
77+ },
78+ Conn: c,
79+ Seed: sessionChecksumSeed,
80+
81+ Files: filesystem,
82+
83+ Logger: logger,
84+ }
85+
86+ if opts.DeleteMode() {
87+ // receive the exclusion list (openrsync’s is always empty)
88+ exclusionList, err := rsyncsender.RecvFilterList(c)
89+ if err != nil {
90+ return err
91+ }
92+ logger.Debug("exclusion list read", "filters", exclusionList.Filters)
93+ }
94+
95+ // receive file list
96+ logger.Debug("receiving file list")
97+ fileList, err := rt.ReceiveFileList()
98+ if err != nil {
99+ return err
100+ }
101+ logger.Debug("received names", "files", fileList)
102+ stats, err := rt.Do(c, fileList, true)
103+ if err != nil {
104+ return err
105+ }
106+
107+ logger.Debug("stats", "stats", stats)
108+ return nil
109+}
A pkg/rsync-receiver/rsyncreceiver/do.go
+94, -0
......@@ -0,0 +1,94 @@
1+package rsyncreceiver
2+
3+import (
4+ "context"
5+
6+ "github.com/picosh/pico/pkg/rsync-receiver/rsyncstats"
7+ "github.com/picosh/pico/pkg/rsync-receiver/rsyncwire"
8+ "github.com/picosh/pico/pkg/rsync-receiver/utils"
9+ "golang.org/x/sync/errgroup"
10+)
11+
12+func (rt *Transfer) deleteFiles(fileList []*utils.ReceiverFile) error {
13+ if rt.IOErrors > 0 {
14+ rt.Logger.Debug("IO error encountered, skipping file deletion")
15+ return nil
16+ }
17+
18+ return rt.Files.Remove(fileList)
19+}
20+
21+// rsync/main.c:do_recv.
22+func (rt *Transfer) Do(c *rsyncwire.Conn, fileList []*utils.ReceiverFile, noReport bool) (*rsyncstats.TransferStats, error) {
23+ if rt.Opts.DeleteMode {
24+ if err := rt.deleteFiles(fileList); err != nil {
25+ return nil, err
26+ }
27+ }
28+
29+ ctx := context.Background()
30+ eg, ctx := errgroup.WithContext(ctx)
31+ eg.Go(func() error {
32+ return rt.GenerateFiles(fileList)
33+ })
34+ eg.Go(func() error {
35+ // Ensure we don’t block on the receiver when the generator returns an
36+ // error.
37+ errChan := make(chan error)
38+ go func() {
39+ errChan <- rt.RecvFiles(fileList)
40+ }()
41+ select {
42+ case <-ctx.Done():
43+ return ctx.Err()
44+ case err := <-errChan:
45+ return err
46+ }
47+ })
48+ if err := eg.Wait(); err != nil {
49+ return nil, err
50+ }
51+
52+ var stats *rsyncstats.TransferStats
53+ if !noReport {
54+ var err error
55+ stats, err = report(c)
56+ rt.Logger.Debug("report", "stats", stats)
57+ if err != nil {
58+ return nil, err
59+ }
60+ }
61+
62+ // send final goodbye message
63+ if err := c.WriteInt32(-1); err != nil {
64+ return nil, err
65+ }
66+
67+ return stats, nil
68+}
69+
70+// rsync/main.c:report.
71+func report(c *rsyncwire.Conn) (*rsyncstats.TransferStats, error) {
72+ // read statistics:
73+ // total bytes read (from network connection)
74+ read, err := c.ReadInt64()
75+ if err != nil {
76+ return nil, err
77+ }
78+ // total bytes written (to network connection)
79+ written, err := c.ReadInt64()
80+ if err != nil {
81+ return nil, err
82+ }
83+ // total size of files
84+ size, err := c.ReadInt64()
85+ if err != nil {
86+ return nil, err
87+ }
88+
89+ return &rsyncstats.TransferStats{
90+ Read: read,
91+ Written: written,
92+ Size: size,
93+ }, nil
94+}
A pkg/rsync-receiver/rsyncreceiver/flist.go
+191, -0
......@@ -0,0 +1,191 @@
1+package rsyncreceiver
2+
3+import (
4+ "fmt"
5+ "io"
6+ "path/filepath"
7+ "time"
8+
9+ "github.com/picosh/pico/pkg/rsync-receiver/rsync"
10+ "github.com/picosh/pico/pkg/rsync-receiver/utils"
11+)
12+
13+// rsync/flist.c:receive_file_entry.
14+func (rt *Transfer) receiveFileEntry(flags uint16, last *utils.ReceiverFile) (*utils.ReceiverFile, error) {
15+ f := &utils.ReceiverFile{}
16+
17+ var l1 int
18+ if flags&rsync.XMIT_SAME_NAME != 0 {
19+ l, err := rt.Conn.ReadByte()
20+ if err != nil {
21+ return nil, err
22+ }
23+ l1 = int(l)
24+ }
25+
26+ var l2 int
27+ if flags&rsync.XMIT_LONG_NAME != 0 {
28+ l, err := rt.Conn.ReadInt32()
29+ if err != nil {
30+ return nil, err
31+ }
32+ l2 = int(l)
33+ } else {
34+ l, err := rt.Conn.ReadByte()
35+ if err != nil {
36+ return nil, err
37+ }
38+ l2 = int(l)
39+ }
40+ // linux/limits.h
41+ const PATH_MAX = 4096
42+ if l2 >= PATH_MAX-l1 {
43+ const lastname = ""
44+ return nil, fmt.Errorf("overflow: flags=0x%x l1=%d l2=%d lastname=%s",
45+ flags, l1, l2, lastname)
46+ }
47+ b := make([]byte, l1+l2)
48+ readb := b
49+ if l1 > 0 {
50+ copy(b, []byte(last.Name))
51+ readb = b[l1:]
52+ }
53+ if _, err := io.ReadFull(rt.Conn.Reader, readb); err != nil {
54+ return nil, err
55+ }
56+ // TODO: does rsync’s clean_fname() and sanitize_path() combination do
57+ // anything more than Go’s filepath.Clean()?
58+ f.Name = filepath.Clean(string(b))
59+
60+ length, err := rt.Conn.ReadInt64()
61+ if err != nil {
62+ return nil, err
63+ }
64+ f.Length = length
65+
66+ if flags&rsync.XMIT_SAME_TIME != 0 {
67+ f.ModTime = last.ModTime
68+ } else {
69+ modTime, err := rt.Conn.ReadInt32()
70+ if err != nil {
71+ return nil, err
72+ }
73+ f.ModTime = time.Unix(int64(modTime), 0)
74+ }
75+
76+ if flags&rsync.XMIT_SAME_MODE != 0 {
77+ f.Mode = last.Mode
78+ } else {
79+ mode, err := rt.Conn.ReadInt32()
80+ if err != nil {
81+ return nil, err
82+ }
83+ f.Mode = mode
84+ }
85+
86+ if rt.Opts.PreserveUid {
87+ if flags&rsync.XMIT_SAME_UID != 0 {
88+ f.Uid = last.Uid
89+ } else {
90+ uid, err := rt.Conn.ReadInt32()
91+ if err != nil {
92+ return nil, err
93+ }
94+ f.Uid = uid
95+ }
96+ }
97+
98+ if rt.Opts.PreserveGid {
99+ if flags&rsync.XMIT_SAME_GID != 0 {
100+ f.Gid = last.Gid
101+ } else {
102+ gid, err := rt.Conn.ReadInt32()
103+ if err != nil {
104+ return nil, err
105+ }
106+ f.Gid = gid
107+ }
108+ }
109+
110+ mode := f.Mode & rsync.S_IFMT
111+ isDev := mode == rsync.S_IFCHR || mode == rsync.S_IFBLK
112+ isSpecial := mode == rsync.S_IFIFO || mode == rsync.S_IFSOCK
113+ isLink := mode == rsync.S_IFLNK
114+
115+ if rt.Opts.PreserveDevices && (isDev || isSpecial) {
116+ // TODO(protocol >= 28): rdev/major/minor handling
117+ if flags&rsync.XMIT_SAME_RDEV_pre28 != 0 {
118+ f.Rdev = last.Rdev
119+ } else {
120+ rdev, err := rt.Conn.ReadInt32()
121+ if err != nil {
122+ return nil, err
123+ }
124+ f.Rdev = rdev
125+ }
126+ }
127+
128+ if rt.Opts.PreserveLinks && isLink {
129+ length, err := rt.Conn.ReadInt32()
130+ if err != nil {
131+ return nil, err
132+ }
133+ b := make([]byte, length)
134+ if _, err := io.ReadFull(rt.Conn.Reader, b); err != nil {
135+ return nil, err
136+ }
137+ f.LinkTarget = string(b)
138+ }
139+
140+ return f, nil
141+}
142+
143+// rsync/flist.c:recv_file_list.
144+func (rt *Transfer) ReceiveFileList() ([]*utils.ReceiverFile, error) {
145+ lastFileEntry := new(utils.ReceiverFile)
146+ var fileList []*utils.ReceiverFile
147+ for {
148+ b, err := rt.Conn.ReadByte()
149+ if err != nil {
150+ return nil, err
151+ }
152+ if b == 0 {
153+ break
154+ }
155+ flags := uint16(b)
156+ // log.Printf("flags: %x", flags)
157+ // TODO(protocol >= 28): extended flags
158+
159+ f, err := rt.receiveFileEntry(flags, lastFileEntry)
160+ if err != nil {
161+ return nil, err
162+ }
163+ lastFileEntry = f
164+ // TODO: include depth in output?
165+ rt.Logger.Debug("recv_file_list", "file", f.Name, "length", f.Length, "mode", f.Mode, "uid", f.Uid, "gid", f.Gid, "flags", flags)
166+
167+ fileList = append(fileList, f)
168+ }
169+
170+ utils.SortFileList(fileList)
171+
172+ if rt.Opts.PreserveUid || rt.Opts.PreserveGid {
173+ // receive the uid/gid list
174+ users, groups, err := rt.RecvIdList()
175+ if err != nil {
176+ return nil, err
177+ }
178+ _ = users
179+ _ = groups
180+ }
181+
182+ // read the i/o error flag
183+ ioErrors, err := rt.Conn.ReadInt32()
184+ if err != nil {
185+ return nil, err
186+ }
187+ rt.Logger.Debug("ioErrors", "errs", ioErrors)
188+ rt.IOErrors = ioErrors
189+
190+ return fileList, nil
191+}
A pkg/rsync-receiver/rsyncreceiver/generator.go
+156, -0
......@@ -0,0 +1,156 @@
1+package rsyncreceiver
2+
3+import (
4+ "fmt"
5+ "io"
6+ "os"
7+
8+ "github.com/picosh/pico/pkg/rsync-receiver/rsync"
9+ "github.com/picosh/pico/pkg/rsync-receiver/rsyncchecksum"
10+ "github.com/picosh/pico/pkg/rsync-receiver/rsynccommon"
11+ "github.com/picosh/pico/pkg/rsync-receiver/utils"
12+)
13+
14+// rsync/generator.c:generate_files().
15+func (rt *Transfer) GenerateFiles(fileList []*utils.ReceiverFile) error {
16+ phase := 0
17+ for idx, f := range fileList {
18+ // TODO: use a copy of f with .Mode |= S_IWUSR for directories, so
19+ // that we can create files within all directories.
20+ if err := rt.recvGenerator(idx, f); err != nil {
21+ return err
22+ }
23+ }
24+ phase++
25+ rt.Logger.Debug("generateFiles", "phase", phase)
26+ if err := rt.Conn.WriteInt32(-1); err != nil {
27+ return err
28+ }
29+
30+ // TODO: re-do any files that failed
31+ phase++
32+ rt.Logger.Debug("generateFiles", "phase", phase)
33+ if err := rt.Conn.WriteInt32(-1); err != nil {
34+ return err
35+ }
36+
37+ rt.Logger.Debug("generateFiles finished")
38+ return nil
39+}
40+
41+// rsync/generator.c:skip_file.
42+func (rt *Transfer) skipFile(f *utils.ReceiverFile, st os.FileInfo) (bool, error) {
43+ if rt.Opts.AlwaysChecksum || rt.Opts.IgnoreTimes {
44+ return false, nil
45+ }
46+
47+ sizeMatch := st.Size() == f.Length
48+ if rt.Opts.SizeOnly {
49+ return sizeMatch, nil
50+ }
51+
52+ timeMatch := st.ModTime().Equal(f.ModTime)
53+ return sizeMatch && timeMatch, nil
54+}
55+
56+// rsync/generator.c:recv_generator.
57+func (rt *Transfer) recvGenerator(idx int, f *utils.ReceiverFile) error {
58+ if rt.listOnly() {
59+ if _, err := fmt.Fprintf(rt.Env.Stdout, "%s %11.0f %s %s\n",
60+ f.FileMode().String(),
61+ float64(f.Length), // TODO: rsync prints decimal separators
62+ f.ModTime.Format("2006/01/02 15:04:05"),
63+ f.Name); err != nil {
64+ return err
65+ }
66+ return nil
67+ }
68+ rt.Logger.Debug("recv_generator", "file", f)
69+
70+ if !f.FileMode().IsRegular() {
71+ // None of the Preserve* options is enabled, so just skip over
72+ // non-regular files.
73+ return nil
74+ }
75+
76+ requestFullFile := func() error {
77+ rt.Logger.Debug("requesting", "file", f)
78+ if err := rt.Conn.WriteInt32(int32(idx)); err != nil {
79+ return err
80+ }
81+ if rt.Opts.DryRun {
82+ return nil
83+ }
84+ var sh rsync.SumHead
85+ if err := sh.WriteTo(rt.Conn); err != nil {
86+ return err
87+ }
88+ return nil
89+ }
90+
91+ st, in, err := rt.Files.Read(&utils.SenderFile{WPath: f.Name})
92+ if err != nil {
93+ rt.Logger.Error("failed to open file", "st", st, "file", f, "err", err)
94+ return requestFullFile()
95+ }
96+
97+ defer func() { _ = in.Close() }()
98+
99+ skip, err := rt.skipFile(f, st)
100+ if err != nil {
101+ return err
102+ }
103+
104+ if skip {
105+ rt.Logger.Debug("skipping", "file", f)
106+ return nil
107+ }
108+
109+ if rt.Opts.DryRun {
110+ if err := rt.Conn.WriteInt32(int32(idx)); err != nil {
111+ return err
112+ }
113+
114+ return nil
115+ }
116+
117+ rt.Logger.Debug("sending sums", "file", f, "st", st)
118+ if err := rt.Conn.WriteInt32(int32(idx)); err != nil {
119+ return err
120+ }
121+
122+ err = rt.generateAndSendSums(in, st.Size())
123+ if err != nil {
124+ rt.Logger.Error("failed to send sums", "file", f, "err", err)
125+ }
126+
127+ return err
128+}
129+
130+// rsync/generator.c:generate_and_send_sums.
131+func (rt *Transfer) generateAndSendSums(in utils.ReaderAtCloser, fileLen int64) error {
132+ sh := rsynccommon.SumSizesSqroot(fileLen)
133+ if err := sh.WriteTo(rt.Conn); err != nil {
134+ return err
135+ }
136+ buf := make([]byte, int(sh.BlockLength))
137+ remaining := fileLen
138+ for i := int32(0); i < sh.ChecksumCount; i++ {
139+ n1 := min(int64(sh.BlockLength), remaining)
140+ b := buf[:n1]
141+ if _, err := io.ReadFull(in, b); err != nil {
142+ return err
143+ }
144+
145+ sum1 := rsyncchecksum.Checksum1(b)
146+ sum2 := rsyncchecksum.Checksum2(rt.Seed, b)
147+ if err := rt.Conn.WriteInt32(int32(sum1)); err != nil {
148+ return err
149+ }
150+ if _, err := rt.Conn.Writer.Write(sum2); err != nil {
151+ return err
152+ }
153+ remaining -= n1
154+ }
155+ return nil
156+}
A pkg/rsync-receiver/rsyncreceiver/receiver.go
+170, -0
......@@ -0,0 +1,170 @@
1+package rsyncreceiver
2+
3+import (
4+ "bytes"
5+ "encoding/binary"
6+ "errors"
7+ "fmt"
8+ "io"
9+ "sync"
10+
11+ "github.com/mmcloughlin/md4"
12+ "github.com/picosh/pico/pkg/rsync-receiver/rsync"
13+ "github.com/picosh/pico/pkg/rsync-receiver/utils"
14+)
15+
16+// rsync/receiver.c:recv_files.
17+func (rt *Transfer) RecvFiles(fileList []*utils.ReceiverFile) error {
18+ phase := 0
19+ for {
20+ idx, err := rt.Conn.ReadInt32()
21+ if err != nil {
22+ return err
23+ }
24+ if idx == -1 {
25+ if phase == 0 {
26+ phase++
27+ rt.Logger.Debug("recvFiles phase", "phase", phase)
28+ // TODO: send done message
29+ continue
30+ }
31+ break
32+ }
33+ rt.Logger.Debug("receiving file", "idx", idx, "file", fileList[idx])
34+ if err := rt.recvFile1(fileList[idx]); err != nil {
35+ return err
36+ }
37+ }
38+ rt.Logger.Debug("recvFiles finished")
39+ return nil
40+}
41+
42+func (rt *Transfer) recvFile1(f *utils.ReceiverFile) error {
43+ if rt.Opts.DryRun {
44+ fmt.Println(f.Name)
45+ return nil
46+ }
47+
48+ localFile, err := rt.openLocalFile(f)
49+ if err != nil {
50+ rt.Logger.Error("opening local file failed, continuing", "err", err, "file", f)
51+ } else {
52+ defer func() { _ = localFile.Close() }()
53+ }
54+
55+ err = rt.receiveData(f, localFile)
56+ if err != nil {
57+ rt.Logger.Error("receiving data failed, continuing", "err", err, "file", f)
58+ }
59+ return err
60+}
61+
62+func (rt *Transfer) openLocalFile(f *utils.ReceiverFile) (utils.ReaderAtCloser, error) {
63+ _, r, err := rt.Files.Read(&utils.SenderFile{
64+ WPath: f.Name,
65+ Regular: true,
66+ })
67+
68+ if err != nil {
69+ return nil, err
70+ }
71+
72+ return r, nil
73+}
74+
75+// rsync/receiver.c:receive_data.
76+func (rt *Transfer) receiveData(f *utils.ReceiverFile, localFile utils.ReaderAtCloser) error {
77+ var sh rsync.SumHead
78+ if err := sh.ReadFrom(rt.Conn); err != nil {
79+ return err
80+ }
81+
82+ r, w := io.Pipe()
83+
84+ f.Reader = r
85+
86+ var wg sync.WaitGroup
87+ wg.Add(1)
88+
89+ go func() {
90+ defer func() {
91+ wg.Done()
92+ if err := r.Close(); err != nil {
93+ return
94+ }
95+ }()
96+
97+ _, err := rt.Files.Put(f)
98+ if err != nil {
99+ return
100+ }
101+ }()
102+
103+ h := md4.New()
104+ _ = binary.Write(h, binary.LittleEndian, rt.Seed) // hash.Hash.Write never fails
105+
106+ for {
107+ token, data, err := rt.recvToken()
108+ if err != nil {
109+ return err
110+ }
111+ if token == 0 {
112+ break
113+ }
114+ if token > 0 {
115+ if _, err := h.Write(data); err != nil {
116+ return err
117+ }
118+
119+ if _, err := w.Write(data); err != nil {
120+ if errors.Is(err, io.ErrClosedPipe) {
121+ continue
122+ }
123+ return err
124+ }
125+ continue
126+ }
127+ if localFile == nil {
128+ return fmt.Errorf("BUG: local file %s not open for copying chunk", localFile)
129+ }
130+ token = -(token + 1)
131+ offset2 := int64(token) * int64(sh.BlockLength)
132+ dataLen := sh.BlockLength
133+ if token == sh.ChecksumCount-1 && sh.RemainderLength != 0 {
134+ dataLen = sh.RemainderLength
135+ }
136+ data = make([]byte, dataLen)
137+ if _, err := localFile.ReadAt(data, offset2); err != nil {
138+ return err
139+ }
140+
141+ if _, err := h.Write(data); err != nil {
142+ return err
143+ }
144+
145+ if _, err := w.Write(data); err != nil {
146+ if errors.Is(err, io.ErrClosedPipe) {
147+ continue
148+ }
149+ return err
150+ }
151+ }
152+
153+ if err := w.Close(); err != nil {
154+ return err
155+ }
156+
157+ wg.Wait()
158+
159+ localSum := h.Sum(nil)
160+ remoteSum := make([]byte, len(localSum))
161+ if _, err := io.ReadFull(rt.Conn.Reader, remoteSum); err != nil {
162+ return err
163+ }
164+ if !bytes.Equal(localSum, remoteSum) {
165+ return fmt.Errorf("file corruption in %s", f.Name)
166+ }
167+ rt.Logger.Debug("checksum matches!", "localSum", localSum)
168+
169+ return nil
170+}
A pkg/rsync-receiver/rsyncreceiver/token.go
+20, -0
......@@ -0,0 +1,20 @@
1+package rsyncreceiver
2+
3+import "io"
4+
5+// rsync/token.c:recvToken.
6+func (rt *Transfer) recvToken() (token int32, data []byte, _ error) {
7+ var err error
8+ token, err = rt.Conn.ReadInt32()
9+ if err != nil {
10+ return 0, nil, err
11+ }
12+ if token <= 0 {
13+ return token, nil, nil
14+ }
15+ data = make([]byte, int(token))
16+ if _, err := io.ReadFull(rt.Conn.Reader, data); err != nil {
17+ return 0, nil, err
18+ }
19+ return token, data, nil
20+}
A pkg/rsync-receiver/rsyncreceiver/transfer.go
+53, -0
......@@ -0,0 +1,53 @@
1+package rsyncreceiver
2+
3+import (
4+ "io"
5+ "log/slog"
6+
7+ "github.com/picosh/pico/pkg/rsync-receiver/rsyncwire"
8+ "github.com/picosh/pico/pkg/rsync-receiver/utils"
9+)
10+
11+type Osenv struct {
12+ Stdin io.Reader
13+ Stdout io.Writer
14+ Stderr io.Writer
15+}
16+
17+// TransferOpts is a subset of Opts which is required for implementing a receiver.
18+type TransferOpts struct {
19+ Verbose bool
20+ DryRun bool
21+
22+ DeleteMode bool
23+ PreserveGid bool
24+ PreserveUid bool
25+ PreserveLinks bool
26+ PreservePerms bool
27+ PreserveDevices bool
28+ PreserveSpecials bool
29+ PreserveTimes bool
30+ PreserveHardlinks bool
31+ IgnoreTimes bool
32+ SizeOnly bool
33+ AlwaysChecksum bool
34+}
35+
36+type Transfer struct {
37+ // config
38+ // Opts *Opts
39+ Opts *TransferOpts
40+ Dest string
41+ Env Osenv
42+
43+ // state
44+ Conn *rsyncwire.Conn
45+ Seed int32
46+ IOErrors int32
47+
48+ Files utils.FS
49+
50+ Logger *slog.Logger
51+}
52+
53+func (rt *Transfer) listOnly() bool { return rt.Dest == "" }
A pkg/rsync-receiver/rsyncreceiver/uidlist.go
+69, -0
......@@ -0,0 +1,69 @@
1+package rsyncreceiver
2+
3+import (
4+ "io"
5+)
6+
7+type mapping struct {
8+ Name string
9+ LocalId int32
10+}
11+
12+func (rt *Transfer) recvIdMapping1(localId func(id int32, name string) int32) (map[int32]mapping, error) {
13+ idMapping := make(map[int32]mapping)
14+ for {
15+ id, err := rt.Conn.ReadInt32()
16+ if err != nil {
17+ return nil, err
18+ }
19+ if id == 0 {
20+ break
21+ }
22+ length, err := rt.Conn.ReadByte()
23+ if err != nil {
24+ return nil, err
25+ }
26+ name := make([]byte, length)
27+ if _, err := io.ReadFull(rt.Conn.Reader, name); err != nil {
28+ return nil, err
29+ }
30+ idMapping[id] = mapping{
31+ Name: string(name),
32+ LocalId: localId(id, string(name)),
33+ }
34+ }
35+ return idMapping, nil
36+}
37+
38+// rsync/uidlist.c:recv_id_list.
39+func (rt *Transfer) RecvIdList() (users map[int32]mapping, groups map[int32]mapping, _ error) {
40+ if rt.Opts.PreserveUid {
41+ var err error
42+ users, err = rt.recvIdMapping1(func(remoteUid int32, remoteUsername string) int32 {
43+ // TODO: look up local uid by username
44+ return remoteUid
45+ })
46+ if err != nil {
47+ return nil, nil, err
48+ }
49+ for remoteUid, mapping := range users {
50+ rt.Logger.Debug("remote uid maps to local uid", "remoteUid", remoteUid, "name", mapping.Name, "localUid", mapping.LocalId)
51+ }
52+ }
53+
54+ if rt.Opts.PreserveGid {
55+ var err error
56+ groups, err = rt.recvIdMapping1(func(remoteGid int32, remoteGroupname string) int32 {
57+ // TODO: look up local gid by groupname
58+ return remoteGid
59+ })
60+ if err != nil {
61+ return nil, nil, err
62+ }
63+ for remoteGid, mapping := range groups {
64+ rt.Logger.Debug("remote gid maps to local gid", "remoteGid", remoteGid, "name", mapping.Name, "localGid", mapping.LocalId)
65+ }
66+ }
67+
68+ return users, groups, nil
69+}
A pkg/rsync-receiver/rsyncsender/client.go
+75, -0
......@@ -0,0 +1,75 @@
1+package rsyncsender
2+
3+import (
4+ "fmt"
5+ "io"
6+ "log/slog"
7+
8+ "github.com/picosh/pico/pkg/rsync-receiver/rsync"
9+ "github.com/picosh/pico/pkg/rsync-receiver/rsyncopts"
10+ "github.com/picosh/pico/pkg/rsync-receiver/rsyncwire"
11+ "github.com/picosh/pico/pkg/rsync-receiver/utils"
12+)
13+
14+func ClientRun(logger *slog.Logger, opts *rsyncopts.Options, conn io.ReadWriter, filesystem utils.FS, paths []string, negotiate bool) error {
15+ var err error
16+
17+ crd, cwr := rsyncwire.CounterPair(conn, conn)
18+
19+ const sessionChecksumSeed = 666
20+
21+ c := &rsyncwire.Conn{
22+ Reader: crd,
23+ Writer: cwr,
24+ }
25+
26+ if negotiate {
27+ remoteProtocol, err := c.ReadInt32()
28+ if err != nil {
29+ return err
30+ }
31+ logger.Debug("remote protocol", "remoteProtocol", remoteProtocol)
32+ if err := c.WriteInt32(rsync.ProtocolVersion); err != nil {
33+ return err
34+ }
35+ }
36+
37+ if err := c.WriteInt32(sessionChecksumSeed); err != nil {
38+ return err
39+ }
40+
41+ // Switch to multiplexing protocol, but only for server-side transmissions.
42+ // Transmissions received from the client are not multiplexed.
43+ mpx := &rsyncwire.MultiplexWriter{Writer: c.Writer}
44+ c.Writer = mpx
45+
46+ defer func() {
47+ if err != nil {
48+ _, _ = mpx.WriteMsg(rsyncwire.MsgError, fmt.Appendf(nil, "gokr-rsync [sender]: %v\n", err))
49+ }
50+ }()
51+
52+ st := &Transfer{
53+ Opts: opts,
54+ Conn: c,
55+ Seed: sessionChecksumSeed,
56+ Files: filesystem,
57+
58+ Logger: logger,
59+ }
60+ // receive the exclusion list (openrsync’s is always empty)
61+ exclusionList, err := RecvFilterList(st.Conn)
62+ if err != nil {
63+ return err
64+ }
65+ logger.Debug("exclusion list read", "filters", exclusionList.Filters)
66+
67+ stats, err := st.Do(crd, cwr, paths, exclusionList)
68+ if err != nil {
69+ return err
70+ }
71+
72+ logger.Debug("handleConnSender done. stats", "stats", stats)
73+
74+ return err
75+}
A pkg/rsync-receiver/rsyncsender/do.go
+68, -0
......@@ -0,0 +1,68 @@
1+package rsyncsender
2+
3+import (
4+ "fmt"
5+ "sort"
6+
7+ "github.com/picosh/pico/pkg/rsync-receiver/rsyncstats"
8+ "github.com/picosh/pico/pkg/rsync-receiver/rsyncwire"
9+)
10+
11+// rsync/main.c:client_run am_sender.
12+func (st *Transfer) Do(crd *rsyncwire.CountingReader, cwr *rsyncwire.CountingWriter, paths []string, exclusionList *filterRuleList) (*rsyncstats.TransferStats, error) {
13+ if exclusionList == nil {
14+ exclusionList = &filterRuleList{}
15+ }
16+
17+ // “Update exchange” as per
18+ // https://github.com/kristapsdz/openrsync/blob/master/rsync.5
19+
20+ // send file list
21+ fileList, err := st.SendFileList(st.Opts, paths, exclusionList)
22+ if err != nil {
23+ return nil, err
24+ }
25+
26+ st.Logger.Debug("file list sent")
27+
28+ // Sort the file list. The client sorts, so we need to sort, too (in the
29+ // same way!), otherwise our indices do not match what the client will
30+ // request.
31+ sort.Slice(fileList.Files, func(i, j int) bool {
32+ return fileList.Files[i].WPath < fileList.Files[j].WPath
33+ })
34+
35+ if err := st.SendFiles(fileList); err != nil {
36+ return nil, err
37+ }
38+
39+ // send statistics:
40+ // total bytes read (from network connection)
41+ if err := st.Conn.WriteInt64(crd.BytesRead); err != nil {
42+ return nil, err
43+ }
44+ // total bytes written (to network connection)
45+ if err := st.Conn.WriteInt64(cwr.BytesWritten); err != nil {
46+ return nil, err
47+ }
48+ // total size of files
49+ if err := st.Conn.WriteInt64(fileList.TotalSize); err != nil {
50+ return nil, err
51+ }
52+
53+ st.Logger.Debug("reading final int32")
54+
55+ finish, err := st.Conn.ReadInt32()
56+ if err != nil {
57+ return nil, err
58+ }
59+ if finish != -1 {
60+ return nil, fmt.Errorf("protocol error: expected final -1, got %d", finish)
61+ }
62+
63+ return &rsyncstats.TransferStats{
64+ Read: crd.BytesRead,
65+ Written: cwr.BytesWritten,
66+ Size: fileList.TotalSize,
67+ }, nil
68+}
A pkg/rsync-receiver/rsyncsender/exclude.go
+109, -0
......@@ -0,0 +1,109 @@
1+package rsyncsender
2+
3+import (
4+ "io"
5+ "path/filepath"
6+ "strings"
7+
8+ "github.com/picosh/pico/pkg/rsync-receiver/rsyncwire"
9+)
10+
11+type filterRuleList struct {
12+ Filters []*filterRule
13+}
14+
15+// exclude.c:add_rule.
16+func (l *filterRuleList) addRule(fr *filterRule) {
17+ if strings.HasSuffix(fr.pattern, "/") {
18+ fr.flag |= filtruleDirectory
19+ fr.pattern = strings.TrimSuffix(fr.pattern, "/")
20+ }
21+ if strings.ContainsFunc(fr.pattern, func(r rune) bool {
22+ return r == '*' || r == '[' || r == '?'
23+ }) {
24+ fr.flag |= filtruleWild
25+ }
26+ l.Filters = append(l.Filters, fr)
27+}
28+
29+// exclude.c:check_filter.
30+func (l *filterRuleList) matches(name string) bool {
31+ for _, fr := range l.Filters {
32+ if fr.matches(name) {
33+ return true
34+ }
35+ }
36+ return false
37+}
38+
39+// exclude.c:recv_filter_list.
40+func RecvFilterList(c *rsyncwire.Conn) (*filterRuleList, error) {
41+ var l filterRuleList
42+ const exclusionListEnd = 0
43+ for {
44+ length, err := c.ReadInt32()
45+ if err != nil {
46+ return nil, err
47+ }
48+ if length == exclusionListEnd {
49+ break
50+ }
51+ line := make([]byte, length)
52+ if _, err := io.ReadFull(c.Reader, line); err != nil {
53+ return nil, err
54+ }
55+ fr, err := parseFilter(string(line))
56+ if err != nil {
57+ return nil, err
58+ }
59+ l.addRule(fr)
60+ }
61+ return &l, nil
62+}
63+
64+const (
65+ filtruleInclude = 1 << iota
66+ filtruleClearList
67+ filtruleDirectory
68+ filtruleWild
69+)
70+
71+type filterRule struct {
72+ flag int
73+ pattern string
74+}
75+
76+// exclude.c:rule_matches.
77+func (fr *filterRule) matches(name string) bool {
78+ if fr.flag&filtruleWild != 0 {
79+ panic("wildcard filter rules not yet implemented")
80+ }
81+ if !strings.ContainsRune(fr.pattern, '/') &&
82+ fr.flag&filtruleWild == 0 {
83+ name = filepath.Base(name)
84+ }
85+ return fr.pattern == name
86+}
87+
88+// exclude.c:parse_filter_str / exclude.c:parse_rule_tok.
89+func parseFilter(line string) (*filterRule, error) {
90+ rule := new(filterRule)
91+
92+ // We only support what rsync calls XFLG_OLD_PREFIXES
93+ if strings.HasPrefix(line, "- ") {
94+ // clear include flag
95+ rule.flag &= ^filtruleInclude
96+ line = strings.TrimPrefix(line, "- ")
97+ } else if strings.HasPrefix(line, "+ ") {
98+ // set include flag
99+ rule.flag |= filtruleInclude
100+ line = strings.TrimPrefix(line, "+ ")
101+ } else if strings.HasPrefix(line, "!") {
102+ // set clear_list flag
103+ rule.flag |= filtruleClearList
104+ }
105+
106+ rule.pattern = line
107+
108+ return rule, nil
109+}
A pkg/rsync-receiver/rsyncsender/fileio.go
+116, -0
......@@ -0,0 +1,116 @@
1+package rsyncsender
2+
3+import (
4+ "io"
5+ "log/slog"
6+ "os"
7+)
8+
9+// rsync.h:map_struct.
10+type mapStruct struct {
11+ fileSize int64 // file size (from stat)
12+ pOffset int64 // window start
13+ pFdOffset int64 // offset of cursor in fd ala lseek
14+ window []byte // window pointer
15+ pSize int64 // largest window we allocated
16+ pLen int64 // latest (rounded) window size
17+ defWindowSize int64 // default window size
18+ f *os.File // file descriptor
19+ err error // first read error
20+}
21+
22+const alignBoundary = 1024
23+
24+func alignedLength(l int64) int64 {
25+ return ((l - 1) | (alignBoundary - 1)) + 1
26+}
27+
28+func alignedOvershoot(off int64) int64 {
29+ return off & (alignBoundary - 1)
30+}
31+
32+func mapFile(f *os.File, len int64, readSize int32, blkSize int32) *mapStruct {
33+ if blkSize > 0 && readSize%blkSize != 0 {
34+ readSize += blkSize - (readSize % blkSize)
35+ }
36+ return &mapStruct{
37+ fileSize: len,
38+ defWindowSize: alignedLength(int64(readSize)),
39+ f: f,
40+ }
41+}
42+
43+func (ms *mapStruct) ptr(offset int64, l int32) []byte {
44+ //log.Printf("ptr(offset=%d, l=%d)", offset, l)
45+ len := int64(l)
46+ if len == 0 {
47+ return nil
48+ }
49+ if len < 0 {
50+ slog.Debug("BUG: invalid len", "len", len)
51+ return nil
52+ }
53+
54+ if offset >= ms.pOffset && offset+int64(len) <= ms.pOffset+int64(ms.pLen) {
55+ //log.Printf("-> already available")
56+ // region already available
57+ off := offset - ms.pOffset
58+ return ms.window[off : off+int64(len)]
59+ }
60+
61+ alignFudge := alignedOvershoot(offset)
62+ windowStart := offset - alignFudge
63+ windowSize := int64(ms.defWindowSize)
64+ if windowStart+windowSize > ms.fileSize {
65+ windowSize = ms.fileSize - windowStart
66+ }
67+ if windowSize < len+alignFudge {
68+ windowSize = alignedLength(len + alignFudge)
69+ }
70+ if windowSize > ms.pSize {
71+ win := make([]byte, windowSize)
72+ copy(win, ms.window)
73+ ms.window = win
74+ ms.pSize = windowSize
75+ }
76+ readStart := windowStart
77+ readSize := windowSize
78+ readOffset := int64(0)
79+
80+ //log.Printf("windowSize: %d, ms=%+v", windowSize, ms)
81+ if windowStart >= ms.pOffset && windowStart < ms.pOffset+ms.pLen &&
82+ windowStart+windowSize >= ms.pOffset+ms.pLen {
83+ readStart = ms.pOffset + ms.pLen
84+ readOffset = readStart - windowStart
85+ readSize = windowSize - readOffset
86+ off := ms.pLen - readOffset
87+ copy(ms.window[:], ms.window[off:off+readOffset])
88+ }
89+ if readSize <= 0 {
90+ slog.Debug("BUG: invalid readSize", "readSize", readSize)
91+ return nil
92+ }
93+ if ms.pFdOffset != readStart {
94+ if _, err := ms.f.Seek(readStart, io.SeekStart); err != nil {
95+ slog.Error("seek error", "err", err)
96+ return nil
97+ }
98+ ms.pFdOffset = readStart
99+ }
100+ ms.pOffset = windowStart
101+ ms.pLen = windowSize
102+ //log.Printf("-> reading %d bytes from %d into buffer at offset=%d", readSize, readStart, readOffset)
103+ for readSize > 0 {
104+ n, err := ms.f.Read(ms.window[readOffset : readOffset+readSize])
105+ if err != nil {
106+ ms.err = err
107+ // TODO: zero the buffer, file has changed mid-transfer
108+ slog.Debug("file has changed mid-transfer")
109+ return nil
110+ }
111+ ms.pFdOffset += int64(n)
112+ readOffset += int64(n)
113+ readSize -= int64(n)
114+ }
115+ return ms.window[alignFudge : alignFudge+len]
116+}
A pkg/rsync-receiver/rsyncsender/flist.go
+236, -0
......@@ -0,0 +1,236 @@
1+package rsyncsender
2+
3+import (
4+ "os"
5+ "os/user"
6+ "strconv"
7+ "sync"
8+
9+ "github.com/picosh/pico/pkg/rsync-receiver/rsync"
10+ "github.com/picosh/pico/pkg/rsync-receiver/rsyncchecksum"
11+ "github.com/picosh/pico/pkg/rsync-receiver/rsyncopts"
12+ "github.com/picosh/pico/pkg/rsync-receiver/rsyncwire"
13+ "github.com/picosh/pico/pkg/rsync-receiver/utils"
14+)
15+
16+type fileList struct {
17+ TotalSize int64
18+ Files []utils.SenderFile
19+}
20+
21+// rsync/rsync.h defines chunkSize as 32 * 1024, but increasing it to 256K
22+// increases throughput with “tridge” rsync as client by 50 Mbit/s.
23+const chunkSize = 256 * 1024
24+
25+var (
26+ lookupOnce sync.Once
27+ lookupGroupOnce sync.Once
28+)
29+
30+// rsync/flist.c:send_file_list.
31+func (st *Transfer) SendFileList(opts *rsyncopts.Options, paths []string, excl *filterRuleList) (*fileList, error) {
32+ var fileList fileList
33+ fec := &rsyncwire.Buffer{}
34+
35+ uidMap := make(map[int32]string)
36+ gidMap := make(map[int32]string)
37+
38+ // TODO: flush in between to keep the pipes filled when traversal takes long
39+
40+ // TODO: handle info == nil case (permission denied?): should set an i/o
41+ // error flag, but traversal should continue
42+
43+ st.Logger.Debug("sendFileList()")
44+ // TODO: handle |root| referring to an individual file, symlink or special (skip)
45+ for _, requested := range paths {
46+ files, err := st.Files.List(requested)
47+ if err != nil {
48+ return nil, err
49+ }
50+
51+ for _, info := range files {
52+ // Only ever transmit long names, like openrsync
53+ flags := byte(rsync.XMIT_LONG_NAME)
54+
55+ // log.Printf("Trim(path=%q, %q) = %q", path, strip, name)
56+ name := info.Name()
57+ path := name
58+ if name == "/" {
59+ name = "."
60+ flags |= rsync.XMIT_TOP_DIR
61+ }
62+ // log.Printf("flags for %q: %v", name, flags)
63+
64+ if excl.matches(name) {
65+ continue
66+ }
67+
68+ fileList.Files = append(fileList.Files, utils.SenderFile{
69+ Path: "/",
70+ Regular: info.Mode().IsRegular(),
71+ WPath: name,
72+ })
73+
74+ // 1. status byte (integer)
75+ _ = fec.WriteByte(flags)
76+
77+ // 2. inherited filename length (optional, byte)
78+ // 3. filename length (integer or byte)
79+ fec.WriteInt32(int32(len(name)))
80+
81+ // 4. file (byte array)
82+ fec.WriteString(name)
83+
84+ // 5. file length (long)
85+ size := info.Size()
86+ if info.Mode().IsDir() {
87+ // tmpfs returns non-4K sizes for directories. Override with
88+ // 4096 to make the tests succeed regardless of the /tmp file
89+ // system type.
90+ size = 4096
91+ }
92+ fec.WriteInt64(size)
93+
94+ fileList.TotalSize += size
95+
96+ // 6. file modification time (optional, integer)
97+ // TODO: this will overflow in 2038! :(
98+ fec.WriteInt32(int32(info.ModTime().Unix()))
99+
100+ // 7. file mode (optional, mode_t, integer)
101+ mode := int32(info.Mode() & os.ModePerm)
102+ isDev := false
103+ isSpecial := false
104+ if info.Mode().IsDir() {
105+ mode |= rsync.S_IFDIR
106+ } else if info.Mode().IsRegular() {
107+ mode |= rsync.S_IFREG
108+ } else if info.Mode().Type()&os.ModeSymlink != 0 {
109+ mode |= rsync.S_IFLNK
110+ // TODO: skip symlink if PreserveSymlinks is not set
111+ }
112+
113+ if info.Mode().Type()&os.ModeCharDevice != 0 {
114+ mode |= rsync.S_IFCHR
115+ isDev = true
116+ } else if info.Mode().Type()&os.ModeDevice != 0 {
117+ mode |= rsync.S_IFBLK
118+ isDev = true
119+ }
120+
121+ if info.Mode().Type()&os.ModeNamedPipe != 0 {
122+ mode |= rsync.S_IFIFO
123+ isSpecial = true
124+ }
125+
126+ if info.Mode().Type()&os.ModeSocket != 0 {
127+ mode |= rsync.S_IFSOCK
128+ isSpecial = true
129+ }
130+
131+ fec.WriteInt32(mode)
132+
133+ if opts.PreserveUid() {
134+ uid, ok := uidFromFileInfo(info)
135+ if ok {
136+ if _, ok := uidMap[uid]; !ok && uid != 0 {
137+ u, err := user.LookupId(strconv.Itoa(int(uid)))
138+ if err != nil {
139+ lookupOnce.Do(func() {
140+ st.Logger.Error("lookup", "uid", uid, "err", err)
141+ })
142+ } else {
143+ uidMap[uid] = u.Username
144+ }
145+ }
146+ }
147+ // 8. if -o, the user id (integer)
148+ fec.WriteInt32(uid)
149+ }
150+
151+ if opts.PreserveGid() {
152+ gid, ok := gidFromFileInfo(info)
153+ if ok {
154+ if _, ok := gidMap[gid]; !ok && gid != 0 {
155+ g, err := user.LookupGroupId(strconv.Itoa(int(gid)))
156+ if err != nil {
157+ lookupGroupOnce.Do(func() {
158+ st.Logger.Error("lookupgroup", "gid", gid, "err", err)
159+ })
160+ } else {
161+ gidMap[gid] = g.Name
162+ }
163+ }
164+ }
165+ // 9. if -g, the group id (integer)
166+ fec.WriteInt32(gid)
167+ }
168+
169+ if (opts.PreserveDevices() && isDev) ||
170+ (opts.PreserveSpecials() && isSpecial) {
171+ // 10. if a special file and -D, the device “rdev” type (integer)
172+ rdev, _ := rdevFromFileInfo(info)
173+ fec.WriteInt32(rdev)
174+ }
175+
176+ if opts.PreserveLinks() && info.Mode().Type()&os.ModeSymlink != 0 {
177+ // 11. if a symbolic link and -l, the link target's length (integer)
178+ // 12. if a symbolic link and -l, the link target (byte array)
179+ target, err := os.Readlink(path)
180+ if err != nil {
181+ continue
182+ }
183+ fec.WriteInt32(int32(len(target)))
184+ fec.WriteString(target)
185+ }
186+
187+ if opts.AlwaysChecksum() {
188+ var emptyChecksum [rsyncchecksum.Size]byte
189+ checksum := emptyChecksum[:]
190+ if info.Mode().IsRegular() {
191+ // TODO: send md4 checksum of this file
192+ checksum, err = rsyncchecksum.FileChecksum(path)
193+ if err != nil {
194+ continue
195+ }
196+ }
197+ // For non-regular files, send empty md4 checksum
198+ fec.WriteString(string(checksum))
199+ }
200+ }
201+ if err != nil {
202+ return nil, err
203+ }
204+ }
205+
206+ const endOfFileList = 0
207+ _ = fec.WriteByte(endOfFileList)
208+
209+ const endOfSet = 0
210+ if opts.PreserveUid() {
211+ for uid, name := range uidMap {
212+ fec.WriteInt32(uid)
213+ _ = fec.WriteByte(byte(len(name)))
214+ fec.WriteString(name)
215+ }
216+ fec.WriteInt32(endOfSet)
217+ }
218+
219+ if opts.PreserveGid() {
220+ for gid, name := range gidMap {
221+ fec.WriteInt32(gid)
222+ _ = fec.WriteByte(byte(len(name)))
223+ fec.WriteString(name)
224+ }
225+ fec.WriteInt32(endOfSet)
226+ }
227+
228+ const ioErrors = 0
229+ fec.WriteInt32(ioErrors)
230+
231+ if err := st.Conn.WriteString(fec.String()); err != nil {
232+ return nil, err
233+ }
234+
235+ return &fileList, nil
236+}
A pkg/rsync-receiver/rsyncsender/match.go
+257, -0
......@@ -0,0 +1,257 @@
1+package rsyncsender
2+
3+import (
4+ "bytes"
5+ "encoding/binary"
6+ "fmt"
7+ "hash"
8+ "os"
9+
10+ "github.com/mmcloughlin/md4"
11+ "github.com/picosh/pico/pkg/rsync-receiver/nofollow"
12+ "github.com/picosh/pico/pkg/rsync-receiver/rsync"
13+ "github.com/picosh/pico/pkg/rsync-receiver/rsyncchecksum"
14+ "github.com/picosh/pico/pkg/rsync-receiver/utils"
15+)
16+
17+type target struct {
18+ index int32
19+ tag uint16
20+}
21+
22+// rsync/match.c:hash_search.
23+func (st *Transfer) hashSearch(targets []target, tagTable map[uint16]int, head rsync.SumHead, fileIndex int32, fl utils.SenderFile) error {
24+ st.Logger.Debug("hashSearch", "file", fl, "head", head)
25+ f, err := os.OpenFile(fl.Path, os.O_RDONLY|nofollow.Maybe, 0)
26+ if err != nil {
27+ return err
28+ }
29+ defer func() { _ = f.Close() }()
30+
31+ fi, err := f.Stat()
32+ if err != nil {
33+ return err
34+ }
35+
36+ readSize := max(3*head.BlockLength, 256*1024)
37+ ms := mapFile(f, fi.Size(), readSize, head.BlockLength)
38+
39+ if err := st.Conn.WriteInt32(fileIndex); err != nil {
40+ return err
41+ }
42+
43+ if err := head.WriteTo(st.Conn); err != nil {
44+ return err
45+ }
46+
47+ // sum_init()
48+ h := md4.New()
49+ _ = binary.Write(h, binary.LittleEndian, st.Seed) // hash.Hash.Write never fails
50+
51+ // The following quotes are citations from
52+ // https://www.samba.org/~tridge/phd_thesis.pdf, section 3.2.6 The
53+ // signature search algorithm (PDF page 64).
54+
55+ // “Once the sorted signature table and the index table have been formed the
56+ // signature search process can begin. For each byte offset in a_i the fast
57+ // signature is computed, along with the 16 bit hash of the fast
58+ // signature. The 16 bit hash is then used to lookup the signature index,
59+ // giving the index in the signature table of the first fast signature with
60+ // that hash.”
61+
62+ var k int
63+ var sum uint32
64+ var s1, s2 uint32
65+ var offset int64
66+ end := fi.Size() + 1 - head.Sums[len(head.Sums)-1].Len
67+ st.Logger.Debug("last block", "len", head.Sums[len(head.Sums)-1].Len, "end", end)
68+
69+ readChunk := func() error {
70+ k = int(head.BlockLength)
71+ if remaining := int(fi.Size() - offset); remaining < k {
72+ k = remaining
73+ }
74+
75+ chunk := ms.ptr(offset, int32(k))
76+ sum = rsyncchecksum.Checksum1(chunk)
77+ s1 = uint32(sum & 0xFFFF)
78+ s2 = uint32(sum >> 16)
79+ return nil
80+ }
81+ if err := readChunk(); err != nil {
82+ return err
83+ }
84+
85+ tagHits := 0
86+Outer:
87+ for {
88+ tag := rsyncchecksum.Tag2(uint16(s1), uint16(s2))
89+ var sum2 []byte
90+ doneCsum2 := false
91+ j, ok := tagTable[tag]
92+ if ok {
93+ // “A linear search is then performed through the signature table, stopping
94+ // when an entry is found with a 16 bit hash which doesn’t match. For each
95+ // entry the current 32 bit fast signature is compared to the entry in the
96+ // signature table, and if that matches then the full 128 bit strong
97+ // signature is computed at the current byte offset and compared to the
98+ // strong signature in the signature table”
99+ sum = (uint32(s1) & 0xFFFF) | (uint32(s2) << 16)
100+ tagHits++
101+ for ; j < int(head.ChecksumCount) && targets[j].tag == tag; j++ {
102+ i := targets[j].index
103+ if sum != head.Sums[i].Sum1 {
104+ continue
105+ }
106+
107+ l := int64(head.BlockLength)
108+ if v := fi.Size() - offset; v < l {
109+ l = v
110+ }
111+ if l != head.Sums[i].Len {
112+ continue
113+ }
114+
115+ // log.Printf("potential match at %d target=%d %d sum=%08x", offset, j, i, sum)
116+
117+ if !doneCsum2 {
118+ buf := ms.ptr(offset, int32(l))
119+ sum2 = rsyncchecksum.Checksum2(st.Seed, buf[:])
120+ doneCsum2 = true
121+ }
122+
123+ if local, remote := sum2[:head.ChecksumLength], head.Sums[i].Sum2[:head.ChecksumLength]; !bytes.Equal(local, remote) {
124+ st.Logger.Debug("false alarm", "local", local, "remote", remote)
125+ //falseAlarms++
126+ continue
127+ }
128+
129+ // TODO(optimization): tridge rsync locates adjacent matches
130+ // here for better run-length encoding, but I’m not sure where
131+ // (if at all) we currently use run-length encoding:
132+ // https://github.com/WayneD/rsync/commit/923fa978088f4c044eec528d9472962d9c9d13c3
133+
134+ // “If the strong signature is found to match then A emits a
135+ // token telling B that a match was found and which block in bi
136+ // was matched12. The search then continues at the byte after
137+ // the matching block.”
138+
139+ if err := st.matched(h, ms, head, offset, i); err != nil {
140+ return err
141+ }
142+
143+ // rsync doesn’t read the next chunk (offset+sums[i].len),
144+ // rsync starts reading one byte before the next chunk
145+ // (offset+sums[i].len-1), because the code path starting at
146+ // “null_tag” removes the chunk’s first byte and adds the
147+ // next byte after the chunk.
148+ offset += head.Sums[i].Len - 1
149+ if err := readChunk(); err != nil {
150+ return fmt.Errorf("readChunk: %v", err)
151+ }
152+
153+ if offset >= end {
154+ break Outer
155+ }
156+
157+ break
158+ }
159+ }
160+
161+ // Update the rolling checksum by removing the oldest byte (update[0])
162+ // and adding the newest byte (update[k]).
163+ backup := max(offset-st.lastMatch, 0)
164+
165+ more := offset+int64(k) < fi.Size()
166+ mmore := int64(0)
167+ if more {
168+ mmore = 1
169+ }
170+ update := ms.ptr(offset-backup, int32(int64(k)+mmore+backup))
171+ update = update[backup:]
172+
173+ s1 -= rsyncchecksum.SignExtend(update[0])
174+ s2 -= uint32(k) * rsyncchecksum.SignExtend(update[0])
175+
176+ if more {
177+ s1 += rsyncchecksum.SignExtend(update[k])
178+ s2 += s1
179+ } else {
180+ k--
181+ }
182+ s1 = uint32(uint16(s1))
183+ s2 = uint32(uint16(s2))
184+
185+ if backup >= int64(head.BlockLength)+chunkSize && end-offset > chunkSize {
186+ // Prevent offset-st.lastMatch from growing too large by flushing
187+ // intermediate chunks.
188+ if err := st.matched(h, ms, head, offset-int64(head.BlockLength), -2); err != nil {
189+ return err
190+ }
191+ }
192+
193+ offset++
194+ if offset >= end {
195+ break
196+ }
197+ }
198+
199+ if err := st.matched(h, ms, head, fi.Size(), -1); err != nil {
200+ return err
201+ }
202+
203+ {
204+ sum := h.Sum(nil)
205+ st.Logger.Debug("sum info", "sum", sum, "len", len(sum))
206+ if _, err := st.Conn.Writer.Write(sum); err != nil {
207+ return err
208+ }
209+ }
210+
211+ return nil
212+
213+}
214+
215+// rsync/match.c:matched.
216+func (st *Transfer) matched(h hash.Hash, ms *mapStruct, head rsync.SumHead, offset int64, i int32) error {
217+ n := offset - st.lastMatch
218+
219+ transmitAccumulated := i < 0
220+
221+ // if !transmitAccumulated {
222+ // log.Printf("match at offset=%d last_match=%d i=%d len=%d n=%d",
223+ // offset, st.lastMatch, i, head.Sums[i].Len, n)
224+ // } else {
225+ // log.Printf("transmit accumulated at offset=%d", offset)
226+ // }
227+
228+ /* FIXME: this is not used
229+ l := int64(0)
230+ if !transmitAccumulated {
231+ l = head.Sums[i].Len
232+ }
233+ */
234+
235+ if err := st.sendToken(ms, i, st.lastMatch, n); err != nil {
236+ return fmt.Errorf("sendToken: %v", err)
237+ }
238+ // TODO: data_transfer += n;
239+
240+ if !transmitAccumulated {
241+ // stats.matched_data += s->sums[i].len;
242+ n += head.Sums[i].Len
243+ }
244+
245+ for j := int64(0); j < n; j += chunkSize {
246+ n1 := min(int64(chunkSize), n-j)
247+ chunk := ms.ptr(st.lastMatch+j, int32(n1))
248+ h.Write(chunk)
249+ }
250+
251+ if !transmitAccumulated {
252+ st.lastMatch = offset + head.Sums[i].Len
253+ } else {
254+ st.lastMatch = offset
255+ }
256+ return nil
257+}
A pkg/rsync-receiver/rsyncsender/sender.go
+216, -0
......@@ -0,0 +1,216 @@
1+package rsyncsender
2+
3+import (
4+ "encoding/binary"
5+ "io"
6+ "os"
7+ "sort"
8+
9+ "github.com/mmcloughlin/md4"
10+ "github.com/picosh/pico/pkg/rsync-receiver/rsync"
11+ "github.com/picosh/pico/pkg/rsync-receiver/rsyncchecksum"
12+ "github.com/picosh/pico/pkg/rsync-receiver/rsynccommon"
13+ "github.com/picosh/pico/pkg/rsync-receiver/utils"
14+)
15+
16+// rsync/sender.c:send_files().
17+func (st *Transfer) SendFiles(fileList *fileList) error {
18+ phase := 0
19+ for {
20+ // receive data about receiver’s copy of the file list contents (not
21+ // ordered)
22+ // see (*rsync.Receiver).Generator()
23+ fileIndex, err := st.Conn.ReadInt32()
24+ if err != nil {
25+ return err
26+ }
27+ if fileIndex == -1 {
28+ if phase == 0 {
29+ phase++
30+ // acknowledge phase change by sending -1
31+ if err := st.Conn.WriteInt32(-1); err != nil {
32+ return err
33+ }
34+ continue
35+ }
36+ break
37+ }
38+
39+ if st.Opts.DryRun() {
40+ if err := st.Conn.WriteInt32(fileIndex); err != nil {
41+ return err
42+ }
43+ continue
44+ }
45+
46+ head, err := st.receiveSums()
47+ if err != nil {
48+ return err
49+ }
50+
51+ // The following quotes are citations from
52+ // https://www.samba.org/~tridge/phd_thesis.pdf, section 3.2.6 The
53+ // signature search algorithm (PDF page 64).
54+
55+ // rsync/match.c:build_hash_table
56+ targets := make([]target, len(head.Sums))
57+ tagTable := make(map[uint16]int) // TODO: or int32 more specifically?
58+ {
59+ // “The first step in the algorithm is to sort the received
60+ // signatures by a 16 bit hash of the fast signature.”
61+ for idx, sum := range head.Sums {
62+ targets[idx] = target{
63+ index: int32(idx),
64+ tag: rsyncchecksum.Tag(sum.Sum1),
65+ }
66+ }
67+ sort.Slice(targets, func(i, j int) bool {
68+ return targets[i].tag < targets[j].tag
69+ })
70+
71+ // “A 16 bit index table is then formed which takes a 16 bit hash
72+ // value and gives an index into the sorted signature table which
73+ // points to the first entry in the table which has a matching
74+ // hash.”
75+ for idx := len(head.Sums) - 1; idx >= 0; idx-- {
76+ tagTable[targets[idx].tag] = idx
77+ }
78+ }
79+
80+ st.lastMatch = 0
81+ if len(head.Sums) == 0 {
82+ // fast path: send the whole file
83+ err = st.sendFile(fileIndex, fileList.Files[fileIndex])
84+ } else {
85+ err = st.hashSearch(targets, tagTable, head, fileIndex, fileList.Files[fileIndex])
86+ }
87+ if err != nil {
88+ if _, ok := err.(*os.PathError); ok {
89+ // OpenFile() failed. Log the error (server side only) and
90+ // proceed. Only starting with protocol 30, an I/O error flag is
91+ // sent after the file transfer phase.
92+ if os.IsNotExist(err) {
93+ st.Logger.Debug("file has vanished", "file", fileList.Files[fileIndex])
94+ } else {
95+ st.Logger.Error("sendFiles", "err", err)
96+ }
97+ continue
98+ } else {
99+ return err
100+ }
101+ }
102+ }
103+
104+ // phase done
105+ if err := st.Conn.WriteInt32(-1); err != nil {
106+ return err
107+ }
108+
109+ return nil
110+}
111+
112+// rsync/sender.c:receive_sums().
113+func (st *Transfer) receiveSums() (rsync.SumHead, error) {
114+ var head rsync.SumHead
115+ if err := head.ReadFrom(st.Conn); err != nil {
116+ return head, err
117+ }
118+ var offset int64
119+ head.Sums = make([]rsync.SumBuf, int(head.ChecksumCount))
120+ for i := int32(0); i < head.ChecksumCount; i++ {
121+ shortChecksum, err := st.Conn.ReadInt32()
122+ if err != nil {
123+ return head, err
124+ }
125+ sb := rsync.SumBuf{
126+ Index: i,
127+ Offset: offset,
128+ Sum1: uint32(shortChecksum),
129+ }
130+ if i == head.ChecksumCount-1 && head.RemainderLength != 0 {
131+ sb.Len = int64(head.RemainderLength)
132+ } else {
133+ sb.Len = int64(head.BlockLength)
134+ }
135+ offset += sb.Len
136+ n, err := io.ReadFull(st.Conn.Reader, sb.Sum2[:head.ChecksumLength])
137+ if err != nil {
138+ return head, err
139+ }
140+ _ = n
141+ // log.Printf("chunk[%d] len=%d offset=%.0f sum1=%08x, sum2=%x",
142+ // i, sb.len, float64(sb.offset), sb.sum1, sb.sum2[:n])
143+ head.Sums[i] = sb
144+ }
145+ return head, nil
146+}
147+
148+func (st *Transfer) sendFile(fileIndex int32, fl utils.SenderFile) error {
149+ // rsync/rsync.h defines CHUNK_SIZE as 32 * 1024. openrsync (tridge)
150+ // uses 256K, but standard rsync rejects tokens larger than 32K.
151+ const chunkSize = 32 * 1024
152+
153+ fi, r, err := st.Files.Read(&fl)
154+ if err != nil {
155+ return err
156+ }
157+ defer func() { _ = r.Close() }()
158+
159+ if err := st.Conn.WriteInt32(fileIndex); err != nil {
160+ return err
161+ }
162+
163+ sh := rsynccommon.SumSizesSqroot(fi.Size())
164+ // log.Printf("sh = %+v", sh)
165+ if err := sh.WriteTo(st.Conn); err != nil {
166+ return err
167+ }
168+
169+ h := md4.New()
170+ _ = binary.Write(h, binary.LittleEndian, st.Seed) // hash.Hash.Write never fails
171+
172+ buf := make([]byte, chunkSize)
173+ for {
174+ shouldBreak := false
175+ n, err := r.Read(buf)
176+ if err != nil {
177+ if err == io.EOF {
178+ shouldBreak = true
179+ } else {
180+ return err
181+ }
182+ }
183+ chunk := buf[:n]
184+
185+ if len(chunk) == 0 {
186+ break
187+ }
188+
189+ _, err = h.Write(chunk)
190+ if err != nil {
191+ return err
192+ }
193+ // chunk size (“rawtok” variable in openrsync)
194+ if err := st.Conn.WriteInt32(int32(len(chunk))); err != nil {
195+ return err
196+ }
197+ if _, err := st.Conn.Writer.Write(chunk); err != nil {
198+ return err
199+ }
200+
201+ if shouldBreak {
202+ break
203+ }
204+ }
205+ // transfer finished:
206+ if err := st.Conn.WriteInt32(0); err != nil {
207+ return err
208+ }
209+
210+ sum := h.Sum(nil)
211+ // log.Printf("sum: %x (len = %d)", sum, len(sum))
212+ if _, err := st.Conn.Writer.Write(sum); err != nil {
213+ return err
214+ }
215+ return nil
216+}
A pkg/rsync-receiver/rsyncsender/stat.go
+15, -0
......@@ -0,0 +1,15 @@
1+package rsyncsender
2+
3+import "io/fs"
4+
5+func uidFromFileInfo(fs.FileInfo) (int32, bool) {
6+ return 1000, false
7+}
8+
9+func gidFromFileInfo(fs.FileInfo) (int32, bool) {
10+ return 1000, false
11+}
12+
13+func rdevFromFileInfo(fs.FileInfo) (int32, bool) {
14+ return 1000, false
15+}
A pkg/rsync-receiver/rsyncsender/token.go
+34, -0
......@@ -0,0 +1,34 @@
1+package rsyncsender
2+
3+// rsync/token.c:simple_send_token.
4+func (st *Transfer) simpleSendToken(ms *mapStruct, token int32, offset int64, n int64) error {
5+ if n > 0 {
6+ st.Logger.Debug("sending unmatched chunks", "offset", offset, "n", n)
7+ l := int64(0)
8+ for l < n {
9+ n1 := min(int64(chunkSize), n-l)
10+
11+ chunk := ms.ptr(offset+l, int32(n1))
12+
13+ if err := st.Conn.WriteInt32(int32(n1)); err != nil {
14+ return err
15+ }
16+
17+ if _, err := st.Conn.Writer.Write(chunk); err != nil {
18+ return err
19+ }
20+
21+ l += n1
22+ }
23+ }
24+ if token != -2 {
25+ return st.Conn.WriteInt32(-(token + 1))
26+ }
27+ return nil
28+}
29+
30+// rsync/token.c:send_token.
31+func (st *Transfer) sendToken(ms *mapStruct, i int32, offset int64, n int64) error {
32+ // TODO(compression): send deflated token
33+ return st.simpleSendToken(ms, i, offset, n)
34+}
A pkg/rsync-receiver/rsyncsender/transfer.go
+49, -0
......@@ -0,0 +1,49 @@
1+package rsyncsender
2+
3+import (
4+ "io"
5+ "log/slog"
6+
7+ "github.com/picosh/pico/pkg/rsync-receiver/rsyncopts"
8+ "github.com/picosh/pico/pkg/rsync-receiver/rsyncwire"
9+ "github.com/picosh/pico/pkg/rsync-receiver/utils"
10+)
11+
12+type Osenv struct {
13+ Stdin io.Reader
14+ Stdout io.Writer
15+ Stderr io.Writer
16+}
17+
18+// TransferOpts is a subset of Opts which is required for implementing a receiver.
19+type TransferOpts struct {
20+ Verbose bool
21+ DryRun bool
22+
23+ DeleteMode bool
24+ PreserveGid bool
25+ PreserveUid bool
26+ PreserveLinks bool
27+ PreservePerms bool
28+ PreserveDevices bool
29+ PreserveSpecials bool
30+ PreserveTimes bool
31+ PreserveHardlinks bool
32+}
33+
34+type Transfer struct {
35+ // config
36+ // Opts *Opts
37+ Opts *rsyncopts.Options
38+
39+ // state
40+ Conn *rsyncwire.Conn
41+ Seed int32
42+ lastMatch int64
43+
44+ Files utils.FS
45+
46+ Logger *slog.Logger
47+}
48+
49+//func (rt *Transfer) listOnly() bool { return rt.Dest == "" }
A pkg/rsync-receiver/rsyncstats/rsyncstats.go
+7, -0
......@@ -0,0 +1,7 @@
1+package rsyncstats
2+
3+type TransferStats struct {
4+ Read int64 // total bytes read (from network connection)
5+ Written int64 // total bytes written (to network connection)
6+ Size int64 // total size of files
7+}
A pkg/rsync-receiver/rsyncwire/wire.go
+214, -0
......@@ -0,0 +1,214 @@
1+package rsyncwire
2+
3+import (
4+ "bytes"
5+ "encoding/binary"
6+ "fmt"
7+ "io"
8+ "log/slog"
9+)
10+
11+const (
12+ MsgData uint8 = 0
13+ MsgInfo uint8 = 2
14+ MsgError uint8 = 1
15+)
16+
17+const mplexBase = 7
18+
19+type MultiplexWriter struct {
20+ Writer io.Writer
21+}
22+
23+func (w *MultiplexWriter) Write(p []byte) (n int, err error) {
24+ return w.WriteMsg(MsgData, p)
25+}
26+
27+func (w *MultiplexWriter) WriteMsg(tag uint8, p []byte) (n int, err error) {
28+ header := uint32(mplexBase+tag)<<24 | uint32(len(p))
29+ // log.Printf("len %d (hex %x)", len(p), uint32(len(p)))
30+ // log.Printf("header=%v (%x)", header, header)
31+ if err := binary.Write(w.Writer, binary.LittleEndian, header); err != nil {
32+ return 0, err
33+ }
34+ return w.Writer.Write(p)
35+}
36+
37+type MultiplexReader struct {
38+ Reader io.Reader
39+}
40+
41+// rsync.h defines IO_BUFFER_SIZE as 32 * 1024, but gokr-rsyncd increases it to
42+// 256K. Since we use this as the maximum message size, too, we need to at least
43+// match it.
44+const ioBufferSize = 256 * 1024
45+const maxMessageSize = ioBufferSize
46+
47+func (w *MultiplexReader) ReadMsg() (tag uint8, p []byte, err error) {
48+ var header uint32
49+ if err := binary.Read(w.Reader, binary.LittleEndian, &header); err != nil {
50+ return 0, nil, err
51+ }
52+
53+ tag = uint8(header>>24) - mplexBase
54+ length := header & 0x00FFFFFF
55+ if length > maxMessageSize {
56+ // NOTE: if you run into this error, one alternative to bumping
57+ // maxMessageSize is to restructure the program to work with i/o buffer
58+ // windowing.
59+ return 0, nil, fmt.Errorf("length %d exceeds max message size (%d)", length, maxMessageSize)
60+ }
61+ p = make([]byte, int(length))
62+ if _, err := io.ReadFull(w.Reader, p); err != nil {
63+ return 0, nil, err
64+ }
65+ // log.Printf("header=%v (%x), tag=%v, length=%v", header, header, tag, length)
66+ // log.Printf("payload=%x / %q", p, p)
67+ return tag, p, nil
68+}
69+
70+func (w *MultiplexReader) Read(p []byte) (n int, err error) {
71+ tag, payload, err := w.ReadMsg()
72+ if err != nil {
73+ return 0, err
74+ }
75+ if tag == MsgError {
76+ return 0, fmt.Errorf("%s", payload)
77+ }
78+ if tag == MsgInfo {
79+ slog.Debug("info", "payload", payload)
80+ }
81+ if tag != MsgData {
82+ return 0, fmt.Errorf("unexpected tag: got %v, want %v", tag, MsgData)
83+ }
84+ if len(p) < len(payload) {
85+ panic(fmt.Sprintf("not enough buffer space! %d < %d", len(p), len(payload)))
86+ }
87+ return copy(p, payload), nil
88+}
89+
90+type Buffer struct {
91+ // buf.Write() never fails, making for a convenient API.
92+ buf bytes.Buffer
93+}
94+
95+func (b *Buffer) WriteByte(data byte) error {
96+ return binary.Write(&b.buf, binary.LittleEndian, data)
97+}
98+
99+func (b *Buffer) WriteInt32(data int32) {
100+ _ = binary.Write(&b.buf, binary.LittleEndian, data)
101+}
102+
103+func (b *Buffer) WriteInt64(data int64) {
104+ // send as a 32-bit integer if possible
105+ if data <= 0x7FFFFFFF && data >= 0 {
106+ b.WriteInt32(int32(data))
107+ return
108+ }
109+ // otherwise, send -1 followed by the 64-bit integer
110+ b.WriteInt32(-1)
111+ _ = binary.Write(&b.buf, binary.LittleEndian, data)
112+}
113+
114+func (b *Buffer) WriteString(data string) {
115+ _, _ = io.WriteString(&b.buf, data)
116+}
117+
118+func (b *Buffer) String() string {
119+ return b.buf.String()
120+}
121+
122+type Conn struct {
123+ Writer io.Writer
124+ Reader io.Reader
125+}
126+
127+func (c *Conn) WriteByte(data byte) error {
128+ return binary.Write(c.Writer, binary.LittleEndian, data)
129+}
130+
131+func (c *Conn) WriteInt32(data int32) error {
132+ return binary.Write(c.Writer, binary.LittleEndian, data)
133+}
134+
135+func (c *Conn) WriteInt64(data int64) error {
136+ // send as a 32-bit integer if possible
137+ if data <= 0x7FFFFFFF && data >= 0 {
138+ return c.WriteInt32(int32(data))
139+ }
140+ // otherwise, send -1 followed by the 64-bit integer
141+ if err := c.WriteInt32(-1); err != nil {
142+ return err
143+ }
144+ return binary.Write(c.Writer, binary.LittleEndian, data)
145+}
146+
147+func (c *Conn) WriteString(data string) error {
148+ _, err := io.WriteString(c.Writer, data)
149+ return err
150+}
151+
152+func (c *Conn) ReadByte() (byte, error) {
153+ var buf [1]byte
154+ if _, err := io.ReadFull(c.Reader, buf[:]); err != nil {
155+ return 0, err
156+ }
157+ return buf[0], nil
158+}
159+
160+func (c *Conn) ReadInt32() (int32, error) {
161+ var buf [4]byte
162+ if _, err := io.ReadFull(c.Reader, buf[:]); err != nil {
163+ return 0, err
164+ }
165+ return int32(binary.LittleEndian.Uint32(buf[:])), nil
166+}
167+
168+func (c *Conn) ReadInt64() (int64, error) {
169+ {
170+ data, err := c.ReadInt32()
171+ if err != nil {
172+ return 0, err
173+ }
174+ if data != -1 {
175+ // The value was small enough to fit into a 32 bit int, so it was
176+ // transferred directly.
177+ return int64(data), nil
178+ }
179+ // Otherwise, -1 was transmitted, followed by the int64.
180+ }
181+ var data int64
182+ if err := binary.Read(c.Reader, binary.LittleEndian, &data); err != nil {
183+ return 0, err
184+ }
185+ return data, nil
186+}
187+
188+type CountingReader struct {
189+ R io.Reader
190+ BytesRead int64
191+}
192+
193+func (r *CountingReader) Read(p []byte) (n int, err error) {
194+ n, err = r.R.Read(p)
195+ r.BytesRead += int64(n)
196+ return n, err
197+}
198+
199+type CountingWriter struct {
200+ W io.Writer
201+ BytesWritten int64
202+}
203+
204+func (w *CountingWriter) Write(p []byte) (n int, err error) {
205+ n, err = w.W.Write(p)
206+ w.BytesWritten += int64(n)
207+ return n, err
208+}
209+
210+func CounterPair(r io.Reader, w io.Writer) (*CountingReader, *CountingWriter) {
211+ crd := &CountingReader{R: r}
212+ cwr := &CountingWriter{W: w}
213+ return crd, cwr
214+}
A pkg/rsync-receiver/utils/file.go
+67, -0
......@@ -0,0 +1,67 @@
1+package utils
2+
3+import (
4+ "io"
5+ "io/fs"
6+ "sort"
7+ "time"
8+
9+ "github.com/picosh/pico/pkg/rsync-receiver/rsync"
10+)
11+
12+type SenderFile struct {
13+ // TODO: store relative to the root to conserve RAM
14+ Path string
15+ WPath string
16+ Regular bool
17+}
18+
19+type ReceiverFile struct {
20+ Name string
21+ Length int64
22+ ModTime time.Time
23+ Mode int32
24+ Uid int32
25+ Gid int32
26+ LinkTarget string
27+ Rdev int32
28+ Reader io.Reader
29+}
30+
31+// FileMode converts from the Linux permission bits to Go’s permission bits.
32+func (f *ReceiverFile) FileMode() fs.FileMode {
33+ ret := fs.FileMode(f.Mode) & fs.ModePerm
34+
35+ mode := f.Mode & rsync.S_IFMT
36+ switch mode {
37+ case rsync.S_IFCHR:
38+ ret |= fs.ModeCharDevice
39+ case rsync.S_IFBLK:
40+ ret |= fs.ModeDevice
41+ case rsync.S_IFIFO:
42+ ret |= fs.ModeNamedPipe
43+ case rsync.S_IFSOCK:
44+ ret |= fs.ModeSocket
45+ case rsync.S_IFLNK:
46+ ret |= fs.ModeSymlink
47+ case rsync.S_IFDIR:
48+ ret |= fs.ModeDir
49+ }
50+
51+ return ret
52+}
53+
54+// rsync/flist.c:flist_sort_and_clean.
55+func SortFileList(fileList []*ReceiverFile) {
56+ sort.Slice(fileList, func(i, j int) bool {
57+ return fileList[i].Name < fileList[j].Name
58+ })
59+}
60+
61+// rsync/receiver.c:delete_files.
62+func FindInFileList(fileList []*ReceiverFile, name string) bool {
63+ i := sort.Search(len(fileList), func(i int) bool {
64+ return fileList[i].Name >= name
65+ })
66+ return i < len(fileList) && fileList[i].Name == name
67+}
A pkg/rsync-receiver/utils/fs.go
+20, -0
......@@ -0,0 +1,20 @@
1+package utils
2+
3+import (
4+ "io"
5+ "os"
6+)
7+
8+type ReaderAtCloser interface {
9+ io.Reader
10+ io.ReaderAt
11+ io.Closer
12+}
13+
14+// File System: need to handle all type of files: regular, folder, symlink, etc.
15+type FS interface {
16+ Put(*ReceiverFile) (int64, error)
17+ List(string) ([]os.FileInfo, error)
18+ Read(*SenderFile) (os.FileInfo, ReaderAtCloser, error)
19+ Remove([]*ReceiverFile) error
20+}
M pkg/send/protocols/rsync/rsync.go
+4, -4
......@@ -9,11 +9,11 @@ import (
99 "slices"
1010 "strings"
1111
12- "github.com/picosh/go-rsync-receiver/rsyncopts"
13- "github.com/picosh/go-rsync-receiver/rsyncreceiver"
14- "github.com/picosh/go-rsync-receiver/rsyncsender"
15- rsyncutils "github.com/picosh/go-rsync-receiver/utils"
1612 "github.com/picosh/pico/pkg/pssh"
13+ "github.com/picosh/pico/pkg/rsync-receiver/rsyncopts"
14+ "github.com/picosh/pico/pkg/rsync-receiver/rsyncreceiver"
15+ "github.com/picosh/pico/pkg/rsync-receiver/rsyncsender"
16+ rsyncutils "github.com/picosh/pico/pkg/rsync-receiver/utils"
1717 "github.com/picosh/pico/pkg/send/utils"
1818 )
1919
M pkg/send/protocols/rsync/rsync_test.go
+1, -1
......@@ -10,8 +10,8 @@ import (
1010 "testing"
1111 "time"
1212
13- rsyncutils "github.com/picosh/go-rsync-receiver/utils"
1413 "github.com/picosh/pico/pkg/pssh"
14+ rsyncutils "github.com/picosh/pico/pkg/rsync-receiver/utils"
1515 "github.com/picosh/pico/pkg/send/utils"
1616 "golang.org/x/crypto/ssh"
1717 )