Commit 9eb8e76
Eric Bower
·
2026-05-31 09:31:45 -0400 EDT
parent 8936652
refactor: move go-rsync-receiver into pico monorepo
M
go.mod
+2,
-5
1@@ -6,8 +6,6 @@ go 1.25.0
2
3 // replace github.com/picosh/send => ../send
4
5-// replace github.com/picosh/go-rsync-receiver => ../go-rsync-receiver
6-
7 // replace github.com/picosh/pobj => ../pobj
8
9 // replace github.com/picosh/pubsub => ../pubsub
10@@ -41,8 +39,8 @@ require (
11 github.com/matryer/is v1.4.1
12 github.com/microcosm-cc/bluemonday v1.0.27
13 github.com/mmcdole/gofeed v1.3.0
14+ github.com/mmcloughlin/md4 v0.1.2
15 github.com/neurosnap/go-exif-remove v0.0.0-20221010134343-50d1e3c35577
16- github.com/picosh/go-rsync-receiver v0.0.0-20250304201040-fcc11dd22d79
17 github.com/picosh/utils v0.0.0-20260125160622-5c3a9e231ec6
18 github.com/pkg/sftp v1.13.10
19 github.com/prometheus/client_golang v1.23.2
20@@ -58,6 +56,7 @@ require (
21 go.abhg.dev/goldmark/hashtag v0.4.0
22 go.abhg.dev/goldmark/toc v0.12.0
23 golang.org/x/crypto v0.50.0
24+ golang.org/x/sync v0.20.0
25 gopkg.in/yaml.v2 v2.4.0
26 modernc.org/sqlite v1.49.1
27 )
28@@ -126,7 +125,6 @@ require (
29 github.com/mattn/go-runewidth v0.0.23 // indirect
30 github.com/mattn/go-sixel v0.0.9 // indirect
31 github.com/mmcdole/goxpp v1.1.1 // indirect
32- github.com/mmcloughlin/md4 v0.1.2 // indirect
33 github.com/moby/docker-image-spec v1.3.1 // indirect
34 github.com/moby/go-archive v0.1.0 // indirect
35 github.com/moby/patternmatcher v0.6.0 // indirect
36@@ -173,7 +171,6 @@ require (
37 go.yaml.in/yaml/v2 v2.4.4 // indirect
38 golang.org/x/image v0.39.0 // indirect
39 golang.org/x/net v0.53.0 // indirect
40- golang.org/x/sync v0.20.0 // indirect
41 golang.org/x/sys v0.43.0 // indirect
42 golang.org/x/text v0.36.0 // indirect
43 golang.org/x/time v0.15.0 // indirect
M
go.sum
+0,
-2
1@@ -260,8 +260,6 @@ github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8
2 github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
3 github.com/opencontainers/image-spec v1.1.1 h1:y0fUlFfIZhPF1W537XOLg0/fcx6zcHCJwooC2xJA040=
4 github.com/opencontainers/image-spec v1.1.1/go.mod h1:qpqAh3Dmcf36wStyyWU+kCeDgrGnAve2nCC8+7h8Q0M=
5-github.com/picosh/go-rsync-receiver v0.0.0-20250304201040-fcc11dd22d79 h1:MyB9P43hlQ6A2FoP9LGeiTBL3WKToW4gcWd6lQPg/Zg=
6-github.com/picosh/go-rsync-receiver v0.0.0-20250304201040-fcc11dd22d79/go.mod h1:4ZICsr6bESoHP8He9DqROlZiMw4hHHjcbDzhtTTDQzA=
7 github.com/picosh/utils v0.0.0-20260125160622-5c3a9e231ec6 h1:9KfCtfcx7vrSyGU1K9whdE1crll9Aq+nAZ6c0FzuzvE=
8 github.com/picosh/utils v0.0.0-20260125160622-5c3a9e231ec6/go.mod h1:HogYEyJ43IGXrOa3D/kjM1pkzNAyh+pejRyv8Eo//pk=
9 github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=
1@@ -0,0 +1,5 @@
2+package nofollow
3+
4+// Maybe resolves to unix.O_NOFOLLOW on unix systems,
5+// 0 on other platforms. TODO(go1.24): use os.Root.
6+const Maybe = 0
+36,
-0
1@@ -0,0 +1,36 @@
2+package rsync
3+
4+// rsync.h.
5+const (
6+ XMIT_TOP_DIR = (1 << 0)
7+ XMIT_SAME_MODE = (1 << 1)
8+ XMIT_EXTENDED_FLAGS = (1 << 2)
9+ XMIT_SAME_RDEV_pre28 = XMIT_EXTENDED_FLAGS /* Only in protocols < 28 */
10+ XMIT_SAME_UID = (1 << 3)
11+ XMIT_SAME_GID = (1 << 4)
12+ XMIT_SAME_NAME = (1 << 5)
13+ XMIT_LONG_NAME = (1 << 6)
14+ XMIT_SAME_TIME = (1 << 7)
15+ XMIT_SAME_RDEV_MAJOR = (1 << 8)
16+ XMIT_HAS_IDEV_DATA = (1 << 9)
17+ XMIT_SAME_DEV = (1 << 10)
18+ XMIT_RDEV_MINOR_IS_SMALL = (1 << 11)
19+)
20+
21+// as per /usr/include/bits/stat.h.
22+const (
23+ S_IFMT = 0o0170000 // bits determining the file type
24+ S_IFDIR = 0o0040000 // Directory
25+ S_IFCHR = 0o0020000 // Character device
26+ S_IFBLK = 0o0060000 // Block device
27+ S_IFREG = 0o0100000 // Regular file
28+ S_IFIFO = 0o0010000 // FIFO
29+ S_IFLNK = 0o0120000 // Symbolic link
30+ S_IFSOCK = 0o0140000 // Socket
31+)
32+
33+// ProtocolVersion defines the currently implemented rsync protocol
34+// version. Protocol version 27 seems to be the safest bet for wide
35+// compatibility: version 27 was introduced by rsync 2.6.0 (released 2004), and
36+// is supported by openrsync and rsyn.
37+const ProtocolVersion = 27
+6,
-0
1@@ -0,0 +1,6 @@
2+// Package rsync contains a native Go rsync implementation.
3+//
4+// The only component currently is gokr-rsyncd, a read-only rsync daemon
5+// sender-only Go implementation of rsyncd. rsync daemon is a custom
6+// (un-standardized) network protocol, running on port 873 by default.
7+package rsync
+7,
-0
1@@ -0,0 +1,7 @@
2+package rsync
3+
4+import "log/slog"
5+
6+// Logger is an interface that allows specifying your own logger.
7+// By default, the Go log package is used, which prints to stderr.
8+type Logger = slog.Logger
+86,
-0
1@@ -0,0 +1,86 @@
2+package rsync
3+
4+import (
5+ "fmt"
6+
7+ "github.com/picosh/pico/pkg/rsync-receiver/rsyncwire"
8+)
9+
10+// rsync/rsync.h:struct sum_buf.
11+type SumBuf struct {
12+ Offset int64
13+ Len int64
14+ Index int32
15+ Sum1 uint32
16+ Sum2 [16]byte
17+}
18+
19+// TODO: remove connection.go:sumHead in favor of this type.
20+type SumHead struct {
21+ // “number of blocks” (openrsync)
22+ // “how many chunks” (rsync)
23+ ChecksumCount int32
24+
25+ // “block length in the file” (openrsync)
26+ // maximum (1 << 29) for older rsync, (1 << 17) for newer
27+ BlockLength int32
28+
29+ // “long checksum length” (openrsync)
30+ ChecksumLength int32
31+
32+ // “terminal (remainder) block length” (openrsync)
33+ // RemainderLength is flength % BlockLength
34+ RemainderLength int32
35+
36+ Sums []SumBuf
37+}
38+
39+func (sh *SumHead) ReadFrom(c *rsyncwire.Conn) error {
40+ // TODO(protocol>=30): update maxBlockLen
41+ const maxBlockLen = 1 << 29 // see rsync.h:OLD_MAX_BLOCK_SIZE
42+
43+ var err error
44+ sh.ChecksumCount, err = c.ReadInt32()
45+ if err != nil {
46+ return err
47+ }
48+ if sh.ChecksumCount < 0 {
49+ return fmt.Errorf("invalid checksum count %d", sh.ChecksumCount)
50+ }
51+
52+ sh.BlockLength, err = c.ReadInt32()
53+ if err != nil {
54+ return err
55+ }
56+ if sh.BlockLength < 0 || sh.BlockLength > maxBlockLen {
57+ return fmt.Errorf("invalid block length %d", sh.BlockLength)
58+ }
59+
60+ sh.ChecksumLength, err = c.ReadInt32()
61+ if err != nil {
62+ return err
63+ }
64+ // TODO(protocol>=27): update max sh.ChecksumLength check
65+ if sh.ChecksumLength < 0 || sh.ChecksumLength > 16 {
66+ return fmt.Errorf("invalid checksum length %d", sh.ChecksumLength)
67+ }
68+
69+ sh.RemainderLength, err = c.ReadInt32()
70+ if err != nil {
71+ return err
72+ }
73+ if sh.RemainderLength < 0 || sh.RemainderLength > sh.BlockLength {
74+ return fmt.Errorf("invalid remainder length %d", sh.RemainderLength)
75+ }
76+
77+ return nil
78+}
79+
80+func (sh *SumHead) WriteTo(c *rsyncwire.Conn) error {
81+ var buf rsyncwire.Buffer
82+ buf.WriteInt32(sh.ChecksumCount)
83+ buf.WriteInt32(sh.BlockLength)
84+ buf.WriteInt32(sh.ChecksumLength)
85+ buf.WriteInt32(sh.RemainderLength)
86+ return c.WriteString(buf.String())
87+}
1@@ -0,0 +1,73 @@
2+package rsyncchecksum_test
3+
4+import (
5+ "bytes"
6+ "os"
7+ "path/filepath"
8+ "testing"
9+
10+ "github.com/picosh/pico/pkg/rsync-receiver/rsyncchecksum"
11+)
12+
13+func constructLargeDataFile(headPattern, bodyPattern, endPattern []byte) []byte {
14+ // create large data file in source directory to be copied
15+ head := bytes.Repeat(headPattern, 1*1024*1024)
16+ body := bytes.Repeat(bodyPattern, 1*1024*1024)
17+ end := bytes.Repeat(endPattern, 1*1024*1024)
18+ return append(append(head, body...), end...)
19+}
20+
21+func writeLargeDataFile(t *testing.T, source string, headPattern, bodyPattern, endPattern []byte) {
22+ // create large data file in source directory to be copied
23+ content := constructLargeDataFile(headPattern, bodyPattern, endPattern)
24+ large := filepath.Join(source, "large-data-file")
25+ if err := os.MkdirAll(filepath.Dir(large), 0755); err != nil {
26+ t.Fatal(err)
27+ }
28+ if err := os.WriteFile(large, content, 0644); err != nil {
29+ t.Fatal(err)
30+ }
31+}
32+
33+func TestSyncExtended(t *testing.T) {
34+ tmp := t.TempDir()
35+ source := filepath.Join(tmp, "source")
36+
37+ writeLargeDataFile(t, source, []byte{0x11}, []byte{0xbb}, []byte{0xee})
38+
39+ // These values are taken from the rsync debug output:
40+ const k = 1768
41+ want := make([]uint32, 1780)
42+ for i := 0; i <= 592; i++ {
43+ want[i] = 0xa5d47568
44+ }
45+ want[593] = 0x23645688
46+ for i := 594; i <= 1185; i++ {
47+ want[i] = 0x8c1c2378
48+ }
49+ want[1186] = 0x12504720
50+ for i := 1187; i <= 1778; i++ {
51+ want[i] = 0x7d9883b0
52+ }
53+ want[1779] = 0x61b8dff0
54+
55+ sourceLarge := filepath.Join(source, "large-data-file")
56+ f, err := os.Open(sourceLarge)
57+ if err != nil {
58+ t.Fatal(err)
59+ }
60+ defer func() { _ = f.Close() }()
61+ buf := make([]byte, k)
62+ for idx, wantChecksum := range want {
63+ n, err := f.Read(buf)
64+ if err != nil {
65+ t.Fatal(err)
66+ }
67+
68+ chunk := buf[:n]
69+ sum := rsyncchecksum.Checksum1(chunk)
70+ if sum != wantChecksum {
71+ t.Fatalf("checksum calculation error: got %08x, want %08x (idx %d), chunk: %#v", sum, wantChecksum, idx, chunk)
72+ }
73+ }
74+}
1@@ -0,0 +1,73 @@
2+package rsyncchecksum
3+
4+import (
5+ "encoding/binary"
6+ "io"
7+ "os"
8+
9+ "github.com/mmcloughlin/md4"
10+)
11+
12+func Tag2(s1, s2 uint16) uint16 {
13+ return (((s1) + (s2)) & 0xFFFF)
14+}
15+
16+func Tag(sum uint32) uint16 {
17+ return Tag2(uint16(sum&0xFFFF), uint16(sum>>16))
18+}
19+
20+// SignExtend mirrors how C converts from (signed char) to uint32, i.e. using
21+// sign extension. get_checksum1 treats the buffer as (signed char*) instead of
22+// (unsigned char*), which likely was not a conscious choice, but here we are.
23+//
24+// This function is exported for use in the rolling checksum in match.go.
25+func SignExtend(b byte) uint32 {
26+ val := uint32(b)
27+ return uint32(int32(val<<24) >> 24)
28+}
29+
30+func Checksum1(buf []byte) uint32 {
31+ bufLen := len(buf)
32+ var s1, s2 uint32
33+ var i int
34+
35+ if bufLen > 4 {
36+ for i = 0; i < (bufLen - 4); i += 4 {
37+ s2 += 4*(s1+SignExtend(buf[i])) +
38+ 3*SignExtend(buf[i+1]) +
39+ 2*SignExtend(buf[i+2]) +
40+ SignExtend(buf[i+3])
41+ s1 += SignExtend(buf[i+0]) +
42+ SignExtend(buf[i+1]) +
43+ SignExtend(buf[i+2]) +
44+ SignExtend(buf[i+3])
45+ }
46+ }
47+ for ; i < bufLen; i++ {
48+ s1 += SignExtend(buf[i])
49+ s2 += s1
50+ }
51+ return (s1 & 0xffff) + (s2 << 16)
52+}
53+
54+func Checksum2(seed int32, buf []byte) []byte {
55+ h := md4.New()
56+ h.Write(buf)
57+ _ = binary.Write(h, binary.LittleEndian, seed) // hash.Hash.Write never fails
58+ return h.Sum(nil)
59+}
60+
61+func FileChecksum(fn string) ([]byte, error) {
62+ f, err := os.Open(fn)
63+ if err != nil {
64+ return nil, err
65+ }
66+ defer func() { _ = f.Close() }()
67+ h := md4.New()
68+ if _, err := io.Copy(h, f); err != nil {
69+ return nil, err
70+ }
71+ return h.Sum(nil), nil
72+}
73+
74+const Size = md4.Size
1@@ -0,0 +1,37 @@
2+// Package rsynccommon contains functionality that both the sender and the
3+// receiver implementation need.
4+package rsynccommon
5+
6+import (
7+ "math"
8+
9+ "github.com/picosh/pico/pkg/rsync-receiver/rsync"
10+)
11+
12+const blockSize = 700 // rsync/rsync.h
13+
14+// Corresponds to rsync/generator.c:sum_sizes_sqroot.
15+func SumSizesSqroot(contentLen int64) rsync.SumHead {
16+ // * The block size is a rounded square root of file length.
17+
18+ // The block size algorithm plays a crucial role in the protocol efficiency. In general, the block size is the rounded square root of the total file size. The minimum block size, however, is 700 B. Otherwise, the square root computation is simply sqrt(3) followed by ceil(3)
19+
20+ // For reasons unknown, the square root result is rounded up to the nearest multiple of eight.
21+
22+ // TODO: round this
23+ blockLength := max(int32(math.Sqrt(float64(contentLen))), blockSize)
24+
25+ // * The checksum size is determined according to:
26+ // * blocksum_bits = BLOCKSUM_EXP + 2*log2(file_len) - log2(block_len)
27+ // * provided by Donovan Baarda which gives a probability of rsync
28+ // * algorithm corrupting data and falling back using the whole md4
29+ // * checksums.
30+ const checksumLength = 16 // TODO?
31+
32+ return rsync.SumHead{
33+ ChecksumCount: int32((contentLen + (int64(blockLength) - 1)) / int64(blockLength)),
34+ RemainderLength: int32(contentLen % int64(blockLength)),
35+ BlockLength: blockLength,
36+ ChecksumLength: checksumLength,
37+ }
38+}
+255,
-0
1@@ -0,0 +1,255 @@
2+package rsyncopts
3+
4+import (
5+ "fmt"
6+ "math"
7+ "strconv"
8+ "strings"
9+)
10+
11+type poptOption struct {
12+ longName string
13+ shortName string
14+ argInfo int
15+ arg any // depends on argInfo
16+ val int // 0 means don't return, just update arg
17+ // descrip string
18+ // argDescrip string
19+}
20+
21+func (o *poptOption) name() string {
22+ if o.longName == "" {
23+ return "-" + o.shortName
24+ }
25+ return "--" + o.longName
26+}
27+
28+// see popt(3).
29+const (
30+ POPT_ARG_NONE = iota // int; No argument expected
31+ POPT_ARG_STRING // char*; No type checking to be performed
32+ POPT_ARG_INT // int; An integer argument is expected
33+ POPT_ARG_LONG // long; A long integer is expected
34+ POPT_ARG_INCLUDE_TABLE // nest another option table
35+ POPT_ARG_CALLBACK // call a function
36+ POPT_ARG_INTL_DOMAIN // <not documented in popt(3)>
37+ POPT_ARG_VAL // int; Integer value taken from val
38+ POPT_ARG_FLOAT // float; A float argument is expected
39+ POPT_ARG_DOUBLE // double; A double argument is expected
40+ POPT_ARG_LONGLONG // long long; A long long integer is expected
41+ POPT_ARG_MAINCALL = 16 + 11
42+ POPT_ARG_ARGV = 12
43+ POPT_ARG_SHORT = 13
44+ POPT_ARG_BITSET = 16 + 14
45+)
46+
47+const POPT_ARG_MASK = 0x000000FF
48+
49+const (
50+ POPT_ARGFLAG_OR = 0x08000000
51+)
52+
53+const (
54+ POPT_BIT_SET = POPT_ARG_VAL | POPT_ARGFLAG_OR
55+)
56+
57+type PoptError struct {
58+ Errno int32
59+ Err error
60+}
61+
62+func (pe *PoptError) Unwrap() error { return pe.Err }
63+
64+func (pe *PoptError) Error() string { return pe.Err.Error() }
65+
66+// TODO(later): turn these into sentinel error values.
67+// which stringify like poptStrerror().
68+const (
69+ POPT_ERROR_NOARG = -10 // missing argument
70+ POPT_ERROR_BADOPT = -11 // unknown option
71+ POPT_ERROR_UNWANTEDARG = -12 // option does not take an argument
72+ POPT_ERROR_OPTSTOODEEP = -13 // aliases nested too deeply
73+ POPT_ERROR_BADQUOTE = -15 // error in parameter quoting
74+ POPT_ERROR_ERRNO = -16 // errno set, use strerror(errno)
75+ POPT_ERROR_BADNUMBER = -17 // invalid numeric value
76+ POPT_ERROR_OVERFLOW = -18 // number too large or too small
77+ POPT_ERROR_BADOPERATION = -19 // mutually exclusive logical operations requested
78+ POPT_ERROR_NULLARG = -20 // opt->arg should not be NULL
79+ POPT_ERROR_MALLOC = -21 // memory allocation failed
80+ POPT_ERROR_BADCONFIG = -22 // config file failed sanity test
81+)
82+
83+type Context struct {
84+ // state
85+ table []poptOption
86+ args []string
87+ nextCharArg string
88+ nextArg string
89+
90+ // output
91+ Options *Options
92+ RemainingArgs []string
93+}
94+
95+func (pc *Context) findOption(longName, shortName string) *poptOption {
96+ for idx, opt := range pc.table {
97+ if longName != "" && opt.longName == longName {
98+ return &pc.table[idx]
99+ }
100+ if shortName != "" && opt.shortName == shortName {
101+ return &pc.table[idx]
102+ }
103+ }
104+ return nil
105+}
106+
107+func (pc *Context) poptSaveInt(opt *poptOption, val int) bool {
108+ intPtr := opt.arg.(*int)
109+ if intPtr == nil {
110+ return false
111+ }
112+ if opt.argInfo&POPT_ARGFLAG_OR != 0 {
113+ *intPtr |= val
114+ } else {
115+ *intPtr = val
116+ }
117+ return true
118+}
119+
120+func (pc *Context) poptSaveArg(opt *poptOption, nextArg string) int32 {
121+ argType := opt.argInfo & POPT_ARG_MASK
122+ switch argType {
123+ case POPT_ARG_INT:
124+ i, err := strconv.ParseInt(nextArg, 0, 64)
125+ if err != nil {
126+ return POPT_ERROR_BADNUMBER
127+ }
128+ if i < math.MinInt32 || i > math.MaxInt32 {
129+ return POPT_ERROR_OVERFLOW
130+ }
131+ pc.poptSaveInt(opt, int(i))
132+ return 0
133+
134+ case POPT_ARG_STRING:
135+ stringPtr := opt.arg.(*string)
136+ if stringPtr == nil {
137+ return 0
138+ }
139+ *stringPtr = nextArg
140+ return 0
141+ }
142+
143+ return POPT_ERROR_BADOPERATION
144+}
145+
146+func (pc *Context) poptGetNextOpt() (int32, error) {
147+ var opt *poptOption
148+ for {
149+ var longArg string
150+ if pc.nextCharArg == "" && len(pc.args) == 0 {
151+ return -1, nil // done
152+ }
153+ if pc.nextCharArg == "" {
154+ // process next long option
155+ origOptString := pc.args[0]
156+ pc.args = pc.args[1:]
157+ if origOptString == "" {
158+ return -1, &PoptError{
159+ Errno: POPT_ERROR_BADOPT,
160+ Err: fmt.Errorf("unknown option: origOptString empty"),
161+ }
162+ }
163+ if origOptString[0] != '-' || origOptString == "-" {
164+ pc.RemainingArgs = append(pc.RemainingArgs, origOptString)
165+ continue
166+ }
167+ before, after, found := strings.Cut(origOptString, "=")
168+ if found {
169+ longArg = after
170+ }
171+ // remove the one dash we ensured is present
172+ before = strings.TrimPrefix(before, "-")
173+ // a second dash is permitted
174+ before = strings.TrimPrefix(before, "-")
175+ opt = pc.findOption(before, "")
176+ if opt == nil {
177+ // try and parse it as a short option
178+ pc.nextCharArg = origOptString[1:]
179+ longArg = ""
180+ }
181+ }
182+ if pc.nextCharArg != "" {
183+ // process next short option
184+ opt = pc.findOption("", pc.nextCharArg[:1])
185+ if opt == nil {
186+ return -1, &PoptError{
187+ Errno: POPT_ERROR_BADOPT,
188+ Err: fmt.Errorf("option %q not found", pc.nextCharArg[:1]),
189+ }
190+ }
191+ pc.nextCharArg = pc.nextCharArg[1:]
192+ }
193+ if opt == nil {
194+ // neither long nor short? how can we end up here?
195+ return -1, &PoptError{
196+ Errno: POPT_ERROR_BADOPT,
197+ Err: fmt.Errorf("neither long nor short option found"),
198+ }
199+ }
200+ argType := opt.argInfo & POPT_ARG_MASK
201+ if argType == POPT_ARG_NONE || argType == POPT_ARG_VAL {
202+ if longArg != "" || strings.HasPrefix(pc.nextCharArg, "=") {
203+ return -1, &PoptError{
204+ Errno: POPT_ERROR_UNWANTEDARG,
205+ Err: fmt.Errorf("option %s does not take an argument", opt.name()),
206+ }
207+ }
208+ if opt.arg != nil {
209+ val := 1
210+ if argType == POPT_ARG_VAL {
211+ val = opt.val
212+ }
213+ if !pc.poptSaveInt(opt, val) {
214+ return -1, &PoptError{
215+ Errno: POPT_ERROR_BADOPERATION,
216+ Err: fmt.Errorf("poptSaveInt"),
217+ }
218+ }
219+ }
220+ } else {
221+ nextArg := longArg
222+ if longArg != "" {
223+ } else if pc.nextCharArg != "" {
224+ nextArg = strings.TrimPrefix(pc.nextCharArg, "=")
225+ pc.nextCharArg = ""
226+ } else {
227+ if len(pc.args) == 0 {
228+ return -1, &PoptError{
229+ Errno: POPT_ERROR_NOARG,
230+ Err: fmt.Errorf("missing argument for option %s", opt.name()),
231+ }
232+ }
233+ nextArg = pc.args[0]
234+ pc.args = pc.args[1:]
235+ }
236+ pc.nextArg = nextArg
237+ if opt.arg != nil {
238+ if errno := pc.poptSaveArg(opt, nextArg); errno != 0 {
239+ return -1, &PoptError{
240+ Errno: errno,
241+ Err: fmt.Errorf("poptSaveArg"),
242+ }
243+ }
244+ }
245+ }
246+ if opt.val != 0 && argType != POPT_ARG_VAL {
247+ return int32(opt.val), nil
248+ }
249+ }
250+}
251+
252+func (pc *Context) poptGetOptArg() string {
253+ ret := pc.nextArg
254+ pc.nextArg = ""
255+ return ret
256+}
+943,
-0
1@@ -0,0 +1,943 @@
2+// Package rsyncopts implements a parser for command-line options that
3+// implements a subset of popt(3) semantics; just enough to parse typical
4+// rsync(1) invocations without the advanced popt features like aliases
5+// or option prefix matching (not --del, only --delete).
6+//
7+// If we encounter arguments that rsync(1) parses differently compared to this
8+// package, then this package should be adjusted to match rsync(1).
9+package rsyncopts
10+
11+import (
12+ "errors"
13+ "fmt"
14+ "log/slog"
15+ "math"
16+ "slices"
17+ "strconv"
18+ "strings"
19+ "syscall"
20+ "unicode"
21+)
22+
23+const (
24+ OPT_SERVER = 1000 + iota
25+ OPT_DAEMON
26+ OPT_SENDER
27+ OPT_EXCLUDE
28+ OPT_EXCLUDE_FROM
29+ OPT_FILTER
30+ OPT_COMPARE_DEST
31+ OPT_COPY_DEST
32+ OPT_LINK_DEST
33+ OPT_HELP
34+ OPT_INCLUDE
35+ OPT_INCLUDE_FROM
36+ OPT_MODIFY_WINDOW
37+ OPT_MIN_SIZE
38+ OPT_CHMOD
39+ OPT_READ_BATCH
40+ OPT_WRITE_BATCH
41+ OPT_ONLY_WRITE_BATCH
42+ OPT_MAX_SIZE
43+ OPT_NO_D
44+ OPT_APPEND
45+ OPT_NO_ICONV
46+ OPT_INFO
47+ OPT_DEBUG
48+ OPT_BLOCK_SIZE
49+ OPT_USERMAP
50+ OPT_GROUPMAP
51+ OPT_CHOWN
52+ OPT_BWLIMIT
53+ OPT_STDERR
54+ OPT_OLD_COMPRESS
55+ OPT_NEW_COMPRESS
56+ OPT_NO_COMPRESS
57+ OPT_OLD_ARGS
58+ OPT_STOP_AFTER
59+ OPT_STOP_AT
60+ OPT_REFUSED_BASE = 9000
61+)
62+
63+type infoLevel int
64+
65+const (
66+ INFO_BACKUP infoLevel = iota
67+ INFO_COPY
68+ INFO_DEL
69+ INFO_FLIST
70+ INFO_MISC
71+ INFO_MOUNT
72+ INFO_NAME
73+ INFO_NONREG
74+ INFO_PROGRESS
75+ INFO_REMOVE
76+ INFO_SKIP
77+ INFO_STATS
78+ INFO_SYMSAFE
79+ COUNT_INFO
80+)
81+
82+// NewOptions returns an Options struct with all options initialized to their
83+// default values. Note that ParseArguments will set some options (that default
84+// to -1) based on the encountered command-line flags and built-in rules.
85+func NewOptions() *Options {
86+ return &Options{
87+ msgs2stderr: 2, // Default: send errors to stderr for local & remote-shell transfers
88+ output_motd: 1,
89+ human_readable: 1,
90+ allow_inc_recurse: 1,
91+ xfer_dirs: -1,
92+ relative_paths: -1,
93+ implied_dirs: 1,
94+ max_delete: math.MinInt32,
95+ whole_file: -1,
96+ do_compression_level: math.MinInt32,
97+ rsync_path: "rsync",
98+ default_af_hint: syscall.AF_INET6,
99+ blocking_io: -1,
100+ protocol_version: 27,
101+ }
102+}
103+
104+// GokrazyOptions contains additional command-line flags, prefixed with
105+// gokr. (like --gokr.modulemap) to not clash with rsync flag names.
106+type GokrazyOptions struct {
107+ Config string
108+ Listen string
109+ MonitoringListen string
110+ AnonSSHListen string
111+ ModuleMap string
112+}
113+
114+func (o *GokrazyOptions) table() []poptOption {
115+ return []poptOption{
116+ /* longName, shortName, argInfo, arg, val */
117+ {"gokr.config", "", POPT_ARG_STRING, &o.Config, 0},
118+ {"gokr.listen", "", POPT_ARG_STRING, &o.Listen, 0},
119+ {"gokr.monitoring_listen", "", POPT_ARG_STRING, &o.MonitoringListen, 0},
120+ {"gokr.anonssh_listen", "", POPT_ARG_STRING, &o.AnonSSHListen, 0},
121+ {"gokr.modulemap", "", POPT_ARG_STRING, &o.ModuleMap, 0},
122+ }
123+}
124+
125+type Options struct {
126+ Gokrazy GokrazyOptions
127+
128+ // not directly referenced in the table, but used in the special case code.
129+ do_compression int
130+ info [COUNT_INFO]uint16
131+ local_server int
132+
133+ // order matches long_options order
134+ verbose int
135+ msgs2stderr int
136+ quiet int
137+ output_motd int
138+ do_stats int
139+ human_readable int
140+ dry_run int
141+ recurse int
142+ allow_inc_recurse int
143+ xfer_dirs int
144+ preserve_perms int
145+ preserve_executability int
146+ preserve_acls int
147+ preserve_xattrs int
148+ preserve_mtimes int
149+ preserve_atimes int
150+ open_noatime int
151+ preserve_crtimes int
152+ omit_dir_times int
153+ omit_link_times int
154+ modify_window int
155+ am_root int // 0 = normal, 1 = root, 2 = --super, -1 = --fake-super
156+ preserve_uid int
157+ preserve_gid int
158+ preserve_devices int
159+ copy_devices int
160+ write_devices int
161+ preserve_specials int
162+ preserve_links int
163+ copy_links int
164+ copy_unsafe_links int
165+ safe_symlinks int
166+ munge_symlinks int
167+ copy_dirlinks int
168+ keep_dirlinks int
169+ preserve_hard_links int
170+ relative_paths int
171+ implied_dirs int
172+ ignore_times int
173+ size_only int
174+ one_file_system int
175+ update_only int
176+ ignore_non_existing int
177+ ignore_existing int
178+ max_size_arg string
179+ min_size_arg string
180+ max_alloc_arg string
181+ sparse_files int
182+ preallocate_files int
183+ inplace int
184+ append_mode int
185+ delete_during int
186+ delete_mode int
187+ delete_before int
188+ delete_after int
189+ delete_excluded int
190+ missing_args int // 0 = FERROR_XFER, 1 = ignore, 2 = delete
191+ remove_source_files int
192+ force_delete int
193+ ignore_errors int
194+ max_delete int
195+ cvs_exclude int
196+ // If 1, send the whole file as literal data rather than trying to create an
197+ // incremental diff.
198+ // If -1, then look at whether we're local or remote and go by that.
199+ // See also disable_deltas_p()
200+ whole_file int
201+ always_checksum int
202+ checksum_choice string
203+ fuzzy_basis int
204+ compress_choice string
205+ skip_compress string
206+ do_compression_level int
207+ do_progress int
208+ keep_partial int
209+ partial_dir string
210+ delay_updates int
211+ prune_empty_dirs int
212+ logfile_name string
213+ logfile_format string
214+ stdout_format string
215+ itemize_changes int
216+ bwlimit_arg string
217+ bwlimit int
218+ make_backups int
219+ backup_dir string
220+ backup_suffix string
221+ list_only int
222+ batch_name string
223+ files_from string
224+ eol_nulls int
225+ old_style_args int // intentionally set to 0; unsupported
226+ protect_args int // intentionally set to 0; currently unsupported
227+ trust_sender int
228+ numeric_ids int
229+ io_timeout int
230+ connect_timeout int
231+ do_fsync int
232+ shell_cmd string
233+ rsync_path string
234+ tmpdir string
235+ iconv_opt string
236+ default_af_hint int
237+ allow_8bit_chars int
238+ mkpath_dest_arg int
239+ use_qsort int
240+ copy_as string
241+ bind_address string // numeric IPv4 or IPv6, or a hostname
242+ rsync_port int
243+ sockopts string
244+ password_file string
245+ early_input_file string
246+ blocking_io int
247+ outbuf_mode string
248+ protocol_version int
249+ checksum_seed int
250+ am_server int
251+ am_sender int
252+ am_daemon int
253+
254+ daemon_bwlimit int
255+ config_file string
256+ daemon_opt int
257+ no_detach int
258+}
259+
260+type priority int
261+
262+const (
263+ DEFAULT_PRIORITY priority = iota
264+ HELP_PRIORITY
265+ USER_PRIORITY
266+ LIMIT_PRIORITY
267+)
268+
269+const (
270+ W_CLI = 1 << iota
271+ W_SRV
272+ W_SND
273+ W_REC
274+)
275+
276+type output struct {
277+ name string
278+ where int
279+ help string
280+}
281+
282+var infoWords = [...]output{
283+ {"BACKUP", W_REC, "Mention files backed up"},
284+ {"COPY", W_REC, "Mention files copied locally on the receiving side"},
285+ {"DEL", W_REC, "Mention deletions on the receiving side"},
286+ {"FLIST", W_CLI, "Mention file-list receiving/sending (levels 1-2)"},
287+ {"MISC", W_SND | W_REC, "Mention miscellaneous information (levels 1-2)"},
288+ {"MOUNT", W_SND | W_REC, "Mention mounts that were found or skipped"},
289+ {"NAME", W_SND | W_REC, "Mention 1) updated file/dir names, 2) unchanged names"},
290+ {"NONREG", W_REC, "Mention skipped non-regular files (default 1, 0 disables)"},
291+ {"PROGRESS", W_CLI, "Mention 1) per-file progress or 2) total transfer progress"},
292+ {"REMOVE", W_SND, "Mention files removed on the sending side"},
293+ {"SKIP", W_REC, "Mention files skipped due to transfer overrides (levels 1-2)"},
294+ {"STATS", W_CLI | W_SRV, "Mention statistics at end of run (levels 1-3)"},
295+ {"SYMSAFE", W_SND | W_REC, "Mention symlinks that are unsafe"},
296+}
297+
298+func parseOutputWords(words []output, levels []uint16, str string, prio priority) {
299+Level:
300+ for s := range strings.SplitSeq(str, ",") {
301+ if strings.TrimSpace(s) == "" {
302+ continue
303+ }
304+ trimmed := strings.TrimRightFunc(s, unicode.IsNumber)
305+ lev := 1
306+ if len(trimmed) < len(s) {
307+ var err error
308+ lev, err = strconv.Atoi(s[len(trimmed):])
309+ if err != nil {
310+ continue
311+ }
312+ }
313+ trimmed = strings.ToLower(trimmed)
314+ all := false
315+ switch trimmed {
316+ case "none":
317+ lev = 0
318+ case "all":
319+ all = true
320+ }
321+ for j := range words {
322+ word := words[j]
323+ if strings.ToLower(word.name) == trimmed || all {
324+ levels[j] = uint16(lev)
325+ if !all {
326+ continue Level
327+ }
328+ }
329+ }
330+ }
331+}
332+
333+func (o *Options) setOutputVerbosity(prio priority) {
334+ // debugVerbosity is reserved for future use with debug logging levels.
335+ // debugVerbosity := [...]string{
336+ // "",
337+ // "",
338+ // "BIND,CMD,CONNECT,DEL,DELTASUM,DUP,FILTER,FLIST,ICONV",
339+ // "ACL,BACKUP,CONNECT2,DELTASUM2,DEL2,EXIT,FILTER2,FLIST2,FUZZY,GENR,OWN,RECV,SEND,TIME",
340+ // "CMD2,DELTASUM3,DEL3,EXIT2,FLIST3,ICONV2,OWN2,PROTO,TIME2",
341+ // "CHDIR,DELTASUM4,FLIST4,FUZZY2,HASH,HLINK",
342+ // }
343+ infoVerbosity := [...]string{
344+ "NONREG",
345+ "COPY,DEL,FLIST,MISC,NAME,STATS,SYMSAFE",
346+ "BACKUP,MISC2,MOUNT,NAME2,REMOVE,SKIP",
347+ }
348+ for j := 0; j <= o.verbose; j++ {
349+ if j < len(infoVerbosity) {
350+ parseOutputWords(infoWords[:], o.info[:], infoVerbosity[j], prio)
351+ }
352+ // TODO: enable debug verbosity when debug logging is implemented
353+ // if j < len(debugVerbosity) {
354+ // parseOutputWords(debugWords[:], o.debug[:], debugVerbosity[j], prio)
355+ // }
356+ }
357+}
358+
359+func (o *Options) Help() string {
360+ return ""
361+}
362+
363+func (o *Options) ShellCommand() string { return o.shell_cmd }
364+func (o *Options) UpdateOnly() bool { return o.update_only != 0 }
365+func (o *Options) DryRun() bool { return o.dry_run != 0 }
366+func (o *Options) PreserveLinks() bool { return o.preserve_links != 0 }
367+func (o *Options) PreserveUid() bool { return o.preserve_uid != 0 }
368+func (o *Options) PreserveGid() bool { return o.preserve_gid != 0 }
369+func (o *Options) PreserveDevices() bool { return o.preserve_devices != 0 }
370+func (o *Options) PreserveMTimes() bool { return o.preserve_mtimes != 0 }
371+func (o *Options) PreservePerms() bool { return o.preserve_perms != 0 }
372+func (o *Options) PreserveSpecials() bool { return o.preserve_specials != 0 }
373+func (o *Options) PreserveHardLinks() bool { return o.preserve_hard_links != 0 }
374+func (o *Options) Recurse() bool { return o.recurse != 0 }
375+func (o *Options) Verbose() bool { return o.verbose != 0 }
376+func (o *Options) DeleteMode() bool { return o.delete_mode != 0 }
377+func (o *Options) Sender() bool { return o.am_sender != 0 }
378+func (o *Options) SetSender() { o.am_sender = 1 }
379+func (o *Options) LocalServer() bool { return o.local_server != 0 }
380+func (o *Options) SetLocalServer() { o.local_server = 1 }
381+func (o *Options) Server() bool { return o.am_server != 0 }
382+func (o *Options) Daemon() bool { return o.am_daemon != 0 }
383+func (o *Options) ConnectTimeoutSeconds() int { return o.connect_timeout }
384+func (o *Options) AlwaysChecksum() bool { return o.always_checksum != 0 }
385+func (o *Options) Compress() bool { return o.do_compression != 0 }
386+func (o *Options) CompressChoice() string { return o.compress_choice }
387+func (o *Options) CompressLevel() int { return o.do_compression_level }
388+func (o *Options) IgnoreTimes() bool { return o.ignore_times == 1 }
389+func (o *Options) SizeOnly() bool { return o.size_only == 1 }
390+
391+func (o *Options) daemonTable() []poptOption {
392+ return []poptOption{
393+ /* longName, shortName, argInfo, arg, val */
394+ {"help", "", POPT_ARG_NONE, nil, OPT_HELP},
395+ {"address", "", POPT_ARG_STRING, &o.bind_address, 0},
396+ {"bwlimit", "", POPT_ARG_INT, &o.daemon_bwlimit, 0},
397+ {"config", "", POPT_ARG_STRING, &o.config_file, 0},
398+ {"daemon", "", POPT_ARG_NONE, &o.daemon_opt, 0},
399+ {"dparam", "M", POPT_ARG_STRING, nil, 'M'},
400+ {"ipv4", "4", POPT_ARG_VAL, &o.default_af_hint, syscall.AF_INET},
401+ {"ipv6", "6", POPT_ARG_VAL, &o.default_af_hint, syscall.AF_INET6},
402+ {"detach", "", POPT_ARG_VAL, &o.no_detach, 0},
403+ {"no-detach", "", POPT_ARG_VAL, &o.no_detach, 1},
404+ {"log-file", "", POPT_ARG_STRING, &o.logfile_name, 0},
405+ {"log-file-format", "", POPT_ARG_STRING, &o.logfile_format, 0},
406+ {"port", "", POPT_ARG_INT, &o.rsync_port, 0},
407+ {"sockopts", "", POPT_ARG_STRING, &o.sockopts, 0},
408+ {"protocol", "", POPT_ARG_INT, &o.protocol_version, 0},
409+ {"server", "", POPT_ARG_NONE, &o.am_server, 0},
410+ {"temp-dir", "T", POPT_ARG_STRING, &o.tmpdir, 0},
411+ {"verbose", "v", POPT_ARG_NONE, 0, 'v'},
412+ {"no-verbose", "", POPT_ARG_VAL, &o.verbose, 0},
413+ {"no-v", "", POPT_ARG_VAL, &o.verbose, 0},
414+ {"help", "h", POPT_ARG_NONE, 0, 'h'},
415+ }
416+}
417+
418+func (o *Options) table() []poptOption {
419+ return []poptOption{
420+ /* longName, shortName, argInfo, arg, val */
421+ {"help", "", POPT_ARG_NONE, nil, OPT_HELP},
422+ {"version", "V", POPT_ARG_NONE, nil, 'V'},
423+ {"verbose", "v", POPT_ARG_NONE, nil, 'v'},
424+ {"no-verbose", "", POPT_ARG_VAL, &o.verbose, 0},
425+ {"no-v", "", POPT_ARG_VAL, &o.verbose, 0},
426+ {"info", "", POPT_ARG_STRING, nil, OPT_INFO},
427+ {"debug", "", POPT_ARG_STRING, nil, OPT_DEBUG},
428+ {"stderr", "", POPT_ARG_STRING, nil, OPT_STDERR},
429+ {"msgs2stderr", "", POPT_ARG_VAL, &o.msgs2stderr, 1},
430+ {"no-msgs2stderr", "", POPT_ARG_VAL, &o.msgs2stderr, 0},
431+ {"quiet", "q", POPT_ARG_NONE, nil, 'q'},
432+ {"motd", "", POPT_ARG_VAL, &o.output_motd, 1},
433+ {"no-motd", "", POPT_ARG_VAL, &o.output_motd, 0},
434+ {"stats", "", POPT_ARG_NONE, &o.do_stats, 0},
435+ {"human-readable", "h", POPT_ARG_NONE, nil, 'h'},
436+ {"no-human-readable", "", POPT_ARG_VAL, &o.human_readable, 0},
437+ {"no-h", "", POPT_ARG_VAL, &o.human_readable, 0},
438+ {"dry-run", "n", POPT_ARG_NONE, &o.dry_run, 0},
439+ {"archive", "a", POPT_ARG_NONE, nil, 'a'},
440+ {"recursive", "r", POPT_ARG_VAL, &o.recurse, 2},
441+ {"no-recursive", "", POPT_ARG_VAL, &o.recurse, 0},
442+ {"no-r", "", POPT_ARG_VAL, &o.recurse, 0},
443+ {"inc-recursive", "", POPT_ARG_VAL, &o.allow_inc_recurse, 1},
444+ {"no-inc-recursive", "", POPT_ARG_VAL, &o.allow_inc_recurse, 0},
445+ {"i-r", "", POPT_ARG_VAL, &o.allow_inc_recurse, 1},
446+ {"no-i-r", "", POPT_ARG_VAL, &o.allow_inc_recurse, 0},
447+ {"dirs", "d", POPT_ARG_VAL, &o.xfer_dirs, 2},
448+ {"no-dirs", "", POPT_ARG_VAL, &o.xfer_dirs, 0},
449+ {"no-d", "", POPT_ARG_VAL, &o.xfer_dirs, 0},
450+ {"old-dirs", "", POPT_ARG_VAL, &o.xfer_dirs, 4},
451+ {"old-d", "", POPT_ARG_VAL, &o.xfer_dirs, 4},
452+ {"perms", "p", POPT_ARG_VAL, &o.preserve_perms, 1},
453+ {"no-perms", "", POPT_ARG_VAL, &o.preserve_perms, 0},
454+ {"no-p", "", POPT_ARG_VAL, &o.preserve_perms, 0},
455+ {"executability", "E", POPT_ARG_NONE, &o.preserve_executability, 0},
456+ {"acls", "A", POPT_ARG_NONE, nil, 'A'},
457+ {"no-acls", "", POPT_ARG_VAL, &o.preserve_acls, 0},
458+ {"no-A", "", POPT_ARG_VAL, &o.preserve_acls, 0},
459+ {"xattrs", "X", POPT_ARG_NONE, nil, 'X'},
460+ {"no-xattrs", "", POPT_ARG_VAL, &o.preserve_xattrs, 0},
461+ {"no-X", "", POPT_ARG_VAL, &o.preserve_xattrs, 0},
462+ {"times", "t", POPT_ARG_VAL, &o.preserve_mtimes, 1},
463+ {"no-times", "", POPT_ARG_VAL, &o.preserve_mtimes, 0},
464+ {"no-t", "", POPT_ARG_VAL, &o.preserve_mtimes, 0},
465+ {"atimes", "U", POPT_ARG_NONE, nil, 'U'},
466+ {"no-atimes", "", POPT_ARG_VAL, &o.preserve_atimes, 0},
467+ {"no-U", "", POPT_ARG_VAL, &o.preserve_atimes, 0},
468+ {"open-noatime", "", POPT_ARG_VAL, &o.open_noatime, 1},
469+ {"no-open-noatime", "", POPT_ARG_VAL, &o.open_noatime, 0},
470+ {"crtimes", "N", POPT_ARG_NONE, &o.preserve_crtimes, 1}, // refused
471+ {"no-crtimes", "", POPT_ARG_VAL, &o.preserve_crtimes, 0},
472+ {"no-N", "", POPT_ARG_VAL, &o.preserve_crtimes, 0},
473+ {"omit-dir-times", "O", POPT_ARG_VAL, &o.omit_dir_times, 1},
474+ {"no-omit-dir-times", "", POPT_ARG_VAL, &o.omit_dir_times, 0},
475+ {"no-O", "", POPT_ARG_VAL, &o.omit_dir_times, 0},
476+ {"omit-link-times", "J", POPT_ARG_VAL, &o.omit_link_times, 1},
477+ {"no-omit-link-times", "", POPT_ARG_VAL, &o.omit_link_times, 0},
478+ {"no-J", "", POPT_ARG_VAL, &o.omit_link_times, 0},
479+ {"modify-window", "@", POPT_ARG_INT, &o.modify_window, OPT_MODIFY_WINDOW},
480+ {"super", "", POPT_ARG_VAL, &o.am_root, 2},
481+ {"no-super", "", POPT_ARG_VAL, &o.am_root, 0},
482+ {"fake-super", "", POPT_ARG_VAL, &o.am_root, -1},
483+ {"owner", "o", POPT_ARG_VAL, &o.preserve_uid, 1},
484+ {"no-owner", "", POPT_ARG_VAL, &o.preserve_uid, 0},
485+ {"no-o", "", POPT_ARG_VAL, &o.preserve_uid, 0},
486+ {"group", "g", POPT_ARG_VAL, &o.preserve_gid, 1},
487+ {"no-group", "", POPT_ARG_VAL, &o.preserve_gid, 0},
488+ {"no-g", "", POPT_ARG_VAL, &o.preserve_gid, 0},
489+ {"", "D", POPT_ARG_NONE, nil, 'D'},
490+ {"no-D", "", POPT_ARG_NONE, nil, OPT_NO_D},
491+ {"devices", "", POPT_ARG_VAL, &o.preserve_devices, 1},
492+ {"no-devices", "", POPT_ARG_VAL, &o.preserve_devices, 0},
493+ {"copy-devices", "", POPT_ARG_NONE, &o.copy_devices, 0},
494+ {"write-devices", "", POPT_ARG_VAL, &o.write_devices, 1},
495+ {"no-write-devices", "", POPT_ARG_VAL, &o.write_devices, 0},
496+ {"specials", "", POPT_ARG_VAL, &o.preserve_specials, 1},
497+ {"no-specials", "", POPT_ARG_VAL, &o.preserve_specials, 0},
498+ {"links", "l", POPT_ARG_VAL, &o.preserve_links, 1},
499+ {"no-links", "", POPT_ARG_VAL, &o.preserve_links, 0},
500+ {"no-l", "", POPT_ARG_VAL, &o.preserve_links, 0},
501+ {"copy-links", "L", POPT_ARG_NONE, &o.copy_links, 0},
502+ {"copy-unsafe-links", "", POPT_ARG_NONE, &o.copy_unsafe_links, 0},
503+ {"safe-links", "", POPT_ARG_NONE, &o.safe_symlinks, 0},
504+ {"munge-links", "", POPT_ARG_VAL, &o.munge_symlinks, 1},
505+ {"no-munge-links", "", POPT_ARG_VAL, &o.munge_symlinks, 0},
506+ {"copy-dirlinks", "k", POPT_ARG_NONE, &o.copy_dirlinks, 0},
507+ {"keep-dirlinks", "K", POPT_ARG_NONE, &o.keep_dirlinks, 0},
508+ {"hard-links", "H", POPT_ARG_NONE, nil, 'H'},
509+ {"no-hard-links", "", POPT_ARG_VAL, &o.preserve_hard_links, 0},
510+ {"no-H", "", POPT_ARG_VAL, &o.preserve_hard_links, 0},
511+ {"relative", "R", POPT_ARG_VAL, &o.relative_paths, 1},
512+ {"no-relative", "", POPT_ARG_VAL, &o.relative_paths, 0},
513+ {"no-R", "", POPT_ARG_VAL, &o.relative_paths, 0},
514+ {"implied-dirs", "", POPT_ARG_VAL, &o.implied_dirs, 1},
515+ {"no-implied-dirs", "", POPT_ARG_VAL, &o.implied_dirs, 0},
516+ {"i-d", "", POPT_ARG_VAL, &o.implied_dirs, 1},
517+ {"no-i-d", "", POPT_ARG_VAL, &o.implied_dirs, 0},
518+ {"chmod", "", POPT_ARG_STRING, nil, OPT_CHMOD},
519+ {"ignore-times", "I", POPT_ARG_NONE, &o.ignore_times, 0},
520+ {"size-only", "", POPT_ARG_NONE, &o.size_only, 0},
521+ {"one-file-system", "x", POPT_ARG_NONE, nil, 'x'},
522+ {"no-one-file-system", "", POPT_ARG_VAL, &o.one_file_system, 0},
523+ {"no-x", "", POPT_ARG_VAL, &o.one_file_system, 0},
524+ {"update", "u", POPT_ARG_NONE, &o.update_only, 0},
525+ {"existing", "", POPT_ARG_NONE, &o.ignore_non_existing, 0},
526+ {"ignore-non-existing", "", POPT_ARG_NONE, &o.ignore_non_existing, 0},
527+ {"ignore-existing", "", POPT_ARG_NONE, &o.ignore_existing, 0},
528+ {"max-size", "", POPT_ARG_STRING, &o.max_size_arg, OPT_MAX_SIZE},
529+ {"min-size", "", POPT_ARG_STRING, &o.min_size_arg, OPT_MIN_SIZE},
530+ {"max-alloc", "", POPT_ARG_STRING, &o.max_alloc_arg, 0},
531+ {"sparse", "S", POPT_ARG_VAL, &o.sparse_files, 1},
532+ {"no-sparse", "", POPT_ARG_VAL, &o.sparse_files, 0},
533+ {"no-S", "", POPT_ARG_VAL, &o.sparse_files, 0},
534+ {"preallocate", "", POPT_ARG_NONE, &o.preallocate_files, 0},
535+ {"inplace", "", POPT_ARG_VAL, &o.inplace, 1},
536+ {"no-inplace", "", POPT_ARG_VAL, &o.inplace, 0},
537+ {"append", "", POPT_ARG_NONE, nil, OPT_APPEND},
538+ {"append-verify", "", POPT_ARG_VAL, &o.append_mode, 2},
539+ {"no-append", "", POPT_ARG_VAL, &o.append_mode, 0},
540+ {"del", "", POPT_ARG_NONE, &o.delete_during, 0},
541+ {"delete", "", POPT_ARG_NONE, &o.delete_mode, 0},
542+ {"delete-before", "", POPT_ARG_NONE, &o.delete_before, 0},
543+ {"delete-during", "", POPT_ARG_VAL, &o.delete_during, 1},
544+ {"delete-delay", "", POPT_ARG_VAL, &o.delete_during, 2},
545+ {"delete-after", "", POPT_ARG_NONE, &o.delete_after, 0},
546+ {"delete-excluded", "", POPT_ARG_NONE, &o.delete_excluded, 0},
547+ {"delete-missing-args", "", POPT_BIT_SET, &o.missing_args, 2},
548+ {"ignore-missing-args", "", POPT_BIT_SET, &o.missing_args, 1},
549+ {"remove-sent-files", "", POPT_ARG_VAL, &o.remove_source_files, 2}, /* deprecated */
550+ {"remove-source-files", "", POPT_ARG_VAL, &o.remove_source_files, 1},
551+ {"force", "", POPT_ARG_VAL, &o.force_delete, 1},
552+ {"no-force", "", POPT_ARG_VAL, &o.force_delete, 0},
553+ {"ignore-errors", "", POPT_ARG_VAL, &o.ignore_errors, 1},
554+ {"no-ignore-errors", "", POPT_ARG_VAL, &o.ignore_errors, 0},
555+ {"max-delete", "", POPT_ARG_INT, &o.max_delete, 0},
556+ {"", "F", POPT_ARG_NONE, nil, 'F'},
557+ {"filter", "f", POPT_ARG_STRING, nil, OPT_FILTER},
558+ {"exclude", "", POPT_ARG_STRING, nil, OPT_EXCLUDE},
559+ {"include", "", POPT_ARG_STRING, nil, OPT_INCLUDE},
560+ {"exclude-from", "", POPT_ARG_STRING, nil, OPT_EXCLUDE_FROM},
561+ {"include-from", "", POPT_ARG_STRING, nil, OPT_INCLUDE_FROM},
562+ {"cvs-exclude", "C", POPT_ARG_NONE, &o.cvs_exclude, 0},
563+ {"whole-file", "W", POPT_ARG_VAL, &o.whole_file, 1},
564+ {"no-whole-file", "", POPT_ARG_VAL, &o.whole_file, 0},
565+ {"no-W", "", POPT_ARG_VAL, &o.whole_file, 0},
566+ {"checksum", "c", POPT_ARG_VAL, &o.always_checksum, 1},
567+ {"no-checksum", "", POPT_ARG_VAL, &o.always_checksum, 0},
568+ {"no-c", "", POPT_ARG_VAL, &o.always_checksum, 0},
569+ {"checksum-choice", "", POPT_ARG_STRING, &o.checksum_choice, 0},
570+ {"cc", "", POPT_ARG_STRING, &o.checksum_choice, 0},
571+ {"block-size", "B", POPT_ARG_STRING, nil, OPT_BLOCK_SIZE},
572+ {"compare-dest", "", POPT_ARG_STRING, nil, OPT_COMPARE_DEST},
573+ {"copy-dest", "", POPT_ARG_STRING, nil, OPT_COPY_DEST},
574+ {"link-dest", "", POPT_ARG_STRING, nil, OPT_LINK_DEST},
575+ {"fuzzy", "y", POPT_ARG_NONE, nil, 'y'},
576+ {"no-fuzzy", "", POPT_ARG_VAL, &o.fuzzy_basis, 0},
577+ {"no-y", "", POPT_ARG_VAL, &o.fuzzy_basis, 0},
578+ {"compress", "z", POPT_ARG_NONE, nil, 'z'},
579+ {"old-compress", "", POPT_ARG_NONE, nil, OPT_OLD_COMPRESS},
580+ {"new-compress", "", POPT_ARG_NONE, nil, OPT_NEW_COMPRESS},
581+ {"no-compress", "", POPT_ARG_NONE, nil, OPT_NO_COMPRESS},
582+ {"no-z", "", POPT_ARG_NONE, nil, OPT_NO_COMPRESS},
583+ {"compress-choice", "", POPT_ARG_STRING, &o.compress_choice, 0},
584+ {"zc", "", POPT_ARG_STRING, &o.compress_choice, 0},
585+ {"skip-compress", "", POPT_ARG_STRING, &o.skip_compress, 0},
586+ {"compress-level", "", POPT_ARG_INT, &o.do_compression_level, 0},
587+ {"zl", "", POPT_ARG_INT, &o.do_compression_level, 0},
588+ {"", "P", POPT_ARG_NONE, nil, 'P'},
589+ {"progress", "", POPT_ARG_VAL, &o.do_progress, 1},
590+ {"no-progress", "", POPT_ARG_VAL, &o.do_progress, 0},
591+ {"partial", "", POPT_ARG_VAL, &o.keep_partial, 1},
592+ {"no-partial", "", POPT_ARG_VAL, &o.keep_partial, 0},
593+ {"partial-dir", "", POPT_ARG_STRING, &o.partial_dir, 0},
594+ {"delay-updates", "", POPT_ARG_VAL, &o.delay_updates, 1},
595+ {"no-delay-updates", "", POPT_ARG_VAL, &o.delay_updates, 0},
596+ {"prune-empty-dirs", "m", POPT_ARG_VAL, &o.prune_empty_dirs, 1},
597+ {"no-prune-empty-dirs", "", POPT_ARG_VAL, &o.prune_empty_dirs, 0},
598+ {"no-m", "", POPT_ARG_VAL, &o.prune_empty_dirs, 0},
599+ {"log-file", "", POPT_ARG_STRING, &o.logfile_name, 0},
600+ {"log-file-format", "", POPT_ARG_STRING, &o.logfile_format, 0},
601+ {"out-format", "", POPT_ARG_STRING, &o.stdout_format, 0},
602+ {"log-format", "", POPT_ARG_STRING, &o.stdout_format, 0}, /* DEPRECATED */
603+ {"itemize-changes", "i", POPT_ARG_NONE, nil, 'i'},
604+ {"no-itemize-changes", "", POPT_ARG_VAL, &o.itemize_changes, 0},
605+ {"no-i", "", POPT_ARG_VAL, &o.itemize_changes, 0},
606+ {"bwlimit", "", POPT_ARG_STRING, &o.bwlimit_arg, OPT_BWLIMIT},
607+ {"no-bwlimit", "", POPT_ARG_VAL, &o.bwlimit, 0},
608+ {"backup", "b", POPT_ARG_VAL, &o.make_backups, 1},
609+ {"no-backup", "", POPT_ARG_VAL, &o.make_backups, 0},
610+ {"backup-dir", "", POPT_ARG_STRING, &o.backup_dir, 0},
611+ {"suffix", "", POPT_ARG_STRING, &o.backup_suffix, 0},
612+ {"list-only", "", POPT_ARG_VAL, &o.list_only, 2},
613+ {"read-batch", "", POPT_ARG_STRING, &o.batch_name, OPT_READ_BATCH},
614+ {"write-batch", "", POPT_ARG_STRING, &o.batch_name, OPT_WRITE_BATCH},
615+ {"only-write-batch", "", POPT_ARG_STRING, &o.batch_name, OPT_ONLY_WRITE_BATCH},
616+ {"files-from", "", POPT_ARG_STRING, &o.files_from, 0},
617+ {"from0", "0", POPT_ARG_VAL, &o.eol_nulls, 1},
618+ {"no-from0", "", POPT_ARG_VAL, &o.eol_nulls, 0},
619+ {"old-args", "", POPT_ARG_NONE, nil, OPT_OLD_ARGS},
620+ {"no-old-args", "", POPT_ARG_VAL, &o.old_style_args, 0},
621+ {"secluded-args", "s", POPT_ARG_VAL, &o.protect_args, 1},
622+ {"no-secluded-args", "", POPT_ARG_VAL, &o.protect_args, 0},
623+ {"protect-args", "", POPT_ARG_VAL, &o.protect_args, 1},
624+ {"no-protect-args", "", POPT_ARG_VAL, &o.protect_args, 0},
625+ {"no-s", "", POPT_ARG_VAL, &o.protect_args, 0},
626+ {"trust-sender", "", POPT_ARG_VAL, &o.trust_sender, 1},
627+ {"numeric-ids", "", POPT_ARG_VAL, &o.numeric_ids, 1},
628+ {"no-numeric-ids", "", POPT_ARG_VAL, &o.numeric_ids, 0},
629+ {"usermap", "", POPT_ARG_STRING, nil, OPT_USERMAP},
630+ {"groupmap", "", POPT_ARG_STRING, nil, OPT_GROUPMAP},
631+ {"chown", "", POPT_ARG_STRING, nil, OPT_CHOWN},
632+ {"timeout", "", POPT_ARG_INT, &o.io_timeout, 0},
633+ {"no-timeout", "", POPT_ARG_VAL, &o.io_timeout, 0},
634+ {"contimeout", "", POPT_ARG_INT, &o.connect_timeout, 0},
635+ {"no-contimeout", "", POPT_ARG_VAL, &o.connect_timeout, 0},
636+ {"fsync", "", POPT_ARG_NONE, &o.do_fsync, 0},
637+ {"stop-after", "", POPT_ARG_STRING, nil, OPT_STOP_AFTER},
638+ {"time-limit", "", POPT_ARG_STRING, nil, OPT_STOP_AFTER}, /* earlier stop-after name */
639+ {"stop-at", "", POPT_ARG_STRING, nil, OPT_STOP_AT},
640+ {"rsh", "e", POPT_ARG_STRING, &o.shell_cmd, 0},
641+ {"rsync-path", "", POPT_ARG_STRING, &o.rsync_path, 0},
642+ {"temp-dir", "T", POPT_ARG_STRING, &o.tmpdir, 0},
643+ {"iconv", "", POPT_ARG_STRING, &o.iconv_opt, 0},
644+ {"no-iconv", "", POPT_ARG_NONE, nil, OPT_NO_ICONV},
645+ {"ipv4", "4", POPT_ARG_VAL, &o.default_af_hint, syscall.AF_INET},
646+ {"ipv6", "6", POPT_ARG_VAL, &o.default_af_hint, syscall.AF_INET6},
647+ {"8-bit-output", "8", POPT_ARG_VAL, &o.allow_8bit_chars, 1},
648+ {"no-8-bit-output", "", POPT_ARG_VAL, &o.allow_8bit_chars, 0},
649+ {"no-8", "", POPT_ARG_VAL, &o.allow_8bit_chars, 0},
650+ {"mkpath", "", POPT_ARG_VAL, &o.mkpath_dest_arg, 1},
651+ {"no-mkpath", "", POPT_ARG_VAL, &o.mkpath_dest_arg, 0},
652+ {"qsort", "", POPT_ARG_NONE, &o.use_qsort, 0},
653+ {"copy-as", "", POPT_ARG_STRING, &o.copy_as, 0},
654+ {"address", "", POPT_ARG_STRING, &o.bind_address, 0},
655+ {"port", "", POPT_ARG_INT, &o.rsync_port, 0},
656+ {"sockopts", "", POPT_ARG_STRING, &o.sockopts, 0},
657+ {"password-file", "", POPT_ARG_STRING, &o.password_file, 0},
658+ {"early-input", "", POPT_ARG_STRING, &o.early_input_file, 0},
659+ {"blocking-io", "", POPT_ARG_VAL, &o.blocking_io, 1},
660+ {"no-blocking-io", "", POPT_ARG_VAL, &o.blocking_io, 0},
661+ {"outbuf", "", POPT_ARG_STRING, &o.outbuf_mode, 0},
662+ {"remote-option", "M", POPT_ARG_STRING, nil, 'M'},
663+ {"protocol", "", POPT_ARG_INT, &o.protocol_version, 0},
664+ {"checksum-seed", "", POPT_ARG_INT, &o.checksum_seed, 0},
665+ {"server", "", POPT_ARG_NONE, nil, OPT_SERVER},
666+ {"sender", "", POPT_ARG_NONE, nil, OPT_SENDER},
667+ /* All the following options switch us into daemon-mode option-parsing. */
668+ {"config", "", POPT_ARG_STRING, nil, OPT_DAEMON},
669+ {"daemon", "", POPT_ARG_NONE, nil, OPT_DAEMON},
670+ {"dparam", "", POPT_ARG_STRING, nil, OPT_DAEMON},
671+ {"detach", "", POPT_ARG_NONE, nil, OPT_DAEMON},
672+ {"no-detach", "", POPT_ARG_NONE, nil, OPT_DAEMON},
673+ }
674+}
675+
676+var errNotYetImplemented = errors.New("option not yet implemented in gokrazy/rsync")
677+
678+// rsync/options.c:parse_arguments.
679+func ParseArguments(args []string, gokrazyTable bool) (*Context, error) {
680+ // NOTE: We do not implement support for refusing options per rsyncd.conf
681+ // here, as we have our own configuration file.
682+
683+ version_opt_cnt := 0
684+
685+ opts := NewOptions()
686+ table := opts.table()
687+ if gokrazyTable {
688+ // We need to make the --gokr.* flags known, otherwise the first parsing
689+ // attempt fails and the daemon mode parsing is never run.
690+ table = slices.Concat(opts.Gokrazy.table(), table)
691+ }
692+ pc := Context{
693+ Options: opts,
694+ table: table,
695+ args: args,
696+ }
697+
698+ for {
699+ opt, err := pc.poptGetNextOpt()
700+ if err != nil {
701+ return nil, err
702+ }
703+ if opt == -1 {
704+ break // done
705+ }
706+ // Most options are handled by poptGetNextOpt, only special cases
707+ // are returned and handled here.
708+ switch opt {
709+ case 'V':
710+ version_opt_cnt++
711+
712+ case OPT_SERVER:
713+ opts.am_server = 1
714+
715+ case OPT_SENDER:
716+ if opts.am_server == 0 {
717+ return nil, fmt.Errorf("--sender only allowed with --server")
718+ }
719+ opts.am_sender = 1
720+
721+ case OPT_DAEMON:
722+ // Parse the whole command-line using the daemon options table.
723+ table := opts.daemonTable()
724+ if gokrazyTable {
725+ table = slices.Concat(opts.Gokrazy.table(), table)
726+ }
727+ pc := Context{
728+ Options: opts,
729+ table: table,
730+ args: args,
731+ }
732+
733+ for {
734+ opt, err := pc.poptGetNextOpt()
735+ if err != nil {
736+ return nil, err
737+ }
738+ if opt == -1 {
739+ break // done
740+ }
741+ // Most options are handled by poptGetNextOpt, only special cases
742+ // are returned and handled here.
743+ switch opt {
744+ case 'M':
745+ return nil, errNotYetImplemented
746+ case 'v':
747+ opts.verbose++
748+ default:
749+ return nil, fmt.Errorf("unhandled special case opt: %v", opt)
750+ }
751+ }
752+
753+ opts.am_daemon = 1
754+
755+ return &pc, nil
756+
757+ case OPT_FILTER,
758+ OPT_EXCLUDE,
759+ OPT_INCLUDE,
760+ OPT_INCLUDE_FROM,
761+ OPT_EXCLUDE_FROM:
762+ return nil, errNotYetImplemented
763+
764+ case 'a':
765+ if opts.recurse == 0 {
766+ opts.recurse = 1
767+ }
768+ opts.preserve_links = 1
769+ opts.preserve_perms = 1
770+ opts.preserve_mtimes = 1
771+ opts.preserve_gid = 1
772+ opts.preserve_uid = 1
773+ opts.preserve_devices = 1
774+ opts.preserve_specials = 1
775+
776+ case 'D':
777+ opts.preserve_devices = 1
778+ opts.preserve_specials = 1
779+
780+ case OPT_NO_D:
781+ opts.preserve_devices = 0
782+ opts.preserve_specials = 0
783+
784+ case 'h':
785+ opts.human_readable++
786+
787+ case 'H':
788+ opts.preserve_hard_links = 1
789+
790+ case 'i':
791+ opts.itemize_changes++
792+
793+ case 'U':
794+ opts.preserve_atimes++
795+ if opts.preserve_atimes > 1 {
796+ opts.open_noatime = 1
797+ }
798+
799+ case 'v':
800+ opts.verbose++
801+
802+ case 'y':
803+ return nil, errNotYetImplemented
804+
805+ case 'q':
806+ opts.quiet++
807+
808+ case 'x':
809+ opts.one_file_system++
810+
811+ case 'F':
812+ return nil, errNotYetImplemented
813+
814+ case 'P':
815+ opts.do_progress = 1
816+ opts.keep_partial = 1
817+
818+ case 'z':
819+ opts.do_compression++
820+
821+ case OPT_OLD_COMPRESS:
822+ opts.compress_choice = "zlib"
823+
824+ case OPT_NEW_COMPRESS:
825+ opts.compress_choice = "zlibx"
826+
827+ case OPT_NO_COMPRESS:
828+ opts.do_compression = 0
829+ opts.compress_choice = ""
830+
831+ case OPT_OLD_ARGS:
832+ return nil, errNotYetImplemented
833+
834+ case 'M': // --remote-option
835+ return nil, errNotYetImplemented
836+
837+ case OPT_WRITE_BATCH,
838+ OPT_ONLY_WRITE_BATCH,
839+ OPT_READ_BATCH:
840+ return nil, errNotYetImplemented
841+
842+ case OPT_BLOCK_SIZE:
843+ return nil, errNotYetImplemented
844+
845+ case OPT_MAX_SIZE, // (needs parse_size_arg)
846+ OPT_MIN_SIZE,
847+ OPT_BWLIMIT:
848+ return nil, errNotYetImplemented
849+
850+ case OPT_APPEND:
851+ return nil, errNotYetImplemented
852+
853+ case OPT_LINK_DEST,
854+ OPT_COPY_DEST,
855+ OPT_COMPARE_DEST:
856+ return nil, errNotYetImplemented
857+
858+ case OPT_CHMOD: // (needs parse_chmod):
859+ return nil, errNotYetImplemented
860+
861+ case OPT_INFO:
862+ parseOutputWords(infoWords[:], opts.info[:], pc.poptGetOptArg(), USER_PRIORITY)
863+
864+ case OPT_DEBUG:
865+ // TODO: plumb the debug level that make sense for our implementation
866+ slog.Info("TODO: set debug level", "to", pc.poptGetOptArg())
867+
868+ case OPT_USERMAP,
869+ OPT_GROUPMAP,
870+ OPT_CHOWN:
871+ return nil, errNotYetImplemented
872+
873+ case 'A':
874+ return nil, fmt.Errorf("ACLs are not supported by gokrazy/rsync")
875+
876+ case 'X':
877+ opts.preserve_xattrs++
878+
879+ case OPT_STOP_AFTER,
880+ OPT_STOP_AT,
881+ OPT_STDERR:
882+ return nil, errNotYetImplemented
883+
884+ default:
885+ return nil, fmt.Errorf("unhandled special case opt: %v", opt)
886+ }
887+ }
888+
889+ // rsync/options.c line 1973 and following set option defaults based on
890+ // other options
891+
892+ opts.setOutputVerbosity(DEFAULT_PRIORITY)
893+
894+ if opts.recurse != 0 {
895+ opts.xfer_dirs = 1
896+ }
897+ if opts.xfer_dirs < 0 {
898+ if opts.list_only != 0 {
899+ opts.xfer_dirs = 1
900+ } else {
901+ opts.xfer_dirs = 0
902+ }
903+ }
904+
905+ if opts.relative_paths < 0 {
906+ if opts.files_from != "" {
907+ opts.relative_paths = 1
908+ } else {
909+ opts.relative_paths = 0
910+ }
911+ }
912+
913+ if opts.relative_paths == 0 {
914+ opts.implied_dirs = 0
915+ }
916+
917+ // NOTE: This simplification means that even if we ignore POPT_ARGFLAG_OR
918+ // and store ints without regards for bit sets, we get the same result.
919+ // Nevertheless, we support bit to be future-proof as new options are added.
920+ if opts.missing_args == 3 {
921+ // simplify if both options were specified
922+ opts.missing_args = 2
923+ }
924+
925+ if opts.backup_suffix == "" && opts.backup_dir == "" {
926+ opts.backup_suffix = "~"
927+ }
928+
929+ if opts.backup_dir != "" {
930+ opts.make_backups = 1 // --backup-dir implies --backup
931+ }
932+
933+ if opts.do_progress != 0 /* && !opts.am_server */ {
934+ if opts.info[INFO_NAME] == 0 {
935+ opts.info[INFO_NAME] = 1
936+ }
937+ }
938+
939+ if opts.info[INFO_NAME] >= 1 && opts.stdout_format == "" {
940+ opts.stdout_format = "%n%L"
941+ }
942+
943+ return &pc, nil
944+}
+109,
-0
1@@ -0,0 +1,109 @@
2+package rsyncreceiver
3+
4+import (
5+ "fmt"
6+ "io"
7+ "log/slog"
8+ "os"
9+
10+ "github.com/picosh/pico/pkg/rsync-receiver/rsync"
11+ "github.com/picosh/pico/pkg/rsync-receiver/rsyncopts"
12+ "github.com/picosh/pico/pkg/rsync-receiver/rsyncsender"
13+ "github.com/picosh/pico/pkg/rsync-receiver/rsyncwire"
14+ "github.com/picosh/pico/pkg/rsync-receiver/utils"
15+)
16+
17+func ClientRun(logger *slog.Logger, opts *rsyncopts.Options, conn io.ReadWriter, filesystem utils.FS, paths []string, negotiate bool) error {
18+ var err error
19+
20+ crd, cwr := rsyncwire.CounterPair(conn, conn)
21+
22+ const sessionChecksumSeed = 666
23+
24+ c := &rsyncwire.Conn{
25+ Reader: crd,
26+ Writer: cwr,
27+ }
28+
29+ if negotiate {
30+ remoteProtocol, err := c.ReadInt32()
31+ if err != nil {
32+ return err
33+ }
34+ logger.Debug("remote protocol", "protocol", remoteProtocol)
35+ if err := c.WriteInt32(rsync.ProtocolVersion); err != nil {
36+ return err
37+ }
38+ }
39+
40+ if err := c.WriteInt32(sessionChecksumSeed); err != nil {
41+ return err
42+ }
43+
44+ // Switch to multiplexing protocol, but only for server-side transmissions.
45+ // Transmissions received from the client are not multiplexed.
46+ mpx := &rsyncwire.MultiplexWriter{Writer: c.Writer}
47+ c.Writer = mpx
48+
49+ defer func() {
50+ if err != nil {
51+ _, _ = mpx.WriteMsg(rsyncwire.MsgError, fmt.Appendf(nil, "gokr-rsync [receiver]: %v\n", err))
52+ }
53+ }()
54+
55+ rt := &Transfer{
56+ Opts: &TransferOpts{
57+ DryRun: opts.DryRun(),
58+
59+ DeleteMode: opts.DeleteMode(),
60+ PreserveGid: opts.PreserveGid(),
61+ PreserveUid: opts.PreserveUid(),
62+ PreserveLinks: opts.PreserveLinks(),
63+ PreservePerms: opts.PreservePerms(),
64+ PreserveDevices: opts.PreserveDevices(),
65+ PreserveSpecials: opts.PreserveSpecials(),
66+ PreserveTimes: opts.PreserveMTimes(),
67+ IgnoreTimes: opts.IgnoreTimes(),
68+ SizeOnly: opts.SizeOnly(),
69+ AlwaysChecksum: opts.AlwaysChecksum(),
70+ // TODO: PreserveHardlinks: opts.PreserveHardlinks,
71+ },
72+ Dest: "/",
73+ // TODO: what is Env used for and can we get rid of it?
74+ Env: Osenv{
75+ Stdout: os.Stdout,
76+ Stderr: os.Stderr,
77+ Stdin: os.Stdin,
78+ },
79+ Conn: c,
80+ Seed: sessionChecksumSeed,
81+
82+ Files: filesystem,
83+
84+ Logger: logger,
85+ }
86+
87+ if opts.DeleteMode() {
88+ // receive the exclusion list (openrsync’s is always empty)
89+ exclusionList, err := rsyncsender.RecvFilterList(c)
90+ if err != nil {
91+ return err
92+ }
93+ logger.Debug("exclusion list read", "filters", exclusionList.Filters)
94+ }
95+
96+ // receive file list
97+ logger.Debug("receiving file list")
98+ fileList, err := rt.ReceiveFileList()
99+ if err != nil {
100+ return err
101+ }
102+ logger.Debug("received names", "files", fileList)
103+ stats, err := rt.Do(c, fileList, true)
104+ if err != nil {
105+ return err
106+ }
107+
108+ logger.Debug("stats", "stats", stats)
109+ return nil
110+}
+94,
-0
1@@ -0,0 +1,94 @@
2+package rsyncreceiver
3+
4+import (
5+ "context"
6+
7+ "github.com/picosh/pico/pkg/rsync-receiver/rsyncstats"
8+ "github.com/picosh/pico/pkg/rsync-receiver/rsyncwire"
9+ "github.com/picosh/pico/pkg/rsync-receiver/utils"
10+ "golang.org/x/sync/errgroup"
11+)
12+
13+func (rt *Transfer) deleteFiles(fileList []*utils.ReceiverFile) error {
14+ if rt.IOErrors > 0 {
15+ rt.Logger.Debug("IO error encountered, skipping file deletion")
16+ return nil
17+ }
18+
19+ return rt.Files.Remove(fileList)
20+}
21+
22+// rsync/main.c:do_recv.
23+func (rt *Transfer) Do(c *rsyncwire.Conn, fileList []*utils.ReceiverFile, noReport bool) (*rsyncstats.TransferStats, error) {
24+ if rt.Opts.DeleteMode {
25+ if err := rt.deleteFiles(fileList); err != nil {
26+ return nil, err
27+ }
28+ }
29+
30+ ctx := context.Background()
31+ eg, ctx := errgroup.WithContext(ctx)
32+ eg.Go(func() error {
33+ return rt.GenerateFiles(fileList)
34+ })
35+ eg.Go(func() error {
36+ // Ensure we don’t block on the receiver when the generator returns an
37+ // error.
38+ errChan := make(chan error)
39+ go func() {
40+ errChan <- rt.RecvFiles(fileList)
41+ }()
42+ select {
43+ case <-ctx.Done():
44+ return ctx.Err()
45+ case err := <-errChan:
46+ return err
47+ }
48+ })
49+ if err := eg.Wait(); err != nil {
50+ return nil, err
51+ }
52+
53+ var stats *rsyncstats.TransferStats
54+ if !noReport {
55+ var err error
56+ stats, err = report(c)
57+ rt.Logger.Debug("report", "stats", stats)
58+ if err != nil {
59+ return nil, err
60+ }
61+ }
62+
63+ // send final goodbye message
64+ if err := c.WriteInt32(-1); err != nil {
65+ return nil, err
66+ }
67+
68+ return stats, nil
69+}
70+
71+// rsync/main.c:report.
72+func report(c *rsyncwire.Conn) (*rsyncstats.TransferStats, error) {
73+ // read statistics:
74+ // total bytes read (from network connection)
75+ read, err := c.ReadInt64()
76+ if err != nil {
77+ return nil, err
78+ }
79+ // total bytes written (to network connection)
80+ written, err := c.ReadInt64()
81+ if err != nil {
82+ return nil, err
83+ }
84+ // total size of files
85+ size, err := c.ReadInt64()
86+ if err != nil {
87+ return nil, err
88+ }
89+
90+ return &rsyncstats.TransferStats{
91+ Read: read,
92+ Written: written,
93+ Size: size,
94+ }, nil
95+}
+191,
-0
1@@ -0,0 +1,191 @@
2+package rsyncreceiver
3+
4+import (
5+ "fmt"
6+ "io"
7+ "path/filepath"
8+ "time"
9+
10+ "github.com/picosh/pico/pkg/rsync-receiver/rsync"
11+ "github.com/picosh/pico/pkg/rsync-receiver/utils"
12+)
13+
14+// rsync/flist.c:receive_file_entry.
15+func (rt *Transfer) receiveFileEntry(flags uint16, last *utils.ReceiverFile) (*utils.ReceiverFile, error) {
16+ f := &utils.ReceiverFile{}
17+
18+ var l1 int
19+ if flags&rsync.XMIT_SAME_NAME != 0 {
20+ l, err := rt.Conn.ReadByte()
21+ if err != nil {
22+ return nil, err
23+ }
24+ l1 = int(l)
25+ }
26+
27+ var l2 int
28+ if flags&rsync.XMIT_LONG_NAME != 0 {
29+ l, err := rt.Conn.ReadInt32()
30+ if err != nil {
31+ return nil, err
32+ }
33+ l2 = int(l)
34+ } else {
35+ l, err := rt.Conn.ReadByte()
36+ if err != nil {
37+ return nil, err
38+ }
39+ l2 = int(l)
40+ }
41+ // linux/limits.h
42+ const PATH_MAX = 4096
43+ if l2 >= PATH_MAX-l1 {
44+ const lastname = ""
45+ return nil, fmt.Errorf("overflow: flags=0x%x l1=%d l2=%d lastname=%s",
46+ flags, l1, l2, lastname)
47+ }
48+ b := make([]byte, l1+l2)
49+ readb := b
50+ if l1 > 0 {
51+ copy(b, []byte(last.Name))
52+ readb = b[l1:]
53+ }
54+ if _, err := io.ReadFull(rt.Conn.Reader, readb); err != nil {
55+ return nil, err
56+ }
57+ // TODO: does rsync’s clean_fname() and sanitize_path() combination do
58+ // anything more than Go’s filepath.Clean()?
59+ f.Name = filepath.Clean(string(b))
60+
61+ length, err := rt.Conn.ReadInt64()
62+ if err != nil {
63+ return nil, err
64+ }
65+ f.Length = length
66+
67+ if flags&rsync.XMIT_SAME_TIME != 0 {
68+ f.ModTime = last.ModTime
69+ } else {
70+ modTime, err := rt.Conn.ReadInt32()
71+ if err != nil {
72+ return nil, err
73+ }
74+ f.ModTime = time.Unix(int64(modTime), 0)
75+ }
76+
77+ if flags&rsync.XMIT_SAME_MODE != 0 {
78+ f.Mode = last.Mode
79+ } else {
80+ mode, err := rt.Conn.ReadInt32()
81+ if err != nil {
82+ return nil, err
83+ }
84+ f.Mode = mode
85+ }
86+
87+ if rt.Opts.PreserveUid {
88+ if flags&rsync.XMIT_SAME_UID != 0 {
89+ f.Uid = last.Uid
90+ } else {
91+ uid, err := rt.Conn.ReadInt32()
92+ if err != nil {
93+ return nil, err
94+ }
95+ f.Uid = uid
96+ }
97+ }
98+
99+ if rt.Opts.PreserveGid {
100+ if flags&rsync.XMIT_SAME_GID != 0 {
101+ f.Gid = last.Gid
102+ } else {
103+ gid, err := rt.Conn.ReadInt32()
104+ if err != nil {
105+ return nil, err
106+ }
107+ f.Gid = gid
108+ }
109+ }
110+
111+ mode := f.Mode & rsync.S_IFMT
112+ isDev := mode == rsync.S_IFCHR || mode == rsync.S_IFBLK
113+ isSpecial := mode == rsync.S_IFIFO || mode == rsync.S_IFSOCK
114+ isLink := mode == rsync.S_IFLNK
115+
116+ if rt.Opts.PreserveDevices && (isDev || isSpecial) {
117+ // TODO(protocol >= 28): rdev/major/minor handling
118+ if flags&rsync.XMIT_SAME_RDEV_pre28 != 0 {
119+ f.Rdev = last.Rdev
120+ } else {
121+ rdev, err := rt.Conn.ReadInt32()
122+ if err != nil {
123+ return nil, err
124+ }
125+ f.Rdev = rdev
126+ }
127+ }
128+
129+ if rt.Opts.PreserveLinks && isLink {
130+ length, err := rt.Conn.ReadInt32()
131+ if err != nil {
132+ return nil, err
133+ }
134+ b := make([]byte, length)
135+ if _, err := io.ReadFull(rt.Conn.Reader, b); err != nil {
136+ return nil, err
137+ }
138+ f.LinkTarget = string(b)
139+ }
140+
141+ return f, nil
142+}
143+
144+// rsync/flist.c:recv_file_list.
145+func (rt *Transfer) ReceiveFileList() ([]*utils.ReceiverFile, error) {
146+ lastFileEntry := new(utils.ReceiverFile)
147+ var fileList []*utils.ReceiverFile
148+ for {
149+ b, err := rt.Conn.ReadByte()
150+ if err != nil {
151+ return nil, err
152+ }
153+ if b == 0 {
154+ break
155+ }
156+ flags := uint16(b)
157+ // log.Printf("flags: %x", flags)
158+ // TODO(protocol >= 28): extended flags
159+
160+ f, err := rt.receiveFileEntry(flags, lastFileEntry)
161+ if err != nil {
162+ return nil, err
163+ }
164+ lastFileEntry = f
165+ // TODO: include depth in output?
166+ rt.Logger.Debug("recv_file_list", "file", f.Name, "length", f.Length, "mode", f.Mode, "uid", f.Uid, "gid", f.Gid, "flags", flags)
167+
168+ fileList = append(fileList, f)
169+ }
170+
171+ utils.SortFileList(fileList)
172+
173+ if rt.Opts.PreserveUid || rt.Opts.PreserveGid {
174+ // receive the uid/gid list
175+ users, groups, err := rt.RecvIdList()
176+ if err != nil {
177+ return nil, err
178+ }
179+ _ = users
180+ _ = groups
181+ }
182+
183+ // read the i/o error flag
184+ ioErrors, err := rt.Conn.ReadInt32()
185+ if err != nil {
186+ return nil, err
187+ }
188+ rt.Logger.Debug("ioErrors", "errs", ioErrors)
189+ rt.IOErrors = ioErrors
190+
191+ return fileList, nil
192+}
1@@ -0,0 +1,156 @@
2+package rsyncreceiver
3+
4+import (
5+ "fmt"
6+ "io"
7+ "os"
8+
9+ "github.com/picosh/pico/pkg/rsync-receiver/rsync"
10+ "github.com/picosh/pico/pkg/rsync-receiver/rsyncchecksum"
11+ "github.com/picosh/pico/pkg/rsync-receiver/rsynccommon"
12+ "github.com/picosh/pico/pkg/rsync-receiver/utils"
13+)
14+
15+// rsync/generator.c:generate_files().
16+func (rt *Transfer) GenerateFiles(fileList []*utils.ReceiverFile) error {
17+ phase := 0
18+ for idx, f := range fileList {
19+ // TODO: use a copy of f with .Mode |= S_IWUSR for directories, so
20+ // that we can create files within all directories.
21+ if err := rt.recvGenerator(idx, f); err != nil {
22+ return err
23+ }
24+ }
25+ phase++
26+ rt.Logger.Debug("generateFiles", "phase", phase)
27+ if err := rt.Conn.WriteInt32(-1); err != nil {
28+ return err
29+ }
30+
31+ // TODO: re-do any files that failed
32+ phase++
33+ rt.Logger.Debug("generateFiles", "phase", phase)
34+ if err := rt.Conn.WriteInt32(-1); err != nil {
35+ return err
36+ }
37+
38+ rt.Logger.Debug("generateFiles finished")
39+ return nil
40+}
41+
42+// rsync/generator.c:skip_file.
43+func (rt *Transfer) skipFile(f *utils.ReceiverFile, st os.FileInfo) (bool, error) {
44+ if rt.Opts.AlwaysChecksum || rt.Opts.IgnoreTimes {
45+ return false, nil
46+ }
47+
48+ sizeMatch := st.Size() == f.Length
49+ if rt.Opts.SizeOnly {
50+ return sizeMatch, nil
51+ }
52+
53+ timeMatch := st.ModTime().Equal(f.ModTime)
54+ return sizeMatch && timeMatch, nil
55+}
56+
57+// rsync/generator.c:recv_generator.
58+func (rt *Transfer) recvGenerator(idx int, f *utils.ReceiverFile) error {
59+ if rt.listOnly() {
60+ if _, err := fmt.Fprintf(rt.Env.Stdout, "%s %11.0f %s %s\n",
61+ f.FileMode().String(),
62+ float64(f.Length), // TODO: rsync prints decimal separators
63+ f.ModTime.Format("2006/01/02 15:04:05"),
64+ f.Name); err != nil {
65+ return err
66+ }
67+ return nil
68+ }
69+ rt.Logger.Debug("recv_generator", "file", f)
70+
71+ if !f.FileMode().IsRegular() {
72+ // None of the Preserve* options is enabled, so just skip over
73+ // non-regular files.
74+ return nil
75+ }
76+
77+ requestFullFile := func() error {
78+ rt.Logger.Debug("requesting", "file", f)
79+ if err := rt.Conn.WriteInt32(int32(idx)); err != nil {
80+ return err
81+ }
82+ if rt.Opts.DryRun {
83+ return nil
84+ }
85+ var sh rsync.SumHead
86+ if err := sh.WriteTo(rt.Conn); err != nil {
87+ return err
88+ }
89+ return nil
90+ }
91+
92+ st, in, err := rt.Files.Read(&utils.SenderFile{WPath: f.Name})
93+ if err != nil {
94+ rt.Logger.Error("failed to open file", "st", st, "file", f, "err", err)
95+ return requestFullFile()
96+ }
97+
98+ defer func() { _ = in.Close() }()
99+
100+ skip, err := rt.skipFile(f, st)
101+ if err != nil {
102+ return err
103+ }
104+
105+ if skip {
106+ rt.Logger.Debug("skipping", "file", f)
107+ return nil
108+ }
109+
110+ if rt.Opts.DryRun {
111+ if err := rt.Conn.WriteInt32(int32(idx)); err != nil {
112+ return err
113+ }
114+
115+ return nil
116+ }
117+
118+ rt.Logger.Debug("sending sums", "file", f, "st", st)
119+ if err := rt.Conn.WriteInt32(int32(idx)); err != nil {
120+ return err
121+ }
122+
123+ err = rt.generateAndSendSums(in, st.Size())
124+ if err != nil {
125+ rt.Logger.Error("failed to send sums", "file", f, "err", err)
126+ }
127+
128+ return err
129+}
130+
131+// rsync/generator.c:generate_and_send_sums.
132+func (rt *Transfer) generateAndSendSums(in utils.ReaderAtCloser, fileLen int64) error {
133+ sh := rsynccommon.SumSizesSqroot(fileLen)
134+ if err := sh.WriteTo(rt.Conn); err != nil {
135+ return err
136+ }
137+ buf := make([]byte, int(sh.BlockLength))
138+ remaining := fileLen
139+ for i := int32(0); i < sh.ChecksumCount; i++ {
140+ n1 := min(int64(sh.BlockLength), remaining)
141+ b := buf[:n1]
142+ if _, err := io.ReadFull(in, b); err != nil {
143+ return err
144+ }
145+
146+ sum1 := rsyncchecksum.Checksum1(b)
147+ sum2 := rsyncchecksum.Checksum2(rt.Seed, b)
148+ if err := rt.Conn.WriteInt32(int32(sum1)); err != nil {
149+ return err
150+ }
151+ if _, err := rt.Conn.Writer.Write(sum2); err != nil {
152+ return err
153+ }
154+ remaining -= n1
155+ }
156+ return nil
157+}
+170,
-0
1@@ -0,0 +1,170 @@
2+package rsyncreceiver
3+
4+import (
5+ "bytes"
6+ "encoding/binary"
7+ "errors"
8+ "fmt"
9+ "io"
10+ "sync"
11+
12+ "github.com/mmcloughlin/md4"
13+ "github.com/picosh/pico/pkg/rsync-receiver/rsync"
14+ "github.com/picosh/pico/pkg/rsync-receiver/utils"
15+)
16+
17+// rsync/receiver.c:recv_files.
18+func (rt *Transfer) RecvFiles(fileList []*utils.ReceiverFile) error {
19+ phase := 0
20+ for {
21+ idx, err := rt.Conn.ReadInt32()
22+ if err != nil {
23+ return err
24+ }
25+ if idx == -1 {
26+ if phase == 0 {
27+ phase++
28+ rt.Logger.Debug("recvFiles phase", "phase", phase)
29+ // TODO: send done message
30+ continue
31+ }
32+ break
33+ }
34+ rt.Logger.Debug("receiving file", "idx", idx, "file", fileList[idx])
35+ if err := rt.recvFile1(fileList[idx]); err != nil {
36+ return err
37+ }
38+ }
39+ rt.Logger.Debug("recvFiles finished")
40+ return nil
41+}
42+
43+func (rt *Transfer) recvFile1(f *utils.ReceiverFile) error {
44+ if rt.Opts.DryRun {
45+ fmt.Println(f.Name)
46+ return nil
47+ }
48+
49+ localFile, err := rt.openLocalFile(f)
50+ if err != nil {
51+ rt.Logger.Error("opening local file failed, continuing", "err", err, "file", f)
52+ } else {
53+ defer func() { _ = localFile.Close() }()
54+ }
55+
56+ err = rt.receiveData(f, localFile)
57+ if err != nil {
58+ rt.Logger.Error("receiving data failed, continuing", "err", err, "file", f)
59+ }
60+ return err
61+}
62+
63+func (rt *Transfer) openLocalFile(f *utils.ReceiverFile) (utils.ReaderAtCloser, error) {
64+ _, r, err := rt.Files.Read(&utils.SenderFile{
65+ WPath: f.Name,
66+ Regular: true,
67+ })
68+
69+ if err != nil {
70+ return nil, err
71+ }
72+
73+ return r, nil
74+}
75+
76+// rsync/receiver.c:receive_data.
77+func (rt *Transfer) receiveData(f *utils.ReceiverFile, localFile utils.ReaderAtCloser) error {
78+ var sh rsync.SumHead
79+ if err := sh.ReadFrom(rt.Conn); err != nil {
80+ return err
81+ }
82+
83+ r, w := io.Pipe()
84+
85+ f.Reader = r
86+
87+ var wg sync.WaitGroup
88+ wg.Add(1)
89+
90+ go func() {
91+ defer func() {
92+ wg.Done()
93+ if err := r.Close(); err != nil {
94+ return
95+ }
96+ }()
97+
98+ _, err := rt.Files.Put(f)
99+ if err != nil {
100+ return
101+ }
102+ }()
103+
104+ h := md4.New()
105+ _ = binary.Write(h, binary.LittleEndian, rt.Seed) // hash.Hash.Write never fails
106+
107+ for {
108+ token, data, err := rt.recvToken()
109+ if err != nil {
110+ return err
111+ }
112+ if token == 0 {
113+ break
114+ }
115+ if token > 0 {
116+ if _, err := h.Write(data); err != nil {
117+ return err
118+ }
119+
120+ if _, err := w.Write(data); err != nil {
121+ if errors.Is(err, io.ErrClosedPipe) {
122+ continue
123+ }
124+ return err
125+ }
126+ continue
127+ }
128+ if localFile == nil {
129+ return fmt.Errorf("BUG: local file %s not open for copying chunk", localFile)
130+ }
131+ token = -(token + 1)
132+ offset2 := int64(token) * int64(sh.BlockLength)
133+ dataLen := sh.BlockLength
134+ if token == sh.ChecksumCount-1 && sh.RemainderLength != 0 {
135+ dataLen = sh.RemainderLength
136+ }
137+ data = make([]byte, dataLen)
138+ if _, err := localFile.ReadAt(data, offset2); err != nil {
139+ return err
140+ }
141+
142+ if _, err := h.Write(data); err != nil {
143+ return err
144+ }
145+
146+ if _, err := w.Write(data); err != nil {
147+ if errors.Is(err, io.ErrClosedPipe) {
148+ continue
149+ }
150+ return err
151+ }
152+ }
153+
154+ if err := w.Close(); err != nil {
155+ return err
156+ }
157+
158+ wg.Wait()
159+
160+ localSum := h.Sum(nil)
161+ remoteSum := make([]byte, len(localSum))
162+ if _, err := io.ReadFull(rt.Conn.Reader, remoteSum); err != nil {
163+ return err
164+ }
165+ if !bytes.Equal(localSum, remoteSum) {
166+ return fmt.Errorf("file corruption in %s", f.Name)
167+ }
168+ rt.Logger.Debug("checksum matches!", "localSum", localSum)
169+
170+ return nil
171+}
1@@ -0,0 +1,20 @@
2+package rsyncreceiver
3+
4+import "io"
5+
6+// rsync/token.c:recvToken.
7+func (rt *Transfer) recvToken() (token int32, data []byte, _ error) {
8+ var err error
9+ token, err = rt.Conn.ReadInt32()
10+ if err != nil {
11+ return 0, nil, err
12+ }
13+ if token <= 0 {
14+ return token, nil, nil
15+ }
16+ data = make([]byte, int(token))
17+ if _, err := io.ReadFull(rt.Conn.Reader, data); err != nil {
18+ return 0, nil, err
19+ }
20+ return token, data, nil
21+}
1@@ -0,0 +1,53 @@
2+package rsyncreceiver
3+
4+import (
5+ "io"
6+ "log/slog"
7+
8+ "github.com/picosh/pico/pkg/rsync-receiver/rsyncwire"
9+ "github.com/picosh/pico/pkg/rsync-receiver/utils"
10+)
11+
12+type Osenv struct {
13+ Stdin io.Reader
14+ Stdout io.Writer
15+ Stderr io.Writer
16+}
17+
18+// TransferOpts is a subset of Opts which is required for implementing a receiver.
19+type TransferOpts struct {
20+ Verbose bool
21+ DryRun bool
22+
23+ DeleteMode bool
24+ PreserveGid bool
25+ PreserveUid bool
26+ PreserveLinks bool
27+ PreservePerms bool
28+ PreserveDevices bool
29+ PreserveSpecials bool
30+ PreserveTimes bool
31+ PreserveHardlinks bool
32+ IgnoreTimes bool
33+ SizeOnly bool
34+ AlwaysChecksum bool
35+}
36+
37+type Transfer struct {
38+ // config
39+ // Opts *Opts
40+ Opts *TransferOpts
41+ Dest string
42+ Env Osenv
43+
44+ // state
45+ Conn *rsyncwire.Conn
46+ Seed int32
47+ IOErrors int32
48+
49+ Files utils.FS
50+
51+ Logger *slog.Logger
52+}
53+
54+func (rt *Transfer) listOnly() bool { return rt.Dest == "" }
1@@ -0,0 +1,69 @@
2+package rsyncreceiver
3+
4+import (
5+ "io"
6+)
7+
8+type mapping struct {
9+ Name string
10+ LocalId int32
11+}
12+
13+func (rt *Transfer) recvIdMapping1(localId func(id int32, name string) int32) (map[int32]mapping, error) {
14+ idMapping := make(map[int32]mapping)
15+ for {
16+ id, err := rt.Conn.ReadInt32()
17+ if err != nil {
18+ return nil, err
19+ }
20+ if id == 0 {
21+ break
22+ }
23+ length, err := rt.Conn.ReadByte()
24+ if err != nil {
25+ return nil, err
26+ }
27+ name := make([]byte, length)
28+ if _, err := io.ReadFull(rt.Conn.Reader, name); err != nil {
29+ return nil, err
30+ }
31+ idMapping[id] = mapping{
32+ Name: string(name),
33+ LocalId: localId(id, string(name)),
34+ }
35+ }
36+ return idMapping, nil
37+}
38+
39+// rsync/uidlist.c:recv_id_list.
40+func (rt *Transfer) RecvIdList() (users map[int32]mapping, groups map[int32]mapping, _ error) {
41+ if rt.Opts.PreserveUid {
42+ var err error
43+ users, err = rt.recvIdMapping1(func(remoteUid int32, remoteUsername string) int32 {
44+ // TODO: look up local uid by username
45+ return remoteUid
46+ })
47+ if err != nil {
48+ return nil, nil, err
49+ }
50+ for remoteUid, mapping := range users {
51+ rt.Logger.Debug("remote uid maps to local uid", "remoteUid", remoteUid, "name", mapping.Name, "localUid", mapping.LocalId)
52+ }
53+ }
54+
55+ if rt.Opts.PreserveGid {
56+ var err error
57+ groups, err = rt.recvIdMapping1(func(remoteGid int32, remoteGroupname string) int32 {
58+ // TODO: look up local gid by groupname
59+ return remoteGid
60+ })
61+ if err != nil {
62+ return nil, nil, err
63+ }
64+ for remoteGid, mapping := range groups {
65+ rt.Logger.Debug("remote gid maps to local gid", "remoteGid", remoteGid, "name", mapping.Name, "localGid", mapping.LocalId)
66+ }
67+ }
68+
69+ return users, groups, nil
70+}
+75,
-0
1@@ -0,0 +1,75 @@
2+package rsyncsender
3+
4+import (
5+ "fmt"
6+ "io"
7+ "log/slog"
8+
9+ "github.com/picosh/pico/pkg/rsync-receiver/rsync"
10+ "github.com/picosh/pico/pkg/rsync-receiver/rsyncopts"
11+ "github.com/picosh/pico/pkg/rsync-receiver/rsyncwire"
12+ "github.com/picosh/pico/pkg/rsync-receiver/utils"
13+)
14+
15+func ClientRun(logger *slog.Logger, opts *rsyncopts.Options, conn io.ReadWriter, filesystem utils.FS, paths []string, negotiate bool) error {
16+ var err error
17+
18+ crd, cwr := rsyncwire.CounterPair(conn, conn)
19+
20+ const sessionChecksumSeed = 666
21+
22+ c := &rsyncwire.Conn{
23+ Reader: crd,
24+ Writer: cwr,
25+ }
26+
27+ if negotiate {
28+ remoteProtocol, err := c.ReadInt32()
29+ if err != nil {
30+ return err
31+ }
32+ logger.Debug("remote protocol", "remoteProtocol", remoteProtocol)
33+ if err := c.WriteInt32(rsync.ProtocolVersion); err != nil {
34+ return err
35+ }
36+ }
37+
38+ if err := c.WriteInt32(sessionChecksumSeed); err != nil {
39+ return err
40+ }
41+
42+ // Switch to multiplexing protocol, but only for server-side transmissions.
43+ // Transmissions received from the client are not multiplexed.
44+ mpx := &rsyncwire.MultiplexWriter{Writer: c.Writer}
45+ c.Writer = mpx
46+
47+ defer func() {
48+ if err != nil {
49+ _, _ = mpx.WriteMsg(rsyncwire.MsgError, fmt.Appendf(nil, "gokr-rsync [sender]: %v\n", err))
50+ }
51+ }()
52+
53+ st := &Transfer{
54+ Opts: opts,
55+ Conn: c,
56+ Seed: sessionChecksumSeed,
57+ Files: filesystem,
58+
59+ Logger: logger,
60+ }
61+ // receive the exclusion list (openrsync’s is always empty)
62+ exclusionList, err := RecvFilterList(st.Conn)
63+ if err != nil {
64+ return err
65+ }
66+ logger.Debug("exclusion list read", "filters", exclusionList.Filters)
67+
68+ stats, err := st.Do(crd, cwr, paths, exclusionList)
69+ if err != nil {
70+ return err
71+ }
72+
73+ logger.Debug("handleConnSender done. stats", "stats", stats)
74+
75+ return err
76+}
+68,
-0
1@@ -0,0 +1,68 @@
2+package rsyncsender
3+
4+import (
5+ "fmt"
6+ "sort"
7+
8+ "github.com/picosh/pico/pkg/rsync-receiver/rsyncstats"
9+ "github.com/picosh/pico/pkg/rsync-receiver/rsyncwire"
10+)
11+
12+// rsync/main.c:client_run am_sender.
13+func (st *Transfer) Do(crd *rsyncwire.CountingReader, cwr *rsyncwire.CountingWriter, paths []string, exclusionList *filterRuleList) (*rsyncstats.TransferStats, error) {
14+ if exclusionList == nil {
15+ exclusionList = &filterRuleList{}
16+ }
17+
18+ // “Update exchange” as per
19+ // https://github.com/kristapsdz/openrsync/blob/master/rsync.5
20+
21+ // send file list
22+ fileList, err := st.SendFileList(st.Opts, paths, exclusionList)
23+ if err != nil {
24+ return nil, err
25+ }
26+
27+ st.Logger.Debug("file list sent")
28+
29+ // Sort the file list. The client sorts, so we need to sort, too (in the
30+ // same way!), otherwise our indices do not match what the client will
31+ // request.
32+ sort.Slice(fileList.Files, func(i, j int) bool {
33+ return fileList.Files[i].WPath < fileList.Files[j].WPath
34+ })
35+
36+ if err := st.SendFiles(fileList); err != nil {
37+ return nil, err
38+ }
39+
40+ // send statistics:
41+ // total bytes read (from network connection)
42+ if err := st.Conn.WriteInt64(crd.BytesRead); err != nil {
43+ return nil, err
44+ }
45+ // total bytes written (to network connection)
46+ if err := st.Conn.WriteInt64(cwr.BytesWritten); err != nil {
47+ return nil, err
48+ }
49+ // total size of files
50+ if err := st.Conn.WriteInt64(fileList.TotalSize); err != nil {
51+ return nil, err
52+ }
53+
54+ st.Logger.Debug("reading final int32")
55+
56+ finish, err := st.Conn.ReadInt32()
57+ if err != nil {
58+ return nil, err
59+ }
60+ if finish != -1 {
61+ return nil, fmt.Errorf("protocol error: expected final -1, got %d", finish)
62+ }
63+
64+ return &rsyncstats.TransferStats{
65+ Read: crd.BytesRead,
66+ Written: cwr.BytesWritten,
67+ Size: fileList.TotalSize,
68+ }, nil
69+}
+109,
-0
1@@ -0,0 +1,109 @@
2+package rsyncsender
3+
4+import (
5+ "io"
6+ "path/filepath"
7+ "strings"
8+
9+ "github.com/picosh/pico/pkg/rsync-receiver/rsyncwire"
10+)
11+
12+type filterRuleList struct {
13+ Filters []*filterRule
14+}
15+
16+// exclude.c:add_rule.
17+func (l *filterRuleList) addRule(fr *filterRule) {
18+ if strings.HasSuffix(fr.pattern, "/") {
19+ fr.flag |= filtruleDirectory
20+ fr.pattern = strings.TrimSuffix(fr.pattern, "/")
21+ }
22+ if strings.ContainsFunc(fr.pattern, func(r rune) bool {
23+ return r == '*' || r == '[' || r == '?'
24+ }) {
25+ fr.flag |= filtruleWild
26+ }
27+ l.Filters = append(l.Filters, fr)
28+}
29+
30+// exclude.c:check_filter.
31+func (l *filterRuleList) matches(name string) bool {
32+ for _, fr := range l.Filters {
33+ if fr.matches(name) {
34+ return true
35+ }
36+ }
37+ return false
38+}
39+
40+// exclude.c:recv_filter_list.
41+func RecvFilterList(c *rsyncwire.Conn) (*filterRuleList, error) {
42+ var l filterRuleList
43+ const exclusionListEnd = 0
44+ for {
45+ length, err := c.ReadInt32()
46+ if err != nil {
47+ return nil, err
48+ }
49+ if length == exclusionListEnd {
50+ break
51+ }
52+ line := make([]byte, length)
53+ if _, err := io.ReadFull(c.Reader, line); err != nil {
54+ return nil, err
55+ }
56+ fr, err := parseFilter(string(line))
57+ if err != nil {
58+ return nil, err
59+ }
60+ l.addRule(fr)
61+ }
62+ return &l, nil
63+}
64+
65+const (
66+ filtruleInclude = 1 << iota
67+ filtruleClearList
68+ filtruleDirectory
69+ filtruleWild
70+)
71+
72+type filterRule struct {
73+ flag int
74+ pattern string
75+}
76+
77+// exclude.c:rule_matches.
78+func (fr *filterRule) matches(name string) bool {
79+ if fr.flag&filtruleWild != 0 {
80+ panic("wildcard filter rules not yet implemented")
81+ }
82+ if !strings.ContainsRune(fr.pattern, '/') &&
83+ fr.flag&filtruleWild == 0 {
84+ name = filepath.Base(name)
85+ }
86+ return fr.pattern == name
87+}
88+
89+// exclude.c:parse_filter_str / exclude.c:parse_rule_tok.
90+func parseFilter(line string) (*filterRule, error) {
91+ rule := new(filterRule)
92+
93+ // We only support what rsync calls XFLG_OLD_PREFIXES
94+ if strings.HasPrefix(line, "- ") {
95+ // clear include flag
96+ rule.flag &= ^filtruleInclude
97+ line = strings.TrimPrefix(line, "- ")
98+ } else if strings.HasPrefix(line, "+ ") {
99+ // set include flag
100+ rule.flag |= filtruleInclude
101+ line = strings.TrimPrefix(line, "+ ")
102+ } else if strings.HasPrefix(line, "!") {
103+ // set clear_list flag
104+ rule.flag |= filtruleClearList
105+ }
106+
107+ rule.pattern = line
108+
109+ return rule, nil
110+}
+116,
-0
1@@ -0,0 +1,116 @@
2+package rsyncsender
3+
4+import (
5+ "io"
6+ "log/slog"
7+ "os"
8+)
9+
10+// rsync.h:map_struct.
11+type mapStruct struct {
12+ fileSize int64 // file size (from stat)
13+ pOffset int64 // window start
14+ pFdOffset int64 // offset of cursor in fd ala lseek
15+ window []byte // window pointer
16+ pSize int64 // largest window we allocated
17+ pLen int64 // latest (rounded) window size
18+ defWindowSize int64 // default window size
19+ f *os.File // file descriptor
20+ err error // first read error
21+}
22+
23+const alignBoundary = 1024
24+
25+func alignedLength(l int64) int64 {
26+ return ((l - 1) | (alignBoundary - 1)) + 1
27+}
28+
29+func alignedOvershoot(off int64) int64 {
30+ return off & (alignBoundary - 1)
31+}
32+
33+func mapFile(f *os.File, len int64, readSize int32, blkSize int32) *mapStruct {
34+ if blkSize > 0 && readSize%blkSize != 0 {
35+ readSize += blkSize - (readSize % blkSize)
36+ }
37+ return &mapStruct{
38+ fileSize: len,
39+ defWindowSize: alignedLength(int64(readSize)),
40+ f: f,
41+ }
42+}
43+
44+func (ms *mapStruct) ptr(offset int64, l int32) []byte {
45+ //log.Printf("ptr(offset=%d, l=%d)", offset, l)
46+ len := int64(l)
47+ if len == 0 {
48+ return nil
49+ }
50+ if len < 0 {
51+ slog.Debug("BUG: invalid len", "len", len)
52+ return nil
53+ }
54+
55+ if offset >= ms.pOffset && offset+int64(len) <= ms.pOffset+int64(ms.pLen) {
56+ //log.Printf("-> already available")
57+ // region already available
58+ off := offset - ms.pOffset
59+ return ms.window[off : off+int64(len)]
60+ }
61+
62+ alignFudge := alignedOvershoot(offset)
63+ windowStart := offset - alignFudge
64+ windowSize := int64(ms.defWindowSize)
65+ if windowStart+windowSize > ms.fileSize {
66+ windowSize = ms.fileSize - windowStart
67+ }
68+ if windowSize < len+alignFudge {
69+ windowSize = alignedLength(len + alignFudge)
70+ }
71+ if windowSize > ms.pSize {
72+ win := make([]byte, windowSize)
73+ copy(win, ms.window)
74+ ms.window = win
75+ ms.pSize = windowSize
76+ }
77+ readStart := windowStart
78+ readSize := windowSize
79+ readOffset := int64(0)
80+
81+ //log.Printf("windowSize: %d, ms=%+v", windowSize, ms)
82+ if windowStart >= ms.pOffset && windowStart < ms.pOffset+ms.pLen &&
83+ windowStart+windowSize >= ms.pOffset+ms.pLen {
84+ readStart = ms.pOffset + ms.pLen
85+ readOffset = readStart - windowStart
86+ readSize = windowSize - readOffset
87+ off := ms.pLen - readOffset
88+ copy(ms.window[:], ms.window[off:off+readOffset])
89+ }
90+ if readSize <= 0 {
91+ slog.Debug("BUG: invalid readSize", "readSize", readSize)
92+ return nil
93+ }
94+ if ms.pFdOffset != readStart {
95+ if _, err := ms.f.Seek(readStart, io.SeekStart); err != nil {
96+ slog.Error("seek error", "err", err)
97+ return nil
98+ }
99+ ms.pFdOffset = readStart
100+ }
101+ ms.pOffset = windowStart
102+ ms.pLen = windowSize
103+ //log.Printf("-> reading %d bytes from %d into buffer at offset=%d", readSize, readStart, readOffset)
104+ for readSize > 0 {
105+ n, err := ms.f.Read(ms.window[readOffset : readOffset+readSize])
106+ if err != nil {
107+ ms.err = err
108+ // TODO: zero the buffer, file has changed mid-transfer
109+ slog.Debug("file has changed mid-transfer")
110+ return nil
111+ }
112+ ms.pFdOffset += int64(n)
113+ readOffset += int64(n)
114+ readSize -= int64(n)
115+ }
116+ return ms.window[alignFudge : alignFudge+len]
117+}
+236,
-0
1@@ -0,0 +1,236 @@
2+package rsyncsender
3+
4+import (
5+ "os"
6+ "os/user"
7+ "strconv"
8+ "sync"
9+
10+ "github.com/picosh/pico/pkg/rsync-receiver/rsync"
11+ "github.com/picosh/pico/pkg/rsync-receiver/rsyncchecksum"
12+ "github.com/picosh/pico/pkg/rsync-receiver/rsyncopts"
13+ "github.com/picosh/pico/pkg/rsync-receiver/rsyncwire"
14+ "github.com/picosh/pico/pkg/rsync-receiver/utils"
15+)
16+
17+type fileList struct {
18+ TotalSize int64
19+ Files []utils.SenderFile
20+}
21+
22+// rsync/rsync.h defines chunkSize as 32 * 1024, but increasing it to 256K
23+// increases throughput with “tridge” rsync as client by 50 Mbit/s.
24+const chunkSize = 256 * 1024
25+
26+var (
27+ lookupOnce sync.Once
28+ lookupGroupOnce sync.Once
29+)
30+
31+// rsync/flist.c:send_file_list.
32+func (st *Transfer) SendFileList(opts *rsyncopts.Options, paths []string, excl *filterRuleList) (*fileList, error) {
33+ var fileList fileList
34+ fec := &rsyncwire.Buffer{}
35+
36+ uidMap := make(map[int32]string)
37+ gidMap := make(map[int32]string)
38+
39+ // TODO: flush in between to keep the pipes filled when traversal takes long
40+
41+ // TODO: handle info == nil case (permission denied?): should set an i/o
42+ // error flag, but traversal should continue
43+
44+ st.Logger.Debug("sendFileList()")
45+ // TODO: handle |root| referring to an individual file, symlink or special (skip)
46+ for _, requested := range paths {
47+ files, err := st.Files.List(requested)
48+ if err != nil {
49+ return nil, err
50+ }
51+
52+ for _, info := range files {
53+ // Only ever transmit long names, like openrsync
54+ flags := byte(rsync.XMIT_LONG_NAME)
55+
56+ // log.Printf("Trim(path=%q, %q) = %q", path, strip, name)
57+ name := info.Name()
58+ path := name
59+ if name == "/" {
60+ name = "."
61+ flags |= rsync.XMIT_TOP_DIR
62+ }
63+ // log.Printf("flags for %q: %v", name, flags)
64+
65+ if excl.matches(name) {
66+ continue
67+ }
68+
69+ fileList.Files = append(fileList.Files, utils.SenderFile{
70+ Path: "/",
71+ Regular: info.Mode().IsRegular(),
72+ WPath: name,
73+ })
74+
75+ // 1. status byte (integer)
76+ _ = fec.WriteByte(flags)
77+
78+ // 2. inherited filename length (optional, byte)
79+ // 3. filename length (integer or byte)
80+ fec.WriteInt32(int32(len(name)))
81+
82+ // 4. file (byte array)
83+ fec.WriteString(name)
84+
85+ // 5. file length (long)
86+ size := info.Size()
87+ if info.Mode().IsDir() {
88+ // tmpfs returns non-4K sizes for directories. Override with
89+ // 4096 to make the tests succeed regardless of the /tmp file
90+ // system type.
91+ size = 4096
92+ }
93+ fec.WriteInt64(size)
94+
95+ fileList.TotalSize += size
96+
97+ // 6. file modification time (optional, integer)
98+ // TODO: this will overflow in 2038! :(
99+ fec.WriteInt32(int32(info.ModTime().Unix()))
100+
101+ // 7. file mode (optional, mode_t, integer)
102+ mode := int32(info.Mode() & os.ModePerm)
103+ isDev := false
104+ isSpecial := false
105+ if info.Mode().IsDir() {
106+ mode |= rsync.S_IFDIR
107+ } else if info.Mode().IsRegular() {
108+ mode |= rsync.S_IFREG
109+ } else if info.Mode().Type()&os.ModeSymlink != 0 {
110+ mode |= rsync.S_IFLNK
111+ // TODO: skip symlink if PreserveSymlinks is not set
112+ }
113+
114+ if info.Mode().Type()&os.ModeCharDevice != 0 {
115+ mode |= rsync.S_IFCHR
116+ isDev = true
117+ } else if info.Mode().Type()&os.ModeDevice != 0 {
118+ mode |= rsync.S_IFBLK
119+ isDev = true
120+ }
121+
122+ if info.Mode().Type()&os.ModeNamedPipe != 0 {
123+ mode |= rsync.S_IFIFO
124+ isSpecial = true
125+ }
126+
127+ if info.Mode().Type()&os.ModeSocket != 0 {
128+ mode |= rsync.S_IFSOCK
129+ isSpecial = true
130+ }
131+
132+ fec.WriteInt32(mode)
133+
134+ if opts.PreserveUid() {
135+ uid, ok := uidFromFileInfo(info)
136+ if ok {
137+ if _, ok := uidMap[uid]; !ok && uid != 0 {
138+ u, err := user.LookupId(strconv.Itoa(int(uid)))
139+ if err != nil {
140+ lookupOnce.Do(func() {
141+ st.Logger.Error("lookup", "uid", uid, "err", err)
142+ })
143+ } else {
144+ uidMap[uid] = u.Username
145+ }
146+ }
147+ }
148+ // 8. if -o, the user id (integer)
149+ fec.WriteInt32(uid)
150+ }
151+
152+ if opts.PreserveGid() {
153+ gid, ok := gidFromFileInfo(info)
154+ if ok {
155+ if _, ok := gidMap[gid]; !ok && gid != 0 {
156+ g, err := user.LookupGroupId(strconv.Itoa(int(gid)))
157+ if err != nil {
158+ lookupGroupOnce.Do(func() {
159+ st.Logger.Error("lookupgroup", "gid", gid, "err", err)
160+ })
161+ } else {
162+ gidMap[gid] = g.Name
163+ }
164+ }
165+ }
166+ // 9. if -g, the group id (integer)
167+ fec.WriteInt32(gid)
168+ }
169+
170+ if (opts.PreserveDevices() && isDev) ||
171+ (opts.PreserveSpecials() && isSpecial) {
172+ // 10. if a special file and -D, the device “rdev” type (integer)
173+ rdev, _ := rdevFromFileInfo(info)
174+ fec.WriteInt32(rdev)
175+ }
176+
177+ if opts.PreserveLinks() && info.Mode().Type()&os.ModeSymlink != 0 {
178+ // 11. if a symbolic link and -l, the link target's length (integer)
179+ // 12. if a symbolic link and -l, the link target (byte array)
180+ target, err := os.Readlink(path)
181+ if err != nil {
182+ continue
183+ }
184+ fec.WriteInt32(int32(len(target)))
185+ fec.WriteString(target)
186+ }
187+
188+ if opts.AlwaysChecksum() {
189+ var emptyChecksum [rsyncchecksum.Size]byte
190+ checksum := emptyChecksum[:]
191+ if info.Mode().IsRegular() {
192+ // TODO: send md4 checksum of this file
193+ checksum, err = rsyncchecksum.FileChecksum(path)
194+ if err != nil {
195+ continue
196+ }
197+ }
198+ // For non-regular files, send empty md4 checksum
199+ fec.WriteString(string(checksum))
200+ }
201+ }
202+ if err != nil {
203+ return nil, err
204+ }
205+ }
206+
207+ const endOfFileList = 0
208+ _ = fec.WriteByte(endOfFileList)
209+
210+ const endOfSet = 0
211+ if opts.PreserveUid() {
212+ for uid, name := range uidMap {
213+ fec.WriteInt32(uid)
214+ _ = fec.WriteByte(byte(len(name)))
215+ fec.WriteString(name)
216+ }
217+ fec.WriteInt32(endOfSet)
218+ }
219+
220+ if opts.PreserveGid() {
221+ for gid, name := range gidMap {
222+ fec.WriteInt32(gid)
223+ _ = fec.WriteByte(byte(len(name)))
224+ fec.WriteString(name)
225+ }
226+ fec.WriteInt32(endOfSet)
227+ }
228+
229+ const ioErrors = 0
230+ fec.WriteInt32(ioErrors)
231+
232+ if err := st.Conn.WriteString(fec.String()); err != nil {
233+ return nil, err
234+ }
235+
236+ return &fileList, nil
237+}
+257,
-0
1@@ -0,0 +1,257 @@
2+package rsyncsender
3+
4+import (
5+ "bytes"
6+ "encoding/binary"
7+ "fmt"
8+ "hash"
9+ "os"
10+
11+ "github.com/mmcloughlin/md4"
12+ "github.com/picosh/pico/pkg/rsync-receiver/nofollow"
13+ "github.com/picosh/pico/pkg/rsync-receiver/rsync"
14+ "github.com/picosh/pico/pkg/rsync-receiver/rsyncchecksum"
15+ "github.com/picosh/pico/pkg/rsync-receiver/utils"
16+)
17+
18+type target struct {
19+ index int32
20+ tag uint16
21+}
22+
23+// rsync/match.c:hash_search.
24+func (st *Transfer) hashSearch(targets []target, tagTable map[uint16]int, head rsync.SumHead, fileIndex int32, fl utils.SenderFile) error {
25+ st.Logger.Debug("hashSearch", "file", fl, "head", head)
26+ f, err := os.OpenFile(fl.Path, os.O_RDONLY|nofollow.Maybe, 0)
27+ if err != nil {
28+ return err
29+ }
30+ defer func() { _ = f.Close() }()
31+
32+ fi, err := f.Stat()
33+ if err != nil {
34+ return err
35+ }
36+
37+ readSize := max(3*head.BlockLength, 256*1024)
38+ ms := mapFile(f, fi.Size(), readSize, head.BlockLength)
39+
40+ if err := st.Conn.WriteInt32(fileIndex); err != nil {
41+ return err
42+ }
43+
44+ if err := head.WriteTo(st.Conn); err != nil {
45+ return err
46+ }
47+
48+ // sum_init()
49+ h := md4.New()
50+ _ = binary.Write(h, binary.LittleEndian, st.Seed) // hash.Hash.Write never fails
51+
52+ // The following quotes are citations from
53+ // https://www.samba.org/~tridge/phd_thesis.pdf, section 3.2.6 The
54+ // signature search algorithm (PDF page 64).
55+
56+ // “Once the sorted signature table and the index table have been formed the
57+ // signature search process can begin. For each byte offset in a_i the fast
58+ // signature is computed, along with the 16 bit hash of the fast
59+ // signature. The 16 bit hash is then used to lookup the signature index,
60+ // giving the index in the signature table of the first fast signature with
61+ // that hash.”
62+
63+ var k int
64+ var sum uint32
65+ var s1, s2 uint32
66+ var offset int64
67+ end := fi.Size() + 1 - head.Sums[len(head.Sums)-1].Len
68+ st.Logger.Debug("last block", "len", head.Sums[len(head.Sums)-1].Len, "end", end)
69+
70+ readChunk := func() error {
71+ k = int(head.BlockLength)
72+ if remaining := int(fi.Size() - offset); remaining < k {
73+ k = remaining
74+ }
75+
76+ chunk := ms.ptr(offset, int32(k))
77+ sum = rsyncchecksum.Checksum1(chunk)
78+ s1 = uint32(sum & 0xFFFF)
79+ s2 = uint32(sum >> 16)
80+ return nil
81+ }
82+ if err := readChunk(); err != nil {
83+ return err
84+ }
85+
86+ tagHits := 0
87+Outer:
88+ for {
89+ tag := rsyncchecksum.Tag2(uint16(s1), uint16(s2))
90+ var sum2 []byte
91+ doneCsum2 := false
92+ j, ok := tagTable[tag]
93+ if ok {
94+ // “A linear search is then performed through the signature table, stopping
95+ // when an entry is found with a 16 bit hash which doesn’t match. For each
96+ // entry the current 32 bit fast signature is compared to the entry in the
97+ // signature table, and if that matches then the full 128 bit strong
98+ // signature is computed at the current byte offset and compared to the
99+ // strong signature in the signature table”
100+ sum = (uint32(s1) & 0xFFFF) | (uint32(s2) << 16)
101+ tagHits++
102+ for ; j < int(head.ChecksumCount) && targets[j].tag == tag; j++ {
103+ i := targets[j].index
104+ if sum != head.Sums[i].Sum1 {
105+ continue
106+ }
107+
108+ l := int64(head.BlockLength)
109+ if v := fi.Size() - offset; v < l {
110+ l = v
111+ }
112+ if l != head.Sums[i].Len {
113+ continue
114+ }
115+
116+ // log.Printf("potential match at %d target=%d %d sum=%08x", offset, j, i, sum)
117+
118+ if !doneCsum2 {
119+ buf := ms.ptr(offset, int32(l))
120+ sum2 = rsyncchecksum.Checksum2(st.Seed, buf[:])
121+ doneCsum2 = true
122+ }
123+
124+ if local, remote := sum2[:head.ChecksumLength], head.Sums[i].Sum2[:head.ChecksumLength]; !bytes.Equal(local, remote) {
125+ st.Logger.Debug("false alarm", "local", local, "remote", remote)
126+ //falseAlarms++
127+ continue
128+ }
129+
130+ // TODO(optimization): tridge rsync locates adjacent matches
131+ // here for better run-length encoding, but I’m not sure where
132+ // (if at all) we currently use run-length encoding:
133+ // https://github.com/WayneD/rsync/commit/923fa978088f4c044eec528d9472962d9c9d13c3
134+
135+ // “If the strong signature is found to match then A emits a
136+ // token telling B that a match was found and which block in bi
137+ // was matched12. The search then continues at the byte after
138+ // the matching block.”
139+
140+ if err := st.matched(h, ms, head, offset, i); err != nil {
141+ return err
142+ }
143+
144+ // rsync doesn’t read the next chunk (offset+sums[i].len),
145+ // rsync starts reading one byte before the next chunk
146+ // (offset+sums[i].len-1), because the code path starting at
147+ // “null_tag” removes the chunk’s first byte and adds the
148+ // next byte after the chunk.
149+ offset += head.Sums[i].Len - 1
150+ if err := readChunk(); err != nil {
151+ return fmt.Errorf("readChunk: %v", err)
152+ }
153+
154+ if offset >= end {
155+ break Outer
156+ }
157+
158+ break
159+ }
160+ }
161+
162+ // Update the rolling checksum by removing the oldest byte (update[0])
163+ // and adding the newest byte (update[k]).
164+ backup := max(offset-st.lastMatch, 0)
165+
166+ more := offset+int64(k) < fi.Size()
167+ mmore := int64(0)
168+ if more {
169+ mmore = 1
170+ }
171+ update := ms.ptr(offset-backup, int32(int64(k)+mmore+backup))
172+ update = update[backup:]
173+
174+ s1 -= rsyncchecksum.SignExtend(update[0])
175+ s2 -= uint32(k) * rsyncchecksum.SignExtend(update[0])
176+
177+ if more {
178+ s1 += rsyncchecksum.SignExtend(update[k])
179+ s2 += s1
180+ } else {
181+ k--
182+ }
183+ s1 = uint32(uint16(s1))
184+ s2 = uint32(uint16(s2))
185+
186+ if backup >= int64(head.BlockLength)+chunkSize && end-offset > chunkSize {
187+ // Prevent offset-st.lastMatch from growing too large by flushing
188+ // intermediate chunks.
189+ if err := st.matched(h, ms, head, offset-int64(head.BlockLength), -2); err != nil {
190+ return err
191+ }
192+ }
193+
194+ offset++
195+ if offset >= end {
196+ break
197+ }
198+ }
199+
200+ if err := st.matched(h, ms, head, fi.Size(), -1); err != nil {
201+ return err
202+ }
203+
204+ {
205+ sum := h.Sum(nil)
206+ st.Logger.Debug("sum info", "sum", sum, "len", len(sum))
207+ if _, err := st.Conn.Writer.Write(sum); err != nil {
208+ return err
209+ }
210+ }
211+
212+ return nil
213+
214+}
215+
216+// rsync/match.c:matched.
217+func (st *Transfer) matched(h hash.Hash, ms *mapStruct, head rsync.SumHead, offset int64, i int32) error {
218+ n := offset - st.lastMatch
219+
220+ transmitAccumulated := i < 0
221+
222+ // if !transmitAccumulated {
223+ // log.Printf("match at offset=%d last_match=%d i=%d len=%d n=%d",
224+ // offset, st.lastMatch, i, head.Sums[i].Len, n)
225+ // } else {
226+ // log.Printf("transmit accumulated at offset=%d", offset)
227+ // }
228+
229+ /* FIXME: this is not used
230+ l := int64(0)
231+ if !transmitAccumulated {
232+ l = head.Sums[i].Len
233+ }
234+ */
235+
236+ if err := st.sendToken(ms, i, st.lastMatch, n); err != nil {
237+ return fmt.Errorf("sendToken: %v", err)
238+ }
239+ // TODO: data_transfer += n;
240+
241+ if !transmitAccumulated {
242+ // stats.matched_data += s->sums[i].len;
243+ n += head.Sums[i].Len
244+ }
245+
246+ for j := int64(0); j < n; j += chunkSize {
247+ n1 := min(int64(chunkSize), n-j)
248+ chunk := ms.ptr(st.lastMatch+j, int32(n1))
249+ h.Write(chunk)
250+ }
251+
252+ if !transmitAccumulated {
253+ st.lastMatch = offset + head.Sums[i].Len
254+ } else {
255+ st.lastMatch = offset
256+ }
257+ return nil
258+}
+216,
-0
1@@ -0,0 +1,216 @@
2+package rsyncsender
3+
4+import (
5+ "encoding/binary"
6+ "io"
7+ "os"
8+ "sort"
9+
10+ "github.com/mmcloughlin/md4"
11+ "github.com/picosh/pico/pkg/rsync-receiver/rsync"
12+ "github.com/picosh/pico/pkg/rsync-receiver/rsyncchecksum"
13+ "github.com/picosh/pico/pkg/rsync-receiver/rsynccommon"
14+ "github.com/picosh/pico/pkg/rsync-receiver/utils"
15+)
16+
17+// rsync/sender.c:send_files().
18+func (st *Transfer) SendFiles(fileList *fileList) error {
19+ phase := 0
20+ for {
21+ // receive data about receiver’s copy of the file list contents (not
22+ // ordered)
23+ // see (*rsync.Receiver).Generator()
24+ fileIndex, err := st.Conn.ReadInt32()
25+ if err != nil {
26+ return err
27+ }
28+ if fileIndex == -1 {
29+ if phase == 0 {
30+ phase++
31+ // acknowledge phase change by sending -1
32+ if err := st.Conn.WriteInt32(-1); err != nil {
33+ return err
34+ }
35+ continue
36+ }
37+ break
38+ }
39+
40+ if st.Opts.DryRun() {
41+ if err := st.Conn.WriteInt32(fileIndex); err != nil {
42+ return err
43+ }
44+ continue
45+ }
46+
47+ head, err := st.receiveSums()
48+ if err != nil {
49+ return err
50+ }
51+
52+ // The following quotes are citations from
53+ // https://www.samba.org/~tridge/phd_thesis.pdf, section 3.2.6 The
54+ // signature search algorithm (PDF page 64).
55+
56+ // rsync/match.c:build_hash_table
57+ targets := make([]target, len(head.Sums))
58+ tagTable := make(map[uint16]int) // TODO: or int32 more specifically?
59+ {
60+ // “The first step in the algorithm is to sort the received
61+ // signatures by a 16 bit hash of the fast signature.”
62+ for idx, sum := range head.Sums {
63+ targets[idx] = target{
64+ index: int32(idx),
65+ tag: rsyncchecksum.Tag(sum.Sum1),
66+ }
67+ }
68+ sort.Slice(targets, func(i, j int) bool {
69+ return targets[i].tag < targets[j].tag
70+ })
71+
72+ // “A 16 bit index table is then formed which takes a 16 bit hash
73+ // value and gives an index into the sorted signature table which
74+ // points to the first entry in the table which has a matching
75+ // hash.”
76+ for idx := len(head.Sums) - 1; idx >= 0; idx-- {
77+ tagTable[targets[idx].tag] = idx
78+ }
79+ }
80+
81+ st.lastMatch = 0
82+ if len(head.Sums) == 0 {
83+ // fast path: send the whole file
84+ err = st.sendFile(fileIndex, fileList.Files[fileIndex])
85+ } else {
86+ err = st.hashSearch(targets, tagTable, head, fileIndex, fileList.Files[fileIndex])
87+ }
88+ if err != nil {
89+ if _, ok := err.(*os.PathError); ok {
90+ // OpenFile() failed. Log the error (server side only) and
91+ // proceed. Only starting with protocol 30, an I/O error flag is
92+ // sent after the file transfer phase.
93+ if os.IsNotExist(err) {
94+ st.Logger.Debug("file has vanished", "file", fileList.Files[fileIndex])
95+ } else {
96+ st.Logger.Error("sendFiles", "err", err)
97+ }
98+ continue
99+ } else {
100+ return err
101+ }
102+ }
103+ }
104+
105+ // phase done
106+ if err := st.Conn.WriteInt32(-1); err != nil {
107+ return err
108+ }
109+
110+ return nil
111+}
112+
113+// rsync/sender.c:receive_sums().
114+func (st *Transfer) receiveSums() (rsync.SumHead, error) {
115+ var head rsync.SumHead
116+ if err := head.ReadFrom(st.Conn); err != nil {
117+ return head, err
118+ }
119+ var offset int64
120+ head.Sums = make([]rsync.SumBuf, int(head.ChecksumCount))
121+ for i := int32(0); i < head.ChecksumCount; i++ {
122+ shortChecksum, err := st.Conn.ReadInt32()
123+ if err != nil {
124+ return head, err
125+ }
126+ sb := rsync.SumBuf{
127+ Index: i,
128+ Offset: offset,
129+ Sum1: uint32(shortChecksum),
130+ }
131+ if i == head.ChecksumCount-1 && head.RemainderLength != 0 {
132+ sb.Len = int64(head.RemainderLength)
133+ } else {
134+ sb.Len = int64(head.BlockLength)
135+ }
136+ offset += sb.Len
137+ n, err := io.ReadFull(st.Conn.Reader, sb.Sum2[:head.ChecksumLength])
138+ if err != nil {
139+ return head, err
140+ }
141+ _ = n
142+ // log.Printf("chunk[%d] len=%d offset=%.0f sum1=%08x, sum2=%x",
143+ // i, sb.len, float64(sb.offset), sb.sum1, sb.sum2[:n])
144+ head.Sums[i] = sb
145+ }
146+ return head, nil
147+}
148+
149+func (st *Transfer) sendFile(fileIndex int32, fl utils.SenderFile) error {
150+ // rsync/rsync.h defines CHUNK_SIZE as 32 * 1024. openrsync (tridge)
151+ // uses 256K, but standard rsync rejects tokens larger than 32K.
152+ const chunkSize = 32 * 1024
153+
154+ fi, r, err := st.Files.Read(&fl)
155+ if err != nil {
156+ return err
157+ }
158+ defer func() { _ = r.Close() }()
159+
160+ if err := st.Conn.WriteInt32(fileIndex); err != nil {
161+ return err
162+ }
163+
164+ sh := rsynccommon.SumSizesSqroot(fi.Size())
165+ // log.Printf("sh = %+v", sh)
166+ if err := sh.WriteTo(st.Conn); err != nil {
167+ return err
168+ }
169+
170+ h := md4.New()
171+ _ = binary.Write(h, binary.LittleEndian, st.Seed) // hash.Hash.Write never fails
172+
173+ buf := make([]byte, chunkSize)
174+ for {
175+ shouldBreak := false
176+ n, err := r.Read(buf)
177+ if err != nil {
178+ if err == io.EOF {
179+ shouldBreak = true
180+ } else {
181+ return err
182+ }
183+ }
184+ chunk := buf[:n]
185+
186+ if len(chunk) == 0 {
187+ break
188+ }
189+
190+ _, err = h.Write(chunk)
191+ if err != nil {
192+ return err
193+ }
194+ // chunk size (“rawtok” variable in openrsync)
195+ if err := st.Conn.WriteInt32(int32(len(chunk))); err != nil {
196+ return err
197+ }
198+ if _, err := st.Conn.Writer.Write(chunk); err != nil {
199+ return err
200+ }
201+
202+ if shouldBreak {
203+ break
204+ }
205+ }
206+ // transfer finished:
207+ if err := st.Conn.WriteInt32(0); err != nil {
208+ return err
209+ }
210+
211+ sum := h.Sum(nil)
212+ // log.Printf("sum: %x (len = %d)", sum, len(sum))
213+ if _, err := st.Conn.Writer.Write(sum); err != nil {
214+ return err
215+ }
216+ return nil
217+}
+15,
-0
1@@ -0,0 +1,15 @@
2+package rsyncsender
3+
4+import "io/fs"
5+
6+func uidFromFileInfo(fs.FileInfo) (int32, bool) {
7+ return 1000, false
8+}
9+
10+func gidFromFileInfo(fs.FileInfo) (int32, bool) {
11+ return 1000, false
12+}
13+
14+func rdevFromFileInfo(fs.FileInfo) (int32, bool) {
15+ return 1000, false
16+}
+34,
-0
1@@ -0,0 +1,34 @@
2+package rsyncsender
3+
4+// rsync/token.c:simple_send_token.
5+func (st *Transfer) simpleSendToken(ms *mapStruct, token int32, offset int64, n int64) error {
6+ if n > 0 {
7+ st.Logger.Debug("sending unmatched chunks", "offset", offset, "n", n)
8+ l := int64(0)
9+ for l < n {
10+ n1 := min(int64(chunkSize), n-l)
11+
12+ chunk := ms.ptr(offset+l, int32(n1))
13+
14+ if err := st.Conn.WriteInt32(int32(n1)); err != nil {
15+ return err
16+ }
17+
18+ if _, err := st.Conn.Writer.Write(chunk); err != nil {
19+ return err
20+ }
21+
22+ l += n1
23+ }
24+ }
25+ if token != -2 {
26+ return st.Conn.WriteInt32(-(token + 1))
27+ }
28+ return nil
29+}
30+
31+// rsync/token.c:send_token.
32+func (st *Transfer) sendToken(ms *mapStruct, i int32, offset int64, n int64) error {
33+ // TODO(compression): send deflated token
34+ return st.simpleSendToken(ms, i, offset, n)
35+}
1@@ -0,0 +1,49 @@
2+package rsyncsender
3+
4+import (
5+ "io"
6+ "log/slog"
7+
8+ "github.com/picosh/pico/pkg/rsync-receiver/rsyncopts"
9+ "github.com/picosh/pico/pkg/rsync-receiver/rsyncwire"
10+ "github.com/picosh/pico/pkg/rsync-receiver/utils"
11+)
12+
13+type Osenv struct {
14+ Stdin io.Reader
15+ Stdout io.Writer
16+ Stderr io.Writer
17+}
18+
19+// TransferOpts is a subset of Opts which is required for implementing a receiver.
20+type TransferOpts struct {
21+ Verbose bool
22+ DryRun bool
23+
24+ DeleteMode bool
25+ PreserveGid bool
26+ PreserveUid bool
27+ PreserveLinks bool
28+ PreservePerms bool
29+ PreserveDevices bool
30+ PreserveSpecials bool
31+ PreserveTimes bool
32+ PreserveHardlinks bool
33+}
34+
35+type Transfer struct {
36+ // config
37+ // Opts *Opts
38+ Opts *rsyncopts.Options
39+
40+ // state
41+ Conn *rsyncwire.Conn
42+ Seed int32
43+ lastMatch int64
44+
45+ Files utils.FS
46+
47+ Logger *slog.Logger
48+}
49+
50+//func (rt *Transfer) listOnly() bool { return rt.Dest == "" }
1@@ -0,0 +1,7 @@
2+package rsyncstats
3+
4+type TransferStats struct {
5+ Read int64 // total bytes read (from network connection)
6+ Written int64 // total bytes written (to network connection)
7+ Size int64 // total size of files
8+}
+214,
-0
1@@ -0,0 +1,214 @@
2+package rsyncwire
3+
4+import (
5+ "bytes"
6+ "encoding/binary"
7+ "fmt"
8+ "io"
9+ "log/slog"
10+)
11+
12+const (
13+ MsgData uint8 = 0
14+ MsgInfo uint8 = 2
15+ MsgError uint8 = 1
16+)
17+
18+const mplexBase = 7
19+
20+type MultiplexWriter struct {
21+ Writer io.Writer
22+}
23+
24+func (w *MultiplexWriter) Write(p []byte) (n int, err error) {
25+ return w.WriteMsg(MsgData, p)
26+}
27+
28+func (w *MultiplexWriter) WriteMsg(tag uint8, p []byte) (n int, err error) {
29+ header := uint32(mplexBase+tag)<<24 | uint32(len(p))
30+ // log.Printf("len %d (hex %x)", len(p), uint32(len(p)))
31+ // log.Printf("header=%v (%x)", header, header)
32+ if err := binary.Write(w.Writer, binary.LittleEndian, header); err != nil {
33+ return 0, err
34+ }
35+ return w.Writer.Write(p)
36+}
37+
38+type MultiplexReader struct {
39+ Reader io.Reader
40+}
41+
42+// rsync.h defines IO_BUFFER_SIZE as 32 * 1024, but gokr-rsyncd increases it to
43+// 256K. Since we use this as the maximum message size, too, we need to at least
44+// match it.
45+const ioBufferSize = 256 * 1024
46+const maxMessageSize = ioBufferSize
47+
48+func (w *MultiplexReader) ReadMsg() (tag uint8, p []byte, err error) {
49+ var header uint32
50+ if err := binary.Read(w.Reader, binary.LittleEndian, &header); err != nil {
51+ return 0, nil, err
52+ }
53+
54+ tag = uint8(header>>24) - mplexBase
55+ length := header & 0x00FFFFFF
56+ if length > maxMessageSize {
57+ // NOTE: if you run into this error, one alternative to bumping
58+ // maxMessageSize is to restructure the program to work with i/o buffer
59+ // windowing.
60+ return 0, nil, fmt.Errorf("length %d exceeds max message size (%d)", length, maxMessageSize)
61+ }
62+ p = make([]byte, int(length))
63+ if _, err := io.ReadFull(w.Reader, p); err != nil {
64+ return 0, nil, err
65+ }
66+ // log.Printf("header=%v (%x), tag=%v, length=%v", header, header, tag, length)
67+ // log.Printf("payload=%x / %q", p, p)
68+ return tag, p, nil
69+}
70+
71+func (w *MultiplexReader) Read(p []byte) (n int, err error) {
72+ tag, payload, err := w.ReadMsg()
73+ if err != nil {
74+ return 0, err
75+ }
76+ if tag == MsgError {
77+ return 0, fmt.Errorf("%s", payload)
78+ }
79+ if tag == MsgInfo {
80+ slog.Debug("info", "payload", payload)
81+ }
82+ if tag != MsgData {
83+ return 0, fmt.Errorf("unexpected tag: got %v, want %v", tag, MsgData)
84+ }
85+ if len(p) < len(payload) {
86+ panic(fmt.Sprintf("not enough buffer space! %d < %d", len(p), len(payload)))
87+ }
88+ return copy(p, payload), nil
89+}
90+
91+type Buffer struct {
92+ // buf.Write() never fails, making for a convenient API.
93+ buf bytes.Buffer
94+}
95+
96+func (b *Buffer) WriteByte(data byte) error {
97+ return binary.Write(&b.buf, binary.LittleEndian, data)
98+}
99+
100+func (b *Buffer) WriteInt32(data int32) {
101+ _ = binary.Write(&b.buf, binary.LittleEndian, data)
102+}
103+
104+func (b *Buffer) WriteInt64(data int64) {
105+ // send as a 32-bit integer if possible
106+ if data <= 0x7FFFFFFF && data >= 0 {
107+ b.WriteInt32(int32(data))
108+ return
109+ }
110+ // otherwise, send -1 followed by the 64-bit integer
111+ b.WriteInt32(-1)
112+ _ = binary.Write(&b.buf, binary.LittleEndian, data)
113+}
114+
115+func (b *Buffer) WriteString(data string) {
116+ _, _ = io.WriteString(&b.buf, data)
117+}
118+
119+func (b *Buffer) String() string {
120+ return b.buf.String()
121+}
122+
123+type Conn struct {
124+ Writer io.Writer
125+ Reader io.Reader
126+}
127+
128+func (c *Conn) WriteByte(data byte) error {
129+ return binary.Write(c.Writer, binary.LittleEndian, data)
130+}
131+
132+func (c *Conn) WriteInt32(data int32) error {
133+ return binary.Write(c.Writer, binary.LittleEndian, data)
134+}
135+
136+func (c *Conn) WriteInt64(data int64) error {
137+ // send as a 32-bit integer if possible
138+ if data <= 0x7FFFFFFF && data >= 0 {
139+ return c.WriteInt32(int32(data))
140+ }
141+ // otherwise, send -1 followed by the 64-bit integer
142+ if err := c.WriteInt32(-1); err != nil {
143+ return err
144+ }
145+ return binary.Write(c.Writer, binary.LittleEndian, data)
146+}
147+
148+func (c *Conn) WriteString(data string) error {
149+ _, err := io.WriteString(c.Writer, data)
150+ return err
151+}
152+
153+func (c *Conn) ReadByte() (byte, error) {
154+ var buf [1]byte
155+ if _, err := io.ReadFull(c.Reader, buf[:]); err != nil {
156+ return 0, err
157+ }
158+ return buf[0], nil
159+}
160+
161+func (c *Conn) ReadInt32() (int32, error) {
162+ var buf [4]byte
163+ if _, err := io.ReadFull(c.Reader, buf[:]); err != nil {
164+ return 0, err
165+ }
166+ return int32(binary.LittleEndian.Uint32(buf[:])), nil
167+}
168+
169+func (c *Conn) ReadInt64() (int64, error) {
170+ {
171+ data, err := c.ReadInt32()
172+ if err != nil {
173+ return 0, err
174+ }
175+ if data != -1 {
176+ // The value was small enough to fit into a 32 bit int, so it was
177+ // transferred directly.
178+ return int64(data), nil
179+ }
180+ // Otherwise, -1 was transmitted, followed by the int64.
181+ }
182+ var data int64
183+ if err := binary.Read(c.Reader, binary.LittleEndian, &data); err != nil {
184+ return 0, err
185+ }
186+ return data, nil
187+}
188+
189+type CountingReader struct {
190+ R io.Reader
191+ BytesRead int64
192+}
193+
194+func (r *CountingReader) Read(p []byte) (n int, err error) {
195+ n, err = r.R.Read(p)
196+ r.BytesRead += int64(n)
197+ return n, err
198+}
199+
200+type CountingWriter struct {
201+ W io.Writer
202+ BytesWritten int64
203+}
204+
205+func (w *CountingWriter) Write(p []byte) (n int, err error) {
206+ n, err = w.W.Write(p)
207+ w.BytesWritten += int64(n)
208+ return n, err
209+}
210+
211+func CounterPair(r io.Reader, w io.Writer) (*CountingReader, *CountingWriter) {
212+ crd := &CountingReader{R: r}
213+ cwr := &CountingWriter{W: w}
214+ return crd, cwr
215+}
+67,
-0
1@@ -0,0 +1,67 @@
2+package utils
3+
4+import (
5+ "io"
6+ "io/fs"
7+ "sort"
8+ "time"
9+
10+ "github.com/picosh/pico/pkg/rsync-receiver/rsync"
11+)
12+
13+type SenderFile struct {
14+ // TODO: store relative to the root to conserve RAM
15+ Path string
16+ WPath string
17+ Regular bool
18+}
19+
20+type ReceiverFile struct {
21+ Name string
22+ Length int64
23+ ModTime time.Time
24+ Mode int32
25+ Uid int32
26+ Gid int32
27+ LinkTarget string
28+ Rdev int32
29+ Reader io.Reader
30+}
31+
32+// FileMode converts from the Linux permission bits to Go’s permission bits.
33+func (f *ReceiverFile) FileMode() fs.FileMode {
34+ ret := fs.FileMode(f.Mode) & fs.ModePerm
35+
36+ mode := f.Mode & rsync.S_IFMT
37+ switch mode {
38+ case rsync.S_IFCHR:
39+ ret |= fs.ModeCharDevice
40+ case rsync.S_IFBLK:
41+ ret |= fs.ModeDevice
42+ case rsync.S_IFIFO:
43+ ret |= fs.ModeNamedPipe
44+ case rsync.S_IFSOCK:
45+ ret |= fs.ModeSocket
46+ case rsync.S_IFLNK:
47+ ret |= fs.ModeSymlink
48+ case rsync.S_IFDIR:
49+ ret |= fs.ModeDir
50+ }
51+
52+ return ret
53+}
54+
55+// rsync/flist.c:flist_sort_and_clean.
56+func SortFileList(fileList []*ReceiverFile) {
57+ sort.Slice(fileList, func(i, j int) bool {
58+ return fileList[i].Name < fileList[j].Name
59+ })
60+}
61+
62+// rsync/receiver.c:delete_files.
63+func FindInFileList(fileList []*ReceiverFile, name string) bool {
64+ i := sort.Search(len(fileList), func(i int) bool {
65+ return fileList[i].Name >= name
66+ })
67+ return i < len(fileList) && fileList[i].Name == name
68+}
+20,
-0
1@@ -0,0 +1,20 @@
2+package utils
3+
4+import (
5+ "io"
6+ "os"
7+)
8+
9+type ReaderAtCloser interface {
10+ io.Reader
11+ io.ReaderAt
12+ io.Closer
13+}
14+
15+// File System: need to handle all type of files: regular, folder, symlink, etc.
16+type FS interface {
17+ Put(*ReceiverFile) (int64, error)
18+ List(string) ([]os.FileInfo, error)
19+ Read(*SenderFile) (os.FileInfo, ReaderAtCloser, error)
20+ Remove([]*ReceiverFile) error
21+}
+4,
-4
1@@ -9,11 +9,11 @@ import (
2 "slices"
3 "strings"
4
5- "github.com/picosh/go-rsync-receiver/rsyncopts"
6- "github.com/picosh/go-rsync-receiver/rsyncreceiver"
7- "github.com/picosh/go-rsync-receiver/rsyncsender"
8- rsyncutils "github.com/picosh/go-rsync-receiver/utils"
9 "github.com/picosh/pico/pkg/pssh"
10+ "github.com/picosh/pico/pkg/rsync-receiver/rsyncopts"
11+ "github.com/picosh/pico/pkg/rsync-receiver/rsyncreceiver"
12+ "github.com/picosh/pico/pkg/rsync-receiver/rsyncsender"
13+ rsyncutils "github.com/picosh/pico/pkg/rsync-receiver/utils"
14 "github.com/picosh/pico/pkg/send/utils"
15 )
16
1@@ -10,8 +10,8 @@ import (
2 "testing"
3 "time"
4
5- rsyncutils "github.com/picosh/go-rsync-receiver/utils"
6 "github.com/picosh/pico/pkg/pssh"
7+ rsyncutils "github.com/picosh/pico/pkg/rsync-receiver/utils"
8 "github.com/picosh/pico/pkg/send/utils"
9 "golang.org/x/crypto/ssh"
10 )