Merge pull request #225 from picosh/am/rsync-protocol

...	...		@@ -88,6 +88,7 @@ PGS_STORAGE_DIR=.storage
88	88		PGS_DEBUG=1
89	89		PGS_CACHE_TTL=600s
90	90		PGS_CACHE_MAX_ITEMS=0
	91	+	PGS_CACHE_MAX_BODY_SIZE=0
91	92
92	93		PICO_CADDYFILE=./caddy/Caddyfile.pico
93	94		PICO_V4=

...	...		@@ -12,9 +12,11 @@ WRITE?=0
12	12		smol:
13	13		curl https://pico.sh/smol.css -o ./pkg/apps/prose/public/smol-v2.css
14	14		cat ./pkg/apps/prose/artifacts/main.css >> ./pkg/apps/prose/public/smol-v2.css
	15	+	curl https://pico.sh/smol.css -o ./pkg/apps/pastes/public/smol.css
15	16		.PHONY: smol
16	17
17	18		css:
	19	+	cp ./syntax.css ./pkg/apps/pastes/public/syntax.css
18	20		cp ./syntax.css ./pkg/apps/prose/public/syntax.css
19	21		.PHONY: css
20	22

...	...		@@ -80,7 +82,7 @@ bp-%: bp-setup
80	82		$(DOCKER_BUILDX_BUILD) "ghcr.io/picosh/pico/$-web:$(DOCKER_TAG)" --build-arg "APP=$" --target release-web .
81	83		.PHONY: bp-%
82	84
83		-	bp-all: bp-prose bp-feeds bp-pgs bp-auth bp-bouncer bp-pipe bp-pgs-cdn
	85	+	bp-all: bp-prose bp-pastes bp-feeds bp-pgs bp-auth bp-bouncer bp-pipe bp-pgs-cdn
84	86		.PHONY: bp-all
85	87
86	88		LDFLAGS?=-s -w

...	...		@@ -102,7 +104,7 @@ build-%:
102	104		go build -ldflags="$(LDFLAGS)" -o "build/$-ssh" "./cmd/$/ssh"
103	105		.PHONY: build-%
104	106
105		-	build: build-prose build-feeds build-pgs build-pgs-cdn build-auth build-pico build-pipe
	107	+	build: build-prose build-pastes build-feeds build-pgs build-pgs-cdn build-auth build-pico build-pipe
106	108		.PHONY: build
107	109
108	110		scripts:

...	...		@@ -162,10 +164,11 @@ migrate:
162	164		$(DOCKER_CMD) exec -i $(DB_CONTAINER) psql -U $(PGUSER) -d $(PGDATABASE) < ./sql/migrations/20260504_add_analytics_summary_indexes.sql
163	165		$(DOCKER_CMD) exec -i $(DB_CONTAINER) psql -U $(PGUSER) -d $(PGDATABASE) < ./sql/migrations/20260716_block_signups.sql
164	166		$(DOCKER_CMD) exec -i $(DB_CONTAINER) psql -U $(PGUSER) -d $(PGDATABASE) < ./sql/migrations/20260925_add_invites.sql
	167	+	$(DOCKER_CMD) exec -i $(DB_CONTAINER) psql -U $(PGUSER) -d $(PGDATABASE) < ./sql/migrations/20261010_add_pastes_invite_feature.sql
165	168		.PHONY: migrate
166	169
167	170		latest:
168		-	$(DOCKER_CMD) exec -i $(DB_CONTAINER) psql -U $(PGUSER) -d $(PGDATABASE) < ./sql/migrations/20260925_add_invites.sql
	171	+	$(DOCKER_CMD) exec -i $(DB_CONTAINER) psql -U $(PGUSER) -d $(PGDATABASE) < ./sql/migrations/20261010_add_pastes_invite_feature.sql
169	172		.PHONY: latest
170	173
171	174		psql:

...	...		@@ -12,7 +12,8 @@ without needing to install anything. We accomplish this with the `ssh` tools you
12	12		already have installed on your system.
13	13
14	14		Want to publish a blog post? Use rsync, scp, or sftp. Want to publish a website?
15		-	Use rsync, scp, or sftp. Hopefully you see the trend.
	15	+	Use rsync, scp, or sftp. Want to share a code snippet with a colleague? Use
	16	+	rsync, scp, or sftp. Hopefully you see the trend.
16	17
17	18		- [pages](https://pico.sh/pgs): A static site hosting platform using `ssh` for
18	19		site deployments. `ssh`.

...	...		@@ -20,7 +21,9 @@ Use rsync, scp, or sftp. Hopefully you see the trend.
20	21		- [pipe](https://pipe.pico.sh): Authenticated `*nix` pipes over `ssh`
21	22		- [prose](https://pico.sh/prose): A blog platform using `ssh` for content
22	23		management.
23		-	- [rss-to-email](https://pico.sh/feeds): An RSS email notification service using `ssh`.
	24	+	- [feeds](https://pico.sh/feeds): An RSS email notification service using `ssh`.
	25	+	- [pastes](https://pico.sh/pastes): Upload code snippets using rsync, scp, and
	26	+	sftp.
24	27
25	28		## Deploy a site with a single command
26	29

...	...		@@ -78,6 +81,16 @@ scp hello-world.md prose.sh:/
78	81		Congrats! You just published a blog article, accessible here:
79	82		https://{user}.prose.sh/hello-world
80	83
	84	+	## Easily share code snippets
	85	+
	86	+	Pipe some stdout to us:
	87	+
	88	+	```bash
	89	+	git diff \| ssh pastes.sh changes.patch
	90	+	```
	91	+
	92	+	And instantly share your code snippets: https://{user}.pastes.sh/changes.patch
	93	+
81	94		## Receive email notifications for your favorite rss feeds
82	95
83	96		Create a blogs.txt file:

...	...		@@ -0,0 +1,7 @@
	1	+	package main
	2	+
	3	+	import "github.com/picosh/pico/pkg/apps/pastes"
	4	+
	5	+	func main() {
	6	+	pastes.StartSshServer()
	7	+	}

...	...		@@ -0,0 +1,7 @@
	1	+	package main
	2	+
	3	+	import "github.com/picosh/pico/pkg/apps/pastes"
	4	+
	5	+	func main() {
	6	+	pastes.StartApiServer()
	7	+	}

...	...		@@ -28,13 +28,13 @@ func main() {
28	28		}()
29	29		cfg := pgs.NewPgsConfig(logger, nil, nil, drain)
30	30		proxy := newProxyServe(cfg.Logger)
31		-	httpCache := pgs.NewPgsHttpCache(cfg, proxy)
	31	+	httpCache, cache := pgs.NewPgsHttpCache(cfg, proxy)
32	32		cacher := &cachedHttp{
33	33		Logger: cfg.Logger,
34	34		Cache: httpCache,
35	35		}
36	36
37		-	go pgs.CacheMgmt(ctx, cfg.CacheClearingQueue, cfg, httpCache.Cache)
	37	+	go pgs.CacheMgmt(ctx, cfg, cache, nil)
38	38
39	39		portStr := fmt.Sprintf(":%s", cfg.WebPort)
40	40		cfg.Logger.Info(

...	...		@@ -0,0 +1,73 @@
	1	+	// clean-keep-dirs removes the ._pico_keep_dir markers pgs used to write to
	2	+	// keep empty directories. Filesystem storage keeps real directories now, so
	3	+	// removing a marker leaves its directory in place.
	4	+	package main
	5	+
	6	+	import (
	7	+	"flag"
	8	+	"fmt"
	9	+	"io/fs"
	10	+	"log/slog"
	11	+	"os"
	12	+	"path/filepath"
	13	+
	14	+	"github.com/picosh/pico/pkg/storage"
	15	+	)
	16	+
	17	+	const keepDirName = "._pico_keep_dir"
	18	+
	19	+	func main() {
	20	+	deleteFlag := flag.Bool("delete", false, "delete the markers found")
	21	+	flag.Parse()
	22	+
	23	+	logger := slog.Default()
	24	+
	25	+	adapter := storage.GetStorageTypeFromEnv()
	26	+	if adapter != "fs" {
	27	+	logger.Error("markers only exist on filesystem storage", "adapter", adapter)
	28	+	os.Exit(1)
	29	+	}
	30	+	st, err := storage.NewStorage(logger, adapter)
	31	+	if err != nil {
	32	+	logger.Error("failed to create storage", "err", err)
	33	+	os.Exit(1)
	34	+	}
	35	+	root := st.(*storage.StorageFS).Dir
	36	+
	37	+	found, deleted, failed := 0, 0, 0
	38	+	err = filepath.WalkDir(root, func(p string, d fs.DirEntry, err error) error {
	39	+	if err != nil {
	40	+	logger.Error("failed to read", "path", p, "err", err)
	41	+	failed++
	42	+	return nil
	43	+	}
	44	+	if d.IsDir() \|\| d.Name() != keepDirName {
	45	+	return nil
	46	+	}
	47	+	found++
	48	+	if !*deleteFlag {
	49	+	fmt.Println(p)
	50	+	return nil
	51	+	}
	52	+	if err := os.Remove(p); err != nil {
	53	+	logger.Error("failed to delete marker", "path", p, "err", err)
	54	+	failed++
	55	+	return nil
	56	+	}
	57	+	deleted++
	58	+	return nil
	59	+	})
	60	+	if err != nil {
	61	+	logger.Error("failed to walk storage", "root", root, "err", err)
	62	+	os.Exit(1)
	63	+	}
	64	+
	65	+	if !*deleteFlag {
	66	+	fmt.Printf("\nFound %d marker(s) under %s. Use --delete to remove them.\n", found, root)
	67	+	return
	68	+	}
	69	+	fmt.Printf("\nDeleted %d marker(s) under %s, %d failed\n", deleted, root, failed)
	70	+	if failed > 0 {
	71	+	os.Exit(1)
	72	+	}
	73	+	}

...	...		@@ -22,6 +22,7 @@ require (
22	22		github.com/alecthomas/chroma/v2 v2.27.0
23	23		github.com/antoniomika/syncmap v1.0.0
24	24		github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de
	25	+	github.com/cespare/xxhash/v2 v2.3.0
25	26		github.com/containerd/console v1.0.5
26	27		github.com/emersion/go-sasl v0.0.0-20241020182733-b788ff22d5a6
27	28		github.com/emersion/go-smtp v0.25.0

...	...		@@ -34,6 +35,7 @@ require (
34	35		github.com/gorilla/websocket v1.5.3
35	36		github.com/hashicorp/golang-lru/v2 v2.0.7
36	37		github.com/jmoiron/sqlx v1.4.0
	38	+	github.com/klauspost/compress v1.19.1
37	39		github.com/lib/pq v1.12.3
38	40		github.com/matryer/is v1.4.1
39	41		github.com/microcosm-cc/bluemonday v1.0.27

...	...		@@ -56,7 +58,6 @@ require (
56	58		go.abhg.dev/goldmark/toc v0.12.0
57	59		go.rockorager.dev/vaxis v0.17.1
58	60		golang.org/x/crypto v0.57.0
59		-	golang.org/x/sync v0.23.0
60	61		gopkg.in/yaml.v2 v2.4.0
61	62		modernc.org/sqlite v1.60.1
62	63		)

...	...		@@ -69,7 +70,6 @@ require (
69	70		github.com/aymerick/douceur v0.2.0 // indirect
70	71		github.com/beorn7/perks v1.0.1 // indirect
71	72		github.com/cenkalti/backoff/v4 v4.3.0 // indirect
72		-	github.com/cespare/xxhash/v2 v2.3.0 // indirect
73	73		github.com/containerd/errdefs v1.0.0 // indirect
74	74		github.com/containerd/errdefs/pkg v0.3.0 // indirect
75	75		github.com/containerd/log v0.1.0 // indirect

...	...		@@ -111,7 +111,6 @@ require (
111	111		github.com/gorilla/css v1.0.1 // indirect
112	112		github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.7 // indirect
113	113		github.com/jackc/pgx/v5 v5.8.0 // indirect
114		-	github.com/klauspost/compress v1.19.1 // indirect
115	114		github.com/kr/fs v0.1.0 // indirect
116	115		github.com/kr/pretty v0.3.1 // indirect
117	116		github.com/kr/text v0.2.0 // indirect

...	...		@@ -164,6 +163,7 @@ require (
164	163		golang.org/x/image v0.46.0 // indirect
165	164		golang.org/x/mod v0.41.0 // indirect
166	165		golang.org/x/net v0.59.0 // indirect
	166	+	golang.org/x/sync v0.23.0 // indirect
167	167		golang.org/x/sys v0.48.0 // indirect
168	168		golang.org/x/telemetry v0.0.0-20260908163034-4bcc4b2ee518 // indirect
169	169		golang.org/x/term v0.46.0 // indirect

...	...		@@ -33,7 +33,7 @@ zmx run cdn docker buildx build --push --platform "$DOCKER_PLATFORM" -t
33	33		zmx run standalone docker buildx build --push --platform "$DOCKER_PLATFORM" -t "ghcr.io/picosh/pgs:$DOCKER_TAG" --target release -f Dockerfile.standalone .
34	34		zmx wait "*"
35	35
36		-	apps=("prose" "pgs" "feeds" "pipe")
	36	+	apps=("prose" "pastes" "pgs" "feeds" "pipe")
37	37		for APP in "${apps[@]}"; do
38	38		zmx run "$APP-ssh" -d docker buildx build --push --platform "$DOCKER_PLATFORM" -t "ghcr.io/picosh/pico/$APP-ssh:$DOCKER_TAG" --build-arg "APP=$APP" --target release-ssh .
39	39		zmx run "$APP-web" -d docker buildx build --push --platform "$DOCKER_PLATFORM" -t "ghcr.io/picosh/pico/$APP-web:$DOCKER_TAG" --build-arg "APP=$APP" --target release-web .

...	...		@@ -160,13 +160,7 @@ func Middleware(dbpool db.DB, cfg *shared.ConfigSite) pssh.SSHServerMiddleware {
160	160		logger.Info("run cmd", "filename", filename)
161	161		fetcher := NewFetcher(dbpool, cfg)
162	162
163		-	isPicoPlus := false
164		-	ff, _ := dbpool.FindFeature(user.ID, "plus")
165		-	if ff != nil {
166		-	if ff.IsValid() {
167		-	isPicoPlus = true
168		-	}
169		-	}
	163	+	isPicoPlus := db.HasPlusOrSpace(dbpool, user.ID, "feeds")
170	164
171	165		err = fetcher.RunPost(logger, user, isPicoPlus, post, true, time.Now().UTC())
172	166		if err != nil {

...	...		@@ -336,16 +336,9 @@ func (f Fetcher) RunUser(user db.User, now time.Time) error {
336	336		if err != nil {
337	337		return err
338	338		}
339		-	isPicoPlus := false
340		-	ff, _ := f.db.FindFeature(user.ID, "plus")
341		-	if ff != nil {
342		-	if ff.IsValid() {
343		-	isPicoPlus = true
344		-	}
345		-	}
346		-
	339	+	isPicoPlus := db.HasPlusOrSpace(f.db, user.ID, "feeds")
347	340		if !isPicoPlus {
348		-	return fmt.Errorf("must be pico+ user to receive rss digests")
	341	+	return fmt.Errorf("must have pico+ or feeds access to receive rss digests")
349	342		}
350	343
351	344		if len(posts.Data) > 0 {

...	...		@@ -37,9 +37,8 @@ func (p FeedHooks) FileValidate(s pssh.SSHServerConnSession, data *filehandler
37	37		return false, err
38	38		}
39	39
40		-	// Because we need to support sshfs, sftp runs our Write handler twice
41		-	// and on the first pass we do not have access to the file data.
42		-	// In that case we should skip the parsing validation
	40	+	// SFTP validates a file when it is opened, before its contents
	41	+	// arrive, so there is nothing to parse yet.
43	42		if data.Text == "" {
44	43		return true, nil
45	44		}

...	...		@@ -0,0 +1,413 @@
	1	+	package pastes
	2	+
	3	+	import (
	4	+	"fmt"
	5	+	"html/template"
	6	+	"net/http"
	7	+	"net/url"
	8	+	"os"
	9	+	"time"
	10	+
	11	+	"github.com/picosh/pico/pkg/db"
	12	+	"github.com/picosh/pico/pkg/db/postgres"
	13	+	"github.com/picosh/pico/pkg/shared"
	14	+	"github.com/picosh/pico/pkg/shared/router"
	15	+	"github.com/prometheus/client_golang/prometheus/promhttp"
	16	+	)
	17	+
	18	+	type PageData struct {
	19	+	Site shared.SitePageData
	20	+	}
	21	+
	22	+	type PostItemData struct {
	23	+	URL template.URL
	24	+	BlogURL template.URL
	25	+	Username string
	26	+	Title string
	27	+	Description string
	28	+	PublishAtISO string
	29	+	PublishAt string
	30	+	UpdatedAtISO string
	31	+	UpdatedTimeAgo string
	32	+	Padding string
	33	+	}
	34	+
	35	+	type BlogPageData struct {
	36	+	Site shared.SitePageData
	37	+	PageTitle string
	38	+	URL template.URL
	39	+	RSSURL template.URL
	40	+	Username string
	41	+	Header *HeaderTxt
	42	+	Posts []PostItemData
	43	+	}
	44	+
	45	+	type PostPageData struct {
	46	+	Site shared.SitePageData
	47	+	PageTitle string
	48	+	URL template.URL
	49	+	RawURL template.URL
	50	+	BlogURL template.URL
	51	+	Title string
	52	+	Description string
	53	+	Username string
	54	+	BlogName string
	55	+	Contents template.HTML
	56	+	Text string
	57	+	PublishAtISO string
	58	+	PublishAt string
	59	+	ExpiresAt string
	60	+	Unlisted bool
	61	+	}
	62	+
	63	+	type Link struct {
	64	+	URL string
	65	+	Text string
	66	+	}
	67	+
	68	+	type HeaderTxt struct {
	69	+	Title string
	70	+	Bio string
	71	+	Nav []Link
	72	+	HasLinks bool
	73	+	}
	74	+
	75	+	func blogHandler(w http.ResponseWriter, r *http.Request) {
	76	+	username := router.GetUsernameFromRequest(r)
	77	+	dbpool := router.GetDB(r)
	78	+	blogger := router.GetLogger(r)
	79	+	logger := blogger.With("user", username)
	80	+	cfg := router.GetCfg(r)
	81	+
	82	+	user, err := dbpool.FindUserByName(username)
	83	+	if err != nil {
	84	+	logger.Info("user not found")
	85	+	http.Error(w, "user not found", http.StatusNotFound)
	86	+	return
	87	+	}
	88	+	logger = shared.LoggerWithUser(blogger, user)
	89	+
	90	+	pager, err := dbpool.FindPostsByUser(&db.Pager{Num: 1000, Page: 0}, user.ID, cfg.Space)
	91	+	if err != nil {
	92	+	logger.Error("could not find posts for user", "err", err.Error())
	93	+	http.Error(w, "could not fetch posts for blog", http.StatusInternalServerError)
	94	+	return
	95	+	}
	96	+
	97	+	posts := pager.Data
	98	+
	99	+	ts, err := router.RenderTemplate(cfg, []string{
	100	+	cfg.StaticPath("html/blog.page.tmpl"),
	101	+	})
	102	+
	103	+	if err != nil {
	104	+	logger.Error("could not render template", "err", err)
	105	+	http.Error(w, err.Error(), http.StatusInternalServerError)
	106	+	return
	107	+	}
	108	+
	109	+	headerTxt := &HeaderTxt{
	110	+	Title: GetBlogName(username),
	111	+	Bio: "",
	112	+	}
	113	+
	114	+	curl := shared.CreateURLFromRequest(cfg, r)
	115	+	postCollection := make([]PostItemData, 0, len(posts))
	116	+	for _, post := range posts {
	117	+	p := PostItemData{
	118	+	URL: template.URL(cfg.FullPostURL(curl, post.Username, post.Slug)),
	119	+	BlogURL: template.URL(cfg.FullBlogURL(curl, post.Username)),
	120	+	Title: post.Filename,
	121	+	PublishAt: post.PublishAt.Format(time.DateOnly),
	122	+	PublishAtISO: post.PublishAt.Format(time.RFC3339),
	123	+	UpdatedTimeAgo: shared.TimeAgo(post.UpdatedAt),
	124	+	UpdatedAtISO: post.UpdatedAt.Format(time.RFC3339),
	125	+	}
	126	+	postCollection = append(postCollection, p)
	127	+	}
	128	+
	129	+	data := BlogPageData{
	130	+	Site: *cfg.GetSiteData(),
	131	+	PageTitle: headerTxt.Title,
	132	+	URL: template.URL(cfg.FullBlogURL(curl, username)),
	133	+	RSSURL: template.URL(cfg.RssBlogURL(curl, username, "")),
	134	+	Header: headerTxt,
	135	+	Username: username,
	136	+	Posts: postCollection,
	137	+	}
	138	+
	139	+	err = ts.Execute(w, data)
	140	+	if err != nil {
	141	+	logger.Error("could not execute tempalte", "err", err)
	142	+	http.Error(w, err.Error(), http.StatusInternalServerError)
	143	+	}
	144	+	}
	145	+
	146	+	func GetPostTitle(post *db.Post) string {
	147	+	if post.Description == "" {
	148	+	return post.Title
	149	+	}
	150	+
	151	+	return fmt.Sprintf("%s: %s", post.Title, post.Description)
	152	+	}
	153	+
	154	+	func GetBlogName(username string) string {
	155	+	return fmt.Sprintf("%s's pastes", username)
	156	+	}
	157	+
	158	+	func postHandler(w http.ResponseWriter, r *http.Request) {
	159	+	username := router.GetUsernameFromRequest(r)
	160	+	subdomain := router.GetSubdomain(r)
	161	+	cfg := router.GetCfg(r)
	162	+
	163	+	var slug string
	164	+	if !cfg.IsSubdomains() \|\| subdomain == "" {
	165	+	slug, _ = url.PathUnescape(router.GetField(r, 1))
	166	+	} else {
	167	+	slug, _ = url.PathUnescape(router.GetField(r, 0))
	168	+	}
	169	+
	170	+	dbpool := router.GetDB(r)
	171	+	blogger := router.GetLogger(r)
	172	+	logger := blogger.With("slug", slug, "user", username)
	173	+
	174	+	user, err := dbpool.FindUserByName(username)
	175	+	if err != nil {
	176	+	logger.Info("paste not found")
	177	+	http.Error(w, "paste not found", http.StatusNotFound)
	178	+	return
	179	+	}
	180	+	logger = shared.LoggerWithUser(logger, user)
	181	+
	182	+	blogName := GetBlogName(username)
	183	+
	184	+	var data PostPageData
	185	+	post, err := dbpool.FindPostWithSlug(slug, user.ID, cfg.Space)
	186	+	if err == nil {
	187	+	logger = logger.With("filename", post.Filename)
	188	+	logger.Info("paste found")
	189	+	expiresAt := "never"
	190	+	unlisted := false
	191	+	parsedText := ""
	192	+	// we dont want to syntax highlight huge files
	193	+	if post.FileSize > 1*shared.MB {
	194	+	logger.Warn("paste too large to parse and apply syntax highlighting")
	195	+	parsedText = "<pre>" + template.HTMLEscapeString(post.Text) + "</pre>"
	196	+	} else {
	197	+	parsedText, err = ParseText(post.Filename, post.Text)
	198	+	if err != nil {
	199	+	logger.Error("could not parse text", "err", err)
	200	+	parsedText = "<pre>" + template.HTMLEscapeString(post.Text) + "</pre>"
	201	+	}
	202	+	if post.ExpiresAt != nil {
	203	+	expiresAt = post.ExpiresAt.Format(time.DateOnly)
	204	+	}
	205	+
	206	+	if post.Hidden {
	207	+	unlisted = true
	208	+	}
	209	+	}
	210	+
	211	+	data = PostPageData{
	212	+	Site: *cfg.GetSiteData(),
	213	+	PageTitle: post.Filename,
	214	+	URL: template.URL(cfg.PostURL(post.Username, post.Slug)),
	215	+	RawURL: template.URL(cfg.RawPostURL(post.Username, post.Slug)),
	216	+	BlogURL: template.URL(cfg.BlogURL(username)),
	217	+	Description: post.Description,
	218	+	Title: post.Filename,
	219	+	PublishAt: post.PublishAt.Format(time.DateOnly),
	220	+	PublishAtISO: post.PublishAt.Format(time.RFC3339),
	221	+	Username: username,
	222	+	BlogName: blogName,
	223	+	Contents: template.HTML(parsedText),
	224	+	Text: post.Text,
	225	+	ExpiresAt: expiresAt,
	226	+	Unlisted: unlisted,
	227	+	}
	228	+	} else {
	229	+	logger.Info("paste not found")
	230	+	data = PostPageData{
	231	+	Site: *cfg.GetSiteData(),
	232	+	PageTitle: "Paste not found",
	233	+	Description: "Paste not found",
	234	+	Title: "Paste not found",
	235	+	BlogURL: template.URL(cfg.BlogURL(username)),
	236	+	PublishAt: time.Now().Format(time.DateOnly),
	237	+	PublishAtISO: time.Now().Format(time.RFC3339),
	238	+	Username: username,
	239	+	BlogName: blogName,
	240	+	Contents: "oops! we can't seem to find this post.",
	241	+	Text: "oops! we can't seem to find this post.",
	242	+	ExpiresAt: "",
	243	+	}
	244	+	}
	245	+
	246	+	ts, err := router.RenderTemplate(cfg, []string{
	247	+	cfg.StaticPath("html/post.page.tmpl"),
	248	+	})
	249	+
	250	+	if err != nil {
	251	+	http.Error(w, err.Error(), http.StatusInternalServerError)
	252	+	}
	253	+
	254	+	logger.Info("serving paste")
	255	+	err = ts.Execute(w, data)
	256	+	if err != nil {
	257	+	logger.Error("could not execute template", "err", err)
	258	+	http.Error(w, err.Error(), http.StatusInternalServerError)
	259	+	}
	260	+	}
	261	+
	262	+	func postHandlerRaw(w http.ResponseWriter, r *http.Request) {
	263	+	username := router.GetUsernameFromRequest(r)
	264	+	subdomain := router.GetSubdomain(r)
	265	+	cfg := router.GetCfg(r)
	266	+
	267	+	var slug string
	268	+	if !cfg.IsSubdomains() \|\| subdomain == "" {
	269	+	slug, _ = url.PathUnescape(router.GetField(r, 1))
	270	+	} else {
	271	+	slug, _ = url.PathUnescape(router.GetField(r, 0))
	272	+	}
	273	+
	274	+	dbpool := router.GetDB(r)
	275	+	blogger := router.GetLogger(r)
	276	+	logger := blogger.With("user", username, "slug", slug)
	277	+
	278	+	user, err := dbpool.FindUserByName(username)
	279	+	if err != nil {
	280	+	logger.Info("user not found")
	281	+	http.Error(w, "user not found", http.StatusNotFound)
	282	+	return
	283	+	}
	284	+	logger = shared.LoggerWithUser(blogger, user)
	285	+
	286	+	post, err := dbpool.FindPostWithSlug(slug, user.ID, cfg.Space)
	287	+	if err != nil {
	288	+	logger.Info("paste not found")
	289	+	http.Error(w, "paste not found", http.StatusNotFound)
	290	+	return
	291	+	}
	292	+	logger = logger.With("filename", post.Filename)
	293	+	logger.Info("raw paste found")
	294	+
	295	+	w.Header().Set("Content-Type", "text/plain")
	296	+	_, err = w.Write([]byte(post.Text))
	297	+	if err != nil {
	298	+	logger.Error("write error", "err", err)
	299	+	}
	300	+	}
	301	+
	302	+	func serveFile(file string, contentType string) http.HandlerFunc {
	303	+	return func(w http.ResponseWriter, r *http.Request) {
	304	+	logger := router.GetLogger(r)
	305	+	cfg := router.GetCfg(r)
	306	+
	307	+	contents, err := os.ReadFile(cfg.StaticPath(fmt.Sprintf("public/%s", file)))
	308	+	if err != nil {
	309	+	logger.Error("could not read file", "err", err)
	310	+	http.Error(w, "file not found", 404)
	311	+	}
	312	+	w.Header().Add("Content-Type", contentType)
	313	+
	314	+	_, err = w.Write(contents)
	315	+	if err != nil {
	316	+	logger.Error("could not write contents", "err", err)
	317	+	http.Error(w, "server error", 500)
	318	+	}
	319	+	}
	320	+	}
	321	+
	322	+	func createStaticRoutes() []router.Route {
	323	+	return []router.Route{
	324	+	router.NewRoute("GET", "/main.css", serveFile("main.css", "text/css")),
	325	+	router.NewRoute("GET", "/smol.css", serveFile("smol.css", "text/css")),
	326	+	router.NewRoute("GET", "/syntax.css", serveFile("syntax.css", "text/css")),
	327	+	router.NewRoute("GET", "/card.png", serveFile("card.png", "image/png")),
	328	+	router.NewRoute("GET", "/favicon-16x16.png", serveFile("favicon-16x16.png", "image/png")),
	329	+	router.NewRoute("GET", "/favicon-32x32.png", serveFile("favicon-32x32.png", "image/png")),
	330	+	router.NewRoute("GET", "/apple-touch-icon.png", serveFile("apple-touch-icon.png", "image/png")),
	331	+	router.NewRoute("GET", "/favicon.ico", serveFile("favicon.ico", "image/x-icon")),
	332	+	router.NewRoute("GET", "/robots.txt", serveFile("robots.txt", "text/plain")),
	333	+	}
	334	+	}
	335	+
	336	+	func createMainRoutes(staticRoutes []router.Route) []router.Route {
	337	+	routes := []router.Route{
	338	+	router.NewRoute("GET", "/", router.CreatePageHandler("html/marketing.page.tmpl")),
	339	+	router.NewRoute("GET", "/check", router.CheckHandler),
	340	+	router.NewRoute("GET", "/_metrics", promhttp.Handler().ServeHTTP),
	341	+	}
	342	+
	343	+	routes = append(
	344	+	routes,
	345	+	staticRoutes...,
	346	+	)
	347	+
	348	+	routes = append(
	349	+	routes,
	350	+	router.NewRoute("GET", "/([^/]+)", blogHandler),
	351	+	router.NewRoute("GET", "/([^/]+)/([^/]+)", postHandler),
	352	+	router.NewRoute("GET", "/([^/]+)/([^/]+)/raw", postHandlerRaw),
	353	+	router.NewRoute("GET", "/raw/([^/]+)/([^/]+)", postHandlerRaw),
	354	+	)
	355	+
	356	+	return routes
	357	+	}
	358	+
	359	+	func createSubdomainRoutes(staticRoutes []router.Route) []router.Route {
	360	+	routes := []router.Route{
	361	+	router.NewRoute("GET", "/", blogHandler),
	362	+	}
	363	+
	364	+	routes = append(
	365	+	routes,
	366	+	staticRoutes...,
	367	+	)
	368	+
	369	+	routes = append(
	370	+	routes,
	371	+	router.NewRoute("GET", "/([^/]+)", postHandler),
	372	+	router.NewRoute("GET", "/([^/]+)/raw", postHandlerRaw),
	373	+	router.NewRoute("GET", "/raw/([^/]+)", postHandlerRaw),
	374	+	)
	375	+
	376	+	return routes
	377	+	}
	378	+
	379	+	func StartApiServer() {
	380	+	cfg := NewConfigSite("pastes-web")
	381	+	db := postgres.NewDB(cfg.DbURL, cfg.Logger)
	382	+	defer func() {
	383	+	_ = db.Close()
	384	+	}()
	385	+	logger := cfg.Logger
	386	+
	387	+	go CronDeleteExpiredPosts(cfg, db)
	388	+
	389	+	staticRoutes := createStaticRoutes()
	390	+
	391	+	if cfg.Debug {
	392	+	staticRoutes = router.CreatePProfRoutes(staticRoutes)
	393	+	}
	394	+
	395	+	mainRoutes := createMainRoutes(staticRoutes)
	396	+	subdomainRoutes := createSubdomainRoutes(staticRoutes)
	397	+
	398	+	apiConfig := &router.ApiConfig{
	399	+	Cfg: cfg,
	400	+	Dbpool: db,
	401	+	}
	402	+	handler := router.CreateServe(mainRoutes, subdomainRoutes, apiConfig)
	403	+	router := http.HandlerFunc(handler)
	404	+
	405	+	portStr := fmt.Sprintf(":%s", cfg.Port)
	406	+	logger.Info(
	407	+	"Starting server on port",
	408	+	"port", cfg.Port,
	409	+	"domain", cfg.Domain,
	410	+	)
	411	+
	412	+	logger.Error(http.ListenAndServe(portStr, router).Error())
	413	+	}

...	...		@@ -0,0 +1,68 @@
	1	+	package pastes
	2	+
	3	+	import (
	4	+	"context"
	5	+	"html/template"
	6	+	"log/slog"
	7	+	"net/http"
	8	+	"net/http/httptest"
	9	+	"strings"
	10	+	"testing"
	11	+	"time"
	12	+
	13	+	"github.com/picosh/pico/pkg/db"
	14	+	"github.com/picosh/pico/pkg/db/stub"
	15	+	"github.com/picosh/pico/pkg/shared"
	16	+	"github.com/picosh/pico/pkg/shared/router"
	17	+	)
	18	+
	19	+	type pasteDB struct {
	20	+	*stub.StubDB
	21	+	post *db.Post
	22	+	}
	23	+
	24	+	func (m pasteDB) FindUserByName(string) (db.User, error) {
	25	+	return &db.User{ID: "user-1", Name: "tester"}, nil
	26	+	}
	27	+
	28	+	func (m pasteDB) FindPostWithSlug(string, string, string) (db.Post, error) {
	29	+	return m.post, nil
	30	+	}
	31	+
	32	+	func TestLargePasteEscapesHTML(t *testing.T) {
	33	+	t.Chdir("../../..")
	34	+	logger := slog.New(slog.DiscardHandler)
	35	+	cfg := &shared.ConfigSite{
	36	+	Domain: "pastes.test", Protocol: "https", Space: "pastes", Logger: logger,
	37	+	}
	38	+	now := time.Now()
	39	+	text := strings.Repeat("x", shared.MB) + "\n<script>alert(1)</script>\n<img src=x onerror=alert(1)>\n& \" '"
	40	+	dbpool := &pasteDB{
	41	+	StubDB: stub.NewStubDB(logger),
	42	+	post: &db.Post{
	43	+	Filename: "test.html", Slug: "test.html", Username: "tester",
	44	+	Text: text, FileSize: len(text), PublishAt: &now, UpdatedAt: &now,
	45	+	},
	46	+	}
	47	+	api := &router.ApiConfig{Cfg: cfg, Dbpool: dbpool}
	48	+	handler := router.CreateServeBasic(createSubdomainRoutes(nil), api.CreateCtx(context.Background(), "tester"))
	49	+
	50	+	page := httptest.NewRecorder()
	51	+	handler(page, httptest.NewRequest(http.MethodGet, "https://tester.pastes.test/test.html", nil))
	52	+	if page.Code != http.StatusOK {
	53	+	t.Fatalf("page status = %d", page.Code)
	54	+	}
	55	+	want := "<pre>" + template.HTMLEscapeString(text) + "</pre>"
	56	+	if !strings.Contains(page.Body.String(), want) {
	57	+	t.Fatal("page does not contain the escaped paste in a pre element")
	58	+	}
	59	+	if strings.Contains(page.Body.String(), "<script>alert(1)</script>") {
	60	+	t.Fatal("page contains the uploaded script as executable HTML")
	61	+	}
	62	+
	63	+	raw := httptest.NewRecorder()
	64	+	handler(raw, httptest.NewRequest(http.MethodGet, "https://tester.pastes.test/test.html/raw", nil))
	65	+	if raw.Code != http.StatusOK \|\| raw.Body.String() != text {
	66	+	t.Fatal("raw response does not preserve the uploaded text")
	67	+	}
	68	+	}

...	...		@@ -0,0 +1,27 @@
	1	+	package pastes
	2	+
	3	+	import (
	4	+	"strings"
	5	+
	6	+	"github.com/picosh/pico/pkg/shared"
	7	+	)
	8	+
	9	+	func NewConfigSite(service string) *shared.ConfigSite {
	10	+	debug := shared.GetEnv("PASTES_DEBUG", "0")
	11	+	domain := shared.GetEnv("PASTES_DOMAIN", "pastes.sh")
	12	+	port := shared.GetEnv("PASTES_WEB_PORT", "3000")
	13	+	dbURL := shared.GetEnv("DATABASE_URL", "")
	14	+	protocol := shared.GetEnv("PASTES_PROTOCOL", "https")
	15	+	withPipe := strings.ToLower(shared.GetEnv("PICO_PIPE_ENABLED", "true")) == "true"
	16	+
	17	+	return &shared.ConfigSite{
	18	+	Debug: debug == "1",
	19	+	Domain: domain,
	20	+	Port: port,
	21	+	Protocol: protocol,
	22	+	DbURL: dbURL,
	23	+	Space: "pastes",
	24	+	Logger: shared.CreateLogger(service, withPipe),
	25	+	MaxAssetSize: int64(3 * shared.MB),
	26	+	}
	27	+	}

...	...		@@ -0,0 +1,39 @@
	1	+	package pastes
	2	+
	3	+	import (
	4	+	"time"
	5	+
	6	+	"github.com/picosh/pico/pkg/db"
	7	+	"github.com/picosh/pico/pkg/shared"
	8	+	)
	9	+
	10	+	func deleteExpiredPosts(cfg *shared.ConfigSite, dbpool db.DB) error {
	11	+	cfg.Logger.Info("checking for expired posts")
	12	+	posts, err := dbpool.FindExpiredPosts(cfg.Space)
	13	+	if err != nil {
	14	+	return err
	15	+	}
	16	+
	17	+	postIds := []string{}
	18	+	for _, post := range posts {
	19	+	postIds = append(postIds, post.ID)
	20	+	}
	21	+
	22	+	cfg.Logger.Info("deleting expired posts", "len", len(postIds))
	23	+	err = dbpool.RemovePosts(postIds)
	24	+	if err != nil {
	25	+	return err
	26	+	}
	27	+
	28	+	return nil
	29	+	}
	30	+
	31	+	func CronDeleteExpiredPosts(cfg *shared.ConfigSite, dbpool db.DB) {
	32	+	for {
	33	+	err := deleteExpiredPosts(cfg, dbpool)
	34	+	if err != nil {
	35	+	cfg.Logger.Error(err.Error())
	36	+	}
	37	+	time.Sleep(12 * time.Hour)
	38	+	}
	39	+	}

...	...		@@ -0,0 +1,19 @@
	1	+	{{define "base"}}
	2	+	<!doctype html>
	3	+	<html lang="en">
	4	+	<head>
	5	+	<meta charset='utf-8'>
	6	+	<meta name="viewport" content="width=device-width, initial-scale=1" />
	7	+	<title>{{template "title" .}}</title>
	8	+
	9	+	<link rel="icon" type="image/png" sizes="16x16" href="/favicon-16x16.png">
	10	+
	11	+	<meta name="keywords" content="pastebin, paste, copy, snippets" />
	12	+	{{template "meta" .}}
	13	+
	14	+	<link rel="stylesheet" href="/smol.css" />
	15	+	<link rel="stylesheet" href="/main.css" />
	16	+	</head>
	17	+	<body {{template "attrs" .}}>{{template "body" .}}</body>
	18	+	</html>
	19	+	{{end}}

...	...		@@ -0,0 +1,46 @@
	1	+	{{template "base" .}}
	2	+
	3	+	{{define "title"}}{{.PageTitle}}{{end}}
	4	+
	5	+	{{define "meta"}}
	6	+	<meta name="description" content="{{if .Header.Bio}}{{.Header.Bio}}{{else}}{{.Header.Title}}{{end}}" />
	7	+
	8	+	<meta property="og:type" content="website">
	9	+	<meta property="og:site_name" content="{{.Site.Domain}}">
	10	+	<meta property="og:url" content="{{.URL}}">
	11	+	<meta property="og:title" content="{{.Header.Title}}">
	12	+	{{if .Header.Bio}}<meta property="og:description" content="{{.Header.Bio}}">{{end}}
	13	+	<meta property="og:image:width" content="300" />
	14	+	<meta property="og:image:height" content="300" />
	15	+	<meta itemprop="image" content="https://{{.Site.Domain}}/card.png" />
	16	+	<meta property="og:image" content="https://{{.Site.Domain}}/card.png" />
	17	+
	18	+	<meta property="twitter:card" content="summary">
	19	+	<meta property="twitter:url" content="{{.URL}}">
	20	+	<meta property="twitter:title" content="{{.Header.Title}}">
	21	+	{{if .Header.Bio}}<meta property="twitter:description" content="{{.Header.Bio}}">{{end}}
	22	+	<meta name="twitter:image" content="https://{{.Site.Domain}}/card.png" />
	23	+	<meta name="twitter:image:src" content="https://{{.Site.Domain}}/card.png" />
	24	+	{{end}}
	25	+
	26	+	{{define "attrs"}}{{end}}
	27	+
	28	+	{{define "body"}}
	29	+	<header class="text-center">
	30	+	<h1 class="text-2xl">{{.Header.Title}}</h1>
	31	+	<hr class="my-2" />
	32	+	</header>
	33	+	<main>
	34	+	<section class="posts">
	35	+	{{range .Posts}}
	36	+	<article class="my">
	37	+	<div class="flex items-center">
	38	+	<time datetime="{{.PublishAtISO}}" class="font-italic text-sm post-date">{{.PublishAt}}</time>
	39	+	<div class="text-md flex-1 transform-none m-0"><a href="{{.URL}}">{{.Title}}</a></div>
	40	+	</div>
	41	+	</article>
	42	+	{{end}}
	43	+	</section>
	44	+	</main>
	45	+	{{template "footer" .}}
	46	+	{{end}}

...	...		@@ -0,0 +1,6 @@
	1	+	{{define "footer"}}
	2	+	<footer>
	3	+	<hr class="my-2" />
	4	+	published with <a href={{.Site.HomeURL}}>{{.Site.Domain}}</a>
	5	+	</footer>
	6	+	{{end}}

...	...		@@ -0,0 +1,6 @@
	1	+	{{define "marketing-footer"}}
	2	+	<footer>
	3	+	<hr />
	4	+	<p class="font-italic">Built and maintained by <a href="https://pico.sh">pico.sh</a>.</p>
	5	+	</footer>
	6	+	{{end}}

...	...		@@ -0,0 +1,39 @@
	1	+	{{template "base" .}}
	2	+
	3	+	{{define "title"}}{{.Site.Domain}} -- a pastebin for hackers{{end}}
	4	+
	5	+	{{define "meta"}}
	6	+	<meta name="description" content="a pastebin for hackers" />
	7	+
	8	+	<meta property="og:type" content="website">
	9	+	<meta property="og:site_name" content="{{.Site.Domain}}">
	10	+	<meta property="og:url" content="https://{{.Site.Domain}}">
	11	+	<meta property="og:title" content="{{.Site.Domain}}">
	12	+	<meta property="og:description" content="a pastebin for hackers">
	13	+
	14	+	<meta name="twitter:card" content="summary" />
	15	+	<meta property="twitter:url" content="https://{{.Site.Domain}}">
	16	+	<meta property="twitter:title" content="{{.Site.Domain}}">
	17	+	<meta property="twitter:description" content="a pastebin for hackers">
	18	+	<meta name="twitter:image" content="https://{{.Site.Domain}}/card.png" />
	19	+	<meta name="twitter:image:src" content="https://{{.Site.Domain}}/card.png" />
	20	+
	21	+	<meta property="og:image:width" content="300" />
	22	+	<meta property="og:image:height" content="300" />
	23	+	<meta itemprop="image" content="https://{{.Site.Domain}}/card.png" />
	24	+	<meta property="og:image" content="https://{{.Site.Domain}}/card.png" />
	25	+	{{end}}
	26	+
	27	+	{{define "attrs"}}{{end}}
	28	+
	29	+	{{define "body"}}
	30	+	<header class="text-center">
	31	+	<h1 class="text-2xl font-bold">{{.Site.Domain}}</h1>
	32	+	<p class="text-lg">a pastebin for hackers</p>
	33	+	<div>
	34	+	<a href="https://pico.sh/pastes" class="btn-link mt inline-block">LEARN MORE</a>
	35	+	</div>
	36	+	</header>
	37	+
	38	+	{{template "marketing-footer" .}}
	39	+	{{end}}

...	...		@@ -0,0 +1,62 @@
	1	+	{{template "base" .}}
	2	+
	3	+	{{define "title"}}{{.PageTitle}}{{end}}
	4	+
	5	+	{{define "meta"}}
	6	+	<meta property="og:type" content="website">
	7	+	<meta property="og:site_name" content="{{.Site.Domain}}">
	8	+	<meta property="og:url" content="{{.URL}}">
	9	+	<meta property="og:title" content="{{.Title}}">
	10	+	<meta property="og:image:width" content="300" />
	11	+	<meta property="og:image:height" content="300" />
	12	+	<meta itemprop="image" content="https://{{.Site.Domain}}/card.png" />
	13	+	<meta property="og:image" content="https://{{.Site.Domain}}/card.png" />
	14	+
	15	+	<meta property="twitter:card" content="summary">
	16	+	<meta property="twitter:url" content="{{.URL}}">
	17	+	<meta property="twitter:title" content="{{.Title}}">
	18	+	<meta name="twitter:image" content="https://{{.Site.Domain}}/card.png" />
	19	+	<meta name="twitter:image:src" content="https://{{.Site.Domain}}/card.png" />
	20	+
	21	+	<link rel="stylesheet" href="/syntax.css" />
	22	+	{{end}}
	23	+
	24	+	{{define "attrs"}}id="post"{{end}}
	25	+
	26	+	{{define "body"}}
	27	+	<header>
	28	+	<h1 class="text-2xl font-bold transform-none">{{.Title}}</h1>
	29	+	<p class="font-bold m-0">
	30	+	{{if .Unlisted}} <code>unlisted</code>{{end}}
	31	+	<time datetime="{{.PublishAtISO}}">{{.PublishAt}}</time>
	32	+	<span> · <span>
	33	+	<a href="{{.BlogURL}}">{{.BlogName}}</a>
	34	+	<span> · <span>
	35	+	<a href="{{.RawURL}}">raw</a>
	36	+	</p>
	37	+	<p class="font-bold m-0">expires: {{.ExpiresAt}}</p>
	38	+	<script>
	39	+	function init() {
	40	+	const btn = document.querySelector("#copy-btn");
	41	+	const res = document.querySelector("#copy-result");
	42	+	const text = "{{.Text}}";
	43	+	btn.addEventListener("click", () => {
	44	+	res.innerHtml = "";
	45	+	navigator.clipboard.writeText(text)
	46	+	.then(() => res.innerHTML = "success!");
	47	+	})
	48	+	}
	49	+	document.addEventListener("DOMContentLoaded", init);
	50	+	</script>
	51	+	</header>
	52	+	<main>
	53	+	<div class="flex gap items-center">
	54	+	<button id="copy-btn" class="btn" style="background-color: transparent;">copy</button>
	55	+	<div id="copy-result" class=""></div>
	56	+	</div>
	57	+	<article>
	58	+	{{.Contents}}
	59	+	</article>
	60	+	</main>
	61	+	{{template "footer" .}}
	62	+	{{end}}

...	...		@@ -0,0 +1,34 @@
	1	+	package pastes
	2	+
	3	+	import (
	4	+	"bytes"
	5	+
	6	+	"github.com/alecthomas/chroma/v2/formatters/html"
	7	+	"github.com/alecthomas/chroma/v2/lexers"
	8	+	"github.com/alecthomas/chroma/v2/styles"
	9	+	)
	10	+
	11	+	func ParseText(filename string, text string) (string, error) {
	12	+	formatter := html.New(
	13	+	html.WithLineNumbers(true),
	14	+	html.WithLinkableLineNumbers(true, ""),
	15	+	html.WithClasses(true),
	16	+	)
	17	+	lexer := lexers.Match(filename)
	18	+	if lexer == nil {
	19	+	lexer = lexers.Analyse(text)
	20	+	}
	21	+	if lexer == nil {
	22	+	lexer = lexers.Get("plaintext")
	23	+	}
	24	+	iterator, err := lexer.Tokenise(nil, text)
	25	+	if err != nil {
	26	+	return text, err
	27	+	}
	28	+	var buf bytes.Buffer
	29	+	err = formatter.Format(&buf, styles.Get("dracula"), iterator)
	30	+	if err != nil {
	31	+	return text, err
	32	+	}
	33	+	return buf.String(), nil
	34	+	}

Binaries are not rendered as diffs.

Binaries are not rendered as diffs.

Binaries are not rendered as diffs.

Binaries are not rendered as diffs.

...	...		@@ -0,0 +1,11 @@
	1	+	body {
	2	+	max-width: 1025px;
	3	+	}
	4	+
	5	+	.post-date {
	6	+	width: 110px;
	7	+	}
	8	+
	9	+	.transform-none {
	10	+	text-transform: none;
	11	+	}

...	...		@@ -0,0 +1,2 @@
	1	+	User-agent: *
	2	+	Allow: /

...	...		@@ -0,0 +1,875 @@
	1	+	*,
	2	+	::before,
	3	+	::after {
	4	+	box-sizing: border-box;
	5	+	}
	6	+
	7	+	::-moz-focus-inner {
	8	+	border-style: none;
	9	+	padding: 0;
	10	+	}
	11	+	:-moz-focusring {
	12	+	outline: 1px dotted ButtonText;
	13	+	}
	14	+	:-moz-ui-invalid {
	15	+	box-shadow: none;
	16	+	}
	17	+
	18	+	:root {
	19	+	--line-height: 1.3rem;
	20	+	--grid-height: 0.65rem;
	21	+	--font-mono: ui-monospace, SFMono-Regular, "SF Mono", Menlo, Consolas, "Liberation Mono", monospace;
	22	+	}
	23	+
	24	+	@media (prefers-color-scheme: light) {
	25	+	:root {
	26	+	--white: #2e3f53;
	27	+	--white-light: #cfe0f4;
	28	+	--white-dark: #6c6a6a;
	29	+	--code: #52576f;
	30	+	--pre: #e1e7ee;
	31	+	--bg-color: #f4f4f4;
	32	+	--text-color: #24292f;
	33	+	--link-color: #005cc5;
	34	+	--visited: #6f42c1;
	35	+	--blockquote: #005cc5;
	36	+	--blockquote-bg: #cfe0f4;
	37	+	--hover: #c11e7a;
	38	+	--grey: #ccc;
	39	+	--grey-light: #6a708e;
	40	+	--green: #1a7f37;
	41	+	}
	42	+	}
	43	+
	44	+	@media (prefers-color-scheme: dark) {
	45	+	:root {
	46	+	--white: #f2f2f2;
	47	+	--white-light: #f2f2f2;
	48	+	--white-dark: #e8e8e8;
	49	+	--code: #414558;
	50	+	--pre: #252525;
	51	+	--bg-color: #282a36;
	52	+	--text-color: #f2f2f2;
	53	+	--link-color: #8be9fd;
	54	+	--visited: #bd93f9;
	55	+	--blockquote: #bd93f9;
	56	+	--blockquote-bg: #353548;
	57	+	--hover: #ff80bf;
	58	+	--grey: #414558;
	59	+	--grey-light: #6a708e;
	60	+	--green: #50fa7b;
	61	+	}
	62	+	}
	63	+
	64	+	html {
	65	+	background-color: var(--bg-color);
	66	+	color: var(--text-color);
	67	+	line-height: var(--line-height);
	68	+	font-family:
	69	+	-apple-system,
	70	+	BlinkMacSystemFont,
	71	+	"Segoe UI",
	72	+	Roboto,
	73	+	Oxygen,
	74	+	Ubuntu,
	75	+	Cantarell,
	76	+	"Fira Sans",
	77	+	"Droid Sans",
	78	+	"Helvetica Neue",
	79	+	Arial,
	80	+	sans-serif,
	81	+	"Apple Color Emoji",
	82	+	"Segoe UI Emoji",
	83	+	"Segoe UI Symbol";
	84	+	-webkit-text-size-adjust: 100%;
	85	+	-moz-tab-size: 4;
	86	+	-o-tab-size: 4;
	87	+	tab-size: 4;
	88	+	}
	89	+
	90	+	body {
	91	+	margin: 0 auto;
	92	+	}
	93	+
	94	+	img {
	95	+	max-width: 100%;
	96	+	height: auto;
	97	+	}
	98	+
	99	+	b,
	100	+	strong {
	101	+	font-weight: bold;
	102	+	}
	103	+
	104	+	code,
	105	+	kbd,
	106	+	samp,
	107	+	pre {
	108	+	font-family: var(--font-mono);
	109	+	font-variant-numeric: tabular-nums;
	110	+	}
	111	+
	112	+	code,
	113	+	kbd,
	114	+	samp {
	115	+	border: 2px solid var(--code);
	116	+	}
	117	+
	118	+	pre > code {
	119	+	background-color: inherit;
	120	+	padding: 0;
	121	+	border: none;
	122	+	border-radius: 0;
	123	+	}
	124	+
	125	+	code {
	126	+	border-radius: 0.3rem;
	127	+	padding: 0.025rem 0.3rem;
	128	+	}
	129	+
	130	+	pre {
	131	+	border-radius: 1px;
	132	+	padding: var(--line-height);
	133	+	overflow-x: auto;
	134	+	background-color: var(--pre) !important;
	135	+	}
	136	+
	137	+	small {
	138	+	font-size: 0.8rem;
	139	+	}
	140	+
	141	+	details {
	142	+	border: 2px solid var(--grey-light);
	143	+	padding: calc(var(--grid-height) - 2px) 1ch;
	144	+	margin-bottom: var(--grid-height);
	145	+	}
	146	+
	147	+	details[open] summary {
	148	+	margin-bottom: var(--grid-height);
	149	+	}
	150	+
	151	+	summary {
	152	+	display: list-item;
	153	+	cursor: pointer;
	154	+	}
	155	+
	156	+	h1,
	157	+	h2,
	158	+	h3,
	159	+	h4,
	160	+	h5,
	161	+	h6 {
	162	+	margin: 0;
	163	+	padding: 0;
	164	+	border: 0;
	165	+	font-family: var(--font-mono);
	166	+	font-weight: bold;
	167	+	}
	168	+
	169	+	h1 {
	170	+	font-size: 1.6rem;
	171	+	line-height: calc(var(--line-height) * 1.5);
	172	+	margin-top: calc(var(--line-height) * 2.5);
	173	+	margin-bottom: var(--grid-height);
	174	+	}
	175	+
	176	+	h2 {
	177	+	font-size: 1.3rem;
	178	+	line-height: var(--line-height);
	179	+	margin-top: calc(var(--line-height) * 2);
	180	+	margin-bottom: var(--grid-height);
	181	+	}
	182	+
	183	+	h3 {
	184	+	font-size: 1.1rem;
	185	+	line-height: var(--line-height);
	186	+	margin-top: calc(var(--line-height) * 1.5);
	187	+	margin-bottom: var(--grid-height);
	188	+	}
	189	+
	190	+	h4,
	191	+	h5,
	192	+	h6 {
	193	+	font-size: 1rem;
	194	+	line-height: var(--line-height);
	195	+	margin-top: var(--line-height);
	196	+	margin-bottom: var(--grid-height);
	197	+	color: var(--white-dark);
	198	+	}
	199	+
	200	+
	201	+	path {
	202	+	fill: var(--text-color);
	203	+	stroke: var(--text-color);
	204	+	}
	205	+
	206	+	hr {
	207	+	color: inherit;
	208	+	border: 0;
	209	+	height: 1px;
	210	+	background: var(--grey-light);
	211	+	margin: calc(var(--grid-height) - 1px) auto;
	212	+	}
	213	+
	214	+	hr.dashed {
	215	+	border: none;
	216	+	border-top: 1px dashed var(--grey-light);
	217	+	background: none;
	218	+	height: 0;
	219	+	margin: calc(var(--grid-height) - 1px) auto;
	220	+	}
	221	+
	222	+	a {
	223	+	color: var(--link-color);
	224	+	text-decoration: underline;
	225	+	text-decoration-thickness: 1px;
	226	+	text-underline-offset: 2px;
	227	+	padding: 0 2px;
	228	+	border-radius: 1px;
	229	+	transition: background-color 0.1s ease, color 0.1s ease;
	230	+	}
	231	+
	232	+	a:hover {
	233	+	background-color: var(--link-color);
	234	+	color: var(--bg-color);
	235	+	text-decoration: none;
	236	+	}
	237	+
	238	+	a:visited {
	239	+	color: var(--visited);
	240	+	}
	241	+
	242	+	a:visited:hover {
	243	+	background-color: var(--visited);
	244	+	color: var(--bg-color);
	245	+	text-decoration: none;
	246	+	}
	247	+
	248	+	section {
	249	+	margin-bottom: 1.4rem;
	250	+	}
	251	+
	252	+	section:last-child {
	253	+	margin-bottom: 0;
	254	+	}
	255	+
	256	+	header {
	257	+	margin: 1rem auto;
	258	+	}
	259	+
	260	+	p {
	261	+	margin-top: var(--line-height);
	262	+	margin-bottom: var(--line-height);
	263	+	}
	264	+
	265	+	article {
	266	+	overflow-wrap: break-word;
	267	+	}
	268	+
	269	+	blockquote {
	270	+	border-left: 5px solid var(--blockquote);
	271	+	background-color: var(--blockquote-bg);
	272	+	padding: var(--grid-height);
	273	+	margin: var(--line-height) 0;
	274	+	}
	275	+
	276	+	blockquote > p {
	277	+	margin: 0;
	278	+	}
	279	+
	280	+	blockquote code {
	281	+	border: 1px solid var(--blockquote);
	282	+	}
	283	+
	284	+	ul,
	285	+	ol {
	286	+	padding: 0 15px;
	287	+	list-style-position: inside;
	288	+	list-style-type: square;
	289	+	margin: var(--grid-height) 0;
	290	+	}
	291	+
	292	+	ul[style*="list-style-type: none;"] {
	293	+	padding: 0;
	294	+	}
	295	+
	296	+	ol ul, ol ol, ul ol, ul ul {
	297	+	padding: 0 0 0 var(--line-height);
	298	+	margin: 0;
	299	+	}
	300	+
	301	+	li {
	302	+	margin: var(--grid-height) 0;
	303	+	padding: 0;
	304	+	}
	305	+
	306	+	li::marker {
	307	+	line-height: 0;
	308	+	color: var(--grey-light);
	309	+	}
	310	+
	311	+	li blockquote,
	312	+	li pre {
	313	+	display: inline-block;
	314	+	margin: 0;
	315	+	width: 100%;
	316	+	}
	317	+
	318	+	li pre {
	319	+	padding: var(--grid-height);
	320	+	}
	321	+
	322	+	[role="list"] {
	323	+	display: flex;
	324	+	flex-direction: column;
	325	+	gap: var(--grid-height);
	326	+	}
	327	+
	328	+	[role="list"] [role="list"] {
	329	+	margin-left: 1.3rem;
	330	+	}
	331	+
	332	+	[role="listitem"] {
	333	+	display: flex;
	334	+	align-items: flex-start;
	335	+	gap: 5px;
	336	+	}
	337	+
	338	+	[role="listitem"]:has(pre, img, blockquote) {
	339	+	align-items: center;
	340	+	}
	341	+
	342	+	table {
	343	+	border-collapse: collapse;
	344	+	margin: var(--line-height) 0;
	345	+	}
	346	+
	347	+	th,
	348	+	td {
	349	+	border: 1px solid var(--white);
	350	+	padding: var(--grid-height);
	351	+	}
	352	+
	353	+	footer {
	354	+	text-align: center;
	355	+	margin-bottom: calc(var(--line-height) * 3);
	356	+	}
	357	+
	358	+	dt {
	359	+	font-weight: bold;
	360	+	}
	361	+
	362	+	dd {
	363	+	margin-left: 0;
	364	+	}
	365	+
	366	+	dd:not(:last-child) {
	367	+	margin-bottom: 0.5rem;
	368	+	}
	369	+
	370	+	figure {
	371	+	margin: 0;
	372	+	}
	373	+
	374	+	sup {
	375	+	line-height: 0;
	376	+	}
	377	+
	378	+	#toc {
	379	+	margin-top: var(--line-height);
	380	+	}
	381	+
	382	+	.container {
	383	+	max-width: 50em;
	384	+	width: 100%;
	385	+	}
	386	+
	387	+	.container-sm {
	388	+	max-width: 40em;
	389	+	width: 100%;
	390	+	}
	391	+
	392	+	.mono,
	393	+	time,
	394	+	nav {
	395	+	font-family: var(--font-mono);
	396	+	font-variant-numeric: tabular-nums;
	397	+	}
	398	+
	399	+	.link-alt-hover,
	400	+	.link-alt-hover:visited,
	401	+	.link-alt-hover:visited:hover,
	402	+	.link-alt-hover:hover {
	403	+	color: var(--hover);
	404	+	text-decoration: none;
	405	+	}
	406	+
	407	+	.link-alt-hover:visited:hover,
	408	+	.link-alt-hover:hover {
	409	+	background-color: transparent;
	410	+	text-decoration: underline;
	411	+	}
	412	+
	413	+	.link-alt,
	414	+	.link-alt:visited,
	415	+	.link-alt:visited:hover,
	416	+	.link-alt:hover {
	417	+	color: var(--white);
	418	+	text-decoration: none;
	419	+	}
	420	+
	421	+	.link-alt:visited:hover,
	422	+	.link-alt:hover {
	423	+	background-color: transparent;
	424	+	text-decoration: underline;
	425	+	}
	426	+
	427	+	.link-grey,
	428	+	.link-grey:visited {
	429	+	color: var(--grey-light);
	430	+	text-decoration: none;
	431	+	}
	432	+
	433	+	.link-grey:hover,
	434	+	.link-grey:visited:hover {
	435	+	background-color: transparent;
	436	+	color: var(--text-color);
	437	+	text-decoration: underline;
	438	+	}
	439	+
	440	+	.text-2xl code, .text-xl code, .text-lg code, .text-md code {
	441	+	text-transform: none;
	442	+	}
	443	+
	444	+	.text-2xl {
	445	+	font-size: 1.6rem;
	446	+	font-weight: bold;
	447	+	line-height: calc(var(--line-height) * 1.5);
	448	+	margin-bottom: var(--grid-height);
	449	+	}
	450	+
	451	+	.text-xl {
	452	+	font-size: 1.3rem;
	453	+	font-weight: bold;
	454	+	line-height: var(--line-height);
	455	+	margin-bottom: var(--grid-height);
	456	+	}
	457	+
	458	+	.text-lg {
	459	+	font-size: 1.1rem;
	460	+	font-weight: bold;
	461	+	line-height: var(--line-height);
	462	+	margin-bottom: var(--grid-height);
	463	+	}
	464	+
	465	+	.text-md {
	466	+	font-size: 1rem;
	467	+	font-weight: bold;
	468	+	line-height: var(--line-height);
	469	+	margin-bottom: var(--grid-height);
	470	+	}
	471	+
	472	+	.text-right {
	473	+	text-align: right;
	474	+	}
	475	+
	476	+	.text-sm {
	477	+	font-size: 0.8rem;
	478	+	}
	479	+
	480	+	.w-full {
	481	+	width: 100%;
	482	+	}
	483	+
	484	+	.border {
	485	+	border: 2px solid var(--grey-light);
	486	+	}
	487	+
	488	+	.text-left {
	489	+	text-align: left;
	490	+	}
	491	+
	492	+	.text-center {
	493	+	text-align: center;
	494	+	}
	495	+
	496	+	.text-underline {
	497	+	text-decoration: underline;
	498	+	text-decoration-thickness: 2px;
	499	+	}
	500	+
	501	+	.text-hdr {
	502	+	color: var(--hover);
	503	+	}
	504	+
	505	+	.font-bold {
	506	+	font-weight: bold;
	507	+	}
	508	+
	509	+	.font-italic {
	510	+	font-style: italic;
	511	+	}
	512	+
	513	+	.inline {
	514	+	display: inline;
	515	+	}
	516	+
	517	+	.inline-block {
	518	+	display: inline-block;
	519	+	}
	520	+
	521	+	.max-w-half {
	522	+	max-width: 50%;
	523	+	}
	524	+
	525	+	.flex {
	526	+	display: flex;
	527	+	}
	528	+
	529	+	.flex-col {
	530	+	flex-direction: column;
	531	+	}
	532	+
	533	+	.flex-wrap {
	534	+	flex-wrap: wrap;
	535	+	}
	536	+
	537	+	.items-center {
	538	+	align-items: center;
	539	+	}
	540	+
	541	+	.m-0 {
	542	+	margin: 0;
	543	+	}
	544	+
	545	+	.mt-0 {
	546	+	margin-top: 0;
	547	+	}
	548	+
	549	+	.mt {
	550	+	margin-top: var(--grid-height);
	551	+	}
	552	+
	553	+	.mt-2 {
	554	+	margin-top: var(--line-height);
	555	+	}
	556	+
	557	+	.mt-4 {
	558	+	margin-top: calc(var(--line-height) * 2);
	559	+	}
	560	+
	561	+	.mb {
	562	+	margin-bottom: var(--grid-height);
	563	+	}
	564	+
	565	+	.mb-2 {
	566	+	margin-bottom: var(--line-height);
	567	+	}
	568	+
	569	+	.mb-4 {
	570	+	margin-bottom: calc(var(--line-height) * 2);
	571	+	}
	572	+
	573	+	.mr {
	574	+	margin-right: 0.5rem;
	575	+	}
	576	+
	577	+	.ml {
	578	+	margin-left: 0.5rem;
	579	+	}
	580	+
	581	+	.my {
	582	+	margin-top: var(--grid-height);
	583	+	margin-bottom: var(--grid-height);
	584	+	}
	585	+
	586	+	.my-2 {
	587	+	margin-top: var(--line-height);
	588	+	margin-bottom: var(--line-height);
	589	+	}
	590	+
	591	+	.my-4 {
	592	+	margin-top: calc(var(--line-height) * 2);
	593	+	margin-bottom: calc(var(--line-height) * 2);
	594	+	}
	595	+
	596	+	.p-0 {
	597	+	padding: 0;
	598	+	}
	599	+
	600	+	.px {
	601	+	padding-left: 0.5rem;
	602	+	padding-right: 0.5rem;
	603	+	}
	604	+
	605	+	.px-4 {
	606	+	padding-left: 2rem;
	607	+	padding-right: 2rem;
	608	+	}
	609	+
	610	+	.py {
	611	+	padding-top: var(--grid-height);
	612	+	padding-bottom: var(--grid-height);
	613	+	}
	614	+
	615	+	.py-4 {
	616	+	padding-top: calc(var(--line-height) * 2);
	617	+	padding-bottom: calc(var(--line-height) * 2);
	618	+	}
	619	+
	620	+	.justify-between {
	621	+	justify-content: space-between;
	622	+	}
	623	+
	624	+	.justify-center {
	625	+	justify-content: center;
	626	+	}
	627	+
	628	+	.gap {
	629	+	gap: var(--grid-height);
	630	+	}
	631	+
	632	+	.gap-2 {
	633	+	gap: var(--line-height);
	634	+	}
	635	+
	636	+	.group {
	637	+	display: flex;
	638	+	flex-direction: column;
	639	+	gap: var(--grid-height);
	640	+	}
	641	+
	642	+	.group-2 {
	643	+	display: flex;
	644	+	flex-direction: column;
	645	+	gap: var(--line-height);
	646	+	}
	647	+
	648	+	.group-h {
	649	+	display: flex;
	650	+	gap: var(--grid-height);
	651	+	align-items: center;
	652	+	}
	653	+
	654	+	.flex-1 {
	655	+	flex: 1;
	656	+	}
	657	+
	658	+	.items-end {
	659	+	align-items: end;
	660	+	}
	661	+
	662	+	.items-start {
	663	+	align-items: start;
	664	+	}
	665	+
	666	+	.justify-end {
	667	+	justify-content: end;
	668	+	}
	669	+
	670	+	.font-grey-light {
	671	+	color: var(--grey-light);
	672	+	}
	673	+
	674	+	.hidden {
	675	+	display: none;
	676	+	}
	677	+
	678	+	.align-right {
	679	+	text-align: right;
	680	+	}
	681	+
	682	+	.text-transform-none {
	683	+	text-transform: none;
	684	+	}
	685	+
	686	+	/* ==== MARKDOWN ==== */
	687	+
	688	+	.md > :first-child {
	689	+	margin-top: 0 !important;
	690	+	}
	691	+
	692	+	.md h1,
	693	+	.md h2,
	694	+	.md h3,
	695	+	.md h4 {
	696	+	padding: 0;
	697	+	font-weight: bold;
	698	+	}
	699	+
	700	+	.md h1 a,
	701	+	.md h2 a,
	702	+	.md h3 a,
	703	+	.md h4 a {
	704	+	color: var(--grey-light);
	705	+	text-decoration: none;
	706	+	}
	707	+
	708	+	.md h1 a:hover,
	709	+	.md h2 a:hover,
	710	+	.md h3 a:hover,
	711	+	.md h4 a:hover {
	712	+	background-color: transparent;
	713	+	color: var(--hover);
	714	+	text-decoration: underline;
	715	+	}
	716	+
	717	+	h1 code, h2 code, h3 code, h4 code {
	718	+	text-transform: none;
	719	+	}
	720	+
	721	+	/* ==== HELPERS ==== */
	722	+
	723	+	.logo-header {
	724	+	line-height: 1;
	725	+	display: inline-block;
	726	+	background-color: #ff79c6;
	727	+	background-image: linear-gradient(to right, #ff5555, #ff79c6, #f8f859);
	728	+	color: transparent;
	729	+	background-clip: text;
	730	+	border: 3px solid #ff79c6;
	731	+	padding: 8px 10px 10px 10px;
	732	+	border-radius: 10px;
	733	+	background-size: 100%;
	734	+	margin: 0;
	735	+	-webkit-background-clip: text;
	736	+	-moz-background-clip: text;
	737	+	-webkit-text-fill-color: transparent;
	738	+	-moz-text-fill-color: transparent;
	739	+	}
	740	+
	741	+	.btn {
	742	+	border: 2px solid var(--link-color);
	743	+	color: var(--link-color);
	744	+	padding: 0.4rem 1rem;
	745	+	font-weight: bold;
	746	+	display: inline-block;
	747	+	}
	748	+
	749	+	.btn:hover {
	750	+	background-color: var(--link-color);
	751	+	color: var(--bg-color);
	752	+	}
	753	+
	754	+	.btn-link,
	755	+	.btn-link:visited {
	756	+	border: 2px solid var(--link-color);
	757	+	color: var(--link-color);
	758	+	padding: var(--grid-height);
	759	+	text-decoration: none;
	760	+	font-weight: bold;
	761	+	display: inline-block;
	762	+	}
	763	+
	764	+	.btn-link:hover,
	765	+	.btn-link:visited:hover {
	766	+	background-color: var(--link-color);
	767	+	color: var(--bg-color);
	768	+	text-decoration: none;
	769	+	}
	770	+
	771	+	.box {
	772	+	border: 1px solid var(--grey-light);
	773	+	padding: var(--grid-height);
	774	+	background-color: var(--pre);
	775	+	}
	776	+
	777	+	.box-sm {
	778	+	border: 1px solid var(--grey-light);
	779	+	padding: var(--grid-height);
	780	+	background-color: var(--pre);
	781	+	}
	782	+
	783	+	.box-dashed {
	784	+	border: 1px dashed var(--grey-light);
	785	+	padding: var(--grid-height);
	786	+	}
	787	+
	788	+	.box-alert {
	789	+	border: 1px solid var(--hover);
	790	+	padding: var(--line-height);
	791	+	}
	792	+
	793	+	.box-sm-alert {
	794	+	border: 1px solid var(--hover);
	795	+	padding: var(--grid-height);
	796	+	}
	797	+
	798	+	.text-green {
	799	+	color: var(--green);
	800	+	}
	801	+
	802	+	.prompt,
	803	+	pre.cmd::before {
	804	+	content: "$ ";
	805	+	color: var(--green);
	806	+	font-weight: bold;
	807	+	}
	808	+
	809	+	.list-none {
	810	+	list-style-type: none;
	811	+	}
	812	+
	813	+	.list-square {
	814	+	list-style-type: square;
	815	+	}
	816	+
	817	+	.list-disc {
	818	+	list-style-type: disc;
	819	+	}
	820	+
	821	+	.list-decimal {
	822	+	list-style-type: decimal;
	823	+	}
	824	+
	825	+	.pill {
	826	+	border: 1px solid var(--link-color);
	827	+	color: var(--link-color);
	828	+	}
	829	+
	830	+	.pill-alert {
	831	+	border: 1px solid var(--hover);
	832	+	color: var(--hover);
	833	+	}
	834	+
	835	+	.pill-info {
	836	+	border: 1px solid var(--visited);
	837	+	color: var(--visited);
	838	+	}
	839	+
	840	+	@media only screen and (max-width: 40em) {
	841	+	body {
	842	+	padding: 0 var(--grid-height);
	843	+	}
	844	+
	845	+	header {
	846	+	margin: 0;
	847	+	}
	848	+
	849	+	.flex-collapse {
	850	+	flex-direction: column;
	851	+	}
	852	+	}
	853	+
	854	+	body {
	855	+	position: relative;
	856	+	}
	857	+
	858	+	/*
	859	+	.debug-grid {
	860	+	width: 100%;
	861	+	height: 100%;
	862	+	position: absolute;
	863	+	top: 0;
	864	+	left: 0;
	865	+	right: 0;
	866	+	bottom: 0;
	867	+	z-index: 1000;
	868	+	pointer-events: none;
	869	+	background-image:
	870	+	repeating-linear-gradient(rgba(255, 121, 198, 0.18) 0 1px, transparent 1px 100%),
	871	+	repeating-linear-gradient(90deg, rgba(139, 233, 253, 0.10) 0 1px, transparent 1px 100%);
	872	+	background-size: 1ch var(--grid-height);
	873	+	margin: 0;
	874	+	}
	875	+	*/

...	...		@@ -0,0 +1,700 @@
	1	+	@media (prefers-color-scheme: light) {
	2	+	/* Background */
	3	+	.bg {
	4	+	background-color: #ffffff;
	5	+	}
	6	+	/* PreWrapper */
	7	+	.chroma {
	8	+	background-color: #ffffff;
	9	+	}
	10	+	/* Other */
	11	+	.chroma .x {
	12	+	}
	13	+	/* Error */
	14	+	.chroma .err {
	15	+	background-color: #a848a8;
	16	+	}
	17	+	/* CodeLine */
	18	+	.chroma .cl {
	19	+	}
	20	+	/* LineTableTD */
	21	+	.chroma .lntd {
	22	+	vertical-align: top;
	23	+	padding: 0;
	24	+	margin: 0;
	25	+	border: 0;
	26	+	}
	27	+	/* LineTable */
	28	+	.chroma .lntable {
	29	+	border-spacing: 0;
	30	+	padding: 0;
	31	+	margin: 0;
	32	+	border: 0;
	33	+	}
	34	+	/* LineHighlight */
	35	+	.chroma .hl {
	36	+	background-color: #ffffcc;
	37	+	}
	38	+	/* LineNumbersTable */
	39	+	.chroma .lnt {
	40	+	white-space: pre;
	41	+	user-select: none;
	42	+	margin-right: 0.4em;
	43	+	padding: 0 0.4em 0 0.4em;
	44	+	color: #7f7f7f;
	45	+	}
	46	+	/* LineNumbers */
	47	+	.chroma .ln {
	48	+	white-space: pre;
	49	+	user-select: none;
	50	+	margin-right: 0.4em;
	51	+	padding: 0 0.4em 0 0.4em;
	52	+	color: #7f7f7f;
	53	+	}
	54	+	/* Line */
	55	+	.chroma .line {
	56	+	display: flex;
	57	+	}
	58	+	/* Keyword */
	59	+	.chroma .k {
	60	+	color: #2838b0;
	61	+	}
	62	+	/* KeywordConstant */
	63	+	.chroma .kc {
	64	+	color: #444444;
	65	+	font-style: italic;
	66	+	}
	67	+	/* KeywordDeclaration */
	68	+	.chroma .kd {
	69	+	color: #2838b0;
	70	+	font-style: italic;
	71	+	}
	72	+	/* KeywordNamespace */
	73	+	.chroma .kn {
	74	+	color: #2838b0;
	75	+	}
	76	+	/* KeywordPseudo */
	77	+	.chroma .kp {
	78	+	color: #2838b0;
	79	+	}
	80	+	/* KeywordReserved */
	81	+	.chroma .kr {
	82	+	color: #2838b0;
	83	+	}
	84	+	/* KeywordType */
	85	+	.chroma .kt {
	86	+	color: #2838b0;
	87	+	font-style: italic;
	88	+	}
	89	+	/* Name */
	90	+	.chroma .n {
	91	+	}
	92	+	/* NameAttribute */
	93	+	.chroma .na {
	94	+	color: #388038;
	95	+	}
	96	+	/* NameBuiltin */
	97	+	.chroma .nb {
	98	+	color: #388038;
	99	+	}
	100	+	/* NameBuiltinPseudo */
	101	+	.chroma .bp {
	102	+	font-style: italic;
	103	+	}
	104	+	/* NameClass */
	105	+	.chroma .nc {
	106	+	color: #287088;
	107	+	}
	108	+	/* NameConstant */
	109	+	.chroma .no {
	110	+	color: #b85820;
	111	+	}
	112	+	/* NameDecorator */
	113	+	.chroma .nd {
	114	+	color: #287088;
	115	+	}
	116	+	/* NameEntity */
	117	+	.chroma .ni {
	118	+	color: #709030;
	119	+	}
	120	+	/* NameException */
	121	+	.chroma .ne {
	122	+	color: #908828;
	123	+	}
	124	+	/* NameFunction */
	125	+	.chroma .nf {
	126	+	color: #785840;
	127	+	}
	128	+	/* NameFunctionMagic */
	129	+	.chroma .fm {
	130	+	color: #b85820;
	131	+	}
	132	+	/* NameLabel */
	133	+	.chroma .nl {
	134	+	color: #289870;
	135	+	}
	136	+	/* NameNamespace */
	137	+	.chroma .nn {
	138	+	color: #289870;
	139	+	}
	140	+	/* NameOther */
	141	+	.chroma .nx {
	142	+	}
	143	+	/* NameProperty */
	144	+	.chroma .py {
	145	+	}
	146	+	/* NameTag */
	147	+	.chroma .nt {
	148	+	color: #2838b0;
	149	+	}
	150	+	/* NameVariable */
	151	+	.chroma .nv {
	152	+	color: #b04040;
	153	+	}
	154	+	/* NameVariableClass */
	155	+	.chroma .vc {
	156	+	}
	157	+	/* NameVariableGlobal */
	158	+	.chroma .vg {
	159	+	color: #908828;
	160	+	}
	161	+	/* NameVariableInstance */
	162	+	.chroma .vi {
	163	+	}
	164	+	/* NameVariableMagic */
	165	+	.chroma .vm {
	166	+	color: #b85820;
	167	+	}
	168	+	/* Literal */
	169	+	.chroma .l {
	170	+	}
	171	+	/* LiteralDate */
	172	+	.chroma .ld {
	173	+	}
	174	+	/* LiteralString */
	175	+	.chroma .s {
	176	+	color: #b83838;
	177	+	}
	178	+	/* LiteralStringAffix */
	179	+	.chroma .sa {
	180	+	color: #444444;
	181	+	}
	182	+	/* LiteralStringBacktick */
	183	+	.chroma .sb {
	184	+	color: #b83838;
	185	+	}
	186	+	/* LiteralStringChar */
	187	+	.chroma .sc {
	188	+	color: #a848a8;
	189	+	}
	190	+	/* LiteralStringDelimiter */
	191	+	.chroma .dl {
	192	+	color: #b85820;
	193	+	}
	194	+	/* LiteralStringDoc */
	195	+	.chroma .sd {
	196	+	color: #b85820;
	197	+	font-style: italic;
	198	+	}
	199	+	/* LiteralStringDouble */
	200	+	.chroma .s2 {
	201	+	color: #b83838;
	202	+	}
	203	+	/* LiteralStringEscape */
	204	+	.chroma .se {
	205	+	color: #709030;
	206	+	}
	207	+	/* LiteralStringHeredoc */
	208	+	.chroma .sh {
	209	+	color: #b83838;
	210	+	}
	211	+	/* LiteralStringInterpol */
	212	+	.chroma .si {
	213	+	color: #b83838;
	214	+	text-decoration: underline;
	215	+	}
	216	+	/* LiteralStringOther */
	217	+	.chroma .sx {
	218	+	color: #a848a8;
	219	+	}
	220	+	/* LiteralStringRegex */
	221	+	.chroma .sr {
	222	+	color: #a848a8;
	223	+	}
	224	+	/* LiteralStringSingle */
	225	+	.chroma .s1 {
	226	+	color: #b83838;
	227	+	}
	228	+	/* LiteralStringSymbol */
	229	+	.chroma .ss {
	230	+	color: #b83838;
	231	+	}
	232	+	/* LiteralNumber */
	233	+	.chroma .m {
	234	+	color: #444444;
	235	+	}
	236	+	/* LiteralNumberBin */
	237	+	.chroma .mb {
	238	+	color: #444444;
	239	+	}
	240	+	/* LiteralNumberFloat */
	241	+	.chroma .mf {
	242	+	color: #444444;
	243	+	}
	244	+	/* LiteralNumberHex */
	245	+	.chroma .mh {
	246	+	color: #444444;
	247	+	}
	248	+	/* LiteralNumberInteger */
	249	+	.chroma .mi {
	250	+	color: #444444;
	251	+	}
	252	+	/* LiteralNumberIntegerLong */
	253	+	.chroma .il {
	254	+	color: #444444;
	255	+	}
	256	+	/* LiteralNumberOct */
	257	+	.chroma .mo {
	258	+	color: #444444;
	259	+	}
	260	+	/* Operator */
	261	+	.chroma .o {
	262	+	color: #666666;
	263	+	}
	264	+	/* OperatorWord */
	265	+	.chroma .ow {
	266	+	color: #a848a8;
	267	+	}
	268	+	/* Punctuation */
	269	+	.chroma .p {
	270	+	color: #888888;
	271	+	}
	272	+	/* Comment */
	273	+	.chroma .c {
	274	+	color: #888888;
	275	+	font-style: italic;
	276	+	}
	277	+	/* CommentHashbang */
	278	+	.chroma .ch {
	279	+	color: #287088;
	280	+	font-style: italic;
	281	+	}
	282	+	/* CommentMultiline */
	283	+	.chroma .cm {
	284	+	color: #888888;
	285	+	font-style: italic;
	286	+	}
	287	+	/* CommentSingle */
	288	+	.chroma .c1 {
	289	+	color: #888888;
	290	+	font-style: italic;
	291	+	}
	292	+	/* CommentSpecial */
	293	+	.chroma .cs {
	294	+	color: #888888;
	295	+	font-style: italic;
	296	+	}
	297	+	/* CommentPreproc */
	298	+	.chroma .cp {
	299	+	color: #289870;
	300	+	}
	301	+	/* CommentPreprocFile */
	302	+	.chroma .cpf {
	303	+	color: #289870;
	304	+	}
	305	+	/* Generic */
	306	+	.chroma .g {
	307	+	}
	308	+	/* GenericDeleted */
	309	+	.chroma .gd {
	310	+	color: #c02828;
	311	+	}
	312	+	/* GenericEmph */
	313	+	.chroma .ge {
	314	+	font-style: italic;
	315	+	}
	316	+	/* GenericError */
	317	+	.chroma .gr {
	318	+	color: #c02828;
	319	+	}
	320	+	/* GenericHeading */
	321	+	.chroma .gh {
	322	+	color: #666666;
	323	+	}
	324	+	/* GenericInserted */
	325	+	.chroma .gi {
	326	+	color: #388038;
	327	+	}
	328	+	/* GenericOutput */
	329	+	.chroma .go {
	330	+	color: #666666;
	331	+	}
	332	+	/* GenericPrompt */
	333	+	.chroma .gp {
	334	+	color: #444444;
	335	+	}
	336	+	/* GenericStrong */
	337	+	.chroma .gs {
	338	+	font-weight: bold;
	339	+	}
	340	+	/* GenericSubheading */
	341	+	.chroma .gu {
	342	+	color: #444444;
	343	+	}
	344	+	/* GenericTraceback */
	345	+	.chroma .gt {
	346	+	color: #2838b0;
	347	+	}
	348	+	/* GenericUnderline */
	349	+	.chroma .gl {
	350	+	text-decoration: underline;
	351	+	}
	352	+	/* TextWhitespace */
	353	+	.chroma .w {
	354	+	color: #a89028;
	355	+	}
	356	+	}
	357	+
	358	+	@media (prefers-color-scheme: dark) {
	359	+	/* Background */
	360	+	.bg {
	361	+	color: #f8f8f2;
	362	+	background-color: #282a36;
	363	+	}
	364	+	/* PreWrapper */
	365	+	.chroma {
	366	+	color: #f8f8f2;
	367	+	background-color: #282a36;
	368	+	}
	369	+	/* Other */
	370	+	.chroma .x {
	371	+	}
	372	+	/* Error */
	373	+	.chroma .err {
	374	+	}
	375	+	/* CodeLine */
	376	+	.chroma .cl {
	377	+	}
	378	+	/* LineTableTD */
	379	+	.chroma .lntd {
	380	+	vertical-align: top;
	381	+	padding: 0;
	382	+	margin: 0;
	383	+	border: 0;
	384	+	}
	385	+	/* LineTable */
	386	+	.chroma .lntable {
	387	+	border-spacing: 0;
	388	+	padding: 0;
	389	+	margin: 0;
	390	+	border: 0;
	391	+	}
	392	+	/* LineHighlight */
	393	+	.chroma .hl {
	394	+	background-color: #ffffcc;
	395	+	}
	396	+	/* LineNumbersTable */
	397	+	.chroma .lnt {
	398	+	white-space: pre;
	399	+	user-select: none;
	400	+	margin-right: 0.4em;
	401	+	padding: 0 0.4em 0 0.4em;
	402	+	color: #7f7f7f;
	403	+	}
	404	+	/* LineNumbers */
	405	+	.chroma .ln {
	406	+	white-space: pre;
	407	+	user-select: none;
	408	+	margin-right: 0.4em;
	409	+	padding: 0 0.4em 0 0.4em;
	410	+	color: #7f7f7f;
	411	+	}
	412	+	/* Line */
	413	+	.chroma .line {
	414	+	display: flex;
	415	+	}
	416	+	/* Keyword */
	417	+	.chroma .k {
	418	+	color: #ff79c6;
	419	+	}
	420	+	/* KeywordConstant */
	421	+	.chroma .kc {
	422	+	color: #ff79c6;
	423	+	}
	424	+	/* KeywordDeclaration */
	425	+	.chroma .kd {
	426	+	color: #8be9fd;
	427	+	font-style: italic;
	428	+	}
	429	+	/* KeywordNamespace */
	430	+	.chroma .kn {
	431	+	color: #ff79c6;
	432	+	}
	433	+	/* KeywordPseudo */
	434	+	.chroma .kp {
	435	+	color: #ff79c6;
	436	+	}
	437	+	/* KeywordReserved */
	438	+	.chroma .kr {
	439	+	color: #ff79c6;
	440	+	}
	441	+	/* KeywordType */
	442	+	.chroma .kt {
	443	+	color: #8be9fd;
	444	+	}
	445	+	/* Name */
	446	+	.chroma .n {
	447	+	}
	448	+	/* NameAttribute */
	449	+	.chroma .na {
	450	+	color: #50fa7b;
	451	+	}
	452	+	/* NameBuiltin */
	453	+	.chroma .nb {
	454	+	color: #8be9fd;
	455	+	font-style: italic;
	456	+	}
	457	+	/* NameBuiltinPseudo */
	458	+	.chroma .bp {
	459	+	}
	460	+	/* NameClass */
	461	+	.chroma .nc {
	462	+	color: #50fa7b;
	463	+	}
	464	+	/* NameConstant */
	465	+	.chroma .no {
	466	+	}
	467	+	/* NameDecorator */
	468	+	.chroma .nd {
	469	+	}
	470	+	/* NameEntity */
	471	+	.chroma .ni {
	472	+	}
	473	+	/* NameException */
	474	+	.chroma .ne {
	475	+	}
	476	+	/* NameFunction */
	477	+	.chroma .nf {
	478	+	color: #50fa7b;
	479	+	}
	480	+	/* NameFunctionMagic */
	481	+	.chroma .fm {
	482	+	}
	483	+	/* NameLabel */
	484	+	.chroma .nl {
	485	+	color: #8be9fd;
	486	+	font-style: italic;
	487	+	}
	488	+	/* NameNamespace */
	489	+	.chroma .nn {
	490	+	}
	491	+	/* NameOther */
	492	+	.chroma .nx {
	493	+	}
	494	+	/* NameProperty */
	495	+	.chroma .py {
	496	+	}
	497	+	/* NameTag */
	498	+	.chroma .nt {
	499	+	color: #ff79c6;
	500	+	}
	501	+	/* NameVariable */
	502	+	.chroma .nv {
	503	+	color: #8be9fd;
	504	+	font-style: italic;
	505	+	}
	506	+	/* NameVariableClass */
	507	+	.chroma .vc {
	508	+	color: #8be9fd;
	509	+	font-style: italic;
	510	+	}
	511	+	/* NameVariableGlobal */
	512	+	.chroma .vg {
	513	+	color: #8be9fd;
	514	+	font-style: italic;
	515	+	}
	516	+	/* NameVariableInstance */
	517	+	.chroma .vi {
	518	+	color: #8be9fd;
	519	+	font-style: italic;
	520	+	}
	521	+	/* NameVariableMagic */
	522	+	.chroma .vm {
	523	+	}
	524	+	/* Literal */
	525	+	.chroma .l {
	526	+	}
	527	+	/* LiteralDate */
	528	+	.chroma .ld {
	529	+	}
	530	+	/* LiteralString */
	531	+	.chroma .s {
	532	+	color: #f1fa8c;
	533	+	}
	534	+	/* LiteralStringAffix */
	535	+	.chroma .sa {
	536	+	color: #f1fa8c;
	537	+	}
	538	+	/* LiteralStringBacktick */
	539	+	.chroma .sb {
	540	+	color: #f1fa8c;
	541	+	}
	542	+	/* LiteralStringChar */
	543	+	.chroma .sc {
	544	+	color: #f1fa8c;
	545	+	}
	546	+	/* LiteralStringDelimiter */
	547	+	.chroma .dl {
	548	+	color: #f1fa8c;
	549	+	}
	550	+	/* LiteralStringDoc */
	551	+	.chroma .sd {
	552	+	color: #f1fa8c;
	553	+	}
	554	+	/* LiteralStringDouble */
	555	+	.chroma .s2 {
	556	+	color: #f1fa8c;
	557	+	}
	558	+	/* LiteralStringEscape */
	559	+	.chroma .se {
	560	+	color: #f1fa8c;
	561	+	}
	562	+	/* LiteralStringHeredoc */
	563	+	.chroma .sh {
	564	+	color: #f1fa8c;
	565	+	}
	566	+	/* LiteralStringInterpol */
	567	+	.chroma .si {
	568	+	color: #f1fa8c;
	569	+	}
	570	+	/* LiteralStringOther */
	571	+	.chroma .sx {
	572	+	color: #f1fa8c;
	573	+	}
	574	+	/* LiteralStringRegex */
	575	+	.chroma .sr {
	576	+	color: #f1fa8c;
	577	+	}
	578	+	/* LiteralStringSingle */
	579	+	.chroma .s1 {
	580	+	color: #f1fa8c;
	581	+	}
	582	+	/* LiteralStringSymbol */
	583	+	.chroma .ss {
	584	+	color: #f1fa8c;
	585	+	}
	586	+	/* LiteralNumber */
	587	+	.chroma .m {
	588	+	color: #bd93f9;
	589	+	}
	590	+	/* LiteralNumberBin */
	591	+	.chroma .mb {
	592	+	color: #bd93f9;
	593	+	}
	594	+	/* LiteralNumberFloat */
	595	+	.chroma .mf {
	596	+	color: #bd93f9;
	597	+	}
	598	+	/* LiteralNumberHex */
	599	+	.chroma .mh {
	600	+	color: #bd93f9;
	601	+	}
	602	+	/* LiteralNumberInteger */
	603	+	.chroma .mi {
	604	+	color: #bd93f9;
	605	+	}
	606	+	/* LiteralNumberIntegerLong */
	607	+	.chroma .il {
	608	+	color: #bd93f9;
	609	+	}
	610	+	/* LiteralNumberOct */
	611	+	.chroma .mo {
	612	+	color: #bd93f9;
	613	+	}
	614	+	/* Operator */
	615	+	.chroma .o {
	616	+	color: #ff79c6;
	617	+	}
	618	+	/* OperatorWord */
	619	+	.chroma .ow {
	620	+	color: #ff79c6;
	621	+	}
	622	+	/* Punctuation */
	623	+	.chroma .p {
	624	+	}
	625	+	/* Comment */
	626	+	.chroma .c {
	627	+	color: #6272a4;
	628	+	}
	629	+	/* CommentHashbang */
	630	+	.chroma .ch {
	631	+	color: #6272a4;
	632	+	}
	633	+	/* CommentMultiline */
	634	+	.chroma .cm {
	635	+	color: #6272a4;
	636	+	}
	637	+	/* CommentSingle */
	638	+	.chroma .c1 {
	639	+	color: #6272a4;
	640	+	}
	641	+	/* CommentSpecial */
	642	+	.chroma .cs {
	643	+	color: #6272a4;
	644	+	}
	645	+	/* CommentPreproc */
	646	+	.chroma .cp {
	647	+	color: #ff79c6;
	648	+	}
	649	+	/* CommentPreprocFile */
	650	+	.chroma .cpf {
	651	+	color: #ff79c6;
	652	+	}
	653	+	/* Generic */
	654	+	.chroma .g {
	655	+	}
	656	+	/* GenericDeleted */
	657	+	.chroma .gd {
	658	+	color: #ff5555;
	659	+	}
	660	+	/* GenericEmph */
	661	+	.chroma .ge {
	662	+	text-decoration: underline;
	663	+	}
	664	+	/* GenericError */
	665	+	.chroma .gr {
	666	+	}
	667	+	/* GenericHeading */
	668	+	.chroma .gh {
	669	+	font-weight: bold;
	670	+	}
	671	+	/* GenericInserted */
	672	+	.chroma .gi {
	673	+	color: #50fa7b;
	674	+	font-weight: bold;
	675	+	}
	676	+	/* GenericOutput */
	677	+	.chroma .go {
	678	+	color: #44475a;
	679	+	}
	680	+	/* GenericPrompt */
	681	+	.chroma .gp {
	682	+	}
	683	+	/* GenericStrong */
	684	+	.chroma .gs {
	685	+	}
	686	+	/* GenericSubheading */
	687	+	.chroma .gu {
	688	+	font-weight: bold;
	689	+	}
	690	+	/* GenericTraceback */
	691	+	.chroma .gt {
	692	+	}
	693	+	/* GenericUnderline */
	694	+	.chroma .gl {
	695	+	text-decoration: underline;
	696	+	}
	697	+	/* TextWhitespace */
	698	+	.chroma .w {
	699	+	}
	700	+	}

...	...		@@ -0,0 +1,116 @@
	1	+	package pastes
	2	+
	3	+	import (
	4	+	"fmt"
	5	+	"strconv"
	6	+	"strings"
	7	+	"time"
	8	+
	9	+	"github.com/araddon/dateparse"
	10	+	"github.com/picosh/pico/pkg/db"
	11	+	"github.com/picosh/pico/pkg/filehandlers"
	12	+	"github.com/picosh/pico/pkg/pssh"
	13	+	"github.com/picosh/pico/pkg/shared"
	14	+	)
	15	+
	16	+	var DEFAULT_EXPIRES_AT = 90
	17	+
	18	+	type FileHooks struct {
	19	+	Cfg *shared.ConfigSite
	20	+	Db db.DB
	21	+	}
	22	+
	23	+	func (p FileHooks) FileValidate(s pssh.SSHServerConnSession, data *filehandlers.PostMetaData) (bool, error) {
	24	+	if !shared.IsTextFile(string(data.Text)) {
	25	+	err := fmt.Errorf(
	26	+	"ERROR: (%s) invalid file must be plain text (utf-8), skipping",
	27	+	data.Filename,
	28	+	)
	29	+	return false, err
	30	+	}
	31	+
	32	+	maxFileSize := int(p.Cfg.MaxAssetSize)
	33	+	if data.FileSize > maxFileSize {
	34	+	return false, fmt.Errorf(
	35	+	"ERROR: file (%s) has exceeded maximum file size (%d bytes)",
	36	+	data.Filename,
	37	+	maxFileSize,
	38	+	)
	39	+	}
	40	+
	41	+	return true, nil
	42	+	}
	43	+
	44	+	func (p FileHooks) FileMeta(s pssh.SSHServerConnSession, data *filehandlers.PostMetaData) error {
	45	+	data.Title = shared.ToUpper(data.Slug)
	46	+	// we want the slug to be the filename for pastes
	47	+	data.Slug = data.Filename
	48	+
	49	+	// mark posts for deletion a X days after creation
	50	+	expiresAt := time.Now().AddDate(0, 0, DEFAULT_EXPIRES_AT)
	51	+	data.ExpiresAt = &expiresAt
	52	+	var hidden bool
	53	+	// A paste written without arguments, as over sftp, keeps its settings.
	54	+	if cur := data.Cur; cur != nil {
	55	+	hidden = cur.Hidden
	56	+	if cur.ExpiresAt == nil \|\| cur.ExpiresAt.After(time.Now()) {
	57	+	data.ExpiresAt = cur.ExpiresAt
	58	+	}
	59	+	}
	60	+
	61	+	var expiresFound bool
	62	+	var expires *time.Time
	63	+
	64	+	cmd := s.Command()
	65	+
	66	+	for _, arg := range cmd {
	67	+	if strings.Contains(arg, "=") {
	68	+	splitArg := strings.Split(arg, "=")
	69	+	if len(splitArg) != 2 {
	70	+	continue
	71	+	}
	72	+
	73	+	switch splitArg[0] {
	74	+	case "hidden":
	75	+	val, err := strconv.ParseBool(splitArg[1])
	76	+	if err != nil {
	77	+	continue
	78	+	}
	79	+
	80	+	hidden = val
	81	+	case "expires":
	82	+	val, err := strconv.ParseBool(splitArg[1])
	83	+	if err == nil {
	84	+	expiresFound = true
	85	+	expires = nil
	86	+	if val {
	87	+	expires = &expiresAt
	88	+	}
	89	+	continue
	90	+	}
	91	+
	92	+	duration, err := time.ParseDuration(splitArg[1])
	93	+	if err == nil {
	94	+	expiresFound = true
	95	+	expireTime := time.Now().Add(duration)
	96	+	expires = &expireTime
	97	+	continue
	98	+	}
	99	+
	100	+	expireTime, err := dateparse.ParseStrict(splitArg[1])
	101	+	if err == nil {
	102	+	expiresFound = true
	103	+	expires = &expireTime
	104	+	}
	105	+	}
	106	+	}
	107	+	}
	108	+
	109	+	data.Hidden = hidden
	110	+
	111	+	if expiresFound {
	112	+	data.ExpiresAt = expires
	113	+	}
	114	+
	115	+	return nil
	116	+	}

...	...		@@ -0,0 +1,52 @@
	1	+	package pastes
	2	+
	3	+	import (
	4	+	"context"
	5	+	"testing"
	6	+	"time"
	7	+
	8	+	"github.com/picosh/pico/pkg/db"
	9	+	"github.com/picosh/pico/pkg/filehandlers"
	10	+	"github.com/picosh/pico/pkg/pssh"
	11	+	)
	12	+
	13	+	func TestFileMetaKeepsSettings(t *testing.T) {
	14	+	soon := time.Now().Add(time.Hour)
	15	+	past := time.Now().Add(-time.Hour)
	16	+	day := 24 * time.Hour
	17	+	tests := []struct {
	18	+	name string
	19	+	cmd []string
	20	+	cur *db.Post
	21	+	hidden bool
	22	+	// expires is how long until the paste expires, or 0 for never.
	23	+	expires time.Duration
	24	+	}{
	25	+	{name: "new", expires: 90 * day},
	26	+	{name: "new with arguments", cmd: []string{"hidden=true", "expires=2h"}, hidden: true, expires: 2 * time.Hour},
	27	+	{name: "existing", cur: &db.Post{Hidden: true, ExpiresAt: &soon}, hidden: true, expires: time.Hour},
	28	+	{name: "existing without expiry", cur: &db.Post{Hidden: true}, hidden: true},
	29	+	{name: "existing past its expiry", cur: &db.Post{Hidden: true, ExpiresAt: &past}, hidden: true, expires: 90 * day},
	30	+	{name: "existing with arguments", cmd: []string{"hidden=false", "expires=true"}, cur: &db.Post{Hidden: true}, expires: 90 * day},
	31	+	{name: "existing made permanent", cmd: []string{"expires=false"}, cur: &db.Post{ExpiresAt: &soon}},
	32	+	}
	33	+	for _, tt := range tests {
	34	+	t.Run(tt.name, func(t *testing.T) {
	35	+	s := &pssh.SSHServerConnSession{Ctx: context.Background()}
	36	+	s.SetValue("command", tt.cmd)
	37	+	data := &filehandlers.PostMetaData{Post: &db.Post{Filename: "a.txt", Slug: "a"}, Cur: tt.cur}
	38	+	if err := (&FileHooks{}).FileMeta(s, data); err != nil {
	39	+	t.Fatal(err)
	40	+	}
	41	+	if data.Hidden != tt.hidden {
	42	+	t.Errorf("hidden = %v, want %v", data.Hidden, tt.hidden)
	43	+	}
	44	+	switch {
	45	+	case tt.expires == 0 && data.ExpiresAt != nil:
	46	+	t.Errorf("expires at %v, want never", data.ExpiresAt)
	47	+	case tt.expires != 0 && (data.ExpiresAt == nil \|\| time.Until(*data.ExpiresAt).Round(time.Minute) != tt.expires):
	48	+	t.Errorf("expires at %v, want in %v", data.ExpiresAt, tt.expires)
	49	+	}
	50	+	})
	51	+	}
	52	+	}

...	...		@@ -0,0 +1,88 @@
	1	+	package pastes
	2	+
	3	+	import (
	4	+	"context"
	5	+	"os"
	6	+	"os/signal"
	7	+	"syscall"
	8	+	"time"
	9	+
	10	+	"github.com/picosh/pico/pkg/db/postgres"
	11	+	"github.com/picosh/pico/pkg/pssh"
	12	+	"github.com/picosh/pico/pkg/send/auth"
	13	+	"github.com/picosh/pico/pkg/send/list"
	14	+	"github.com/picosh/pico/pkg/send/pipe"
	15	+	"github.com/picosh/pico/pkg/send/protocols/rsync"
	16	+	"github.com/picosh/pico/pkg/send/protocols/scp"
	17	+	"github.com/picosh/pico/pkg/send/protocols/sftp"
	18	+	"github.com/picosh/pico/pkg/shared"
	19	+	)
	20	+
	21	+	func StartSshServer() {
	22	+	appName := "pastes-ssh"
	23	+
	24	+	host := shared.GetEnv("PASTES_HOST", "0.0.0.0")
	25	+	port := shared.GetEnv("PASTES_SSH_PORT", "2222")
	26	+	promPort := shared.GetEnv("PASTES_PROM_PORT", "9222")
	27	+	cfg := NewConfigSite(appName)
	28	+	logger := cfg.Logger
	29	+
	30	+	ctx, cancel := context.WithCancel(context.Background())
	31	+	defer cancel()
	32	+
	33	+	dbh := postgres.NewDB(cfg.DbURL, cfg.Logger)
	34	+	defer func() {
	35	+	_ = dbh.Close()
	36	+	}()
	37	+	handler := NewFileHandler(cfg, dbh)
	38	+	sshAuth := shared.NewSshAuthHandler(dbh, logger, "pastes")
	39	+
	40	+	// Create a new SSH server
	41	+	server, err := pssh.NewSSHServerWithConfig(
	42	+	ctx,
	43	+	logger,
	44	+	appName,
	45	+	host,
	46	+	port,
	47	+	promPort,
	48	+	"ssh_data/term_info_ed25519",
	49	+	sshAuth.PubkeyAuthHandler,
	50	+	[]pssh.SSHServerMiddleware{
	51	+	pipe.Middleware(handler, ""),
	52	+	list.Middleware(handler),
	53	+	scp.Middleware(handler),
	54	+	rsync.Middleware(handler),
	55	+	auth.Middleware(handler),
	56	+	pssh.PtyMdw(pssh.DeprecatedNotice(), 200*time.Millisecond),
	57	+	pssh.LogMiddleware(handler, dbh),
	58	+	},
	59	+	[]pssh.SSHServerMiddleware{
	60	+	sftp.Middleware(handler),
	61	+	pssh.LogMiddleware(handler, dbh),
	62	+	},
	63	+	nil,
	64	+	)
	65	+
	66	+	if err != nil {
	67	+	logger.Error("failed to create ssh server", "err", err.Error())
	68	+	os.Exit(1)
	69	+	}
	70	+
	71	+	done := make(chan os.Signal, 1)
	72	+	signal.Notify(done, os.Interrupt, syscall.SIGINT, syscall.SIGTERM)
	73	+	logger.Info("Starting SSH server", "addr", server.Config.ListenAddr)
	74	+	go func() {
	75	+	if err = server.ListenAndServe(); err != nil {
	76	+	logger.Error("serve", "err", err.Error())
	77	+	os.Exit(1)
	78	+	}
	79	+	}()
	80	+
	81	+	exit := func() {
	82	+	logger.Info("stopping ssh server")
	83	+	cancel()
	84	+	}
	85	+
	86	+	<-done
	87	+	exit()
	88	+	}

...	...		@@ -0,0 +1,85 @@
	1	+	package pastes
	2	+
	3	+	import (
	4	+	"fmt"
	5	+	"log/slog"
	6	+
	7	+	"github.com/picosh/pico/pkg/db"
	8	+	"github.com/picosh/pico/pkg/filehandlers"
	9	+	"github.com/picosh/pico/pkg/pssh"
	10	+	sendutils "github.com/picosh/pico/pkg/send/utils"
	11	+	"github.com/picosh/pico/pkg/shared"
	12	+	)
	13	+
	14	+	func checkAccess(dbpool db.DB, userID string) error {
	15	+	ffPlus, plusErr := dbpool.FindFeature(userID, "plus")
	16	+	if plusErr == nil && ffPlus.IsValid() {
	17	+	return nil
	18	+	}
	19	+
	20	+	ffPastes, pastesErr := dbpool.FindFeature(userID, "pastes")
	21	+	if pastesErr == nil {
	22	+	if ffPastes.IsValid() {
	23	+	return nil
	24	+	}
	25	+	return fmt.Errorf("ERROR: your pastes access has expired: https://blog.pico.sh/ann-038-pico-invite-system")
	26	+	}
	27	+
	28	+	if plusErr == nil {
	29	+	return fmt.Errorf("ERROR: your pico+ has expired: https://blog.pico.sh/ann-038-pico-invite-system")
	30	+	}
	31	+	return fmt.Errorf("ERROR: uploading to pastes requires an invitation or pico+: https://blog.pico.sh/ann-038-pico-invite-system")
	32	+	}
	33	+
	34	+	type UploadHandler struct {
	35	+	*filehandlers.FileHandlerRouter
	36	+	Cfg *shared.ConfigSite
	37	+	DB db.DB
	38	+	}
	39	+
	40	+	var _ sendutils.CopyFromClientHandler = (*UploadHandler)(nil)
	41	+
	42	+	func NewUploadHandler(cfg shared.ConfigSite, dbpool db.DB, fileMap map[string]filehandlers.ReadWriteHandler) UploadHandler {
	43	+	router := filehandlers.NewFileHandlerRouter(cfg, dbpool, fileMap)
	44	+	return &UploadHandler{
	45	+	FileHandlerRouter: router,
	46	+	Cfg: cfg,
	47	+	DB: dbpool,
	48	+	}
	49	+	}
	50	+
	51	+	// NewFileHandler stores every file as a paste.
	52	+	func NewFileHandler(cfg shared.ConfigSite, dbpool db.DB) UploadHandler {
	53	+	hooks := &FileHooks{Cfg: cfg, Db: dbpool}
	54	+	return NewUploadHandler(cfg, dbpool, map[string]filehandlers.ReadWriteHandler{
	55	+	"fallback": filehandlers.NewScpPostHandler(dbpool, cfg, hooks),
	56	+	})
	57	+	}
	58	+
	59	+	func (h UploadHandler) GetLogger(s pssh.SSHServerConnSession) *slog.Logger {
	60	+	logger := pssh.GetLogger(s)
	61	+	if logger == nil {
	62	+	if h.Cfg != nil && h.Cfg.Logger != nil {
	63	+	return h.Cfg.Logger
	64	+	}
	65	+	return slog.Default()
	66	+	}
	67	+	return logger
	68	+	}
	69	+
	70	+	func (h UploadHandler) Validate(s pssh.SSHServerConnSession) error {
	71	+	logger := h.GetLogger(s)
	72	+	user := pssh.GetUser(s)
	73	+
	74	+	if user == nil {
	75	+	err := fmt.Errorf("could not get user from ctx")
	76	+	logger.Error("error getting user from ctx", "err", err)
	77	+	return err
	78	+	}
	79	+
	80	+	if err := checkAccess(h.DB, user.ID); err != nil {
	81	+	return err
	82	+	}
	83	+
	84	+	return h.FileHandlerRouter.Validate(s)
	85	+	}

...	...		@@ -0,0 +1,130 @@
	1	+	package pastes
	2	+
	3	+	import (
	4	+	"context"
	5	+	"fmt"
	6	+	"log/slog"
	7	+	"strings"
	8	+	"testing"
	9	+	"time"
	10	+
	11	+	"github.com/picosh/pico/pkg/db"
	12	+	"github.com/picosh/pico/pkg/db/stub"
	13	+	"github.com/picosh/pico/pkg/filehandlers"
	14	+	"github.com/picosh/pico/pkg/pssh"
	15	+	"golang.org/x/crypto/ssh"
	16	+	)
	17	+
	18	+	type mockFeatureDB struct {
	19	+	*stub.StubDB
	20	+	features map[string]*db.FeatureFlag
	21	+	}
	22	+
	23	+	func newMockFeatureDB() *mockFeatureDB {
	24	+	return &mockFeatureDB{
	25	+	StubDB: stub.NewStubDB(slog.Default()),
	26	+	features: make(map[string]*db.FeatureFlag),
	27	+	}
	28	+	}
	29	+
	30	+	func (m mockFeatureDB) FindFeature(userID, name string) (db.FeatureFlag, error) {
	31	+	if ff, ok := m.features[userID+":"+name]; ok {
	32	+	return ff, nil
	33	+	}
	34	+	return nil, fmt.Errorf("feature flag %s not found for user %s", name, userID)
	35	+	}
	36	+
	37	+	func (m *mockFeatureDB) setFeature(userID, name string, expiresAt time.Time) {
	38	+	m.features[userID+":"+name] = &db.FeatureFlag{UserID: userID, Name: name, ExpiresAt: &expiresAt}
	39	+	}
	40	+
	41	+	func TestUploadHandlerValidate(t *testing.T) {
	42	+	cfg := NewConfigSite("pastes-test")
	43	+	userID := "user-1"
	44	+	future := time.Now().Add(24 * time.Hour)
	45	+	past := time.Now().Add(-24 * time.Hour)
	46	+
	47	+	createSession := func(user db.User) pssh.SSHServerConnSession {
	48	+	conn := &pssh.SSHServerConn{
	49	+	Conn: &ssh.ServerConn{
	50	+	Permissions: &ssh.Permissions{
	51	+	Extensions: map[string]string{},
	52	+	},
	53	+	},
	54	+	Logger: slog.Default(),
	55	+	}
	56	+	sesh := &pssh.SSHServerConnSession{
	57	+	SSHServerConn: conn,
	58	+	Ctx: context.Background(),
	59	+	}
	60	+	if user != nil {
	61	+	pssh.SetUser(sesh, user)
	62	+	}
	63	+	return sesh
	64	+	}
	65	+
	66	+	tests := []struct {
	67	+	name string
	68	+	setup func(m *mockFeatureDB)
	69	+	user *db.User
	70	+	wantErr string
	71	+	}{
	72	+	{
	73	+	name: "nil user",
	74	+	setup: func(m *mockFeatureDB) {},
	75	+	wantErr: "could not get user",
	76	+	},
	77	+	{
	78	+	name: "no invite or pico+",
	79	+	setup: func(m *mockFeatureDB) {},
	80	+	user: &db.User{ID: userID, Name: "tester"},
	81	+	wantErr: "requires an invitation or pico+",
	82	+	},
	83	+	{
	84	+	name: "pastes feature",
	85	+	setup: func(m *mockFeatureDB) { m.setFeature(userID, "pastes", future) },
	86	+	user: &db.User{ID: userID, Name: "tester"},
	87	+	},
	88	+	{
	89	+	name: "pico+ feature",
	90	+	setup: func(m *mockFeatureDB) { m.setFeature(userID, "plus", future) },
	91	+	user: &db.User{ID: userID, Name: "tester"},
	92	+	},
	93	+	{
	94	+	name: "expired pico+ with active pastes",
	95	+	setup: func(m *mockFeatureDB) { m.setFeature(userID, "plus", past); m.setFeature(userID, "pastes", future) },
	96	+	user: &db.User{ID: userID, Name: "tester"},
	97	+	},
	98	+	{
	99	+	name: "expired pastes",
	100	+	setup: func(m *mockFeatureDB) { m.setFeature(userID, "pastes", past) },
	101	+	user: &db.User{ID: userID, Name: "tester"},
	102	+	wantErr: "pastes access has expired",
	103	+	},
	104	+	{
	105	+	name: "expired pico+",
	106	+	setup: func(m *mockFeatureDB) { m.setFeature(userID, "plus", past) },
	107	+	user: &db.User{ID: userID, Name: "tester"},
	108	+	wantErr: "pico+ has expired",
	109	+	},
	110	+	}
	111	+
	112	+	for _, tt := range tests {
	113	+	t.Run(tt.name, func(t *testing.T) {
	114	+	mockDB := newMockFeatureDB()
	115	+	tt.setup(mockDB)
	116	+	handler := NewUploadHandler(cfg, mockDB, map[string]filehandlers.ReadWriteHandler{})
	117	+
	118	+	err := handler.Validate(createSession(tt.user))
	119	+	if tt.wantErr == "" {
	120	+	if err != nil {
	121	+	t.Fatalf("unexpected error: %v", err)
	122	+	}
	123	+	return
	124	+	}
	125	+	if err == nil \|\| !strings.Contains(err.Error(), tt.wantErr) {
	126	+	t.Fatalf("expected error containing %q, got %v", tt.wantErr, err)
	127	+	}
	128	+	})
	129	+	}
	130	+	}

...	...		@@ -233,8 +233,11 @@ func calcRoutes(projectName, fp string, userRedirects []RedirectRule) []HttpRe
233	233		case "wildcard":
234	234		fallthrough
235	235		case "variable":
236		-	rr := regexp.MustCompile(fromMatcher)
237		-	match = rr.FindStringSubmatch(fp)
	236	+	// The pattern comes from the user's _redirects, so an invalid one
	237	+	// is a rule that never matches.
	238	+	if rr, err := regexp.Compile(fromMatcher); err == nil {
	239	+	match = rr.FindStringSubmatch(fp)
	240	+	}
238	241		case "none":
239	242		fallthrough
240	243		default:

...	...		@@ -7,6 +7,7 @@ import (
7	7		"io"
8	8		"log/slog"
9	9		"path/filepath"
	10	+	"slices"
10	11		"strings"
11	12		"text/tabwriter"
12	13		"time"

...	...		@@ -80,6 +81,12 @@ func (c *Cmd) notice() {
80	81		}
81	82		}
82	83
	84	+	// purge clears what the web servers cache about a project after a command
	85	+	// changed it.
	86	+	func (c *Cmd) purge(projectName string) {
	87	+	c.Cfg.CacheClearingQueue <- getSurrogateKey(c.User.Name, projectName)
	88	+	}
	89	+
83	90		func (c *Cmd) RmProjectAssets(projectName string) error {
84	91		bucketName := shared.GetAssetBucketName(c.User.ID)
85	92		bucket, err := c.Store.GetBucket(bucketName)

...	...		@@ -95,9 +102,15 @@ func (c *Cmd) RmProjectAssets(projectName string) error {
95	102
96	103		if len(fileList) == 0 {
97	104		c.output(fmt.Sprintf("no assets found for project (%s)", projectName))
	105	+	if c.Write {
	106	+	return c.Store.DeleteObject(bucket, projectName)
	107	+	}
98	108		return nil
99	109		}
100	110		c.output(fmt.Sprintf("found (%d) assets for project (%s), removing", len(fileList), projectName))
	111	+	if c.Write {
	112	+	defer c.purge(projectName)
	113	+	}
101	114
102	115		for _, file := range fileList {
103	116		if file.IsDir() {

...	...		@@ -124,6 +137,25 @@ func (c *Cmd) RmProjectAssets(projectName string) error {
124	137		c.output(intent)
125	138		}
126	139		}
	140	+	if !c.Write {
	141	+	return nil
	142	+	}
	143	+
	144	+	// Directories outlive their files, so remove them deepest first.
	145	+	dirs := []string{projectName}
	146	+	for _, file := range fileList {
	147	+	if file.IsDir() {
	148	+	dirs = append(dirs, filepath.Join(projectName, file.Name()))
	149	+	}
	150	+	}
	151	+	slices.SortStableFunc(dirs, func(a, b string) int {
	152	+	return strings.Count(b, "/") - strings.Count(a, "/")
	153	+	})
	154	+	for _, dir := range dirs {
	155	+	if err := c.Store.DeleteObject(bucket, dir); err != nil {
	156	+	return err
	157	+	}
	158	+	}
127	159		return nil
128	160		}
129	161

...	...		@@ -291,6 +323,9 @@ func (c *Cmd) unlink(projectName string) error {
291	323		if err != nil {
292	324		return err
293	325		}
	326	+	if c.Write {
	327	+	c.purge(project.Name)
	328	+	}
294	329		c.output(fmt.Sprintf("(%s) unlinked", project.Name))
295	330
296	331		return nil

...	...		@@ -313,9 +348,6 @@ func (c *Cmd) fzf(projectName string) error {
313	348		}
314	349
315	350		for _, obj := range objs {
316		-	if strings.Contains(obj.Name(), "._pico_keep_dir") {
317		-	continue
318		-	}
319	351		url := c.Cfg.AssetURL(
320	352		c.User.Name,
321	353		project.Name,

...	...		@@ -365,6 +397,9 @@ func (c *Cmd) link(projectName, linkTo string) error {
365	397		if err != nil {
366	398		return err
367	399		}
	400	+	if c.Write {
	401	+	c.purge(projectName)
	402	+	}
368	403
369	404		out := fmt.Sprintf("(%s) might have orphaned assets, removing", projectName)
370	405		c.output(out)

...	...		@@ -463,6 +498,7 @@ func (c *Cmd) prune(prefix string, keepNumLatest int) error {
463	498		if err != nil {
464	499		return err
465	500		}
	501	+	c.purge(project.Name)
466	502		}
467	503		}
468	504

...	...		@@ -500,6 +536,7 @@ func (c *Cmd) rm(projectName string) error {
500	536		if err != nil {
501	537		return err
502	538		}
	539	+	c.purge(project.Name)
503	540		}
504	541		} else {
505	542		msg := fmt.Sprintf("(%s) project record not found for user (%s)", projectName, c.User.Name)

...	...		@@ -524,7 +561,10 @@ func (c *Cmd) acl(projectName, aclType string, acls []string) error {
524	561		Data: acls,
525	562		}
526	563		if c.Write {
527		-	return c.Dbpool.UpdateProjectAcl(c.User.ID, projectName, acl)
	564	+	if err := c.Dbpool.UpdateProjectAcl(c.User.ID, projectName, acl); err != nil {
	565	+	return err
	566	+	}
	567	+	c.purge(projectName)
528	568		}
529	569		return nil
530	570		}

...	...		@@ -0,0 +1,102 @@
	1	+	package pgs
	2	+
	3	+	import (
	4	+	"bytes"
	5	+	"io"
	6	+	"log/slog"
	7	+	"os"
	8	+	"path/filepath"
	9	+	"strings"
	10	+	"testing"
	11	+
	12	+	"github.com/picosh/pico/pkg/shared"
	13	+	"github.com/picosh/pico/pkg/storage"
	14	+	)
	15	+
	16	+	type fakeCmdSession struct{ bytes.Buffer }
	17	+
	18	+	func (s *fakeCmdSession) Exit(int) error { return nil }
	19	+	func (s *fakeCmdSession) Close() error { return nil }
	20	+	func (s *fakeCmdSession) Stderr() io.ReadWriter { return &s.Buffer }
	21	+
	22	+	// Commands that change how a project is served purge its cached pages and
	23	+	// records.
	24	+	func TestCliPurges(t *testing.T) {
	25	+	logger := slog.New(slog.DiscardHandler)
	26	+	dbpool := NewPgsDb(logger)
	27	+	user := dbpool.Users[0]
	28	+	if _, err := dbpool.InsertProject(user.ID, "other", "other"); err != nil {
	29	+	t.Fatal(err)
	30	+	}
	31	+	st, err := storage.NewStorageMemory(map[string]map[string]string{
	32	+	shared.GetAssetBucketName(user.ID): {"/test/index.html": "hello"},
	33	+	})
	34	+	if err != nil {
	35	+	t.Fatal(err)
	36	+	}
	37	+	cfg := NewPgsConfig(logger, dbpool, st, discardPubsub{})
	38	+	cmd := &Cmd{User: user, Session: &fakeCmdSession{}, Log: logger, Store: st, Dbpool: dbpool, Write: true, Cfg: cfg}
	39	+
	40	+	tests := []struct {
	41	+	name string
	42	+	run func() error
	43	+	}{
	44	+	{"acl", func() error { return cmd.acl("test", "private", nil) }},
	45	+	{"link", func() error { return cmd.link("test", "other") }},
	46	+	{"unlink", func() error { return cmd.unlink("test") }},
	47	+	{"rm", func() error { return cmd.rm("test") }},
	48	+	}
	49	+	want := getSurrogateKey(user.Name, "test")
	50	+	for _, tt := range tests {
	51	+	if err := tt.run(); err != nil {
	52	+	t.Fatalf("%s: %v", tt.name, err)
	53	+	}
	54	+	select {
	55	+	case got := <-cfg.CacheClearingQueue:
	56	+	if got != want {
	57	+	t.Fatalf("%s purged %q, want %q", tt.name, got, want)
	58	+	}
	59	+	default:
	60	+	t.Fatalf("%s purged nothing", tt.name)
	61	+	}
	62	+	for len(cfg.CacheClearingQueue) > 0 {
	63	+	<-cfg.CacheClearingQueue
	64	+	}
	65	+	}
	66	+	}
	67	+
	68	+	// Removing a project's assets also removes its directories, which storage
	69	+	// keeps after their last file is deleted.
	70	+	func TestRmProjectAssetsRemovesDirs(t *testing.T) {
	71	+	logger := slog.New(slog.DiscardHandler)
	72	+	dbpool := NewPgsDb(logger)
	73	+	user := dbpool.Users[0]
	74	+	st, err := storage.NewStorageFS(logger, t.TempDir())
	75	+	if err != nil {
	76	+	t.Fatal(err)
	77	+	}
	78	+	bucket, err := st.UpsertBucket(shared.GetAssetBucketName(user.ID))
	79	+	if err != nil {
	80	+	t.Fatal(err)
	81	+	}
	82	+	for _, name := range []string{"test/index.html", "test/css/deep/site.css", "other/index.html"} {
	83	+	if _, _, err := st.PutObject(bucket, name, strings.NewReader("x"), &storage.ObjectInfo{}); err != nil {
	84	+	t.Fatal(err)
	85	+	}
	86	+	}
	87	+	if err := st.PutDir(bucket, "test/empty/nested"); err != nil {
	88	+	t.Fatal(err)
	89	+	}
	90	+
	91	+	cfg := NewPgsConfig(logger, dbpool, st, discardPubsub{})
	92	+	cmd := &Cmd{User: user, Session: &fakeCmdSession{}, Log: logger, Store: st, Dbpool: dbpool, Write: true, Cfg: cfg}
	93	+	if err := cmd.RmProjectAssets("test"); err != nil {
	94	+	t.Fatal(err)
	95	+	}
	96	+	if _, err := os.Stat(filepath.Join(bucket.Path, "test")); !os.IsNotExist(err) {
	97	+	t.Fatalf("project directory still exists: %v", err)
	98	+	}
	99	+	if _, err := os.Stat(filepath.Join(bucket.Path, "other/index.html")); err != nil {
	100	+	t.Fatalf("another project was touched: %v", err)
	101	+	}
	102	+	}

...	...		@@ -15,6 +15,7 @@ import (
15	15		type PgsConfig struct {
16	16		CacheTTL time.Duration
17	17		CacheMaxItems int
	18	+	CacheMaxBodySize int64
18	19		Domain string
19	20		MaxAssetSize int64
20	21		MaxSize uint64

...	...		@@ -65,6 +66,10 @@ func (c *PgsConfig) StaticPath(fname string) string {
65	66		var maxSize = uint64(50 * shared.MB)
66	67		var maxAssetSize = int64(10 * shared.MB)
67	68
	69	+	// Larger responses stream from storage instead of being held in the cache.
	70	+	// Zero means no limit.
	71	+	var cacheMaxBodySize = int64(0)
	72	+
68	73		// Needs to be small for caching files like _headers and _redirects.
69	74		var maxSpecialFileSize = int64(5 * shared.KB)
70	75

...	...		@@ -81,6 +86,10 @@ func NewPgsConfig(logger *slog.Logger, dbpool pgsdb.PgsDB, st storage.StorageSer
81	86		if cacheMaxItemsStr != "" {
82	87		cacheMaxItems, _ = strconv.Atoi(cacheMaxItemsStr)
83	88		}
	89	+	cacheMaxBody, err := strconv.ParseInt(shared.GetEnv("PGS_CACHE_MAX_BODY_SIZE", ""), 10, 64)
	90	+	if err != nil {
	91	+	cacheMaxBody = cacheMaxBodySize
	92	+	}
84	93
85	94		sshHost := shared.GetEnv("PGS_SSH_HOST", "0.0.0.0")
86	95		sshPort := shared.GetEnv("PGS_SSH_PORT", "2222")

...	...		@@ -88,6 +97,7 @@ func NewPgsConfig(logger *slog.Logger, dbpool pgsdb.PgsDB, st storage.StorageSer
88	97		cfg := PgsConfig{
89	98		CacheTTL: cacheTTL,
90	99		CacheMaxItems: cacheMaxItems,
	100	+	CacheMaxBodySize: cacheMaxBody,
91	101		Domain: domain,
92	102		MaxAssetSize: maxAssetSize,
93	103		MaxSize: maxSize,

...	...		@@ -3,6 +3,7 @@ package pgsdb
3	3		import (
4	4		"fmt"
5	5		"log/slog"
	6	+	"slices"
6	7		"time"
7	8
8	9		"github.com/google/uuid"

...	...		@@ -149,11 +150,22 @@ func (me MemoryDB) UpsertProject(userID, projectName, projectDir string) (db.P
149	150		}
150	151
151	152		func (me *MemoryDB) LinkToProject(userID, projectID, projectDir string, commit bool) error {
152		-	return errNotImpl
	153	+	for _, project := range me.Projects {
	154	+	if project.ID == projectID && project.UserID == userID {
	155	+	if commit {
	156	+	project.ProjectDir = projectDir
	157	+	}
	158	+	return nil
	159	+	}
	160	+	}
	161	+	return fmt.Errorf("project not found by id %s", projectID)
153	162		}
154	163
155	164		func (me *MemoryDB) RemoveProject(projectID string) error {
156		-	return errNotImpl
	165	+	me.Projects = slices.DeleteFunc(me.Projects, func(project *db.Project) bool {
	166	+	return project.ID == projectID
	167	+	})
	168	+	return nil
157	169		}
158	170
159	171		func (me MemoryDB) FindProjectByName(userID, name string) (db.Project, error) {

...	...		@@ -172,7 +184,13 @@ func (me MemoryDB) FindProjectByName(userID, name string) (db.Project, error)
172	184		}
173	185
174	186		func (me MemoryDB) FindProjectLinks(userID, name string) ([]db.Project, error) {
175		-	return []*db.Project{}, errNotImpl
	187	+	links := []*db.Project{}
	188	+	for _, project := range me.Projects {
	189	+	if project.UserID == userID && project.ProjectDir == name && project.Name != name {
	190	+	links = append(links, project)
	191	+	}
	192	+	}
	193	+	return links, nil
176	194		}
177	195
178	196		func (me MemoryDB) FindProjectsByPrefix(userID, prefix string) ([]db.Project, error) {

...	...		@@ -195,7 +213,12 @@ func (me MemoryDB) FindProjects(userID string) ([]db.Project, error) {
195	213		}
196	214
197	215		func (me *MemoryDB) UpdateProjectAcl(userID, name string, acl db.ProjectAcl) error {
198		-	return errNotImpl
	216	+	project, err := me.FindProjectByName(userID, name)
	217	+	if err != nil {
	218	+	return err
	219	+	}
	220	+	project.Acl = acl
	221	+	return nil
199	222		}
200	223
201	224		func (me *MemoryDB) RegisterAdmin(username, pubkey, pubkeyName string) error {

...	...		@@ -92,24 +92,20 @@ func toDisplayEntries(entries []os.FileInfo) []dirEntryDisplay {
92	92		return displayEntries
93	93		}
94	94
95		-	func shouldGenerateListing(st storage.StorageServe, bucket storage.Bucket, projectDir string, path string) bool {
	95	+	// dirListing returns the entries to list for a path, and false when the path
	96	+	// is not a directory or has an index.html to serve instead.
	97	+	func dirListing(st storage.StorageServe, bucket storage.Bucket, projectDir string, path string) ([]os.FileInfo, bool) {
96	98		dirPath := projectDir + path
97		-	if path == "/" {
98		-	dirPath = projectDir + "/"
99		-	}
100	99
101	100		entries, err := st.ListObjects(bucket, dirPath, false)
102	101		if err != nil \|\| len(entries) == 0 {
103		-	return false
	102	+	return nil, false
104	103		}
105	104
106		-	indexPath := dirPath + "index.html"
107		-	obj, _, err := st.GetObject(bucket, indexPath)
108		-	if err != nil {
109		-	return true
	105	+	if _, err := st.StatObject(bucket, dirPath+"index.html"); err == nil {
	106	+	return nil, false
110	107		}
111		-	_ = obj.Close()
112		-	return false
	108	+	return entries, true
113	109		}
114	110
115	111		func generateDirectoryHTML(path string, entries []os.FileInfo) string {

...	...		@@ -211,10 +211,10 @@ func TestShouldGenerateListing(t *testing.T) {
211	211		st, _ := storage.NewStorageMemory(fixture.Storage)
212	212		bucket := storage.Bucket{Name: "testbucket", Path: "testbucket"}
213	213
214		-	result := shouldGenerateListing(st, bucket, "project", fixture.Path)
	214	+	_, result := dirListing(st, bucket, "project", fixture.Path)
215	215
216	216		if result != fixture.Expected {
217		-	t.Errorf("shouldGenerateListing(%q) = %v, want %v", fixture.Path, result, fixture.Expected)
	217	+	t.Errorf("dirListing(%q) = %v, want %v", fixture.Path, result, fixture.Expected)
218	218		}
219	219		})
220	220		}

...	...		@@ -2,6 +2,8 @@ package pgs
2	2
3	3		import (
4	4		"fmt"
	5	+	"log/slog"
	6	+	"regexp"
5	7		"slices"
6	8		"strings"
7	9		)

...	...		@@ -82,6 +84,27 @@ func parseHeaderText(text string) ([]*HeaderRule, error) {
82	84		return rules, nil
83	85		}
84	86
	87	+	// headerMatcher is a _headers rule with its path compiled.
	88	+	type headerMatcher struct {
	89	+	path *regexp.Regexp
	90	+	headers []*HeaderLine
	91	+	}
	92	+
	93	+	// compileHeaderRules compiles each rule's path once, when the rules are
	94	+	// cached, and drops rules whose path is not a valid pattern.
	95	+	func compileHeaderRules(rules []HeaderRule, logger slog.Logger) []headerMatcher {
	96	+	matchers := make([]headerMatcher, 0, len(rules))
	97	+	for _, rule := range rules {
	98	+	re, err := regexp.Compile(rule.Path)
	99	+	if err != nil {
	100	+	logger.Error("skipping _headers rule with invalid path", "path", rule.Path, "err", err)
	101	+	continue
	102	+	}
	103	+	matchers = append(matchers, headerMatcher{path: re, headers: rule.Headers})
	104	+	}
	105	+	return matchers
	106	+	}
	107	+
85	108		func parseLine(line string) (*HeaderLine, error) {
86	109		rule := &HeaderLine{}
87	110

...	...		@@ -0,0 +1,561 @@
	1	+	package pgs
	2	+
	3	+	import (
	4	+	"bytes"
	5	+	"crypto/rand"
	6	+	"encoding/pem"
	7	+	"errors"
	8	+	"fmt"
	9	+	"io"
	10	+	"log/slog"
	11	+	"net"
	12	+	"os"
	13	+	"os/exec"
	14	+	"path/filepath"
	15	+	"regexp"
	16	+	"strconv"
	17	+	"strings"
	18	+	"testing"
	19	+	"time"
	20	+
	21	+	pgsdb "github.com/picosh/pico/pkg/apps/pgs/db"
	22	+	"github.com/picosh/pico/pkg/db"
	23	+	"github.com/picosh/pico/pkg/pssh"
	24	+	"github.com/picosh/pico/pkg/shared"
	25	+	"github.com/picosh/pico/pkg/storage"
	26	+	"github.com/picosh/pico/pkg/storage/storagetest"
	27	+	"github.com/prometheus/client_golang/prometheus"
	28	+	)
	29	+
	30	+	// rsyncFileMax lifts pgs's per-file cap so the test can move files larger
	31	+	// than the default allows.
	32	+	const rsyncFileMax = 24 << 20
	33	+
	34	+	// rsyncEnv is a pgs SSH server backed by one storage implementation, and a
	35	+	// real rsync client set up to talk to it.
	36	+	type rsyncEnv struct {
	37	+	t *testing.T
	38	+	st storage.StorageServe
	39	+	bucket storage.Bucket
	40	+	rsh string
	41	+	feature *db.FeatureFlag
	42	+	// sshArgs are the client options scp and sftp need to reach the server.
	43	+	sshArgs []string
	44	+	addr string
	45	+	user UserSSH
	46	+	}
	47	+
	48	+	func newRsyncEnv(t testing.T, st storage.StorageServe) rsyncEnv {
	49	+	if _, err := exec.LookPath("rsync"); err != nil {
	50	+	t.Skip("rsync not installed")
	51	+	}
	52	+	logger := slog.New(slog.DiscardHandler)
	53	+	dbpool := pgsdb.NewDBMemory(logger)
	54	+	dbpool.SetupTestData()
	55	+	dbpool.Feature.Data.FileMax = rsyncFileMax
	56	+	dbpool.Feature.Data.StorageMax = 256 << 20
	57	+	bucket := storagetest.Bucket(t, st, shared.GetAssetBucketName(dbpool.Users[0].ID))
	58	+
	59	+	t.Setenv("PGS_SSH_PORT", "0")
	60	+	cfg := NewPgsConfig(logger, dbpool, st, discardPubsub{})
	61	+	done := make(chan error)
	62	+	readyCh := make(chan *pssh.SSHServer)
	63	+	prometheus.DefaultRegisterer = prometheus.NewRegistry()
	64	+	go StartSshServerForTesting(cfg, done, readyCh)
	65	+	t.Cleanup(func() { close(done) })
	66	+
	67	+	server := <-readyCh
	68	+	if server == nil {
	69	+	t.Fatal("failed to create ssh server")
	70	+	}
	71	+	var addr string
	72	+	for range 100 {
	73	+	server.Mu.Lock()
	74	+	if server.Listener != nil {
	75	+	addr = server.Listener.Addr().String()
	76	+	}
	77	+	server.Mu.Unlock()
	78	+	if addr != "" {
	79	+	break
	80	+	}
	81	+	time.Sleep(10 * time.Millisecond)
	82	+	}
	83	+	if addr == "" {
	84	+	t.Fatal("server listener not ready")
	85	+	}
	86	+
	87	+	user := GenerateUser()
	88	+	dbpool.Pubkeys = append(dbpool.Pubkeys, &db.PublicKey{
	89	+	ID: "rsync-pubkey",
	90	+	UserID: dbpool.Users[0].ID,
	91	+	Key: shared.KeyForKeyText(user.signer.PublicKey()),
	92	+	})
	93	+	keyFile := filepath.Join(t.TempDir(), "id_ed25519")
	94	+	block := &pem.Block{Type: "OPENSSH PRIVATE KEY", Bytes: user.privateKey}
	95	+	if err := os.WriteFile(keyFile, pem.EncodeToMemory(block), 0o600); err != nil {
	96	+	t.Fatal(err)
	97	+	}
	98	+	_, port, _ := net.SplitHostPort(addr)
	99	+	rsh := fmt.Sprintf(
	100	+	"ssh -p %s -i %s -o IdentitiesOnly=yes -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o LogLevel=ERROR",
	101	+	port, keyFile,
	102	+	)
	103	+	sshArgs := []string{"-P", port, "-i", keyFile, "-o", "IdentitiesOnly=yes", "-o", "StrictHostKeyChecking=no", "-o", "UserKnownHostsFile=/dev/null", "-o", "LogLevel=ERROR"}
	104	+	return &rsyncEnv{t: t, st: st, bucket: bucket, rsh: rsh, feature: dbpool.Feature, sshArgs: sshArgs, addr: addr, user: user}
	105	+	}
	106	+
	107	+	// discardPubsub drops cache purges, which pgs sends from goroutines that can
	108	+	// outlive the test.
	109	+	type discardPubsub struct{}
	110	+
	111	+	func (discardPubsub) Read([]byte) (int, error) { return 0, io.EOF }
	112	+	func (discardPubsub) Write(b []byte) (int, error) { return len(b), nil }
	113	+	func (discardPubsub) Close() error { return nil }
	114	+
	115	+	type rsyncStats struct {
	116	+	output string
	117	+	transferred int
	118	+	deleted int
	119	+	literal int
	120	+	matched int
	121	+	}
	122	+
	123	+	var statPatterns = map[string]*regexp.Regexp{
	124	+	"transferred": regexp.MustCompile(`Number of regular files transferred: ([\d,]+)`),
	125	+	"deleted": regexp.MustCompile(`Number of deleted files: ([\d,]+)`),
	126	+	"literal": regexp.MustCompile(`Literal data: ([\d,]+)`),
	127	+	"matched": regexp.MustCompile(`Matched data: ([\d,]+)`),
	128	+	}
	129	+
	130	+	func (e *rsyncEnv) rsync(args ...string) rsyncStats {
	131	+	e.t.Helper()
	132	+	return e.rsyncExit(0, args...)
	133	+	}
	134	+
	135	+	func (e *rsyncEnv) rsyncExit(want int, args ...string) rsyncStats {
	136	+	e.t.Helper()
	137	+	args = append([]string{"-e", e.rsh, "--stats"}, args...)
	138	+	out, err := exec.Command("rsync", args...).CombinedOutput()
	139	+	code := 0
	140	+	if err != nil {
	141	+	var exit *exec.ExitError
	142	+	if !errors.As(err, &exit) {
	143	+	e.t.Fatalf("rsync %q: %v\n%s", args, err, out)
	144	+	}
	145	+	code = exit.ExitCode()
	146	+	}
	147	+	if code != want {
	148	+	e.t.Fatalf("rsync %q: exit %d, want %d\n%s", args, code, want, out)
	149	+	}
	150	+	stat := func(name string) int {
	151	+	m := statPatterns[name].FindSubmatch(out)
	152	+	if m == nil {
	153	+	e.t.Fatalf("no %s in rsync output:\n%s", name, out)
	154	+	}
	155	+	n, _ := strconv.Atoi(strings.ReplaceAll(string(m[1]), ",", ""))
	156	+	return n
	157	+	}
	158	+	return rsyncStats{
	159	+	output: string(out),
	160	+	transferred: stat("transferred"),
	161	+	deleted: stat("deleted"),
	162	+	literal: stat("literal"),
	163	+	matched: stat("matched"),
	164	+	}
	165	+	}
	166	+
	167	+	func (e *rsyncEnv) stored(name string) ([]byte, time.Time, bool) {
	168	+	e.t.Helper()
	169	+	r, info, err := e.st.GetObject(e.bucket, name)
	170	+	if err != nil {
	171	+	return nil, time.Time{}, false
	172	+	}
	173	+	defer func() { _ = r.Close() }()
	174	+	data, err := io.ReadAll(r)
	175	+	if err != nil {
	176	+	e.t.Fatalf("read %s: %v", name, err)
	177	+	}
	178	+	return data, info.LastModified, true
	179	+	}
	180	+
	181	+	// tree is a local directory of files with fixed contents and mtimes.
	182	+	type tree struct {
	183	+	t *testing.T
	184	+	dir string
	185	+	}
	186	+
	187	+	func (tr tree) write(name string, data []byte, mtime time.Time) {
	188	+	tr.t.Helper()
	189	+	p := filepath.Join(tr.dir, name)
	190	+	if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil {
	191	+	tr.t.Fatal(err)
	192	+	}
	193	+	if err := os.WriteFile(p, data, 0o644); err != nil {
	194	+	tr.t.Fatal(err)
	195	+	}
	196	+	tr.touch(name, mtime)
	197	+	}
	198	+
	199	+	func (tr tree) touch(name string, mtime time.Time) {
	200	+	tr.t.Helper()
	201	+	if err := os.Chtimes(filepath.Join(tr.dir, name), mtime, mtime); err != nil {
	202	+	tr.t.Fatal(err)
	203	+	}
	204	+	}
	205	+
	206	+	func (tr tree) read(name string) ([]byte, time.Time) {
	207	+	tr.t.Helper()
	208	+	p := filepath.Join(tr.dir, name)
	209	+	data, err := os.ReadFile(p)
	210	+	if err != nil {
	211	+	tr.t.Fatal(err)
	212	+	}
	213	+	info, err := os.Stat(p)
	214	+	if err != nil {
	215	+	tr.t.Fatal(err)
	216	+	}
	217	+	return data, info.ModTime()
	218	+	}
	219	+
	220	+	func randomBytes(n int) []byte {
	221	+	b := make([]byte, n)
	222	+	_, _ = rand.Read(b)
	223	+	return b
	224	+	}
	225	+
	226	+	func rsyncStorageBackends() map[string]func(t *testing.T) storage.StorageServe {
	227	+	return map[string]func(t *testing.T) storage.StorageServe{
	228	+	"fs": func(t *testing.T) storage.StorageServe {
	229	+	st, err := storage.NewStorageFS(slog.New(slog.DiscardHandler), t.TempDir())
	230	+	if err != nil {
	231	+	t.Fatal(err)
	232	+	}
	233	+	return st
	234	+	},
	235	+	"memory": func(t *testing.T) storage.StorageServe {
	236	+	st, err := storage.NewStorageMemory(map[string]map[string]string{})
	237	+	if err != nil {
	238	+	t.Fatal(err)
	239	+	}
	240	+	return st
	241	+	},
	242	+	}
	243	+	}
	244	+
	245	+	// TestRsyncStorage runs a real rsync client against pgs on each storage
	246	+	// backend and checks that quick checks, deltas, checksums and deletes are
	247	+	// driven by what storage holds.
	248	+	func TestRsyncStorage(t *testing.T) {
	249	+	for name, newStorage := range rsyncStorageBackends() {
	250	+	t.Run(name, func(t *testing.T) {
	251	+	env := newRsyncEnv(t, newStorage(t))
	252	+	src := tree{t, t.TempDir()}
	253	+	remote := "localhost:/site"
	254	+
	255	+	t0 := time.Unix(1_600_000_000, 0)
	256	+	t1 := time.Unix(1_650_000_000, 0)
	257	+	big := randomBytes(20 << 20)
	258	+	src.write("index.html", []byte("<html>index</html>"), t0)
	259	+	src.write("big.bin", big, t0)
	260	+	src.write("css/site.css", []byte("body{}"), t0)
	261	+	src.write("css/deep/x.css", []byte("a{}"), t1)
	262	+	src.write("img/logo.svg", []byte("<svg/>"), t1)
	263	+	src.write("keep.txt", []byte("excluded from deletes"), t0)
	264	+
	265	+	s := env.rsync("-rtz", src.dir+"/", remote)
	266	+	if s.transferred != 6 {
	267	+	t.Fatalf("initial upload transferred %d files\n%s", s.transferred, s.output)
	268	+	}
	269	+	for name, mtime := range map[string]time.Time{"index.html": t0, "big.bin": t0, "css/deep/x.css": t1} {
	270	+	data, stored, ok := env.stored("/site/" + name)
	271	+	local, _ := src.read(name)
	272	+	if !ok \|\| !bytes.Equal(data, local) {
	273	+	t.Fatalf("%s not stored correctly\n%s", name, s.output)
	274	+	}
	275	+	if !stored.Equal(mtime) {
	276	+	t.Errorf("%s stored with mtime %v, want %v", name, stored, mtime)
	277	+	}
	278	+	}
	279	+
	280	+	t.Run("unchanged upload sends nothing", func(t *testing.T) {
	281	+	s := env.rsync("-rtz", src.dir+"/", remote)
	282	+	if s.transferred != 0 \|\| s.literal != 0 {
	283	+	t.Fatalf("transferred %d files, %d literal bytes\n%s", s.transferred, s.literal, s.output)
	284	+	}
	285	+	})
	286	+
	287	+	t.Run("changed file is sent as a delta", func(t *testing.T) {
	288	+	copy(big[1<<20:], "a small change in the middle of the file")
	289	+	src.write("big.bin", big, t1)
	290	+	s := env.rsync("-rtz", src.dir+"/", remote)
	291	+	if s.transferred != 1 \|\| s.literal > 16<<10 \|\| s.matched < len(big)-16<<10 {
	292	+	t.Fatalf("transferred %d, literal %d, matched %d\n%s", s.transferred, s.literal, s.matched, s.output)
	293	+	}
	294	+	data, mtime, _ := env.stored("/site/big.bin")
	295	+	if !bytes.Equal(data, big) \|\| !mtime.Equal(t1) {
	296	+	t.Fatalf("stored big.bin wrong after delta (mtime %v)", mtime)
	297	+	}
	298	+	})
	299	+
	300	+	t.Run("grown and shrunk files", func(t *testing.T) {
	301	+	src.write("index.html", []byte("<html>index, now with more content</html>"), t1)
	302	+	src.write("css/site.css", []byte("b{}"), t1)
	303	+	env.rsync("-rtz", src.dir+"/", remote)
	304	+	for _, name := range []string{"index.html", "css/site.css"} {
	305	+	data, _, _ := env.stored("/site/" + name)
	306	+	local, _ := src.read(name)
	307	+	if !bytes.Equal(data, local) {
	308	+	t.Errorf("%s stored as %q, want %q", name, data, local)
	309	+	}
	310	+	}
	311	+	})
	312	+
	313	+	t.Run("checksum mode ignores mtimes", func(t *testing.T) {
	314	+	src.touch("css/deep/x.css", t0)
	315	+	s := env.rsync("-rcz", src.dir+"/", remote)
	316	+	if s.transferred != 0 {
	317	+	t.Fatalf("-c transferred %d files with unchanged contents\n%s", s.transferred, s.output)
	318	+	}
	319	+	s = env.rsync("-rtz", src.dir+"/", remote)
	320	+	if s.transferred != 1 \|\| s.literal != 0 {
	321	+	t.Fatalf("mtime-only change transferred %d files, %d literal bytes\n%s", s.transferred, s.literal, s.output)
	322	+	}
	323	+	if _, mtime, _ := env.stored("/site/css/deep/x.css"); !mtime.Equal(t0) {
	324	+	t.Fatalf("mtime-only change stored mtime %v, want %v", mtime, t0)
	325	+	}
	326	+	})
	327	+
	328	+	t.Run("oversized file is rejected", func(t *testing.T) {
	329	+	src.write("huge.bin", randomBytes(rsyncFileMax+1), t0)
	330	+	s := env.rsyncExit(23, "-rt", src.dir+"/", remote)
	331	+	if !strings.Contains(s.output, "huge.bin") {
	332	+	t.Fatalf("no warning about huge.bin\n%s", s.output)
	333	+	}
	334	+	if _, _, ok := env.stored("/site/huge.bin"); ok {
	335	+	t.Fatal("rejected file was stored")
	336	+	}
	337	+	if err := os.Remove(filepath.Join(src.dir, "huge.bin")); err != nil {
	338	+	t.Fatal(err)
	339	+	}
	340	+	})
	341	+
	342	+	t.Run("delete removes what the source lacks", func(t *testing.T) {
	343	+	if err := os.RemoveAll(filepath.Join(src.dir, "css/deep")); err != nil {
	344	+	t.Fatal(err)
	345	+	}
	346	+	if err := os.Remove(filepath.Join(src.dir, "img/logo.svg")); err != nil {
	347	+	t.Fatal(err)
	348	+	}
	349	+	if err := os.Remove(filepath.Join(src.dir, "keep.txt")); err != nil {
	350	+	t.Fatal(err)
	351	+	}
	352	+	s := env.rsync("-rt", "--delete", "--exclude=keep.txt", src.dir+"/", remote)
	353	+	if s.deleted < 2 {
	354	+	t.Fatalf("deleted %d files\n%s", s.deleted, s.output)
	355	+	}
	356	+	for name, want := range map[string]bool{
	357	+	"css/deep/x.css": false,
	358	+	"img/logo.svg": false,
	359	+	"keep.txt": true,
	360	+	"css/site.css": true,
	361	+	"big.bin": true,
	362	+	} {
	363	+	if _, _, ok := env.stored("/site/" + name); ok != want {
	364	+	t.Errorf("%s stored=%v after delete, want %v", name, ok, want)
	365	+	}
	366	+	}
	367	+	})
	368	+
	369	+	dst := tree{t, t.TempDir()}
	370	+
	371	+	t.Run("download", func(t *testing.T) {
	372	+	s := env.rsync("-rtz", remote+"/", dst.dir+"/")
	373	+	if s.transferred != 4 {
	374	+	t.Fatalf("download transferred %d files\n%s", s.transferred, s.output)
	375	+	}
	376	+	for _, name := range []string{"index.html", "big.bin", "css/site.css", "keep.txt"} {
	377	+	got, mtime := dst.read(name)
	378	+	want, storedMtime, _ := env.stored("/site/" + name)
	379	+	if !bytes.Equal(got, want) {
	380	+	t.Errorf("downloaded %s differs from storage", name)
	381	+	}
	382	+	if !mtime.Equal(storedMtime) {
	383	+	t.Errorf("downloaded %s has mtime %v, storage has %v", name, mtime, storedMtime)
	384	+	}
	385	+	}
	386	+	})
	387	+
	388	+	t.Run("unchanged download sends nothing", func(t *testing.T) {
	389	+	s := env.rsync("-rtz", remote+"/", dst.dir+"/")
	390	+	if s.transferred != 0 \|\| s.literal != 0 {
	391	+	t.Fatalf("transferred %d files, %d literal bytes\n%s", s.transferred, s.literal, s.output)
	392	+	}
	393	+	})
	394	+
	395	+	t.Run("download delta against a local copy", func(t *testing.T) {
	396	+	local, _ := dst.read("big.bin")
	397	+	copy(local[2<<20:], "local edit")
	398	+	dst.write("big.bin", local, t0)
	399	+	s := env.rsync("-rtz", remote+"/", dst.dir+"/")
	400	+	if s.transferred != 1 \|\| s.literal > 16<<10 \|\| s.matched < len(big)-16<<10 {
	401	+	t.Fatalf("transferred %d, literal %d, matched %d\n%s", s.transferred, s.literal, s.matched, s.output)
	402	+	}
	403	+	if got, _ := dst.read("big.bin"); !bytes.Equal(got, big) {
	404	+	t.Fatal("big.bin wrong after delta download")
	405	+	}
	406	+	})
	407	+
	408	+	t.Run("download single file", func(t *testing.T) {
	409	+	one := tree{t, t.TempDir()}
	410	+	env.rsync("-t", remote+"/css/site.css", one.dir+"/")
	411	+	if got, _ := one.read("site.css"); string(got) != "b{}" {
	412	+	t.Fatalf("site.css is %q", got)
	413	+	}
	414	+	})
	415	+	})
	416	+	}
	417	+	}
	418	+
	419	+	func TestRsyncStorageRejections(t *testing.T) {
	420	+	for name, newStorage := range rsyncStorageBackends() {
	421	+	t.Run(name, func(t *testing.T) {
	422	+	for _, reason := range []string{"ignore", "file limit", "storage quota"} {
	423	+	t.Run(reason, func(t *testing.T) {
	424	+	env := newRsyncEnv(t, newStorage(t))
	425	+	src := tree{t, t.TempDir()}
	426	+	remote := "localhost:/site"
	427	+	mtime := time.Unix(1_600_000_000, 0)
	428	+	wantExit := 23
	429	+	accepted := map[string]string{"z.txt": "ok"}
	430	+	switch reason {
	431	+	case "ignore":
	432	+	src.write("_pgs_ignore", []byte("a.txt\n"), mtime)
	433	+	env.rsync("-rt", src.dir+"/", remote)
	434	+	wantExit = 0
	435	+	case "file limit":
	436	+	env.feature.Data.FileMax = 3
	437	+	case "storage quota":
	438	+	src.write("z.txt", []byte("full"), mtime)
	439	+	env.rsync("-rt", src.dir+"/", remote)
	440	+	env.feature.Data.StorageMax = 4
	441	+	accepted["zz.txt"] = "x"
	442	+	src.write("zz.txt", []byte("x"), mtime)
	443	+	}
	444	+	src.write("a.txt", []byte("rejected"), mtime)
	445	+	src.write("z.txt", []byte("ok"), mtime)
	446	+	s := env.rsyncExit(wantExit, "-rt", "--remove-source-files", src.dir+"/", remote)
	447	+	if !strings.Contains(s.output, "a.txt") {
	448	+	t.Fatalf("missing per-file rejection message\n%s", s.output)
	449	+	}
	450	+	if reason == "ignore" && !strings.Contains(s.output, "a.txt skipped: file ignored: (/site/a.txt), https://pico.sh/pgs#-pgs-ignore\n") {
	451	+	t.Fatalf("unexpected ignore message\n%s", s.output)
	452	+	}
	453	+	if _, _, ok := env.stored("/site/a.txt"); ok {
	454	+	t.Fatal("rejected file was stored")
	455	+	}
	456	+	if data, _ := src.read("a.txt"); string(data) != "rejected" {
	457	+	t.Fatal("rejected source file changed")
	458	+	}
	459	+	for file, contents := range accepted {
	460	+	if data, _, ok := env.stored("/site/" + file); !ok \|\| string(data) != contents {
	461	+	t.Errorf("accepted file %s not stored correctly", file)
	462	+	}
	463	+	if _, err := os.Stat(filepath.Join(src.dir, file)); !errors.Is(err, os.ErrNotExist) {
	464	+	t.Errorf("accepted file %s not removed from source: %v", file, err)
	465	+	}
	466	+	}
	467	+	})
	468	+	}
	469	+	})
	470	+	}
	471	+	}
	472	+
	473	+	// TestRsyncStorageQuota checks that shrinking and deleting files frees quota
	474	+	// for files later in the same run.
	475	+	func TestRsyncStorageQuota(t *testing.T) {
	476	+	for name, newStorage := range rsyncStorageBackends() {
	477	+	t.Run(name, func(t *testing.T) {
	478	+	env := newRsyncEnv(t, newStorage(t))
	479	+	env.feature.Data.StorageMax = 3 << 20
	480	+	src := tree{t, t.TempDir()}
	481	+	remote := "localhost:/site"
	482	+	t0 := time.Unix(1_600_000_000, 0)
	483	+
	484	+	src.write("a.bin", randomBytes(2<<20), t0)
	485	+	env.rsync("-rt", src.dir+"/", remote)
	486	+
	487	+	// a.bin is sent before b.bin, which only fits once a.bin's
	488	+	// shrink is counted.
	489	+	src.write("a.bin", randomBytes(512<<10), t0)
	490	+	src.write("b.bin", randomBytes(2<<20), t0)
	491	+	if s := env.rsync("-rt", src.dir+"/", remote); s.transferred != 2 {
	492	+	t.Fatalf("transferred %d files, want 2\n%s", s.transferred, s.output)
	493	+	}
	494	+
	495	+	if err := os.Remove(filepath.Join(src.dir, "a.bin")); err != nil {
	496	+	t.Fatal(err)
	497	+	}
	498	+	src.write("c.bin", randomBytes(768<<10), t0)
	499	+	if s := env.rsync("-rt", "--delete-before", src.dir+"/", remote); s.deleted != 1 \|\| s.transferred != 1 {
	500	+	t.Fatalf("deleted %d and transferred %d files, want 1 and 1\n%s", s.deleted, s.transferred, s.output)
	501	+	}
	502	+	if _, _, ok := env.stored("/site/c.bin"); !ok {
	503	+	t.Fatal("c.bin was not stored")
	504	+	}
	505	+	})
	506	+	}
	507	+	}
	508	+
	509	+	// TestRsyncStorageMergeFilters checks rsync's per-directory filter files
	510	+	// against pgs storage, where dotfiles are only stored when _pgs_ignore
	511	+	// allows them.
	512	+	func TestRsyncStorageMergeFilters(t *testing.T) {
	513	+	for name, newStorage := range rsyncStorageBackends() {
	514	+	t.Run(name, func(t *testing.T) {
	515	+	for _, allowDotfiles := range []bool{true, false} {
	516	+	t.Run(fmt.Sprintf("dotfiles=%v", allowDotfiles), func(t *testing.T) {
	517	+	env := newRsyncEnv(t, newStorage(t))
	518	+	src := tree{t, t.TempDir()}
	519	+	remote := "localhost:/site"
	520	+	mtime := time.Unix(1_600_000_000, 0)
	521	+	if allowDotfiles {
	522	+	src.write("_pgs_ignore", []byte("_private\n"), mtime)
	523	+	env.rsync("-rt", src.dir+"/", remote)
	524	+	}
	525	+	src.write(".rsync-filter", []byte("- keep.txt\n"), mtime)
	526	+	src.write("sub/.rsync-filter", []byte("- *.tmp\n"), mtime)
	527	+	for _, f := range []string{"index.html", "keep.txt", "stale.txt", "sub/a.tmp", "sub/b.html"} {
	528	+	src.write(f, []byte(f), mtime)
	529	+	}
	530	+	env.rsync("-rt", src.dir+"/", remote)
	531	+	_, _, stored := env.stored("/site/.rsync-filter")
	532	+	if stored != allowDotfiles {
	533	+	t.Fatalf(".rsync-filter stored=%v with dotfiles allowed=%v", stored, allowDotfiles)
	534	+	}
	535	+
	536	+	dst := tree{t, t.TempDir()}
	537	+	env.rsync("-rt", "-F", remote+"/", dst.dir+"/")
	538	+	for f, filtered := range map[string]bool{"keep.txt": true, "sub/a.tmp": true, "index.html": false, "sub/b.html": false} {
	539	+	_, err := os.Stat(filepath.Join(dst.dir, f))
	540	+	if got := err == nil; got != (!filtered \|\| !allowDotfiles) {
	541	+	t.Errorf("download: %s present=%v", f, got)
	542	+	}
	543	+	}
	544	+
	545	+	for _, f := range []string{"keep.txt", "stale.txt", "sub/a.tmp"} {
	546	+	if err := os.Remove(filepath.Join(src.dir, f)); err != nil {
	547	+	t.Fatal(err)
	548	+	}
	549	+	}
	550	+	s := env.rsync("-rt", "-F", "--delete", src.dir+"/", remote)
	551	+	for f, protected := range map[string]bool{"keep.txt": true, "sub/a.tmp": true, "stale.txt": false} {
	552	+	_, _, ok := env.stored("/site/" + f)
	553	+	if ok != (protected && allowDotfiles) {
	554	+	t.Errorf("upload: %s stored=%v\n%s", f, ok, s.output)
	555	+	}
	556	+	}
	557	+	})
	558	+	}
	559	+	})
	560	+	}
	561	+	}

...	...		@@ -0,0 +1,632 @@
	1	+	package pgs
	2	+
	3	+	import (
	4	+	"errors"
	5	+	"io"
	6	+	"os"
	7	+	"os/exec"
	8	+	"path/filepath"
	9	+	"strings"
	10	+	"testing"
	11	+	"time"
	12	+
	13	+	"github.com/pkg/sftp"
	14	+	)
	15	+
	16	+	// uploadModes upload the contents of a local directory to /site. OpenSSH's
	17	+	// scp uses SFTP unless -O asks for the legacy protocol.
	18	+	var uploadModes = map[string]func(e rsyncEnv, dir string) exec.Cmd{
	19	+	"scp-legacy": func(e rsyncEnv, dir string) exec.Cmd {
	20	+	return exec.Command("scp", append(append([]string{"-O", "-r"}, e.sshArgs...), dir+"/.", "localhost:/site")...)
	21	+	},
	22	+	"scp-sftp": func(e rsyncEnv, dir string) exec.Cmd {
	23	+	return exec.Command("scp", append(append([]string{"-r"}, e.sshArgs...), dir+"/.", "localhost:/site")...)
	24	+	},
	25	+	"sftp": func(e rsyncEnv, dir string) exec.Cmd {
	26	+	return e.sftp("-mkdir /site", "put -r "+dir+"/. /site")
	27	+	},
	28	+	}
	29	+
	30	+	func (e rsyncEnv) sftp(cmds ...string) exec.Cmd {
	31	+	cmd := exec.Command("sftp", append(append([]string{"-b", "-"}, e.sshArgs...), "localhost")...)
	32	+	cmd.Stdin = strings.NewReader(strings.Join(cmds, "\n") + "\n")
	33	+	return cmd
	34	+	}
	35	+
	36	+	func runCmd(t testing.T, cmd exec.Cmd) (string, int) {
	37	+	t.Helper()
	38	+	out, err := cmd.CombinedOutput()
	39	+	var exit *exec.ExitError
	40	+	switch {
	41	+	case errors.As(err, &exit):
	42	+	return string(out), exit.ExitCode()
	43	+	case err != nil:
	44	+	t.Fatalf("%s: %v", cmd.Path, err)
	45	+	}
	46	+	return string(out), 0
	47	+	}
	48	+
	49	+	func (e *rsyncEnv) assertStored(files map[string]string) {
	50	+	e.t.Helper()
	51	+	for name, want := range files {
	52	+	if got, _, ok := e.stored("/site/" + name); !ok \|\| string(got) != want {
	53	+	e.t.Errorf("%s stored=%v contents=%q, want %q", name, ok, got, want)
	54	+	}
	55	+	}
	56	+	}
	57	+
	58	+	// waitStored waits for files to be stored. sshfs closes a file after the
	59	+	// program writing it has exited, and the file is stored on close.
	60	+	func (e *rsyncEnv) waitStored(files map[string]string) {
	61	+	e.t.Helper()
	62	+	for range 50 {
	63	+	done := true
	64	+	for name, want := range files {
	65	+	if got, _, ok := e.stored("/site/" + name); !ok \|\| string(got) != want {
	66	+	done = false
	67	+	}
	68	+	}
	69	+	if done {
	70	+	return
	71	+	}
	72	+	time.Sleep(20 * time.Millisecond)
	73	+	}
	74	+	e.assertStored(files)
	75	+	}
	76	+
	77	+	func TestScpSftpUpload(t *testing.T) {
	78	+	newStorage := rsyncStorageBackends()["memory"]
	79	+	mtime := time.Unix(1_600_000_000, 0)
	80	+	for mode, upload := range uploadModes {
	81	+	t.Run(mode, func(t *testing.T) {
	82	+	t.Run("tree and overwrite", func(t *testing.T) {
	83	+	env := newRsyncEnv(t, newStorage(t))
	84	+	src := tree{t, t.TempDir()}
	85	+	files := map[string]string{"index.html": "index", "css/site.css": "css", "img/a/b.png": "png"}
	86	+	for name, contents := range files {
	87	+	src.write(name, []byte(contents), mtime)
	88	+	}
	89	+	if out, code := runCmd(t, upload(env, src.dir)); code != 0 {
	90	+	t.Fatalf("exit %d\n%s", code, out)
	91	+	}
	92	+	env.assertStored(files)
	93	+
	94	+	files["index.html"] = "changed"
	95	+	src.write("index.html", []byte("changed"), mtime)
	96	+	if out, code := runCmd(t, upload(env, src.dir)); code != 0 {
	97	+	t.Fatalf("exit %d\n%s", code, out)
	98	+	}
	99	+	env.assertStored(files)
	100	+	})
	101	+
	102	+	t.Run("rejected file fails without stopping the rest", func(t *testing.T) {
	103	+	env := newRsyncEnv(t, newStorage(t))
	104	+	env.feature.Data.FileMax = 100
	105	+	src := tree{t, t.TempDir()}
	106	+	ok := map[string]string{"a1.txt": "ok", "c3.txt": "ok", "sub/d4.txt": "ok"}
	107	+	for name, contents := range ok {
	108	+	src.write(name, []byte(contents), mtime)
	109	+	}
	110	+	big := strings.Repeat("x", 200)
	111	+	src.write("b2.bin", []byte(big), mtime)
	112	+	out, code := runCmd(t, upload(env, src.dir))
	113	+	if code == 0 {
	114	+	t.Errorf("rejected upload exited 0\n%s", out)
	115	+	}
	116	+	if !strings.Contains(out, "stream size exceeded") {
	117	+	t.Errorf("no rejection message\n%s", out)
	118	+	}
	119	+	env.assertStored(ok)
	120	+	if got, _, _ := env.stored("/site/b2.bin"); string(got) == big {
	121	+	t.Error("oversized file was stored")
	122	+	}
	123	+	})
	124	+
	125	+	t.Run("ignored dotfile", func(t *testing.T) {
	126	+	env := newRsyncEnv(t, newStorage(t))
	127	+	src := tree{t, t.TempDir()}
	128	+	ok := map[string]string{"a1.txt": "ok", "z9.txt": "ok"}
	129	+	for name, contents := range ok {
	130	+	src.write(name, []byte(contents), mtime)
	131	+	}
	132	+	src.write(".secret", []byte("dot"), mtime)
	133	+	out, code := runCmd(t, upload(env, src.dir))
	134	+	if !strings.Contains(out, "file ignored: (/site/.secret)") {
	135	+	t.Errorf("no ignore message\n%s", out)
	136	+	}
	137	+	if code != 0 {
	138	+	t.Errorf("ignored file failed the upload: exit %d\n%s", code, out)
	139	+	}
	140	+	env.assertStored(ok)
	141	+	if _, _, stored := env.stored("/site/.secret"); stored {
	142	+	t.Error("ignored file was stored")
	143	+	}
	144	+	})
	145	+	})
	146	+	}
	147	+	}
	148	+
	149	+	func TestScpSftpDownloadAndCommands(t *testing.T) {
	150	+	env := newRsyncEnv(t, rsyncStorageBackends()["memory"](t))
	151	+	src := tree{t, t.TempDir()}
	152	+	files := map[string]string{"index.html": "index", "css/site.css": "css", "old.txt": "old"}
	153	+	for name, contents := range files {
	154	+	src.write(name, []byte(contents), time.Unix(1_600_000_000, 0))
	155	+	}
	156	+	if out, code := runCmd(t, uploadModes["sftp"](env, src.dir)); code != 0 {
	157	+	t.Fatalf("exit %d\n%s", code, out)
	158	+	}
	159	+
	160	+	t.Run("sftp get", func(t *testing.T) {
	161	+	dst := t.TempDir()
	162	+	if out, code := runCmd(t, env.sftp("get -r /site "+dst)); code != 0 {
	163	+	t.Fatalf("exit %d\n%s", code, out)
	164	+	}
	165	+	for name, want := range files {
	166	+	if got, err := os.ReadFile(filepath.Join(dst, "site", name)); err != nil \|\| string(got) != want {
	167	+	t.Errorf("%s: %q %v", name, got, err)
	168	+	}
	169	+	}
	170	+	})
	171	+
	172	+	t.Run("scp download over sftp", func(t *testing.T) {
	173	+	dst := t.TempDir()
	174	+	cmd := exec.Command("scp", append(append([]string{"-r"}, env.sshArgs...), "localhost:/site/css", dst)...)
	175	+	if out, code := runCmd(t, cmd); code != 0 {
	176	+	t.Fatalf("exit %d\n%s", code, out)
	177	+	}
	178	+	if got, err := os.ReadFile(filepath.Join(dst, "css/site.css")); err != nil \|\| string(got) != "css" {
	179	+	t.Errorf("site.css: %q %v", got, err)
	180	+	}
	181	+	})
	182	+
	183	+	t.Run("legacy scp download is refused", func(t *testing.T) {
	184	+	cmd := exec.Command("scp", append(append([]string{"-O"}, env.sshArgs...), "localhost:/site/index.html", t.TempDir())...)
	185	+	out, code := runCmd(t, cmd)
	186	+	if code == 0 \|\| !strings.Contains(out, "unsupported, use rsync or sftp") {
	187	+	t.Fatalf("exit %d\n%s", code, out)
	188	+	}
	189	+	})
	190	+
	191	+	t.Run("sftp ls rm", func(t *testing.T) {
	192	+	out, code := runCmd(t, env.sftp("ls /site", "rm /site/old.txt"))
	193	+	if code != 0 {
	194	+	t.Fatalf("exit %d\n%s", code, out)
	195	+	}
	196	+	for _, name := range []string{"index.html", "old.txt", "css"} {
	197	+	if !strings.Contains(out, name) {
	198	+	t.Errorf("ls is missing %s\n%s", name, out)
	199	+	}
	200	+	}
	201	+	if _, _, ok := env.stored("/site/old.txt"); ok {
	202	+	t.Error("rm did not delete the file")
	203	+	}
	204	+	env.assertStored(map[string]string{"index.html": "index", "css/site.css": "css"})
	205	+	})
	206	+	}
	207	+
	208	+	func TestSftpIgnoredFileKeepsBatchGoing(t *testing.T) {
	209	+	env := newRsyncEnv(t, rsyncStorageBackends()["memory"](t))
	210	+	src := tree{t, t.TempDir()}
	211	+	src.write(".secret", []byte("dot"), time.Unix(1_600_000_000, 0))
	212	+	src.write("after.txt", []byte("after"), time.Unix(1_600_000_000, 0))
	213	+	out, code := runCmd(t, env.sftp(
	214	+	"-mkdir /site",
	215	+	"put "+filepath.Join(src.dir, ".secret")+" /site/.secret",
	216	+	"put "+filepath.Join(src.dir, "after.txt")+" /site/after.txt",
	217	+	))
	218	+	if code != 0 {
	219	+	t.Fatalf("exit %d\n%s", code, out)
	220	+	}
	221	+	env.assertStored(map[string]string{"after.txt": "after"})
	222	+	if _, _, ok := env.stored("/site/.secret"); ok {
	223	+	t.Error("ignored file was stored")
	224	+	}
	225	+	}
	226	+
	227	+	func TestSftpRejectedUploadKeepsStoredFile(t *testing.T) {
	228	+	for _, mode := range []string{"scp-sftp", "sftp"} {
	229	+	t.Run(mode, func(t *testing.T) {
	230	+	env := newRsyncEnv(t, rsyncStorageBackends()["memory"](t))
	231	+	src := tree{t, t.TempDir()}
	232	+	src.write("page.html", []byte("original"), time.Unix(1_600_000_000, 0))
	233	+	if out, code := runCmd(t, uploadModes[mode](env, src.dir)); code != 0 {
	234	+	t.Fatalf("exit %d\n%s", code, out)
	235	+	}
	236	+	env.feature.Data.FileMax = 10
	237	+	src.write("page.html", []byte(strings.Repeat("x", 100)), time.Unix(1_600_000_000, 0))
	238	+	src.write("new.html", []byte(strings.Repeat("y", 100)), time.Unix(1_600_000_000, 0))
	239	+	if out, code := runCmd(t, uploadModes[mode](env, src.dir)); code == 0 {
	240	+	t.Fatalf("oversized upload exited 0\n%s", out)
	241	+	}
	242	+	env.assertStored(map[string]string{"page.html": "original"})
	243	+	if _, _, ok := env.stored("/site/new.html"); ok {
	244	+	t.Error("rejected new file left an object in storage")
	245	+	}
	246	+	})
	247	+	}
	248	+	}
	249	+
	250	+	// TestSftpOpenFiles drives the calls sshfs makes for touch and editors.
	251	+	func TestSftpOpenFiles(t *testing.T) {
	252	+	env := newRsyncEnv(t, rsyncStorageBackends()["memory"](t))
	253	+	conn, err := env.user.NewClientAddr(env.addr)
	254	+	if err != nil {
	255	+	t.Fatal(err)
	256	+	}
	257	+	defer func() { _ = conn.Close() }()
	258	+	client, err := sftp.NewClient(conn)
	259	+	if err != nil {
	260	+	t.Fatal(err)
	261	+	}
	262	+	defer func() { _ = client.Close() }()
	263	+	_ = client.Mkdir("/site")
	264	+
	265	+	put := func(name, contents string) {
	266	+	t.Helper()
	267	+	f, err := client.Create(name)
	268	+	if err != nil {
	269	+	t.Fatal(err)
	270	+	}
	271	+	if _, err := f.Write([]byte(contents)); err != nil {
	272	+	t.Fatal(err)
	273	+	}
	274	+	if err := f.Close(); err != nil {
	275	+	t.Fatal(err)
	276	+	}
	277	+	}
	278	+
	279	+	t.Run("touch keeps an existing file", func(t *testing.T) {
	280	+	put("/site/a.txt", "hello")
	281	+	f, err := client.OpenFile("/site/a.txt", os.O_WRONLY\|os.O_CREATE)
	282	+	if err != nil {
	283	+	t.Fatal(err)
	284	+	}
	285	+	if err := f.Close(); err != nil {
	286	+	t.Fatal(err)
	287	+	}
	288	+	env.assertStored(map[string]string{"a.txt": "hello"})
	289	+	})
	290	+
	291	+	t.Run("touch creates an empty file", func(t *testing.T) {
	292	+	f, err := client.OpenFile("/site/empty.txt", os.O_WRONLY\|os.O_CREATE)
	293	+	if err != nil {
	294	+	t.Fatal(err)
	295	+	}
	296	+	if _, err := client.Stat("/site/empty.txt"); err != nil {
	297	+	t.Errorf("open file not visible to stat: %v", err)
	298	+	}
	299	+	if _, _, ok := env.stored("/site/empty.txt"); ok {
	300	+	t.Error("file stored before close")
	301	+	}
	302	+	if err := f.Close(); err != nil {
	303	+	t.Fatal(err)
	304	+	}
	305	+	env.assertStored(map[string]string{"empty.txt": ""})
	306	+	})
	307	+
	308	+	t.Run("open file is listed and readable", func(t *testing.T) {
	309	+	f, err := client.OpenFile("/site/draft.txt", os.O_WRONLY\|os.O_CREATE\|os.O_TRUNC)
	310	+	if err != nil {
	311	+	t.Fatal(err)
	312	+	}
	313	+	if _, err := f.Write([]byte("draft")); err != nil {
	314	+	t.Fatal(err)
	315	+	}
	316	+	infos, err := client.ReadDir("/site")
	317	+	if err != nil {
	318	+	t.Fatal(err)
	319	+	}
	320	+	listed := false
	321	+	for _, info := range infos {
	322	+	if info.Name() == "draft.txt" && info.Size() == 5 {
	323	+	listed = true
	324	+	}
	325	+	}
	326	+	if !listed {
	327	+	t.Error("open file missing from the listing")
	328	+	}
	329	+	r, err := client.Open("/site/draft.txt")
	330	+	if err != nil {
	331	+	t.Fatal(err)
	332	+	}
	333	+	got, _ := io.ReadAll(r)
	334	+	_ = r.Close()
	335	+	if string(got) != "draft" {
	336	+	t.Errorf("read %q from the open file", got)
	337	+	}
	338	+	if err := f.Close(); err != nil {
	339	+	t.Fatal(err)
	340	+	}
	341	+	env.assertStored(map[string]string{"draft.txt": "draft"})
	342	+	})
	343	+
	344	+	t.Run("write at an offset keeps the rest", func(t *testing.T) {
	345	+	put("/site/b.txt", "hello world")
	346	+	f, err := client.OpenFile("/site/b.txt", os.O_WRONLY)
	347	+	if err != nil {
	348	+	t.Fatal(err)
	349	+	}
	350	+	if _, err := f.WriteAt([]byte("WORLD"), 6); err != nil {
	351	+	t.Fatal(err)
	352	+	}
	353	+	if err := f.Close(); err != nil {
	354	+	t.Fatal(err)
	355	+	}
	356	+	env.assertStored(map[string]string{"b.txt": "hello WORLD"})
	357	+	})
	358	+
	359	+	t.Run("truncate keeps nothing", func(t *testing.T) {
	360	+	put("/site/c.txt", "old")
	361	+	f, err := client.OpenFile("/site/c.txt", os.O_WRONLY\|os.O_TRUNC)
	362	+	if err != nil {
	363	+	t.Fatal(err)
	364	+	}
	365	+	if err := f.Close(); err != nil {
	366	+	t.Fatal(err)
	367	+	}
	368	+	env.assertStored(map[string]string{"c.txt": ""})
	369	+	})
	370	+
	371	+	t.Run("rename", func(t *testing.T) {
	372	+	put("/site/r1.txt", "one")
	373	+	put("/site/r2.txt", "two")
	374	+	if err := client.Rename("/site/r1.txt", "/site/r2.txt"); err == nil {
	375	+	t.Error("rename replaced an existing file")
	376	+	}
	377	+	if err := client.PosixRename("/site/r1.txt", "/site/r2.txt"); err != nil {
	378	+	t.Fatal(err)
	379	+	}
	380	+	env.assertStored(map[string]string{"r2.txt": "one"})
	381	+	if _, _, ok := env.stored("/site/r1.txt"); ok {
	382	+	t.Error("source survived the rename")
	383	+	}
	384	+	if err := client.Rename("/site/r2.txt", "/site/.r2"); err == nil {
	385	+	t.Error("rename to an ignored name succeeded")
	386	+	}
	387	+	env.assertStored(map[string]string{"r2.txt": "one"})
	388	+
	389	+	f, err := client.OpenFile("/site/r2.txt", os.O_WRONLY)
	390	+	if err != nil {
	391	+	t.Fatal(err)
	392	+	}
	393	+	if _, err := f.WriteAt([]byte("ONE"), 0); err != nil {
	394	+	t.Fatal(err)
	395	+	}
	396	+	if err := client.Rename("/site/r2.txt", "/site/r3.txt"); err != nil {
	397	+	t.Fatal(err)
	398	+	}
	399	+	if err := f.Close(); err != nil {
	400	+	t.Fatal(err)
	401	+	}
	402	+	env.assertStored(map[string]string{"r3.txt": "ONE"})
	403	+	if _, _, ok := env.stored("/site/r2.txt"); ok {
	404	+	t.Error("open file's old path is still stored after it closed")
	405	+	}
	406	+	})
	407	+
	408	+	t.Run("truncate then write past the end leaves zeros", func(t *testing.T) {
	409	+	put("/site/z.txt", "secret")
	410	+	f, err := client.OpenFile("/site/z.txt", os.O_WRONLY)
	411	+	if err != nil {
	412	+	t.Fatal(err)
	413	+	}
	414	+	if err := f.Truncate(0); err != nil {
	415	+	t.Fatal(err)
	416	+	}
	417	+	if _, err := f.WriteAt([]byte("x"), 5); err != nil {
	418	+	t.Fatal(err)
	419	+	}
	420	+	if err := f.Close(); err != nil {
	421	+	t.Fatal(err)
	422	+	}
	423	+	env.assertStored(map[string]string{"z.txt": "\x00\x00\x00\x00\x00x"})
	424	+	})
	425	+
	426	+	t.Run("renaming an open file keeps the source until it is stored", func(t *testing.T) {
	427	+	openAndWrite := func(name, contents string) *sftp.File {
	428	+	t.Helper()
	429	+	f, err := client.OpenFile(name, os.O_WRONLY)
	430	+	if err != nil {
	431	+	t.Fatal(err)
	432	+	}
	433	+	if _, err := f.WriteAt([]byte(contents), 0); err != nil {
	434	+	t.Fatal(err)
	435	+	}
	436	+	return f
	437	+	}
	438	+
	439	+	put("/site/k.txt", "keep")
	440	+	f := openAndWrite("/site/k.txt", "KEEP")
	441	+	if err := client.PosixRename("/site/k.txt", "/site/.ignored"); err == nil {
	442	+	t.Error("renamed an open file to an ignored name")
	443	+	}
	444	+	_ = f.Close()
	445	+	env.assertStored(map[string]string{"k.txt": "KEEP"})
	446	+
	447	+	put("/site/k.txt", "keep")
	448	+	f = openAndWrite("/site/k.txt", strings.Repeat("x", 200))
	449	+	if err := client.PosixRename("/site/k.txt", "/site/big.txt"); err != nil {
	450	+	t.Fatal(err)
	451	+	}
	452	+	if _, err := client.Stat("/site/k.txt"); !errors.Is(err, os.ErrNotExist) {
	453	+	t.Errorf("renamed source still visible: %v", err)
	454	+	}
	455	+	env.feature.Data.FileMax = 100
	456	+	if err := f.Close(); err == nil {
	457	+	t.Error("oversized close succeeded")
	458	+	}
	459	+	env.feature.Data.FileMax = rsyncFileMax
	460	+	env.assertStored(map[string]string{"k.txt": "keep"})
	461	+	if _, _, ok := env.stored("/site/big.txt"); ok {
	462	+	t.Error("rejected destination was stored")
	463	+	}
	464	+
	465	+	f = openAndWrite("/site/k.txt", "NEW!")
	466	+	if err := client.PosixRename("/site/k.txt", "/site/moved.txt"); err != nil {
	467	+	t.Fatal(err)
	468	+	}
	469	+	env.assertStored(map[string]string{"k.txt": "keep"})
	470	+	put("/site/k.txt", "replacement")
	471	+	if err := f.Close(); err != nil {
	472	+	t.Fatal(err)
	473	+	}
	474	+	env.assertStored(map[string]string{"moved.txt": "NEW!", "k.txt": "replacement"})
	475	+
	476	+	put("/site/k2.txt", "k2")
	477	+	f = openAndWrite("/site/k2.txt", "K2")
	478	+	if err := client.PosixRename("/site/k2.txt", "/site/moved2.txt"); err != nil {
	479	+	t.Fatal(err)
	480	+	}
	481	+	if err := f.Close(); err != nil {
	482	+	t.Fatal(err)
	483	+	}
	484	+	env.assertStored(map[string]string{"moved2.txt": "K2"})
	485	+	if _, _, ok := env.stored("/site/k2.txt"); ok {
	486	+	t.Error("source kept after the renamed file was stored")
	487	+	}
	488	+	})
	489	+
	490	+	t.Run("renamed source replaced by another session survives", func(t *testing.T) {
	491	+	conn2, err := env.user.NewClientAddr(env.addr)
	492	+	if err != nil {
	493	+	t.Fatal(err)
	494	+	}
	495	+	defer func() { _ = conn2.Close() }()
	496	+	other, err := sftp.NewClient(conn2)
	497	+	if err != nil {
	498	+	t.Fatal(err)
	499	+	}
	500	+	defer func() { _ = other.Close() }()
	501	+
	502	+	put("/site/s1.txt", "orig")
	503	+	f, err := client.OpenFile("/site/s1.txt", os.O_WRONLY)
	504	+	if err != nil {
	505	+	t.Fatal(err)
	506	+	}
	507	+	if _, err := f.WriteAt([]byte("ORIG"), 0); err != nil {
	508	+	t.Fatal(err)
	509	+	}
	510	+	if err := client.PosixRename("/site/s1.txt", "/site/s2.txt"); err != nil {
	511	+	t.Fatal(err)
	512	+	}
	513	+	g, err := other.Create("/site/s1.txt")
	514	+	if err != nil {
	515	+	t.Fatal(err)
	516	+	}
	517	+	if _, err := g.Write([]byte("from another session")); err != nil {
	518	+	t.Fatal(err)
	519	+	}
	520	+	if err := g.Close(); err != nil {
	521	+	t.Fatal(err)
	522	+	}
	523	+	if err := f.Close(); err != nil {
	524	+	t.Fatal(err)
	525	+	}
	526	+	env.assertStored(map[string]string{"s1.txt": "from another session", "s2.txt": "ORIG"})
	527	+	})
	528	+
	529	+	t.Run("removing a file renamed over a stored one removes both", func(t *testing.T) {
	530	+	put("/site/old.txt", "old")
	531	+	put("/site/src.txt", "src")
	532	+	f, err := client.OpenFile("/site/src.txt", os.O_WRONLY)
	533	+	if err != nil {
	534	+	t.Fatal(err)
	535	+	}
	536	+	if _, err := f.WriteAt([]byte("SRC"), 0); err != nil {
	537	+	t.Fatal(err)
	538	+	}
	539	+	if err := client.PosixRename("/site/src.txt", "/site/old.txt"); err != nil {
	540	+	t.Fatal(err)
	541	+	}
	542	+	if err := client.Remove("/site/old.txt"); err != nil {
	543	+	t.Fatal(err)
	544	+	}
	545	+	if err := f.Close(); err != nil {
	546	+	t.Fatal(err)
	547	+	}
	548	+	for _, name := range []string{"old.txt", "src.txt"} {
	549	+	if _, _, ok := env.stored("/site/" + name); ok {
	550	+	t.Errorf("%s is still stored", name)
	551	+	}
	552	+	}
	553	+	})
	554	+
	555	+	t.Run("removed while open is not stored", func(t *testing.T) {
	556	+	f, err := client.OpenFile("/site/swap.txt", os.O_WRONLY\|os.O_CREATE\|os.O_TRUNC)
	557	+	if err != nil {
	558	+	t.Fatal(err)
	559	+	}
	560	+	if _, err := f.Write([]byte("swap")); err != nil {
	561	+	t.Fatal(err)
	562	+	}
	563	+	if err := client.Remove("/site/swap.txt"); err != nil {
	564	+	t.Fatal(err)
	565	+	}
	566	+	if err := f.Close(); err != nil {
	567	+	t.Fatal(err)
	568	+	}
	569	+	if _, _, ok := env.stored("/site/swap.txt"); ok {
	570	+	t.Error("file removed while open was stored")
	571	+	}
	572	+	})
	573	+	}
	574	+
	575	+	// TestSshfs mounts pgs with sshfs and edits files the way shells and
	576	+	// editors do.
	577	+	func TestSshfs(t *testing.T) {
	578	+	if _, err := exec.LookPath("sshfs"); err != nil {
	579	+	t.Skip("sshfs not installed")
	580	+	}
	581	+	for name, newStorage := range rsyncStorageBackends() {
	582	+	t.Run(name, func(t *testing.T) { testSshfs(t, newRsyncEnv(t, newStorage(t))) })
	583	+	}
	584	+	}
	585	+
	586	+	func testSshfs(t testing.T, env rsyncEnv) {
	587	+	mnt := t.TempDir()
	588	+	opts := []string{"-p", env.sshArgs[1], "-o", "IdentityFile=" + env.sshArgs[3], "-o", "IdentitiesOnly=yes",
	589	+	"-o", "StrictHostKeyChecking=no", "-o", "UserKnownHostsFile=/dev/null", "-o", "LogLevel=ERROR"}
	590	+	if out, err := exec.Command("sshfs", append(opts, "localhost:/", mnt)...).CombinedOutput(); err != nil {
	591	+	t.Skipf("sshfs mount failed: %v\n%s", err, out)
	592	+	}
	593	+	t.Cleanup(func() { _ = exec.Command("fusermount3", "-u", "-z", mnt).Run() })
	594	+
	595	+	home := t.TempDir()
	596	+	steps := []struct {
	597	+	cmd string
	598	+	fails bool
	599	+	stored map[string]string
	600	+	}{
	601	+	{cmd: "mkdir site && echo hello > site/a.txt", stored: map[string]string{"a.txt": "hello\n"}},
	602	+	{cmd: "touch site/a.txt", stored: map[string]string{"a.txt": "hello\n"}},
	603	+	{cmd: "touch site/new.txt", stored: map[string]string{"new.txt": ""}},
	604	+	{cmd: "vim -n -es -c 'normal! ihi' -c wq site/vim.txt", stored: map[string]string{"vim.txt": "hi\n"}},
	605	+	{cmd: "vim -es -c '%s/hello/bye/' -c wq site/a.txt", stored: map[string]string{"a.txt": "bye\n"}},
	606	+	{cmd: "echo more >> site/a.txt", stored: map[string]string{"a.txt": "bye\nmore\n"}},
	607	+	{cmd: "printf abc > site/t.txt && truncate -s 1 site/t.txt", stored: map[string]string{"t.txt": "a"}},
	608	+	{cmd: "rm site/new.txt", stored: map[string]string{"a.txt": "bye\nmore\n"}},
	609	+	{cmd: "echo secret > site/.dot", fails: true},
	610	+	{cmd: "mv site/t.txt site/moved.txt", stored: map[string]string{"moved.txt": "a"}},
	611	+	{cmd: "mv site/moved.txt site/a.txt", stored: map[string]string{"a.txt": "a"}},
	612	+	{cmd: "mv site/a.txt site/.dot", fails: true, stored: map[string]string{"a.txt": "a"}},
	613	+	{cmd: "mkdir -p site/dir/sub site/empty && echo y > site/dir/sub/y.txt && mv site/dir site/dir2 && mv site/a.txt site/dir2/", stored: map[string]string{"dir2/sub/y.txt": "y\n", "dir2/a.txt": "a"}},
	614	+	{cmd: "mv site/empty site/empty2 && test -d site/empty2 && ! test -e site/empty && ! test -e site/dir"},
	615	+	{cmd: "echo q > site/q.txt && mv site/q.txt site/r.txt && cat site/r.txt", stored: map[string]string{"r.txt": "q\n"}},
	616	+	}
	617	+	for _, step := range steps {
	618	+	cmd := exec.Command("bash", "-c", step.cmd)
	619	+	cmd.Dir = mnt
	620	+	cmd.Env = append(os.Environ(), "HOME="+home)
	621	+	out, err := cmd.CombinedOutput()
	622	+	if (err != nil) != step.fails {
	623	+	t.Fatalf("%s: err=%v\n%s", step.cmd, err, out)
	624	+	}
	625	+	env.waitStored(step.stored)
	626	+	}
	627	+	for _, name := range []string{"new.txt", ".dot", "t.txt", "moved.txt", "a.txt", "dir/sub/y.txt", "q.txt"} {
	628	+	if _, _, ok := env.stored("/site/" + name); ok {
	629	+	t.Errorf("%s is stored", name)
	630	+	}
	631	+	}
	632	+	}

...	...		@@ -1,16 +1,12 @@
1	1		package pgs
2	2
3	3		import (
4		-	"bytes"
5	4		"context"
6	5		"fmt"
7	6		"io"
8		-	"io/fs"
9	7		"log/slog"
10	8		"os"
11		-	"path"
12	9		"path/filepath"
13		-	"slices"
14	10		"strings"
15	11		"sync"
16	12		"time"

...	...		@@ -23,82 +19,41 @@ import (
23	19		ignore "github.com/sabhiram/go-gitignore"
24	20		)
25	21
26		-	type ctxBucketKey struct{}
27		-	type ctxStorageSizeKey struct{}
28		-	type ctxProjectKey struct{}
29		-	type ctxFeatureFlagKey struct{}
30		-	type ctxDenylistKey struct{}
31		-
32		-	type DenyList struct {
33		-	Denylist string
34		-	}
35		-
36		-	func getDenylist(s pssh.SSHServerConnSession) DenyList {
37		-	v := s.Context().Value(ctxDenylistKey{})
38		-	if v == nil {
39		-	return nil
40		-	}
41		-	denylist := s.Context().Value(ctxDenylistKey{}).(*DenyList)
42		-	return denylist
43		-	}
44		-
45		-	func setDenylist(s *pssh.SSHServerConnSession, denylist string) {
46		-	s.SetValue(ctxDenylistKey{}, &DenyList{Denylist: denylist})
47		-	}
48		-
49		-	func getProject(s pssh.SSHServerConnSession) db.Project {
50		-	v := s.Context().Value(ctxProjectKey{})
51		-	if v == nil {
52		-	return nil
53		-	}
54		-	project := s.Context().Value(ctxProjectKey{}).(*db.Project)
55		-	return project
	22	+	type ctxUploadStateKey struct{}
	23	+
	24	+	// uploadState is what a session's uploads share. It is set on the session
	25	+	// once because every SetValue nests the session context one level deeper,
	26	+	// and SFTP can run several writes at a time.
	27	+	type uploadState struct {
	28	+	mu sync.Mutex
	29	+	bucket storage.Bucket
	30	+	featureFlag *db.FeatureFlag
	31	+	storageSize int64
	32	+	project *db.Project
	33	+	denylist *ignore.GitIgnore
56	34		}
57	35
58		-	func setProject(s pssh.SSHServerConnSession, project db.Project) {
59		-	s.SetValue(ctxProjectKey{}, project)
60		-	}
61		-
62		-	func getFeatureFlag(s pssh.SSHServerConnSession) db.FeatureFlag {
63		-	v := s.Context().Value(ctxFeatureFlagKey{})
64		-	if v == nil {
65		-	return nil
	36	+	func getUploadState(s pssh.SSHServerConnSession) (uploadState, error) {
	37	+	state, ok := s.Context().Value(ctxUploadStateKey{}).(*uploadState)
	38	+	if !ok {
	39	+	return nil, fmt.Errorf("upload state not set on `ssh.Context()` for connection")
66	40		}
67		-	ff := s.Context().Value(ctxFeatureFlagKey{}).(*db.FeatureFlag)
68		-	return ff
	41	+	return state, nil
69	42		}
70	43
71		-	func setFeatureFlag(s pssh.SSHServerConnSession, ff db.FeatureFlag) {
72		-	s.SetValue(ctxFeatureFlagKey{}, ff)
73		-	}
74		-
75		-	func getBucket(s *pssh.SSHServerConnSession) (storage.Bucket, error) {
76		-	bucket := s.Context().Value(ctxBucketKey{}).(storage.Bucket)
77		-	if bucket.Name == "" {
78		-	return bucket, fmt.Errorf("bucket not set on `ssh.Context()` for connection")
79		-	}
80		-	return bucket, nil
	44	+	func (u *uploadState) getStorageSize() int64 {
	45	+	u.mu.Lock()
	46	+	defer u.mu.Unlock()
	47	+	return u.storageSize
81	48		}
82	49
83		-	func getStorageSize(s *pssh.SSHServerConnSession) uint64 {
84		-	return s.Context().Value(ctxStorageSizeKey{}).(uint64)
85		-	}
86		-
87		-	func incrementStorageSize(s *pssh.SSHServerConnSession, fileSize int64) uint64 {
88		-	curSize := getStorageSize(s)
89		-	var nextStorageSize uint64
90		-	if fileSize < 0 {
91		-	nextStorageSize = curSize - uint64(fileSize)
92		-	} else {
93		-	nextStorageSize = curSize + uint64(fileSize)
94		-	}
95		-	s.SetValue(ctxStorageSizeKey{}, nextStorageSize)
96		-	return nextStorageSize
97		-	}
98		-
99		-	func shouldIgnoreFile(fp, ignoreStr string) bool {
100		-	object := ignore.CompileIgnoreLines(strings.Split(ignoreStr, "\n")...)
101		-	return object.MatchesPath(fp)
	50	+	// addStorageSize adds delta bytes to the bucket's size and returns the new
	51	+	// size.
	52	+	func (u *uploadState) addStorageSize(delta int64) int64 {
	53	+	u.mu.Lock()
	54	+	defer u.mu.Unlock()
	55	+	u.storageSize = max(u.storageSize+delta, 0)
	56	+	return u.storageSize
102	57		}
103	58
104	59		type FileData struct {

...	...		@@ -106,7 +61,7 @@ type FileData struct {
106	61		User *db.User
107	62		Bucket storage.Bucket
108	63		Project *db.Project
109		-	DenyList string
	64	+	DenyList *ignore.GitIgnore
110	65		}
111	66
112	67		type UploadAssetHandler struct {

...	...		@@ -222,20 +177,22 @@ func (h UploadAssetHandler) Validate(s pssh.SSHServerConnSession) error {
222	177		if err != nil {
223	178		return err
224	179		}
225		-	setFeatureFlag(s, ff)
226	180
227	181		assetBucket := shared.GetAssetBucketName(user.ID)
228	182		bucket, err := h.Cfg.Storage.UpsertBucket(assetBucket)
229	183		if err != nil {
230	184		return err
231	185		}
232		-	s.SetValue(ctxBucketKey{}, bucket)
233	186
234	187		totalStorageSize, err := h.Cfg.Storage.GetBucketQuota(bucket)
235	188		if err != nil {
236	189		return err
237	190		}
238		-	s.SetValue(ctxStorageSizeKey{}, totalStorageSize)
	191	+	s.SetValue(ctxUploadStateKey{}, &uploadState{
	192	+	bucket: bucket,
	193	+	featureFlag: ff,
	194	+	storageSize: int64(totalStorageSize),
	195	+	})
239	196
240	197		logger.Info(
241	198		"bucket size",

...	...		@@ -252,6 +209,33 @@ func (h UploadAssetHandler) Validate(s pssh.SSHServerConnSession) error {
252	209		return nil
253	210		}
254	211
	212	+	// sessionProject returns the project being uploaded to and its compiled
	213	+	// _pgs_ignore, finding or creating the project when it changes. A session
	214	+	// can upload to many projects because SFTP connections are kept alive.
	215	+	func (h UploadAssetHandler) sessionProject(state uploadState, user db.User, projectName string, logger slog.Logger) (db.Project, ignore.GitIgnore, error) {
	216	+	state.mu.Lock()
	217	+	defer state.mu.Unlock()
	218	+	if state.project != nil && state.project.Name == projectName {
	219	+	return state.project, state.denylist, nil
	220	+	}
	221	+
	222	+	project, err := h.Cfg.DB.UpsertProject(user.ID, projectName, projectName)
	223	+	if err != nil {
	224	+	logger.Error("upsert project", "err", err.Error())
	225	+	return nil, nil, err
	226	+	}
	227	+
	228	+	dlist, err := h.findDenylist(state.bucket, project, logger)
	229	+	if err != nil {
	230	+	logger.Info("failed to get denylist, setting default (.*)", "err", err.Error())
	231	+	dlist = ".*"
	232	+	}
	233	+
	234	+	state.project = project
	235	+	state.denylist = ignore.CompileIgnoreLines(strings.Split(dlist, "\n")...)
	236	+	return state.project, state.denylist, nil
	237	+	}
	238	+
255	239		func (h UploadAssetHandler) findDenylist(bucket storage.Bucket, project db.Project, logger *slog.Logger) (string, error) {
256	240		fp, _, err := h.Cfg.Storage.GetObject(bucket, filepath.Join(project.ProjectDir, "_pgs_ignore"))
257	241		if err != nil {

...	...		@@ -281,6 +265,19 @@ func mtimeToTime(entry *sendutils.FileEntry) time.Time {
281	265		}
282	266
283	267		func (h UploadAssetHandler) Write(s pssh.SSHServerConnSession, entry *sendutils.FileEntry) (string, error) {
	268	+	return h.write(s, entry, false)
	269	+	}
	270	+
	271	+	// ValidateWrite runs the checks Write makes before reading the file.
	272	+	func (h UploadAssetHandler) ValidateWrite(s pssh.SSHServerConnSession, entry *sendutils.FileEntry) error {
	273	+	if entry.Mode.IsDir() {
	274	+	return nil
	275	+	}
	276	+	_, err := h.write(s, entry, true)
	277	+	return err
	278	+	}
	279	+
	280	+	func (h UploadAssetHandler) write(s pssh.SSHServerConnSession, entry *sendutils.FileEntry, validateOnly bool) (string, error) {
284	281		logger := pssh.GetLogger(s)
285	282		user := pssh.GetUser(s)
286	283

...	...		@@ -299,27 +296,19 @@ func (h UploadAssetHandler) Write(s pssh.SSHServerConnSession, entry *sendutil
299	296		"size", entry.Size,
300	297		)
301	298
302		-	bucket, err := getBucket(s)
	299	+	state, err := getUploadState(s)
303	300		if err != nil {
304		-	logger.Error("could not find bucket in ctx", "err", err.Error())
	301	+	logger.Error("could not find upload state in ctx", "err", err.Error())
305	302		return "", err
306	303		}
	304	+	bucket := state.bucket
307	305
308		-	project := getProject(s)
309	306		projectName := shared.GetProjectName(entry)
310	307		logger = logger.With("project", projectName)
311	308
312		-	// find, create, or update project if we haven't already done it
313		-	// we need to also check if the project stored in ctx is the same project
314		-	// being uploaded since users can keep an ssh connection alive via sftp
315		-	// and created many projects in a single session
316		-	if project == nil \|\| project.Name != projectName {
317		-	project, err = h.Cfg.DB.UpsertProject(user.ID, projectName, projectName)
318		-	if err != nil {
319		-	logger.Error("upsert project", "err", err.Error())
320		-	return "", err
321		-	}
322		-	setProject(s, project)
	309	+	project, denylist, err := h.sessionProject(state, user, projectName, logger)
	310	+	if err != nil {
	311	+	return "", err
323	312		}
324	313
325	314		if project.Blocked != "" {

...	...		@@ -327,49 +316,23 @@ func (h UploadAssetHandler) Write(s pssh.SSHServerConnSession, entry *sendutil
327	316		return "", fmt.Errorf(msg, project.Blocked)
328	317		}
329	318
330		-	info := &storage.ObjectInfo{
331		-	LastModified: mtimeToTime(entry),
332		-	}
333	319		if entry.Mode.IsDir() {
334		-	_, _, err := h.Cfg.Storage.PutObject(
335		-	bucket,
336		-	path.Join(shared.GetAssetFileName(entry), "._pico_keep_dir"),
337		-	bytes.NewReader([]byte{}),
338		-	info,
339		-	)
340		-	return "", err
	320	+	return "", h.Cfg.Storage.PutDir(bucket, shared.GetAssetFileName(entry))
341	321		}
342	322
343	323		// calculate the filsize difference between the same file already
344	324		// stored and the updated file being uploaded
345	325		assetFilename := shared.GetAssetFileName(entry)
346		-	obj, info, _ := h.Cfg.Storage.GetObject(bucket, assetFilename)
347	326		var curFileSize int64
348		-	if info != nil {
	327	+	if info, err := h.Cfg.Storage.StatObject(bucket, assetFilename); err == nil {
349	328		curFileSize = info.Size
350	329		}
351		-	if obj != nil {
352		-	defer func() {
353		-	_ = obj.Close()
354		-	}()
355		-	}
356		-
357		-	denylist := getDenylist(s)
358		-	if denylist == nil {
359		-	dlist, err := h.findDenylist(bucket, project, logger)
360		-	if err != nil {
361		-	logger.Info("failed to get denylist, setting default (.*)", "err", err.Error())
362		-	dlist = ".*"
363		-	}
364		-	setDenylist(s, dlist)
365		-	denylist = &DenyList{Denylist: dlist}
366		-	}
367	330
368	331		data := &FileData{
369	332		FileEntry: entry,
370	333		User: user,
371	334		Bucket: bucket,
372		-	DenyList: denylist.Denylist,
	335	+	DenyList: denylist,
373	336		Project: project,
374	337		}
375	338

...	...		@@ -378,26 +341,22 @@ func (h UploadAssetHandler) Write(s pssh.SSHServerConnSession, entry *sendutil
378	341		return "", err
379	342		}
380	343
381		-	featureFlag := getFeatureFlag(s)
382		-	if featureFlag == nil {
383		-	return "", fmt.Errorf("pico+ feature flag ctx not set")
384		-	}
	344	+	featureFlag := state.featureFlag
385	345		// SFTP does not report file size so the more performant way to
386	346		// check filesize constraints is to try and upload the file to s3
387	347		// with a specialized reader that raises an error if the filesize limit
388	348		// has been reached
389	349		storageMax := featureFlag.Data.StorageMax
390	350		fileMax := featureFlag.Data.FileMax
391		-	curStorageSize := getStorageSize(s)
392		-	remaining := int64(storageMax) - int64(curStorageSize)
	351	+	curStorageSize := state.getStorageSize()
	352	+	remaining := int64(storageMax) - curStorageSize
393	353		sizeRemaining := min(remaining+curFileSize, fileMax)
394	354		if sizeRemaining <= 0 {
395		-	_, _ = fmt.Fprintln(s.Stderr(), "storage quota reached")
396		-	_, _ = fmt.Fprintf(s.Stderr(), "\r")
397		-	_ = s.Exit(1)
398		-	_ = s.Close()
399	355		return "", fmt.Errorf("storage quota reached")
400	356		}
	357	+	if validateOnly {
	358	+	return "", nil
	359	+	}
401	360		logger = logger.With(
402	361		"storageMax", storageMax,
403	362		"currentStorageMax", curStorageSize,

...	...		@@ -428,8 +387,7 @@ func (h UploadAssetHandler) Write(s pssh.SSHServerConnSession, entry *sendutil
428	387		return "", cerr
429	388		}
430	389
431		-	deltaFileSize := curFileSize - fsize
432		-	nextStorageSize := incrementStorageSize(s, deltaFileSize)
	390	+	nextStorageSize := state.addStorageSize(fsize - curFileSize)
433	391
434	392		url := h.Cfg.AssetURL(
435	393		user.Name,

...	...		@@ -477,93 +435,23 @@ func (h UploadAssetHandler) Delete(s pssh.SSHServerConnSession, entry *senduti
477	435		"file", assetFilepath,
478	436		)
479	437
480		-	bucket, err := getBucket(s)
	438	+	state, err := getUploadState(s)
481	439		if err != nil {
482		-	logger.Error("could not find bucket in ctx", "err", err.Error())
	440	+	logger.Error("could not find upload state in ctx", "err", err.Error())
483	441		return err
484	442		}
	443	+	bucket := state.bucket
485	444
486	445		projectName := shared.GetProjectName(entry)
487	446		logger = logger.With("project", projectName)
488	447
489	448		logger.Info("deleting file")
490	449
491		-	// Check if this path represents a directory (has a . _pico_keep_dir marker)
492		-	keepDirPath := filepath.Join(assetFilepath, "._pico_keep_dir")
493		-	keepDirReader, _, keepDirErr := h.Cfg.Storage.GetObject(bucket, keepDirPath)
494		-	if keepDirReader != nil {
495		-	defer func() {
496		-	_ = keepDirReader.Close()
497		-	}()
498		-	}
499		-
500		-	if keepDirErr == nil {
501		-	// This is a directory being deleted. We must delete all nested
502		-	// . _pico_keep_dir files first, otherwise os.Remove() on the
503		-	// directory will fail with "directory not empty".
504		-	nested, err := h.Cfg.Storage.ListObjects(bucket, assetFilepath+"/", true)
505		-	if err != nil {
506		-	return err
507		-	}
508		-
509		-	baseDepth := strings.Count(assetFilepath, string(os.PathSeparator))
510		-	for _, nestedEntry := range nested {
511		-	if filepath.Base(nestedEntry.Name()) != "._pico_keep_dir" {
512		-	continue
513		-	}
514		-	// Only delete keep_dir files that are direct or nested children
515		-	nestedDepth := strings.Count(nestedEntry.Name(), string(os.PathSeparator))
516		-	if nestedDepth <= baseDepth {
517		-	continue
518		-	}
519		-	nestedKeepDirPath := filepath.Join(assetFilepath, nestedEntry.Name())
520		-	if delErr := h.Cfg.Storage.DeleteObject(bucket, nestedKeepDirPath); delErr != nil {
521		-	return delErr
522		-	}
523		-	}
524		-
525		-	// Delete this directory's own . _pico_keep_dir
526		-	err = h.Cfg.Storage.DeleteObject(bucket, keepDirPath)
527		-	if err != nil {
528		-	return err
529		-	}
530		-
531		-	// Delete the directory itself (no-op for S3-style storage, removes the dir for fs storage)
532		-	_ = h.Cfg.Storage.DeleteObject(bucket, assetFilepath)
533		-
534		-	surrogate := getSurrogateKey(user.Name, projectName)
535		-	h.Cfg.CacheClearingQueue <- surrogate
536		-	return nil
537		-	}
538		-
539		-	// Regular file deletion: create . _pico_keep_dir if the directory becomes empty
540		-	pathDir := filepath.Dir(assetFilepath)
541		-	fileName := filepath.Base(assetFilepath)
542		-
543		-	sibs, err := h.Cfg.Storage.ListObjects(bucket, pathDir+"/", false)
544		-	if err != nil {
545		-	return err
546		-	}
547		-
548		-	sibs = slices.DeleteFunc(sibs, func(sib fs.FileInfo) bool {
549		-	return sib.Name() == fileName
550		-	})
551		-
552		-	if len(sibs) == 0 {
553		-	info := &storage.ObjectInfo{
554		-	LastModified: mtimeToTime(entry),
555		-	}
556		-	_, _, err := h.Cfg.Storage.PutObject(
557		-	bucket,
558		-	filepath.Join(pathDir, "._pico_keep_dir"),
559		-	bytes.NewReader([]byte{}),
560		-	info,
561		-	)
562		-	if err != nil {
563		-	return err
564		-	}
565		-	}
	450	+	info, statErr := h.Cfg.Storage.StatObject(bucket, assetFilepath)
566	451		err = h.Cfg.Storage.DeleteObject(bucket, assetFilepath)
	452	+	if err == nil && statErr == nil {
	453	+	state.addStorageSize(-info.Size)
	454	+	}
567	455
568	456		surrogate := getSurrogateKey(user.Name, projectName)
569	457		h.Cfg.CacheClearingQueue <- surrogate

...	...		@@ -589,9 +477,10 @@ func (h UploadAssetHandler) validateAsset(data FileData) (bool, error) {
589	477		}
590	478
591	479		fpath := strings.Replace(data.Filepath, "/"+projectName, "", 1)
592		-	if shouldIgnoreFile(fpath, data.DenyList) {
	480	+	if data.DenyList.MatchesPath(fpath) {
593	481		err := fmt.Errorf(
594		-	"ERROR: (%s) file rejected, https://pico.sh/pgs#-pgs-ignore",
	482	+	"%w: (%s), https://pico.sh/pgs#-pgs-ignore",
	483	+	sendutils.ErrFileIgnored,
595	484		data.Filepath,
596	485		)
597	486		return false, err

...	...		@@ -4,6 +4,7 @@ import (
4	4		"encoding/pem"
5	5		"fmt"
6	6		"io"
	7	+	"io/fs"
7	8		"log/slog"
8	9		"net"
9	10		"os"

...	...		@@ -22,13 +23,9 @@ import (
22	23		"github.com/prometheus/client_golang/prometheus"
23	24		)
24	25
25		-	// TestRsyncDeleteDirectoryWithKeepDir verifies that rsync --delete can
26		-	// successfully delete a directory that contains . _pico_keep_dir markers.
27		-	//
28		-	// Regression test for: "remove /storage/.../project/... directory not empty"
29		-	// The bug occurred because . _pico_keep_dir files were not cleaned up when
30		-	// their parent directory was explicitly deleted, causing os.Remove() to fail.
31		-	func TestRsyncDeleteDirectoryWithKeepDir(t *testing.T) {
	26	+	// TestRsyncDeleteDirectory verifies that rsync --delete removes a directory
	27	+	// tree on filesystem storage without failing with "directory not empty".
	28	+	func TestRsyncDeleteDirectory(t *testing.T) {
32	29		opts := &slog.HandlerOptions{
33	30		AddSource: true,
34	31		Level: slog.LevelDebug,

...	...		@@ -205,12 +202,6 @@ func TestRsyncDeleteDirectoryWithKeepDir(t *testing.T) {
205	202		t.Fatalf("expected 'not found' error, got: %v", err)
206	203		}
207	204
208		-	// Verify the . _pico_keep_dir markers are also cleaned up
209		-	_, err = client.Lstat("/testdir/subdir/._pico_keep_dir")
210		-	if err == nil {
211		-	t.Fatal("subdir/._pico_keep_dir should have been deleted but still exists")
212		-	}
213		-
214	205		// Verify index.html still exists (it wasn't deleted)
215	206		fi, err := client.Lstat("/testdir/index.html")
216	207		if err != nil {

...	...		@@ -225,8 +216,7 @@ func TestRsyncDeleteDirectoryWithKeepDir(t *testing.T) {
225	216		}
226	217
227	218		// TestRsyncDeleteNestedEmptyDirectories verifies that rsync --delete handles
228		-	// deeply nested directory structures where intermediate directories become empty
229		-	// and need . _pico_keep_dir markers managed correctly.
	219	+	// deeply nested directory structures where intermediate directories become empty.
230	220		func TestRsyncDeleteNestedEmptyDirectories(t *testing.T) {
231	221		opts := &slog.HandlerOptions{
232	222		AddSource: true,

...	...		@@ -720,3 +710,96 @@ func TestPgsFlagAllowsUpload(t *testing.T) {
720	710		t.Errorf("uploaded file has zero size")
721	711		}
722	712		}
	713	+
	714	+	// SFTP directories behave like a filesystem's on storage that has real
	715	+	// directories, with no marker files left behind.
	716	+	func TestSftpDirectories(t *testing.T) {
	717	+	logger := slog.New(slog.DiscardHandler)
	718	+	dbpool := pgsdb.NewDBMemory(logger)
	719	+	dbpool.SetupTestData()
	720	+	storageDir := t.TempDir()
	721	+	st, err := storage.NewStorageFS(logger, storageDir)
	722	+	if err != nil {
	723	+	t.Fatal(err)
	724	+	}
	725	+	_ = os.Setenv("PGS_SSH_PORT", "0")
	726	+	cfg := NewPgsConfig(logger, dbpool, st, discardPubsub{})
	727	+	done := make(chan error)
	728	+	defer close(done)
	729	+	readyCh := make(chan *pssh.SSHServer)
	730	+	prometheus.DefaultRegisterer = prometheus.NewRegistry()
	731	+	go StartSshServerForTesting(cfg, done, readyCh)
	732	+	server := <-readyCh
	733	+	if server == nil {
	734	+	t.Fatal("failed to create ssh server")
	735	+	}
	736	+	var addr string
	737	+	for range 100 {
	738	+	server.Mu.Lock()
	739	+	listener := server.Listener
	740	+	server.Mu.Unlock()
	741	+	if listener != nil {
	742	+	addr = listener.Addr().String()
	743	+	break
	744	+	}
	745	+	time.Sleep(10 * time.Millisecond)
	746	+	}
	747	+
	748	+	user := GenerateUser()
	749	+	dbpool.Pubkeys = append(dbpool.Pubkeys, &db.PublicKey{
	750	+	ID: "test-pubkey-dirs",
	751	+	UserID: dbpool.Users[0].ID,
	752	+	Key: shared.KeyForKeyText(user.signer.PublicKey()),
	753	+	})
	754	+	conn, err := user.NewClientAddr(addr)
	755	+	if err != nil {
	756	+	t.Fatal(err)
	757	+	}
	758	+	defer func() { _ = conn.Close() }()
	759	+	client, err := sftp.NewClient(conn)
	760	+	if err != nil {
	761	+	t.Fatal(err)
	762	+	}
	763	+	defer func() { _ = client.Close() }()
	764	+
	765	+	if err := client.MkdirAll("/site/empty"); err != nil {
	766	+	t.Fatal(err)
	767	+	}
	768	+	f, err := client.Create("/site/css/site.css")
	769	+	if err != nil {
	770	+	t.Fatal(err)
	771	+	}
	772	+	if _, err := f.Write([]byte("body{}")); err != nil {
	773	+	t.Fatal(err)
	774	+	}
	775	+	_ = f.Close()
	776	+
	777	+	if err := client.RemoveDirectory("/site/css"); err == nil {
	778	+	t.Fatal("removing a non-empty directory succeeded")
	779	+	}
	780	+	if err := client.Remove("/site/css/site.css"); err != nil {
	781	+	t.Fatal(err)
	782	+	}
	783	+	for _, dir := range []string{"/site/empty", "/site/css"} {
	784	+	fi, err := client.Stat(dir)
	785	+	if err != nil \|\| !fi.IsDir() {
	786	+	t.Fatalf("%s should be an empty directory: %v", dir, err)
	787	+	}
	788	+	if err := client.RemoveDirectory(dir); err != nil {
	789	+	t.Fatalf("rmdir %s: %v", dir, err)
	790	+	}
	791	+	if _, err := client.Stat(dir); err == nil {
	792	+	t.Fatalf("%s still exists after rmdir", dir)
	793	+	}
	794	+	}
	795	+
	796	+	err = filepath.WalkDir(storageDir, func(p string, d fs.DirEntry, err error) error {
	797	+	if err == nil && !d.IsDir() {
	798	+	t.Errorf("unexpected file left in storage: %s", p)
	799	+	}
	800	+	return err
	801	+	})
	802	+	if err != nil {
	803	+	t.Fatal(err)
	804	+	}
	805	+	}

...	...		@@ -109,10 +109,10 @@ func (p *PromCacheMetrics) AddUpstreamRequest() {
109	109		p.UpstreamReq.Add(1)
110	110		}
111	111
112		-	func NewPgsHttpCache(cfg PgsConfig, upstream http.Handler) httpcache.HttpCache {
	112	+	func NewPgsHttpCache(cfg PgsConfig, upstream http.Handler) (httpcache.HttpCache, *SiteCache) {
113	113		ttl := cfg.CacheTTL
114	114		metrics := NewPromCacheMetrics(cfg.Logger, prometheus.DefaultRegisterer)
115		-	cache := expirable.NewLRU(cfg.CacheMaxItems, metrics.EvictCacheItem, ttl)
	115	+	cache := NewSiteCache(cfg.CacheMaxItems, ttl, metrics.EvictCacheItem)
116	116		httpCache := &httpcache.HttpCache{
117	117		Ttl: ttl,
118	118		Logger: cfg.Logger,

...	...		@@ -123,22 +123,24 @@ func NewPgsHttpCache(cfg PgsConfig, upstream http.Handler) httpcache.HttpCache
123	123		TxtPrefix: cfg.TxtPrefix,
124	124		},
125	125		CacheMetrics: metrics,
	126	+	MaxBodySize: cfg.CacheMaxBodySize,
126	127		}
127	128		httpCache.Logger.Info(
128	129		"httpcache initiated",
129	130		"storageType", "expirable.LRU",
130	131		"ttl", ttl,
131	132		"maxItems", cfg.CacheMaxItems,
	133	+	"maxBodySize", cfg.CacheMaxBodySize,
132	134		)
133		-	return httpCache
	135	+	return httpCache, cache
134	136		}
135	137
136	138		func StartApiServer(cfg *PgsConfig) {
137	139		ctx := context.Background()
138	140
139	141		router := NewWebRouter(cfg)
140		-	httpCache := NewPgsHttpCache(router.Cfg, router)
141		-	go CacheMgmt(ctx, cfg.CacheClearingQueue, cfg, httpCache.Cache)
	142	+	httpCache, cache := NewPgsHttpCache(router.Cfg, router)
	143	+	go CacheMgmt(ctx, cfg, cache, router.clearSiteCache)
142	144
143	145		portStr := fmt.Sprintf(":%s", cfg.WebPort)
144	146		cfg.Logger.Info(

...	...		@@ -160,20 +162,32 @@ type WebRouter struct {
160	162		RootRouter *http.ServeMux
161	163		UserRouter *http.ServeMux
162	164		RedirectsCache expirable.LRU[string, []RedirectRule]
163		-	HeadersCache expirable.LRU[string, []HeaderRule]
	165	+	HeadersCache *expirable.LRU[string, []headerMatcher]
	166	+	SiteCache expirable.LRU[string, siteInfo]
	167	+	}
	168	+
	169	+	// siteInfo is what serving a site needs from the database and storage.
	170	+	type siteInfo struct {
	171	+	user *db.User
	172	+	project *db.Project
	173	+	bucket storage.Bucket
	174	+	plus *db.FeatureFlag
	175	+	}
	176	+
	177	+	func (s *siteInfo) hasPicoPlus() bool {
	178	+	return s.plus != nil && s.plus.ExpiresAt.After(time.Now())
164	179		}
165	180
166	181		func NewWebRouter(cfg PgsConfig) WebRouter {
167		-	router := newWebRouter(cfg)
168		-	go router.WatchCacheClear()
169		-	return router
	182	+	return newWebRouter(cfg)
170	183		}
171	184
172	185		func newWebRouter(cfg PgsConfig) WebRouter {
173	186		router := &WebRouter{
174	187		Cfg: cfg,
175	188		RedirectsCache: expirable.NewLRU[string, []*RedirectRule](2048, nil, shared.CacheTimeout),
176		-	HeadersCache: expirable.NewLRU[string, []*HeaderRule](2048, nil, shared.CacheTimeout),
	189	+	HeadersCache: expirable.NewLRU[string, []headerMatcher](2048, nil, shared.CacheTimeout),
	190	+	SiteCache: expirable.NewLRU[string, *siteInfo](2048, nil, shared.CacheTimeout),
177	191		}
178	192		router.initRouters()
179	193		return router

...	...		@@ -199,14 +213,51 @@ func (web WebRouter) ServeHTTP(w http.ResponseWriter, r http.Request) {
199	213		mux.ServeHTTP(w, r.WithContext(ctx))
200	214		}
201	215
202		-	func (web *WebRouter) WatchCacheClear() {
203		-	for key := range web.Cfg.CacheClearingQueue {
204		-	web.Cfg.Logger.Info("lru cache clear request", "key", key)
205		-	rKey := filepath.Join(key, "_redirects")
206		-	web.RedirectsCache.Remove(rKey)
207		-	hKey := filepath.Join(key, "_headers")
208		-	web.HeadersCache.Remove(hKey)
	216	+	// clearSiteCache drops what is cached about a site: its database records and
	217	+	// its parsed _redirects and _headers. The key is the surrogate key that
	218	+	// uploads and the CLI purge, or "*" for every site.
	219	+	func (web *WebRouter) clearSiteCache(key string) {
	220	+	web.Cfg.Logger.Info("lru cache clear request", "key", key)
	221	+	if key == "*" {
	222	+	web.SiteCache.Purge()
	223	+	web.RedirectsCache.Purge()
	224	+	web.HeadersCache.Purge()
	225	+	return
209	226		}
	227	+	web.SiteCache.Remove(key)
	228	+	web.RedirectsCache.Remove(filepath.Join(key, "_redirects"))
	229	+	web.HeadersCache.Remove(filepath.Join(key, "_headers"))
	230	+	}
	231	+
	232	+	// findSite looks up the user, project, bucket and pico+ flag behind a
	233	+	// subdomain. Only complete results are cached, so a site that is created or
	234	+	// fixed later is found on its next request.
	235	+	func (web WebRouter) findSite(props router.SubdomainProps, logger slog.Logger) (siteInfo, int, string) {
	236	+	key := getSurrogateKey(props.Username, props.ProjectName)
	237	+	if site, ok := web.SiteCache.Get(key); ok {
	238	+	return site, http.StatusOK, ""
	239	+	}
	240	+
	241	+	user, err := web.Cfg.DB.FindUserByName(props.Username)
	242	+	if err != nil {
	243	+	logger.Info("user not found")
	244	+	return nil, http.StatusNotFound, "user not found"
	245	+	}
	246	+	project, err := web.Cfg.DB.FindProjectByName(user.ID, props.ProjectName)
	247	+	if err != nil {
	248	+	logger.Info("project not found")
	249	+	return nil, http.StatusNotFound, "project not found"
	250	+	}
	251	+	bucket, err := web.Cfg.Storage.GetBucket(shared.GetAssetBucketName(user.ID))
	252	+	if err != nil {
	253	+	logger.Error("bucket not found", "err", err)
	254	+	return nil, http.StatusNotFound, "bucket not found"
	255	+	}
	256	+	plus, _ := web.Cfg.DB.FindFeature(user.ID, "plus")
	257	+
	258	+	site := &siteInfo{user: user, project: project, bucket: bucket, plus: plus}
	259	+	web.SiteCache.Add(key, site)
	260	+	return site, http.StatusOK, ""
210	261		}
211	262
212	263		func (web *WebRouter) initRouters() {

...	...		@@ -376,7 +427,10 @@ func (web WebRouter) checkHandler(w http.ResponseWriter, r http.Request) {
376	427		w.WriteHeader(http.StatusNotFound)
377	428		}
378	429
379		-	func CacheMgmt(ctx context.Context, notify chan string, cfg *PgsConfig, cacher httpcache.Cacher) {
	430	+	// CacheMgmt applies the purges published on the cache-drain pipe. onPurge,
	431	+	// when set, clears whatever else is cached about the site; it runs first so
	432	+	// the http cache isn't refilled from stale records.
	433	+	func CacheMgmt(ctx context.Context, cfg PgsConfig, cache SiteCache, onPurge func(site string)) {
380	434		cfg.Logger.Info("cache mgmt initiated")
381	435		for {
382	436		scanner := bufio.NewScanner(cfg.Pubsub)

...	...		@@ -384,20 +438,16 @@ func CacheMgmt(ctx context.Context, notify chan string, cfg *PgsConfig, cacher h
384	438		for scanner.Scan() {
385	439		subdomain := strings.TrimSpace(scanner.Text())
386	440		cfg.Logger.Info("received cache-drain item", "subdomain", subdomain)
387		-	notify <- subdomain
	441	+	if onPurge != nil {
	442	+	onPurge(subdomain)
	443	+	}
388	444
389	445		if subdomain == "*" {
390		-	cacher.Purge()
	446	+	cache.Purge()
391	447		cfg.Logger.Info("successfully cleared cache from remote cli request")
392	448		continue
393	449		}
394		-
395		-	for _, key := range cacher.Keys() {
396		-	if strings.HasPrefix(key, subdomain) {
397		-	cfg.Logger.Info("deleting cache item", "subdomain", subdomain, "key", key)
398		-	_ = cacher.Remove(key)
399		-	}
400		-	}
	450	+	cache.PurgeSite(subdomain)
401	451		}
402	452		}
403	453		}

...	...		@@ -533,27 +583,15 @@ func (web WebRouter) ServeAsset(fname string, opts storage.ImgProcessOpts, has
533	583		"user", props.Username,
534	584		)
535	585
536		-	user, err := web.Cfg.DB.FindUserByName(props.Username)
537		-	if err != nil {
538		-	logger.Info("user not found")
539		-	http.Error(w, "user not found", http.StatusNotFound)
	586	+	site, status, msg := web.findSite(props, logger)
	587	+	if site == nil {
	588	+	http.Error(w, msg, status)
540	589		return
541	590		}
	591	+	user, project, bucket := site.user, site.project, site.bucket
542	592
543	593		logger = logger.With(
544	594		"userId", user.ID,
545		-	)
546		-
547		-	var bucket storage.Bucket
548		-	bucket, err = web.Cfg.Storage.GetBucket(shared.GetAssetBucketName(user.ID))
549		-	project, perr := web.Cfg.DB.FindProjectByName(user.ID, props.ProjectName)
550		-	if perr != nil {
551		-	logger.Info("project not found")
552		-	http.Error(w, "project not found", http.StatusNotFound)
553		-	return
554		-	}
555		-
556		-	logger = logger.With(
557	595		"projectId", project.ID,
558	596		"project", project.Name,
559	597		)

...	...		@@ -589,20 +627,6 @@ func (web WebRouter) ServeAsset(fname string, opts storage.ImgProcessOpts, has
589	627		return
590	628		}
591	629
592		-	if err != nil {
593		-	logger.Error("bucket not found", "err", err)
594		-	http.Error(w, "bucket not found", http.StatusNotFound)
595		-	return
596		-	}
597		-
598		-	hasPicoPlus := false
599		-	ff, _ := web.Cfg.DB.FindFeature(user.ID, "plus")
600		-	if ff != nil {
601		-	if ff.ExpiresAt.After(time.Now()) {
602		-	hasPicoPlus = true
603		-	}
604		-	}
605		-
606	630		asset := &ApiAssetHandler{
607	631		WebRouter: web,
608	632		Logger: logger,

...	...		@@ -615,7 +639,7 @@ func (web WebRouter) ServeAsset(fname string, opts storage.ImgProcessOpts, has
615	639		Filepath: fname,
616	640		Bucket: bucket,
617	641		ImgProcessOpts: opts,
618		-	HasPicoPlus: hasPicoPlus,
	642	+	HasPicoPlus: site.hasPicoPlus(),
619	643		HttpPass: project.Acl.Type == "http-pass",
620	644		}
621	645

...	...		@@ -7,7 +7,6 @@ import (
7	7		"net/http"
8	8		"net/url"
9	9		"path/filepath"
10		-	"regexp"
11	10		"strconv"
12	11		"strings"
13	12

...	...		@@ -42,7 +41,7 @@ func (h ApiAssetHandler) ServeHTTP(w http.ResponseWriter, r http.Request) {
42	41		logger := h.Logger
43	42		var redirects []*RedirectRule
44	43
45		-	redirectsCacheKey := filepath.Join(getSurrogateKey(h.UserID, h.ProjectDir), "_redirects")
	44	+	redirectsCacheKey := filepath.Join(getSurrogateKey(h.Username, h.ProjectDir), "_redirects")
46	45		logger.Info("looking for _redirects in lru cache", "key", redirectsCacheKey)
47	46		if cachedRedirects, found := h.RedirectsCache.Get(redirectsCacheKey); found {
48	47		logger.Info("_redirects found in lru cache", "key", redirectsCacheKey)

...	...		@@ -104,11 +103,9 @@ func (h ApiAssetHandler) ServeHTTP(w http.ResponseWriter, r http.Request) {
104	103		// before redirecting, this saves a hop that will just end up a 404
105	104		if !hasProtocol(fp.Filepath) && strings.HasSuffix(fp.Filepath, "/") {
106	105		next := filepath.Join(h.ProjectDir, fp.Filepath, "index.html")
107		-	obj, _, err := h.Cfg.Storage.GetObject(h.Bucket, next)
108		-	if err != nil {
	106	+	if _, err := h.Cfg.Storage.StatObject(h.Bucket, next); err != nil {
109	107		continue
110	108		}
111		-	_ = obj.Close()
112	109		}
113	110		logger.Info(
114	111		"redirecting request",

...	...		@@ -177,25 +174,21 @@ func (h ApiAssetHandler) ServeHTTP(w http.ResponseWriter, r http.Request) {
177	174		}
178	175
179	176		if assetFilepath == "" {
180		-	if shouldGenerateListing(h.Cfg.Storage, h.Bucket, h.ProjectDir, "/"+fpath) {
	177	+	if entries, ok := dirListing(h.Cfg.Storage, h.Bucket, h.ProjectDir, "/"+fpath); ok {
181	178		logger.Info(
182	179		"generating directory listing",
183	180		"path", fpath,
184	181		)
185		-	dirPath := h.ProjectDir + "/" + fpath
186		-	entries, err := h.Cfg.Storage.ListObjects(h.Bucket, dirPath, false)
187		-	if err == nil {
188		-	requestPath := "/" + fpath
189		-	if !strings.HasSuffix(requestPath, "/") {
190		-	requestPath += "/"
191		-	}
192		-
193		-	html := generateDirectoryHTML(requestPath, entries)
194		-	w.Header().Set("content-type", "text/html")
195		-	w.WriteHeader(http.StatusOK)
196		-	_, _ = w.Write([]byte(html))
197		-	return
	182	+	requestPath := "/" + fpath
	183	+	if !strings.HasSuffix(requestPath, "/") {
	184	+	requestPath += "/"
198	185		}
	186	+
	187	+	html := generateDirectoryHTML(requestPath, entries)
	188	+	w.Header().Set("content-type", "text/html")
	189	+	w.WriteHeader(http.StatusOK)
	190	+	_, _ = w.Write([]byte(html))
	191	+	return
199	192		}
200	193
201	194		logger.Info(

...	...		@@ -210,9 +203,9 @@ func (h ApiAssetHandler) ServeHTTP(w http.ResponseWriter, r http.Request) {
210	203		_ = contents.Close()
211	204		}()
212	205
213		-	var headers []*HeaderRule
	206	+	var headers []headerMatcher
214	207
215		-	headersCacheKey := filepath.Join(getSurrogateKey(h.UserID, h.ProjectDir), "_headers")
	208	+	headersCacheKey := filepath.Join(getSurrogateKey(h.Username, h.ProjectDir), "_headers")
216	209		logger.Info("looking for _headers in lru cache", "key", headersCacheKey)
217	210		if cachedHeaders, found := h.HeadersCache.Get(headersCacheKey); found {
218	211		logger.Info("_headers found in lru", "key", headersCacheKey)

...	...		@@ -238,21 +231,20 @@ func (h ApiAssetHandler) ServeHTTP(w http.ResponseWriter, r http.Request) {
238	231		return
239	232		}
240	233
241		-	headers, err = parseHeaderText(buf.String())
	234	+	rules, err := parseHeaderText(buf.String())
242	235		if err != nil {
243	236		logger.Error("could not parse header text", "err", err.Error())
244	237		}
	238	+	headers = compileHeaderRules(rules, logger)
245	239		}
246	240
247	241		h.HeadersCache.Add(headersCacheKey, headers)
248	242		}
249	243
250	244		userHeaders := []*HeaderLine{}
251		-	for _, headerRule := range headers {
252		-	rr := regexp.MustCompile(headerRule.Path)
253		-	match := rr.FindStringSubmatch(assetFilepath)
254		-	if len(match) > 0 {
255		-	userHeaders = headerRule.Headers
	245	+	for _, rule := range headers {
	246	+	if rule.path.MatchString(assetFilepath) {
	247	+	userHeaders = rule.headers
256	248		}
257	249		}
258	250

...	...		@@ -5,8 +5,11 @@ import (
5	5		"fmt"
6	6		"io"
7	7		"log/slog"
	8	+	"strings"
	9	+	"sync"
8	10		"time"
9	11
	12	+	"github.com/hashicorp/golang-lru/v2/expirable"
10	13		"github.com/picosh/pico/pkg/shared"
11	14		"github.com/picosh/utils/pipe"
12	15		)

...	...		@@ -104,7 +107,6 @@ func CreateSubCacheDrain(ctx context.Context, logger slog.Logger) pipe.Reconne
104	107		// from the cache).
105	108		func purgeCache(cfg *PgsConfig, writer io.Writer, surrogate string) error {
106	109		cfg.Logger.Info("purging cache", "surrogate", surrogate)
107		-	time.Sleep(1 * time.Second)
108	110		_, err := writer.Write([]byte(surrogate + "\n"))
109	111		return err
110	112		}

...	...		@@ -112,3 +114,64 @@ func purgeCache(cfg *PgsConfig, writer io.Writer, surrogate string) error {
112	114		func purgeAllCache(cfg *PgsConfig, writer io.Writer) error {
113	115		return purgeCache(cfg, writer, "*")
114	116		}
	117	+
	118	+	// SiteCache holds the http cache's responses and indexes their keys by site,
	119	+	// so purging one site doesn't scan the whole cache.
	120	+	type SiteCache struct {
	121	+	*expirable.LRU[string, []byte]
	122	+	mu sync.Mutex
	123	+	sites map[string]map[string]struct{}
	124	+	}
	125	+
	126	+	func NewSiteCache(size int, ttl time.Duration, onEvict func(key string, value []byte)) *SiteCache {
	127	+	c := &SiteCache{sites: map[string]map[string]struct{}{}}
	128	+	c.LRU = expirable.NewLRU(size, func(key string, value []byte) {
	129	+	c.unindex(key)
	130	+	if onEvict != nil {
	131	+	onEvict(key, value)
	132	+	}
	133	+	}, ttl)
	134	+	return c
	135	+	}
	136	+
	137	+	// cacheKeySite returns the site a PgsCacheKey belongs to.
	138	+	func cacheKeySite(key string) string {
	139	+	site, _, _ := strings.Cut(key, "__")
	140	+	return site
	141	+	}
	142	+
	143	+	func (c *SiteCache) Add(key string, value []byte) bool {
	144	+	evicted := c.LRU.Add(key, value)
	145	+	site := cacheKeySite(key)
	146	+	c.mu.Lock()
	147	+	defer c.mu.Unlock()
	148	+	if c.sites[site] == nil {
	149	+	c.sites[site] = map[string]struct{}{}
	150	+	}
	151	+	c.sites[site][key] = struct{}{}
	152	+	return evicted
	153	+	}
	154	+
	155	+	// PurgeSite removes every cached response for a site. The index lock is
	156	+	// released before touching the LRU, whose eviction callback takes it.
	157	+	func (c *SiteCache) PurgeSite(site string) {
	158	+	c.mu.Lock()
	159	+	keys := c.sites[site]
	160	+	delete(c.sites, site)
	161	+	c.mu.Unlock()
	162	+	for key := range keys {
	163	+	c.Remove(key)
	164	+	}
	165	+	}
	166	+
	167	+	func (c *SiteCache) unindex(key string) {
	168	+	site := cacheKeySite(key)
	169	+	c.mu.Lock()
	170	+	defer c.mu.Unlock()
	171	+	if keys, ok := c.sites[site]; ok {
	172	+	delete(keys, key)
	173	+	if len(keys) == 0 {
	174	+	delete(c.sites, site)
	175	+	}
	176	+	}
	177	+	}

...	...		@@ -0,0 +1,42 @@
	1	+	package pgs
	2	+
	3	+	import (
	4	+	"slices"
	5	+	"testing"
	6	+	"time"
	7	+	)
	8	+
	9	+	func TestSiteCachePurgeSite(t *testing.T) {
	10	+	evicted := []string{}
	11	+	c := NewSiteCache(0, time.Minute, func(key string, _ []byte) { evicted = append(evicted, key) })
	12	+	keys := []string{
	13	+	"antonio-blog__GET__/",
	14	+	"antonio-blog__GET__/about",
	15	+	"antonio-blog2__GET__/",
	16	+	"antonio-blog-v2__GET__/",
	17	+	}
	18	+	for _, key := range keys {
	19	+	c.Add(key, []byte("x"))
	20	+	}
	21	+
	22	+	c.PurgeSite("antonio-blog")
	23	+	got := c.Keys()
	24	+	slices.Sort(got)
	25	+	want := []string{"antonio-blog-v2__GET__/", "antonio-blog2__GET__/"}
	26	+	if !slices.Equal(got, want) {
	27	+	t.Fatalf("after purge got %q, want %q", got, want)
	28	+	}
	29	+	if len(evicted) != 2 {
	30	+	t.Fatalf("eviction callback ran %d times, want 2", len(evicted))
	31	+	}
	32	+
	33	+	// Entries that leave the cache on their own leave the index too.
	34	+	c.Remove("antonio-blog2__GET__/")
	35	+	if _, ok := c.sites["antonio-blog2"]; ok {
	36	+	t.Fatal("removed entry still indexed")
	37	+	}
	38	+	c.Purge()
	39	+	if len(c.sites) != 0 {
	40	+	t.Fatalf("index holds %d sites after a full purge", len(c.sites))
	41	+	}
	42	+	}

...	...		@@ -6,6 +6,7 @@ import (
6	6		"net/http"
7	7		"net/http/httptest"
8	8		"strings"
	9	+	"sync/atomic"
9	10		"testing"
10	11		"time"
11	12

...	...		@@ -413,6 +414,34 @@ func TestApiBasic(t *testing.T) {
413	414		},
414	415		},
415	416		},
	417	+	{
	418	+	name: "headers-invalid-path-skipped",
	419	+	path: "/test.html",
	420	+	want: "hello world!",
	421	+	status: http.StatusOK,
	422	+	contentType: "text/html",
	423	+	wantCacheCtrl: "public, max-age=31536000, immutable",
	424	+
	425	+	storage: map[string]map[string]string{
	426	+	bucketName: {
	427	+	"/test/test.html": "hello world!",
	428	+	"/test/_headers": "/[\n\tx-broken: yes\n/*\n\tcache-control: public, max-age=31536000, immutable",
	429	+	},
	430	+	},
	431	+	},
	432	+	{
	433	+	name: "redirects-invalid-pattern-skipped",
	434	+	path: "/foo(",
	435	+	want: "404 not found",
	436	+	status: http.StatusNotFound,
	437	+	contentType: "text/plain; charset=utf-8",
	438	+
	439	+	storage: map[string]map[string]string{
	440	+	bucketName: {
	441	+	"/test/_redirects": "/foo( /index.html 301",
	442	+	},
	443	+	},
	444	+	},
416	445		}
417	446
418	447		for _, tc := range tt {

...	...		@@ -748,3 +777,106 @@ func TestDirectoryListing(t *testing.T) {
748	777		// })
749	778		// }
750	779		// }
	780	+
	781	+	// The purge an upload sends must drop the site's parsed _redirects, or the
	782	+	// old rules keep being served and cached.
	783	+	func TestRulesCacheClearedByUploadPurge(t *testing.T) {
	784	+	logger := slog.New(slog.DiscardHandler)
	785	+	dbpool := NewPgsDb(logger)
	786	+	user := dbpool.Users[0]
	787	+	bucketName := shared.GetAssetBucketName(user.ID)
	788	+	st, err := storage.NewStorageMemory(map[string]map[string]string{
	789	+	bucketName: {
	790	+	"/test/old.html": "old",
	791	+	"/test/new.html": "new",
	792	+	"/test/_redirects": "/page /old.html 200",
	793	+	},
	794	+	})
	795	+	if err != nil {
	796	+	t.Fatal(err)
	797	+	}
	798	+	cfg := NewPgsConfig(logger, dbpool, st, discardPubsub{})
	799	+	cfg.Domain = "pgs.test"
	800	+	router := newWebRouter(cfg)
	801	+
	802	+	get := func() string {
	803	+	rec := httptest.NewRecorder()
	804	+	router.ServeHTTP(rec, httptest.NewRequest("GET", dbpool.mkpath("/page"), nil))
	805	+	return strings.TrimSpace(rec.Body.String())
	806	+	}
	807	+	if got := get(); got != "old" {
	808	+	t.Fatalf("before upload got %q", got)
	809	+	}
	810	+
	811	+	bucket, err := st.GetBucket(bucketName)
	812	+	if err != nil {
	813	+	t.Fatal(err)
	814	+	}
	815	+	if _, _, err := st.PutObject(bucket, "/test/_redirects", strings.NewReader("/page /new.html 200"), &storage.ObjectInfo{}); err != nil {
	816	+	t.Fatal(err)
	817	+	}
	818	+	router.clearSiteCache(getSurrogateKey(user.Name, "test"))
	819	+	if got := get(); got != "new" {
	820	+	t.Fatalf("after upload got %q", got)
	821	+	}
	822	+	}
	823	+
	824	+	// countingDB counts the lookups serving a site makes.
	825	+	type countingDB struct {
	826	+	*PgsDb
	827	+	lookups atomic.Int64
	828	+	}
	829	+
	830	+	func (c countingDB) FindUserByName(name string) (db.User, error) {
	831	+	c.lookups.Add(1)
	832	+	return c.PgsDb.FindUserByName(name)
	833	+	}
	834	+
	835	+	func (c countingDB) FindProjectByName(userID, name string) (db.Project, error) {
	836	+	c.lookups.Add(1)
	837	+	return c.PgsDb.FindProjectByName(userID, name)
	838	+	}
	839	+
	840	+	func (c countingDB) FindFeature(userID, name string) (db.FeatureFlag, error) {
	841	+	c.lookups.Add(1)
	842	+	return c.PgsDb.FindFeature(userID, name)
	843	+	}
	844	+
	845	+	// A site's database records are looked up once and again only after a purge.
	846	+	func TestSiteCache(t *testing.T) {
	847	+	logger := slog.New(slog.DiscardHandler)
	848	+	dbpool := &countingDB{PgsDb: NewPgsDb(logger)}
	849	+	user := dbpool.Users[0]
	850	+	st, err := storage.NewStorageMemory(map[string]map[string]string{
	851	+	shared.GetAssetBucketName(user.ID): {"/test/index.html": "hello"},
	852	+	})
	853	+	if err != nil {
	854	+	t.Fatal(err)
	855	+	}
	856	+	cfg := NewPgsConfig(logger, dbpool, st, discardPubsub{})
	857	+	cfg.Domain = "pgs.test"
	858	+	router := newWebRouter(cfg)
	859	+
	860	+	get := func() {
	861	+	t.Helper()
	862	+	rec := httptest.NewRecorder()
	863	+	router.ServeHTTP(rec, httptest.NewRequest("GET", dbpool.mkpath("/"), nil))
	864	+	if rec.Code != http.StatusOK {
	865	+	t.Fatalf("got status %d", rec.Code)
	866	+	}
	867	+	}
	868	+	get()
	869	+	first := dbpool.lookups.Load()
	870	+	if first == 0 {
	871	+	t.Fatal("serving made no lookups")
	872	+	}
	873	+	get()
	874	+	if n := dbpool.lookups.Load(); n != first {
	875	+	t.Fatalf("second request made %d more lookups", n-first)
	876	+	}
	877	+	router.clearSiteCache(getSurrogateKey(user.Name, "test"))
	878	+	get()
	879	+	if n := dbpool.lookups.Load(); n != 2*first {
	880	+	t.Fatalf("request after purge made %d lookups, want %d", n-first, first)
	881	+	}
	882	+	}

...	...		@@ -42,6 +42,8 @@ type Cmd struct {
42	42		Log *slog.Logger
43	43		Dbpool db.DB
44	44		Write bool
	45	+
	46	+	lookupTXT func(host, space string) ([]string, error)
45	47		}
46	48
47	49		func (c *Cmd) output(out string) {

...	...		@@ -49,13 +51,14 @@ func (c *Cmd) output(out string) {
49	51		}
50	52
51	53		func (c *Cmd) help() {
52		-	helpStr := "Commands: [help, user, logs, access_logs, chat, not-found]\n"
	54	+	helpStr := "Commands: [help, user, logs, access-logs, chat, not-found, dns-check]\n"
53	55		helpStr += "help - this message\n"
54	56		helpStr += "user - display user information (returns name, id, account created, pico+ expiration)\n"
55	57		helpStr += "logs - stream user logs\n"
56		-	helpStr += "access_logs - fetch access logs from the last 30 days\n"
	58	+	helpStr += "access-logs - fetch access logs from the last 30 days\n"
57	59		helpStr += "chat - IRC chat (must enable pty with `-t` to the SSH command)\n"
58	60		helpStr += "not-found - return all status 404 requests for a host (hostname.com [year\|month])\n"
	61	+	helpStr += "dns-check - check a domain's TXT records for pgs, prose and tuns custom domains (hostname.com or *.hostname.com)\n"
59	62		c.output(helpStr)
60	63		}
61	64

...	...		@@ -96,7 +99,7 @@ func (c *Cmd) notFound(host, interval string) error {
96	99		return nil
97	100		}
98	101
99		-	func (c *Cmd) access_logs(_ context.Context) error {
	102	+	func (c *Cmd) accessLogs(_ context.Context) error {
100	103		fromDate := time.Now().AddDate(0, 0, -30)
101	104		logs, err := c.Dbpool.FindAccessLogs(c.User.ID, &fromDate)
102	105		if err != nil {

...	...		@@ -264,8 +267,12 @@ func Middleware(handler *CliHandler) pssh.SSHServerMiddleware {
264	267		sesh.Fatal(err)
265	268		}
266	269		return nil
267		-	case "access_logs":
268		-	err = opts.access_logs(sesh.Context())
	270	+	case "dns-check":
	271	+	sesh.Fatal(fmt.Errorf("must provide a domain name (`dns-check example.com`)"))
	272	+	return nil
	273	+	// access_logs is the command's old name.
	274	+	case "access-logs", "access_logs":
	275	+	err = opts.accessLogs(sesh.Context())
269	276		if err != nil {
270	277		sesh.Fatal(err)
271	278		}

...	...		@@ -275,6 +282,14 @@ func Middleware(handler *CliHandler) pssh.SSHServerMiddleware {
275	282		}
276	283		}
277	284
	285	+	if cmd == "dns-check" {
	286	+	err = opts.dnsCheck(args[1])
	287	+	if err != nil {
	288	+	sesh.Fatal(err)
	289	+	}
	290	+	return nil
	291	+	}
	292	+
278	293		if cmd == "not-found" {
279	294		if len(args) < 3 {
280	295		sesh.Fatal(fmt.Errorf("must provide host name and interval (`month` or `year`)"))

...	...		@@ -0,0 +1,149 @@
	1	+	package pico
	2	+
	3	+	import (
	4	+	"crypto/rand"
	5	+	"encoding/hex"
	6	+	"errors"
	7	+	"fmt"
	8	+	"strings"
	9	+
	10	+	"github.com/picosh/pico/pkg/db"
	11	+	"github.com/picosh/pico/pkg/shared/router"
	12	+	"golang.org/x/crypto/ssh"
	13	+	)
	14	+
	15	+	// customDomainService is a service that serves a domain whose TXT record
	16	+	// names it. check judges the records the way the service does and describes
	17	+	// the result.
	18	+	type customDomainService struct {
	19	+	space string
	20	+	check func(c *Cmd, records []string) string
	21	+	wildcard bool
	22	+	}
	23	+
	24	+	var customDomainServices = []customDomainService{
	25	+	{space: "pgs", check: siteCheck(resolvePgsSite)},
	26	+	{space: "prose", check: siteCheck(resolveProseSite)},
	27	+	// tuns accepts a match under either prefix; _sish is what sish uses.
	28	+	{space: "tuns", check: checkTunsRecords, wildcard: true},
	29	+	{space: "sish", check: checkTunsRecords, wildcard: true},
	30	+	}
	31	+
	32	+	// siteCheck checks a record whose value names a site. Like the services,
	33	+	// it reads only the first record.
	34	+	func siteCheck(resolve func(dbpool db.DB, value string) (db.User, string, error)) func(c Cmd, records []string) string {
	35	+	return func(c *Cmd, records []string) string {
	36	+	value := strings.TrimSpace(records[0])
	37	+	user, site, err := resolve(c.Dbpool, value)
	38	+	if err != nil {
	39	+	return fmt.Sprintf("%q is invalid: %v", value, err)
	40	+	}
	41	+	line := fmt.Sprintf("%q is valid, serving %s", value, site)
	42	+	if user.ID != c.User.ID {
	43	+	line += " (not your account)"
	44	+	}
	45	+	return line
	46	+	}
	47	+	}
	48	+
	49	+	func resolvePgsSite(dbpool db.DB, value string) (*db.User, string, error) {
	50	+	props, err := router.GetProjectFromSubdomain(value)
	51	+	if err != nil {
	52	+	return nil, "", err
	53	+	}
	54	+	user, err := dbpool.FindUserByName(props.Username)
	55	+	if err != nil {
	56	+	return nil, "", fmt.Errorf("user %q not found", props.Username)
	57	+	}
	58	+	if _, err := dbpool.FindProjectByName(user.ID, props.ProjectName); err != nil {
	59	+	return user, "", fmt.Errorf("user %q has no project %q", user.Name, props.ProjectName)
	60	+	}
	61	+	return user, fmt.Sprintf("pgs project %q of %s", props.ProjectName, user.Name), nil
	62	+	}
	63	+
	64	+	func resolveProseSite(dbpool db.DB, value string) (*db.User, string, error) {
	65	+	user, err := dbpool.FindUserByName(value)
	66	+	if err != nil {
	67	+	return nil, "", fmt.Errorf("user %q not found", value)
	68	+	}
	69	+	return user, fmt.Sprintf("prose blog of %s", user.Name), nil
	70	+	}
	71	+
	72	+	// checkTunsRecords looks for a record holding the user's name or the SHA256
	73	+	// fingerprint of one of their keys. tuns compares the whole record, so it
	74	+	// must match exactly.
	75	+	func checkTunsRecords(c *Cmd, records []string) string {
	76	+	keys, err := c.Dbpool.FindKeysByUser(c.User)
	77	+	if err != nil {
	78	+	return fmt.Sprintf("could not load your keys: %v", err)
	79	+	}
	80	+	names := map[string]string{}
	81	+	for _, k := range keys {
	82	+	pk, _, _, _, err := ssh.ParseAuthorizedKey([]byte(k.Key))
	83	+	if err != nil {
	84	+	continue
	85	+	}
	86	+	names[ssh.FingerprintSHA256(pk)] = k.Name
	87	+	}
	88	+
	89	+	for _, r := range records {
	90	+	if r == c.User.Name {
	91	+	return fmt.Sprintf("%q is valid, tunnels from any of your keys can use this domain", r)
	92	+	}
	93	+	if name, ok := names[r]; ok {
	94	+	return fmt.Sprintf("%q is valid, tunnels from your key %q can use this domain", r, name)
	95	+	}
	96	+	}
	97	+	for _, r := range records {
	98	+	trimmed := strings.TrimSpace(r)
	99	+	if _, ok := names[trimmed]; ok \|\| trimmed == c.User.Name {
	100	+	return fmt.Sprintf("%q is invalid: it has surrounding whitespace, and tuns needs an exact match", r)
	101	+	}
	102	+	}
	103	+	quoted := make([]string, len(records))
	104	+	for i, r := range records {
	105	+	quoted[i] = fmt.Sprintf("%q", r)
	106	+	}
	107	+	return fmt.Sprintf("%s is invalid: no record is your username or the SHA256 fingerprint of one of your keys", strings.Join(quoted, ", "))
	108	+	}
	109	+
	110	+	// dnsCheck reports, for each service, whether domain has a TXT record and
	111	+	// whether the service would accept it.
	112	+	func (c *Cmd) dnsCheck(domain string) error {
	113	+	domain = strings.TrimSuffix(strings.ToLower(strings.TrimSpace(domain)), ".")
	114	+	base, wildcard := strings.CutPrefix(domain, "*.")
	115	+	if base == "" \|\| strings.ContainsAny(base, "/:@ *") {
	116	+	return fmt.Errorf("expected a domain name like example.com or *.example.com")
	117	+	}
	118	+	lookup := c.lookupTXT
	119	+	if lookup == nil {
	120	+	lookup = router.LookupTXTRecords
	121	+	}
	122	+
	123	+	host := domain
	124	+	if wildcard {
	125	+	// tuns verifies a wildcard by looking up a random name under it.
	126	+	label := make([]byte, 6)
	127	+	_, _ = rand.Read(label)
	128	+	host = hex.EncodeToString(label) + "." + base
	129	+	}
	130	+
	131	+	for _, svc := range customDomainServices {
	132	+	name := router.TXTRecordName(host, svc.space)
	133	+	if wildcard && !svc.wildcard {
	134	+	c.output(fmt.Sprintf("%s: wildcard domains are not supported", router.TXTRecordName(domain, svc.space)))
	135	+	continue
	136	+	}
	137	+	records, err := lookup(host, svc.space)
	138	+	switch {
	139	+	case errors.Is(err, router.ErrNoTXTRecord):
	140	+	c.output(fmt.Sprintf("%s: no TXT record", name))
	141	+	case err != nil:
	142	+	c.output(fmt.Sprintf("%s: lookup failed: %v", name, err))
	143	+	default:
	144	+	c.output(fmt.Sprintf("%s: %s", name, svc.check(c, records)))
	145	+	}
	146	+	}
	147	+	c.output("pgs and prose remember a missing record for 30 seconds and a found record for 2 minutes. tuns checks when a tunnel connects.")
	148	+	return nil
	149	+	}

...	...		@@ -0,0 +1,162 @@
	1	+	package pico
	2	+
	3	+	import (
	4	+	"bytes"
	5	+	"crypto/ed25519"
	6	+	"fmt"
	7	+	"io"
	8	+	"log/slog"
	9	+	"strings"
	10	+	"testing"
	11	+
	12	+	"github.com/picosh/pico/pkg/db"
	13	+	"github.com/picosh/pico/pkg/shared"
	14	+	"github.com/picosh/pico/pkg/shared/router"
	15	+	"golang.org/x/crypto/ssh"
	16	+	)
	17	+
	18	+	type fakeSession struct{ bytes.Buffer }
	19	+
	20	+	func (s *fakeSession) Exit(int) error { return nil }
	21	+	func (s *fakeSession) Close() error { return nil }
	22	+	func (s *fakeSession) Stderr() io.ReadWriter { return &s.Buffer }
	23	+
	24	+	type dnsCheckDB struct {
	25	+	db.DB
	26	+	users []*db.User
	27	+	projects map[string]bool
	28	+	keys []*db.PublicKey
	29	+	}
	30	+
	31	+	func (d dnsCheckDB) FindKeysByUser(user db.User) ([]*db.PublicKey, error) {
	32	+	keys := []*db.PublicKey{}
	33	+	for _, k := range d.keys {
	34	+	if k.UserID == user.ID {
	35	+	keys = append(keys, k)
	36	+	}
	37	+	}
	38	+	return keys, nil
	39	+	}
	40	+
	41	+	func (d dnsCheckDB) FindUserByName(name string) (db.User, error) {
	42	+	for _, u := range d.users {
	43	+	if u.Name == name {
	44	+	return u, nil
	45	+	}
	46	+	}
	47	+	return nil, fmt.Errorf("user not found")
	48	+	}
	49	+
	50	+	func (d dnsCheckDB) FindProjectByName(userID, name string) (db.Project, error) {
	51	+	if d.projects[userID+"/"+name] {
	52	+	return &db.Project{UserID: userID, Name: name}, nil
	53	+	}
	54	+	return nil, fmt.Errorf("project not found")
	55	+	}
	56	+
	57	+	func TestDNSCheck(t *testing.T) {
	58	+	me := &db.User{ID: "1", Name: "antonio"}
	59	+	other := &db.User{ID: "2", Name: "erock"}
	60	+	edKey, _, err := ed25519.GenerateKey(nil)
	61	+	if err != nil {
	62	+	t.Fatal(err)
	63	+	}
	64	+	pk, err := ssh.NewPublicKey(edKey)
	65	+	if err != nil {
	66	+	t.Fatal(err)
	67	+	}
	68	+	fingerprint := ssh.FingerprintSHA256(pk)
	69	+	dbpool := &dnsCheckDB{
	70	+	users: []*db.User{me, other},
	71	+	projects: map[string]bool{"1/blog": true},
	72	+	keys: []*db.PublicKey{{UserID: me.ID, Name: "laptop", Key: shared.KeyForKeyText(pk)}},
	73	+	}
	74	+
	75	+	tests := []struct {
	76	+	name string
	77	+	domain string
	78	+	records map[string][]string
	79	+	want []string
	80	+	}{
	81	+	{"no records", "", map[string][]string{}, []string{
	82	+	"_pgs.example.com: no TXT record",
	83	+	"_prose.example.com: no TXT record",
	84	+	"_tuns.example.com: no TXT record",
	85	+	"_sish.example.com: no TXT record",
	86	+	}},
	87	+	{"valid", "", map[string][]string{
	88	+	"pgs": {"antonio-blog"},
	89	+	"prose": {"erock"},
	90	+	"sish": {"SHA256:someone-else", fingerprint},
	91	+	}, []string{
	92	+	`_pgs.example.com: "antonio-blog" is valid, serving pgs project "blog" of antonio`,
	93	+	`_prose.example.com: "erock" is valid, serving prose blog of erock (not your account)`,
	94	+	fmt.Sprintf(`_sish.example.com: %q is valid, tunnels from your key "laptop" can use this domain`, fingerprint),
	95	+	}},
	96	+	{"invalid", "", map[string][]string{
	97	+	"pgs": {"antonio-blgo"},
	98	+	"prose": {"nobody"},
	99	+	"sish": {"SHA256:someone-else"},
	100	+	}, []string{
	101	+	`_pgs.example.com: "antonio-blgo" is invalid: user "antonio" has no project "blgo"`,
	102	+	`_prose.example.com: "nobody" is invalid: user "nobody" not found`,
	103	+	`_sish.example.com: "SHA256:someone-else" is invalid: no record is your username or the SHA256 fingerprint of one of your keys`,
	104	+	}},
	105	+	{"tuns record with whitespace", "", map[string][]string{"sish": {fingerprint + " "}}, []string{
	106	+	fmt.Sprintf(`_sish.example.com: %q is invalid: it has surrounding whitespace, and tuns needs an exact match`, fingerprint+" "),
	107	+	}},
	108	+	{"tuns username", "", map[string][]string{"tuns": {"antonio"}}, []string{
	109	+	`_tuns.example.com: "antonio" is valid, tunnels from any of your keys can use this domain`,
	110	+	"_sish.example.com: no TXT record",
	111	+	}},
	112	+	{"wildcard", "*.example.com", map[string][]string{"pgs": {"antonio-blog"}, "sish": {"antonio"}}, []string{
	113	+	"_pgs.*.example.com: wildcard domains are not supported\r\n",
	114	+	"_prose.*.example.com: wildcard domains are not supported\r\n",
	115	+	`.example.com: "antonio" is valid, tunnels from any of your keys can use this domain`,
	116	+	"_tuns.",
	117	+	}},
	118	+	}
	119	+	for _, tt := range tests {
	120	+	t.Run(tt.name, func(t *testing.T) {
	121	+	session := &fakeSession{}
	122	+	cmd := &Cmd{
	123	+	User: me,
	124	+	Session: session,
	125	+	Log: slog.New(slog.DiscardHandler),
	126	+	Dbpool: dbpool,
	127	+	lookupTXT: func(host, space string) ([]string, error) {
	128	+	if tt.domain == "" && host != "example.com" {
	129	+	t.Fatalf("looked up %q", host)
	130	+	}
	131	+	if tt.domain != "" && (!strings.HasSuffix(host, ".example.com") \|\| strings.Contains(host, "*")) {
	132	+	t.Fatalf("looked up %q for %q", host, tt.domain)
	133	+	}
	134	+	if v, ok := tt.records[space]; ok {
	135	+	return v, nil
	136	+	}
	137	+	return nil, router.ErrNoTXTRecord
	138	+	},
	139	+	}
	140	+	domain := tt.domain
	141	+	if domain == "" {
	142	+	domain = " Example.COM. "
	143	+	}
	144	+	if err := cmd.dnsCheck(domain); err != nil {
	145	+	t.Fatal(err)
	146	+	}
	147	+	out := session.String()
	148	+	for _, line := range tt.want {
	149	+	if !strings.Contains(out, line) {
	150	+	t.Errorf("missing %q in output:\n%s", line, out)
	151	+	}
	152	+	}
	153	+	})
	154	+	}
	155	+
	156	+	cmd := &Cmd{User: me, Session: &fakeSession{}, Dbpool: dbpool}
	157	+	for _, bad := range []string{"https://example.com/", ".", "a..example.com"} {
	158	+	if err := cmd.dnsCheck(bad); err == nil {
	159	+	t.Errorf("%q was accepted as a domain", bad)
	160	+	}
	161	+	}
	162	+	}

...	...		@@ -8,8 +8,6 @@ import (
8	8		"time"
9	9
10	10		"github.com/picosh/pico/pkg/db/postgres"
11		-	"github.com/picosh/pico/pkg/filehandlers"
12		-	uploadimgs "github.com/picosh/pico/pkg/filehandlers/imgs"
13	11		"github.com/picosh/pico/pkg/pssh"
14	12		"github.com/picosh/pico/pkg/send/auth"
15	13		"github.com/picosh/pico/pkg/send/list"

...	...		@@ -38,11 +36,6 @@ func StartSshServer() {
38	36		_ = dbh.Close()
39	37		}()
40	38
41		-	hooks := &MarkdownHooks{
42		-	Cfg: cfg,
43		-	Db: dbh,
44		-	}
45		-
46	39		adapter := storage.GetStorageTypeFromEnv()
47	40		st, err := storage.NewStorage(cfg.Logger, adapter)
48	41		if err != nil {

...	...		@@ -50,14 +43,7 @@ func StartSshServer() {
50	43		return
51	44		}
52	45
53		-	fileMap := map[string]filehandlers.ReadWriteHandler{
54		-	".md": filehandlers.NewScpPostHandler(dbh, cfg, hooks),
55		-	".txt": filehandlers.NewScpPostHandler(dbh, cfg, hooks),
56		-	".css": filehandlers.NewScpPostHandler(dbh, cfg, hooks),
57		-	".lxt": filehandlers.NewScpPostHandler(dbh, cfg, hooks),
58		-	"fallback": uploadimgs.NewUploadImgHandler(dbh, cfg, st),
59		-	}
60		-	handler := NewUploadHandler(cfg, dbh, fileMap)
	46	+	handler := NewFileHandler(cfg, dbh, st)
61	47
62	48		sshAuth := shared.NewSshAuthHandler(dbh, logger, "prose")
63	49

...	...		@@ -7,9 +7,11 @@ import (
7	7
8	8		"github.com/picosh/pico/pkg/db"
9	9		"github.com/picosh/pico/pkg/filehandlers"
	10	+	uploadimgs "github.com/picosh/pico/pkg/filehandlers/imgs"
10	11		"github.com/picosh/pico/pkg/pssh"
11	12		sendutils "github.com/picosh/pico/pkg/send/utils"
12	13		"github.com/picosh/pico/pkg/shared"
	14	+	"github.com/picosh/pico/pkg/storage"
13	15		)
14	16
15	17		type ctxFeatureFlagKey struct{}

...	...		@@ -79,6 +81,21 @@ func NewUploadHandler(cfg *shared.ConfigSite, dbpool db.DB, fileMap map[string]f
79	81		}
80	82		}
81	83
	84	+	// NewFileHandler routes prose's text files to posts and the rest to images.
	85	+	func NewFileHandler(cfg shared.ConfigSite, dbpool db.DB, st storage.StorageServe) UploadHandler {
	86	+	hooks := &MarkdownHooks{Cfg: cfg, Db: dbpool}
	87	+	// One post handler serves every text extension, so a post can be
	88	+	// renamed in place from one to another.
	89	+	posts := filehandlers.NewScpPostHandler(dbpool, cfg, hooks)
	90	+	return NewUploadHandler(cfg, dbpool, map[string]filehandlers.ReadWriteHandler{
	91	+	".md": posts,
	92	+	".txt": posts,
	93	+	".css": posts,
	94	+	".lxt": posts,
	95	+	"fallback": uploadimgs.NewUploadImgHandler(dbpool, cfg, st),
	96	+	})
	97	+	}
	98	+
82	99		func (h UploadHandler) GetLogger(s pssh.SSHServerConnSession) *slog.Logger {
83	100		logger := pssh.GetLogger(s)
84	101		if logger == nil {

...	...		@@ -14,6 +14,7 @@ var ErrNameTaken = errors.New("username has already been claimed")
14	14		var ErrNameDenied = errors.New("username is on the denylist")
15	15		var ErrNameInvalid = errors.New("username has invalid characters in it")
16	16		var ErrPublicKeyTaken = errors.New("public key is already associated with another user")
	17	+	var ErrPostChanged = errors.New("post changed while it was being updated")
17	18
18	19		// sqlite uses string to BLOB type and postgres uses []uint8 for JSONB.
19	20		func tcast(value any) ([]byte, error) {

...	...		@@ -385,9 +386,10 @@ func (m *ErrMultiplePublicKeys) Error() string {
385	386		}
386	387
387	388		type UserStats struct {
388		-	Prose UserServiceStats
389		-	Feeds UserServiceStats
390		-	Pages UserServiceStats
	389	+	Prose UserServiceStats
	390	+	Pastes UserServiceStats
	391	+	Feeds UserServiceStats
	392	+	Pages UserServiceStats
391	393		}
392	394
393	395		type UserServiceStats struct {

...	...		@@ -603,6 +605,12 @@ type DB interface {
603	605		FindPostWithSlug(slug string, userID string, space string) (*Post, error)
604	606		InsertPost(post Post) (Post, error)
605	607		UpdatePost(post Post) (Post, error)
	608	+	// RenamePost moves cur to next's filename, with the slug, title,
	609	+	// description, publish date, hidden flag, expiry, mime type, tags and
	610	+	// aliases that name gives it, and removes replaced when it is set, all
	611	+	// in one transaction. It returns ErrPostChanged when either post no
	612	+	// longer matches what the caller read.
	613	+	RenamePost(cur, next, replaced Post, tags, aliases []string) (Post, error)
606	614		RemovePosts(postIDs []string) error
607	615
608	616		FindPopularPosts() ([]*Post, error)

...	...		@@ -659,3 +667,7 @@ type DB interface {
659	667
660	668		Close() error
661	669		}
	670	+
	671	+	func HasPlusOrSpace(dbpool DB, userID, space string) bool {
	672	+	return dbpool.HasFeatureByUser(userID, "plus") \|\| dbpool.HasFeatureByUser(userID, space)
	673	+	}

...	...		@@ -693,6 +693,95 @@ func (me PsqlDB) UpdatePost(post db.Post) (*db.Post, error) {
693	693		return me.FindPost(post.ID)
694	694		}
695	695
	696	+	func (me PsqlDB) RenamePost(cur, next, replaced db.Post, tags, aliases []string) (*db.Post, error) {
	697	+	tx, err := me.Db.Beginx()
	698	+	if err != nil {
	699	+	return nil, err
	700	+	}
	701	+	defer func() {
	702	+	_ = tx.Rollback()
	703	+	}()
	704	+
	705	+	affected := func(res sql.Result, err error) error {
	706	+	if err != nil {
	707	+	return err
	708	+	}
	709	+	n, err := res.RowsAffected()
	710	+	if err != nil {
	711	+	return err
	712	+	}
	713	+	if n == 0 {
	714	+	return db.ErrPostChanged
	715	+	}
	716	+	return nil
	717	+	}
	718	+
	719	+	// A save of the replaced post since it was read is kept.
	720	+	if replaced != nil {
	721	+	err := affected(tx.Exec(
	722	+	`DELETE FROM posts WHERE id = $1 AND filename = $2 AND text = $3 AND updated_at IS NOT DISTINCT FROM $4`,
	723	+	replaced.ID, replaced.Filename, replaced.Text, replaced.UpdatedAt,
	724	+	))
	725	+	if err != nil {
	726	+	return nil, err
	727	+	}
	728	+	}
	729	+
	730	+	// The update only applies to the post as the caller read it, so a save
	731	+	// in between is never overwritten. It sets only what a new name can
	732	+	// change, which leaves data that other writers keep, like feed digests.
	733	+	query := `
	734	+	UPDATE posts
	735	+	SET filename = $1, slug = $2, title = $3, description = $4, publish_at = $5, hidden = $6,
	736	+	expires_at = $7, mime_type = $8
	737	+	WHERE id = $9 AND filename = $10 AND slug = $11 AND title = $12 AND text = $13
	738	+	AND description = $14 AND updated_at IS NOT DISTINCT FROM $15
	739	+	AND publish_at IS NOT DISTINCT FROM $16 AND hidden = $17
	740	+	AND expires_at IS NOT DISTINCT FROM $18`
	741	+	err = affected(tx.Exec(
	742	+	query,
	743	+	next.Filename,
	744	+	next.Slug,
	745	+	next.Title,
	746	+	next.Description,
	747	+	next.PublishAt,
	748	+	next.Hidden,
	749	+	next.ExpiresAt,
	750	+	next.MimeType,
	751	+	cur.ID,
	752	+	cur.Filename,
	753	+	cur.Slug,
	754	+	cur.Title,
	755	+	cur.Text,
	756	+	cur.Description,
	757	+	cur.UpdatedAt,
	758	+	cur.PublishAt,
	759	+	cur.Hidden,
	760	+	cur.ExpiresAt,
	761	+	))
	762	+	if err != nil {
	763	+	return nil, err
	764	+	}
	765	+
	766	+	if err := me.removeTagsByPost(tx, cur.ID); err != nil {
	767	+	return nil, err
	768	+	}
	769	+	if _, err := me.insertTagsByPost(tx, tags, cur.ID); err != nil {
	770	+	return nil, err
	771	+	}
	772	+	if err := me.removeAliasesByPost(tx, cur.ID); err != nil {
	773	+	return nil, err
	774	+	}
	775	+	if _, err := me.insertAliasesByPost(tx, aliases, cur.ID); err != nil {
	776	+	return nil, err
	777	+	}
	778	+	if err := tx.Commit(); err != nil {
	779	+	return nil, err
	780	+	}
	781	+
	782	+	return me.FindPost(cur.ID)
	783	+	}
	784	+
696	785		func (me *PsqlDB) RemovePosts(postIDs []string) error {
697	786		param := "{" + strings.Join(postIDs, ",") + "}"
698	787		_, err := me.Db.Exec(`DELETE FROM posts WHERE id = ANY($1::uuid[])`, param)

...	...		@@ -2041,13 +2130,18 @@ func (me PsqlDB) findPagesStats(userID string) (db.UserServiceStats, error) {
2041	2130		stats := db.UserServiceStats{
2042	2131		Service: "pgs",
2043	2132		}
	2133	+	// the aggregates are NULL when the user has no projects
	2134	+	var first, latestCreated, latestUpdated sql.NullTime
2044	2135		err := me.Db.QueryRow(
2045	2136		`SELECT count(id), min(created_at), max(created_at), max(updated_at) FROM projects WHERE user_id=$1`,
2046	2137		userID,
2047		-	).Scan(&stats.Num, &stats.FirstCreatedAt, &stats.LastestCreatedAt, &stats.LatestUpdatedAt)
	2138	+	).Scan(&stats.Num, &first, &latestCreated, &latestUpdated)
2048	2139		if err != nil {
2049	2140		return nil, err
2050	2141		}
	2142	+	stats.FirstCreatedAt = first.Time
	2143	+	stats.LastestCreatedAt = latestCreated.Time
	2144	+	stats.LatestUpdatedAt = latestUpdated.Time
2051	2145
2052	2146		return &stats, nil
2053	2147		}

...	...		@@ -2101,6 +2195,8 @@ func (me PsqlDB) FindUserStats(userID string) (db.UserStats, error) {
2101	2195		switch stat.Service {
2102	2196		case "prose":
2103	2197		stats.Prose = stat
	2198	+	case "pastes":
	2199	+	stats.Pastes = stat
2104	2200		case "feeds":
2105	2201		stats.Feeds = stat
2106	2202		}

...	...		@@ -2260,16 +2356,35 @@ func (me PsqlDB) FindInvitesByUser(userID string) ([]db.Invite, error) {
2260	2356		return invites, nil
2261	2357		}
2262	2358
2263		-	func (me *PsqlDB) InviteUser(fromUserID string, toUserID string) error {
2264		-	ff, _ := me.FindFeature(fromUserID, "plus")
2265		-	hasPlus := ff != nil && ff.IsValid()
	2359	+	// inviteFeatures are the services an invite grants access to.
	2360	+	var inviteFeatures = []string{"prose", "pgs", "pastes"}
2266	2361
2267		-	var invite *db.Invite
2268		-	_ = me.Db.Select(&invite, "SELECT * from invites WHERE to_user_id=?", fromUserID)
2269		-	hasBeenInvited := invite != nil
	2362	+	func (me *PsqlDB) hasActiveInvite(userID string) (bool, error) {
	2363	+	var invited bool
	2364	+	err := me.Db.Get(&invited, `SELECT EXISTS (SELECT 1 FROM invites WHERE to_user_id = $1)`, userID)
	2365	+	if err != nil \|\| !invited {
	2366	+	return false, err
	2367	+	}
	2368	+	for _, name := range inviteFeatures {
	2369	+	if me.HasFeatureByUser(userID, name) {
	2370	+	return true, nil
	2371	+	}
	2372	+	}
	2373	+	return false, nil
	2374	+	}
	2375	+
	2376	+	func (me *PsqlDB) InviteUser(fromUserID string, toUserID string) error {
	2377	+	canInvite := me.HasFeatureByUser(fromUserID, "plus")
	2378	+	if !canInvite {
	2379	+	invited, err := me.hasActiveInvite(fromUserID)
	2380	+	if err != nil {
	2381	+	return err
	2382	+	}
	2383	+	canInvite = invited
	2384	+	}
2270	2385
2271		-	if !hasBeenInvited && !hasPlus {
2272		-	return fmt.Errorf("must have valid pico+ membership or have already been invited by someone")
	2386	+	if !canInvite {
	2387	+	return fmt.Errorf("must have valid pico+ membership or an active invite from someone")
2273	2388		}
2274	2389
2275	2390		_, err := me.Db.Exec(

...	...		@@ -2281,10 +2396,10 @@ func (me *PsqlDB) InviteUser(fromUserID string, toUserID string) error {
2281	2396		}
2282	2397
2283	2398		expiresAt := time.Now().Add(5 * 365 * 24 * time.Hour)
2284		-	_, err = me.InsertFeature(toUserID, "prose", expiresAt)
2285		-	if err != nil {
2286		-	return err
	2399	+	for _, name := range inviteFeatures {
	2400	+	if _, err := me.InsertFeature(toUserID, name, expiresAt); err != nil {
	2401	+	return err
	2402	+	}
2287	2403		}
2288		-	_, err = me.InsertFeature(toUserID, "pgs", expiresAt)
2289		-	return err
	2404	+	return nil
2290	2405		}

...	...		@@ -2,12 +2,14 @@ package postgres
2	2
3	3		import (
4	4		"context"
	5	+	"errors"
5	6		"fmt"
6	7		"log/slog"
7	8		"os"
8	9		"os/exec"
9	10		"path/filepath"
10	11		"runtime"
	12	+	"slices"
11	13		"sort"
12	14		"testing"
13	15		"time"

...	...		@@ -578,6 +580,125 @@ func TestUpdatePost(t *testing.T) {
578	580		}
579	581		}
580	582
	583	+	func TestRenamePost(t *testing.T) {
	584	+	cleanupTestData(t)
	585	+
	586	+	user, _ := testDB.RegisterUser("renamepostowner", "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAI renamepostowner", "comment", "")
	587	+	mustInsertPost(t, &db.Post{UserID: user.ID, Filename: "a.md", Slug: "a", Text: "a", Space: "prose", Hidden: true})
	588	+	mustInsertPost(t, &db.Post{UserID: user.ID, Filename: "b.md", Slug: "b", Text: "b", Space: "prose"})
	589	+	third := mustInsertPost(t, &db.Post{UserID: user.ID, Filename: "c.md", Slug: "c", Text: "c", Space: "prose"})
	590	+	// Posts made before data was written on insert have the column default.
	591	+	if _, err := testDB.Db.Exec(`UPDATE posts SET data = '{}'::jsonb`); err != nil {
	592	+	t.Fatal(err)
	593	+	}
	594	+	find := func(filename string) *db.Post {
	595	+	t.Helper()
	596	+	p, err := testDB.FindPostWithFilename(filename, user.ID, "prose")
	597	+	if err != nil {
	598	+	t.Fatalf("%s: %v", filename, err)
	599	+	}
	600	+	return p
	601	+	}
	602	+	renamed := func(cur db.Post, filename, slug string) db.Post {
	603	+	next := *cur
	604	+	next.Filename, next.Slug, next.MimeType = filename, slug, "text/plain"
	605	+	return &next
	606	+	}
	607	+
	608	+	a := find("a.md")
	609	+	got, err := testDB.RenamePost(a, renamed(a, "a.lxt", "a"), nil, []string{"x", "y"}, []string{"old-a"})
	610	+	if err != nil {
	611	+	t.Fatalf("RenamePost failed: %v", err)
	612	+	}
	613	+	if got.ID != a.ID \|\| got.Filename != "a.lxt" \|\| got.MimeType != "text/plain" \|\| !got.Hidden {
	614	+	t.Errorf("unexpected post after rename: %+v", got)
	615	+	}
	616	+	a = find("a.lxt")
	617	+	if slices.Sort(a.Tags); !slices.Equal(a.Tags, []string{"x", "y"}) {
	618	+	t.Errorf("tags = %q", a.Tags)
	619	+	}
	620	+	if p, err := testDB.FindPostWithSlug("old-a", user.ID, "prose"); err != nil \|\| p.ID != a.ID {
	621	+	t.Errorf("alias old-a = %+v, %v", p, err)
	622	+	}
	623	+
	624	+	// A save after the post was read stops the rename and is kept.
	625	+	stale := *a
	626	+	saved := *a
	627	+	saved.Text = "saved"
	628	+	if _, err := testDB.UpdatePost(&saved); err != nil {
	629	+	t.Fatal(err)
	630	+	}
	631	+	if _, err := testDB.RenamePost(&stale, renamed(&stale, "d.md", "d"), nil, nil, nil); !errors.Is(err, db.ErrPostChanged) {
	632	+	t.Fatalf("rename of a changed post: %v", err)
	633	+	}
	634	+	if p := find("a.lxt"); p.Text != "saved" {
	635	+	t.Errorf("a.lxt text = %q, want the save", p.Text)
	636	+	}
	637	+
	638	+	// So does a save to the post it replaces.
	639	+	a = find("a.lxt")
	640	+	b := find("b.md")
	641	+	bSaved := *b
	642	+	bSaved.Text = "b saved"
	643	+	if _, err := testDB.UpdatePost(&bSaved); err != nil {
	644	+	t.Fatal(err)
	645	+	}
	646	+	if _, err := testDB.RenamePost(a, renamed(a, "b.md", "b"), b, nil, nil); !errors.Is(err, db.ErrPostChanged) {
	647	+	t.Fatalf("rename over a changed post: %v", err)
	648	+	}
	649	+	if p := find("b.md"); p.Text != "b saved" {
	650	+	t.Errorf("b.md text = %q, want the save", p.Text)
	651	+	}
	652	+	if p := find("a.lxt"); p.ID != a.ID {
	653	+	t.Errorf("a.lxt moved by a failed rename")
	654	+	}
	655	+
	656	+	// Taking another post's slug fails and keeps the post it would replace.
	657	+	b = find("b.md")
	658	+	if _, err := testDB.RenamePost(a, renamed(a, "c2.md", "c"), b, nil, nil); err == nil {
	659	+	t.Fatal("expected a slug conflict")
	660	+	}
	661	+	find("b.md")
	662	+
	663	+	if _, err := testDB.RenamePost(a, renamed(a, "b.md", "b"), b, nil, nil); err != nil {
	664	+	t.Fatalf("RenamePost over b.md failed: %v", err)
	665	+	}
	666	+	if p := find("b.md"); p.ID != a.ID \|\| p.Text != "saved" {
	667	+	t.Errorf("b.md = %+v", p)
	668	+	}
	669	+	if _, err := testDB.FindPost(b.ID); err == nil {
	670	+	t.Error("replaced post still exists")
	671	+	}
	672	+	if _, err := testDB.FindPost(third.ID); err != nil {
	673	+	t.Errorf("unrelated post removed: %v", err)
	674	+	}
	675	+
	676	+	// Data written after the post was read, like a feed digest, is kept.
	677	+	moved := find("b.md")
	678	+	if _, err := testDB.Db.Exec(`UPDATE posts SET data = '{"attempts": 2}'::jsonb WHERE id = $1`, moved.ID); err != nil {
	679	+	t.Fatal(err)
	680	+	}
	681	+	if _, err := testDB.RenamePost(moved, renamed(moved, "e.md", "e"), nil, nil, nil); err != nil {
	682	+	t.Fatalf("rename after a data change: %v", err)
	683	+	}
	684	+	if p := find("e.md"); p.Data.Attempts != 2 {
	685	+	t.Errorf("data = %+v, want the later write", p.Data)
	686	+	}
	687	+
	688	+	// A post moved to another name after it was read isn't replaced.
	689	+	e := find("e.md")
	690	+	c := find("c.md")
	691	+	if _, err := testDB.Db.Exec(`UPDATE posts SET filename = 'f.md' WHERE id = $1`, c.ID); err != nil {
	692	+	t.Fatal(err)
	693	+	}
	694	+	if _, err := testDB.RenamePost(e, renamed(e, "c.md", "c"), c, nil, nil); !errors.Is(err, db.ErrPostChanged) {
	695	+	t.Fatalf("rename over a moved post: %v", err)
	696	+	}
	697	+	if p := find("f.md"); p.ID != c.ID {
	698	+	t.Errorf("f.md = %+v", p)
	699	+	}
	700	+	}
	701	+
581	702		func TestRemovePosts(t *testing.T) {
582	703		cleanupTestData(t)
583	704

...	...		@@ -748,11 +869,11 @@ func TestFindExpiredPosts(t *testing.T) {
748	869		Filename: "expired.txt",
749	870		Slug: "expired",
750	871		Title: "Expired",
751		-	Space: "prose",
	872	+	Space: "pastes",
752	873		ExpiresAt: &expired,
753	874		})
754	875
755		-	posts, err := testDB.FindExpiredPosts("prose")
	876	+	posts, err := testDB.FindExpiredPosts("pastes")
756	877		if err != nil {
757	878		t.Fatalf("FindExpiredPosts failed: %v", err)
758	879		}

...	...		@@ -1278,6 +1399,23 @@ func TestFindUserStats(t *testing.T) {
1278	1399		}
1279	1400		}
1280	1401
	1402	+	func TestFindUserStats_NoProjects(t *testing.T) {
	1403	+	cleanupTestData(t)
	1404	+
	1405	+	user, _ := testDB.RegisterUser("statsempty", "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAI statsempty", "comment", "")
	1406	+
	1407	+	stats, err := testDB.FindUserStats(user.ID)
	1408	+	if err != nil {
	1409	+	t.Fatalf("FindUserStats failed: %v", err)
	1410	+	}
	1411	+	if stats.Pages.Num != 0 {
	1412	+	t.Errorf("expected 0 projects, got %d", stats.Pages.Num)
	1413	+	}
	1414	+	if !stats.Pages.FirstCreatedAt.IsZero() {
	1415	+	t.Errorf("expected zero first created at, got %s", stats.Pages.FirstCreatedAt)
	1416	+	}
	1417	+	}
	1418	+
1281	1419		// ============ Tuns Event Logs Tests ============
1282	1420
1283	1421		func TestInsertTunsEventLog(t *testing.T) {

...	...		@@ -1716,3 +1854,83 @@ func TestFindPipeMonitorsByUser_Empty(t *testing.T) {
1716	1854		t.Errorf("expected 0 monitors for user with none, got %d", len(monitors))
1717	1855		}
1718	1856		}
	1857	+
	1858	+	func TestInviteUser_PlusMember(t *testing.T) {
	1859	+	cleanupTestData(t)
	1860	+
	1861	+	from, _ := testDB.RegisterUser("inviteplus", "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAI inviteplus", "comment", "")
	1862	+	to, _ := testDB.RegisterUser("inviteplusto", "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAI inviteplusto", "comment", "")
	1863	+	_, _ = testDB.InsertFeature(from.ID, "plus", time.Now().Add(36524time.Hour))
	1864	+
	1865	+	if err := testDB.InviteUser(from.ID, to.ID); err != nil {
	1866	+	t.Fatalf("InviteUser failed: %v", err)
	1867	+	}
	1868	+
	1869	+	for _, name := range []string{"prose", "pgs", "pastes"} {
	1870	+	if !testDB.HasFeatureByUser(to.ID, name) {
	1871	+	t.Errorf("expected invitee to have %s", name)
	1872	+	}
	1873	+	}
	1874	+
	1875	+	inviter, err := testDB.FindWhoInvitedUser(to.ID)
	1876	+	if err != nil {
	1877	+	t.Fatalf("FindWhoInvitedUser failed: %v", err)
	1878	+	}
	1879	+	if inviter != "inviteplus" {
	1880	+	t.Errorf("expected inviter inviteplus, got %s", inviter)
	1881	+	}
	1882	+	}
	1883	+
	1884	+	func TestInviteUser_InvitedUserCanInvite(t *testing.T) {
	1885	+	cleanupTestData(t)
	1886	+
	1887	+	plus, _ := testDB.RegisterUser("invitechaina", "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAI invitechaina", "comment", "")
	1888	+	mid, _ := testDB.RegisterUser("invitechainb", "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAI invitechainb", "comment", "")
	1889	+	last, _ := testDB.RegisterUser("invitechainc", "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAI invitechainc", "comment", "")
	1890	+	_, _ = testDB.InsertFeature(plus.ID, "plus", time.Now().Add(36524time.Hour))
	1891	+
	1892	+	if err := testDB.InviteUser(plus.ID, mid.ID); err != nil {
	1893	+	t.Fatalf("InviteUser from plus failed: %v", err)
	1894	+	}
	1895	+	if err := testDB.InviteUser(mid.ID, last.ID); err != nil {
	1896	+	t.Fatalf("InviteUser from invited user failed: %v", err)
	1897	+	}
	1898	+	if !testDB.HasFeatureByUser(last.ID, "pastes") {
	1899	+	t.Error("expected second invitee to have pastes")
	1900	+	}
	1901	+	}
	1902	+
	1903	+	func TestInviteUser_Unauthorized(t *testing.T) {
	1904	+	cleanupTestData(t)
	1905	+
	1906	+	from, _ := testDB.RegisterUser("invitenone", "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAI invitenone", "comment", "")
	1907	+	to, _ := testDB.RegisterUser("invitenoneto", "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAI invitenoneto", "comment", "")
	1908	+
	1909	+	if err := testDB.InviteUser(from.ID, to.ID); err == nil {
	1910	+	t.Fatal("expected error for user without pico+ or invite")
	1911	+	}
	1912	+	if testDB.HasFeatureByUser(to.ID, "prose") {
	1913	+	t.Error("expected invitee not to have prose")
	1914	+	}
	1915	+	}
	1916	+
	1917	+	func TestInviteUser_ExpiredInvite(t *testing.T) {
	1918	+	cleanupTestData(t)
	1919	+
	1920	+	plus, _ := testDB.RegisterUser("inviteexpa", "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAI inviteexpa", "comment", "")
	1921	+	expired, _ := testDB.RegisterUser("inviteexpb", "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAI inviteexpb", "comment", "")
	1922	+	to, _ := testDB.RegisterUser("inviteexpc", "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAI inviteexpc", "comment", "")
	1923	+
	1924	+	_, err := testDB.Db.Exec(`INSERT INTO invites (from_user_id, to_user_id) VALUES ($1, $2)`, plus.ID, expired.ID)
	1925	+	if err != nil {
	1926	+	t.Fatalf("insert invite failed: %v", err)
	1927	+	}
	1928	+	past := time.Now().Add(-24 * time.Hour)
	1929	+	for _, name := range []string{"prose", "pgs", "pastes"} {
	1930	+	_, _ = testDB.InsertFeature(expired.ID, name, past)
	1931	+	}
	1932	+
	1933	+	if err := testDB.InviteUser(expired.ID, to.ID); err == nil {
	1934	+	t.Fatal("expected error for user whose invite access expired")
	1935	+	}
	1936	+	}

...	...		@@ -0,0 +1,223 @@
	1	+	package stub
	2	+
	3	+	import (
	4	+	"database/sql"
	5	+	"fmt"
	6	+	"log/slog"
	7	+	"slices"
	8	+	"sync"
	9	+	"time"
	10	+
	11	+	"github.com/picosh/pico/pkg/db"
	12	+	)
	13	+
	14	+	// PostsDB keeps one user's posts in memory, with the unique filename and
	15	+	// slug constraints of the posts table.
	16	+	type PostsDB struct {
	17	+	*StubDB
	18	+	User *db.User
	19	+	Pubkey string
	20	+
	21	+	mu sync.Mutex
	22	+	posts []*db.Post
	23	+	aliases map[string][]string
	24	+	n int
	25	+	}
	26	+
	27	+	func NewPostsDB(logger slog.Logger, user db.User) *PostsDB {
	28	+	return &PostsDB{StubDB: NewStubDB(logger), User: user}
	29	+	}
	30	+
	31	+	func (m PostsDB) FindUserByPubkey(key string) (db.User, error) {
	32	+	if key != m.Pubkey {
	33	+	return nil, sql.ErrNoRows
	34	+	}
	35	+	return m.User, nil
	36	+	}
	37	+
	38	+	func (m *PostsDB) ReplaceTagsByPost(tags []string, id string) error {
	39	+	m.mu.Lock()
	40	+	defer m.mu.Unlock()
	41	+	if i := slices.IndexFunc(m.posts, func(p *db.Post) bool { return p.ID == id }); i >= 0 {
	42	+	m.posts[i].Tags = slices.Clone(tags)
	43	+	}
	44	+	return nil
	45	+	}
	46	+
	47	+	func (m *PostsDB) ReplaceAliasesByPost(aliases []string, id string) error {
	48	+	m.mu.Lock()
	49	+	defer m.mu.Unlock()
	50	+	m.setAliases(id, aliases)
	51	+	return nil
	52	+	}
	53	+
	54	+	// Aliases returns the aliases of the post with id.
	55	+	func (m *PostsDB) Aliases(id string) []string {
	56	+	m.mu.Lock()
	57	+	defer m.mu.Unlock()
	58	+	return slices.Clone(m.aliases[id])
	59	+	}
	60	+
	61	+	// setAliases stores a post's aliases. The caller holds m.mu.
	62	+	func (m *PostsDB) setAliases(id string, aliases []string) {
	63	+	if m.aliases == nil {
	64	+	m.aliases = map[string][]string{}
	65	+	}
	66	+	m.aliases[id] = slices.Clone(aliases)
	67	+	}
	68	+
	69	+	func (m PostsDB) FindUser(string) (db.User, error) { return m.User, nil }
	70	+	func (m PostsDB) FindUserByName(string) (db.User, error) { return m.User, nil }
	71	+	func (m PostsDB) InsertAccessLog(db.AccessLog) error { return nil }
	72	+
	73	+	func (m PostsDB) FindFeature(userID, name string) (db.FeatureFlag, error) {
	74	+	if name != "plus" {
	75	+	return nil, sql.ErrNoRows
	76	+	}
	77	+	expires := time.Now().Add(time.Hour)
	78	+	return &db.FeatureFlag{UserID: userID, Name: name, ExpiresAt: &expires}, nil
	79	+	}
	80	+
	81	+	func (m PostsDB) find(match func(db.Post) bool) (*db.Post, error) {
	82	+	m.mu.Lock()
	83	+	defer m.mu.Unlock()
	84	+	i := slices.IndexFunc(m.posts, match)
	85	+	if i < 0 {
	86	+	return nil, sql.ErrNoRows
	87	+	}
	88	+	p := *m.posts[i]
	89	+	return &p, nil
	90	+	}
	91	+
	92	+	func (m PostsDB) FindPost(id string) (db.Post, error) {
	93	+	return m.find(func(p *db.Post) bool { return p.ID == id })
	94	+	}
	95	+
	96	+	func (m PostsDB) FindPostWithFilename(filename, userID, space string) (db.Post, error) {
	97	+	return m.find(func(p *db.Post) bool { return p.UserID == userID && p.Space == space && p.Filename == filename })
	98	+	}
	99	+
	100	+	func (m PostsDB) FindPostWithSlug(slug, userID, space string) (db.Post, error) {
	101	+	return m.find(func(p *db.Post) bool { return p.UserID == userID && p.Space == space && p.Slug == slug })
	102	+	}
	103	+
	104	+	func (m PostsDB) FindAllPostsByUser(userID, space string) ([]db.Post, error) {
	105	+	m.mu.Lock()
	106	+	defer m.mu.Unlock()
	107	+	var out []*db.Post
	108	+	for _, p := range m.posts {
	109	+	if p.UserID == userID && p.Space == space {
	110	+	c := *p
	111	+	out = append(out, &c)
	112	+	}
	113	+	}
	114	+	return out, nil
	115	+	}
	116	+
	117	+	// conflict reports a post other than the ones in skip that p's filename or
	118	+	// slug would collide with. The caller holds m.mu.
	119	+	func (m PostsDB) conflict(p db.Post, skip ...string) error {
	120	+	for _, o := range m.posts {
	121	+	if slices.Contains(skip, o.ID) \|\| o.UserID != p.UserID \|\| o.Space != p.Space {
	122	+	continue
	123	+	}
	124	+	if o.Filename == p.Filename \|\| o.Slug == p.Slug {
	125	+	return fmt.Errorf("duplicate key value violates unique constraint: %s", o.Filename)
	126	+	}
	127	+	}
	128	+	return nil
	129	+	}
	130	+
	131	+	func (m PostsDB) InsertPost(post db.Post) (*db.Post, error) {
	132	+	m.mu.Lock()
	133	+	defer m.mu.Unlock()
	134	+	if err := m.conflict(post); err != nil {
	135	+	return nil, err
	136	+	}
	137	+	m.n++
	138	+	p := *post
	139	+	p.ID = fmt.Sprintf("post-%d", m.n)
	140	+	now := time.Now()
	141	+	p.CreatedAt = &now
	142	+	m.posts = append(m.posts, &p)
	143	+	c := p
	144	+	return &c, nil
	145	+	}
	146	+
	147	+	func (m PostsDB) update(post db.Post, filename bool, skip ...string) (*db.Post, error) {
	148	+	i := slices.IndexFunc(m.posts, func(p *db.Post) bool { return p.ID == post.ID })
	149	+	if i < 0 {
	150	+	return nil, sql.ErrNoRows
	151	+	}
	152	+	next := *m.posts[i]
	153	+	if filename {
	154	+	next.Filename, next.MimeType = post.Filename, post.MimeType
	155	+	}
	156	+	next.Slug, next.Title, next.Text, next.Description = post.Slug, post.Title, post.Text, post.Description
	157	+	next.UpdatedAt, next.PublishAt, next.FileSize, next.Shasum = post.UpdatedAt, post.PublishAt, post.FileSize, post.Shasum
	158	+	next.Data, next.Hidden, next.ExpiresAt = post.Data, post.Hidden, post.ExpiresAt
	159	+	if err := m.conflict(&next, append(skip, post.ID)...); err != nil {
	160	+	return nil, err
	161	+	}
	162	+	m.posts[i] = &next
	163	+	c := next
	164	+	return &c, nil
	165	+	}
	166	+
	167	+	func (m PostsDB) UpdatePost(post db.Post) (*db.Post, error) {
	168	+	m.mu.Lock()
	169	+	defer m.mu.Unlock()
	170	+	return m.update(post, false)
	171	+	}
	172	+
	173	+	func (m PostsDB) RenamePost(cur, next, replaced db.Post, tags, aliases []string) (*db.Post, error) {
	174	+	m.mu.Lock()
	175	+	defer m.mu.Unlock()
	176	+	unchanged := func(read *db.Post, all bool) bool {
	177	+	i := slices.IndexFunc(m.posts, func(p *db.Post) bool { return p.ID == read.ID })
	178	+	if i < 0 {
	179	+	return false
	180	+	}
	181	+	p := m.posts[i]
	182	+	same := p.Filename == read.Filename && p.Text == read.Text && equalTime(p.UpdatedAt, read.UpdatedAt)
	183	+	if all {
	184	+	same = same && p.Slug == read.Slug && p.Title == read.Title && p.Description == read.Description &&
	185	+	equalTime(p.PublishAt, read.PublishAt) && p.Hidden == read.Hidden && equalTime(p.ExpiresAt, read.ExpiresAt)
	186	+	}
	187	+	return same
	188	+	}
	189	+	if !unchanged(cur, true) \|\| (replaced != nil && !unchanged(replaced, false)) {
	190	+	return nil, db.ErrPostChanged
	191	+	}
	192	+	// Only what a new name can change is written.
	193	+	update := m.posts[slices.IndexFunc(m.posts, func(p db.Post) bool { return p.ID == cur.ID })]
	194	+	update.Filename, update.Slug, update.Title, update.Description = next.Filename, next.Slug, next.Title, next.Description
	195	+	update.PublishAt, update.Hidden, update.ExpiresAt, update.MimeType = next.PublishAt, next.Hidden, next.ExpiresAt, next.MimeType
	196	+	var skip []string
	197	+	if replaced != nil {
	198	+	skip = append(skip, replaced.ID)
	199	+	}
	200	+	p, err := m.update(&update, true, skip...)
	201	+	if err != nil {
	202	+	return nil, err
	203	+	}
	204	+	if replaced != nil {
	205	+	m.posts = slices.DeleteFunc(m.posts, func(p *db.Post) bool { return p.ID == replaced.ID })
	206	+	}
	207	+	i := slices.IndexFunc(m.posts, func(p *db.Post) bool { return p.ID == cur.ID })
	208	+	m.posts[i].Tags = slices.Clone(tags)
	209	+	m.setAliases(cur.ID, aliases)
	210	+	p.Tags = slices.Clone(tags)
	211	+	return p, nil
	212	+	}
	213	+
	214	+	func equalTime(a, b *time.Time) bool {
	215	+	return (a == nil && b == nil) \|\| (a != nil && b != nil && a.Equal(*b))
	216	+	}
	217	+
	218	+	func (m *PostsDB) RemovePosts(ids []string) error {
	219	+	m.mu.Lock()
	220	+	defer m.mu.Unlock()
	221	+	m.posts = slices.DeleteFunc(m.posts, func(p *db.Post) bool { return slices.Contains(ids, p.ID) })
	222	+	return nil
	223	+	}

...	...		@@ -88,6 +88,10 @@ func (me StubDB) UpdatePost(post db.Post) (*db.Post, error) {
88	88		return nil, errNotImpl
89	89		}
90	90
	91	+	func (me StubDB) RenamePost(cur, next, replaced db.Post, tags, aliases []string) (*db.Post, error) {
	92	+	return nil, errNotImpl
	93	+	}
	94	+
91	95		func (me *StubDB) RemovePosts(postIDs []string) error {
92	96		return errNotImpl
93	97		}

...	...		@@ -1,7 +1,9 @@
1	1		package filehandlers
2	2
3	3		import (
	4	+	"database/sql"
4	5		"encoding/binary"
	6	+	"errors"
5	7		"fmt"
6	8		"io"
7	9		"net/http"

...	...		@@ -81,6 +83,28 @@ func (h ScpUploadHandler) Read(s pssh.SSHServerConnSession, entry *sendutils.F
81	83		return fileInfo, reader, nil
82	84		}
83	85
	86	+	// ValidateWrite runs the hooks' validation on the file before its
	87	+	// contents arrive.
	88	+	func (h ScpUploadHandler) ValidateWrite(s pssh.SSHServerConnSession, entry *sendutils.FileEntry) error {
	89	+	user := pssh.GetUser(s)
	90	+	if user == nil {
	91	+	return fmt.Errorf("could not get user from ctx")
	92	+	}
	93	+	if entry.Mode.IsDir() {
	94	+	return fmt.Errorf("file entry is directory, but only files are supported: %s", entry.Filepath)
	95	+	}
	96	+	filename := filepath.Base(entry.Filepath)
	97	+	metadata := PostMetaData{
	98	+	Post: &db.Post{Filename: filename, Slug: shared.SanitizeFileExt(filename)},
	99	+	User: user,
	100	+	FileEntry: entry,
	101	+	}
	102	+	if valid, err := h.Hooks.FileValidate(s, &metadata); !valid {
	103	+	return err
	104	+	}
	105	+	return nil
	106	+	}
	107	+
84	108		func (h ScpUploadHandler) Write(s pssh.SSHServerConnSession, entry *sendutils.FileEntry) (string, error) {
85	109		logger := pssh.GetLogger(s)
86	110		user := pssh.GetUser(s)

...	...		@@ -102,33 +126,12 @@ func (h ScpUploadHandler) Write(s pssh.SSHServerConnSession, entry *sendutils.
102	126		return "", fmt.Errorf("file entry is directory, but only files are supported: %s", filename)
103	127		}
104	128
105		-	var origText []byte
106		-	if b, err := io.ReadAll(entry.Reader); err == nil {
107		-	origText = b
108		-	}
109		-
110		-	mimeType := http.DetectContentType(origText)
111		-	ext := filepath.Ext(filename)
112		-	// DetectContentType does not detect markdown
113		-	if ext == ".md" {
114		-	mimeType = "text/markdown; charset=UTF-8"
115		-	}
116		-
117		-	now := time.Now()
118		-	slug := shared.SanitizeFileExt(filename)
119		-	fileSize := binary.Size(origText)
120		-	shasum := shared.Shasum(origText)
121		-
122		-	nextPost := db.Post{
123		-	Filename: filename,
124		-	Slug: slug,
125		-	PublishAt: &now,
126		-	Text: string(origText),
127		-	MimeType: mimeType,
128		-	FileSize: fileSize,
129		-	Shasum: shasum,
	129	+	origText, err := io.ReadAll(entry.Reader)
	130	+	if err != nil {
	131	+	return "", fmt.Errorf("could not read file (%s): %w", filename, err)
130	132		}
131	133
	134	+	nextPost := newPost(filename, origText)
132	135		metadata := PostMetaData{
133	136		Post: &nextPost,
134	137		User: user,

...	...		@@ -269,6 +272,94 @@ func (h ScpUploadHandler) Write(s pssh.SSHServerConnSession, entry *sendutils.
269	272		return h.Cfg.FullPostURL(curl, user.Name, metadata.Slug), nil
270	273		}
271	274
	275	+	// newPost is the post a file makes before the hooks fill in the rest.
	276	+	func newPost(filename string, text []byte) db.Post {
	277	+	mimeType := http.DetectContentType(text)
	278	+	// DetectContentType does not detect markdown
	279	+	if filepath.Ext(filename) == ".md" {
	280	+	mimeType = "text/markdown; charset=UTF-8"
	281	+	}
	282	+	now := time.Now()
	283	+	return db.Post{
	284	+	Filename: filename,
	285	+	Slug: shared.SanitizeFileExt(filename),
	286	+	PublishAt: &now,
	287	+	Text: string(text),
	288	+	MimeType: mimeType,
	289	+	FileSize: binary.Size(text),
	290	+	Shasum: shared.Shasum(text),
	291	+	}
	292	+	}
	293	+
	294	+	// Rename moves a post to a new filename in place, so it keeps its ID,
	295	+	// publish date and settings. A post at the new filename is replaced.
	296	+	func (h ScpUploadHandler) Rename(s pssh.SSHServerConnSession, from, to *sendutils.FileEntry) error {
	297	+	user := pssh.GetUser(s)
	298	+	if user == nil {
	299	+	return fmt.Errorf("could not get user from ctx")
	300	+	}
	301	+	fromName, toName := filepath.Base(from.Filepath), filepath.Base(to.Filepath)
	302	+	// A save to either post while it is being renamed starts it over.
	303	+	var err error
	304	+	for range 3 {
	305	+	if err = h.rename(s, user, fromName, toName, to); !errors.Is(err, db.ErrPostChanged) {
	306	+	break
	307	+	}
	308	+	}
	309	+	if err != nil {
	310	+	return err
	311	+	}
	312	+	pssh.GetLogger(s).Info("renamed post", "from", fromName, "to", toName)
	313	+	return nil
	314	+	}
	315	+
	316	+	func (h ScpUploadHandler) rename(s pssh.SSHServerConnSession, user db.User, fromName, toName string, to sendutils.FileEntry) error {
	317	+	post, err := h.DBPool.FindPostWithFilename(fromName, user.ID, h.Cfg.Space)
	318	+	if errors.Is(err, sql.ErrNoRows) {
	319	+	return os.ErrNotExist
	320	+	}
	321	+	if err != nil {
	322	+	return err
	323	+	}
	324	+	if fromName == toName {
	325	+	return nil
	326	+	}
	327	+	replaced, err := h.DBPool.FindPostWithFilename(toName, user.ID, h.Cfg.Space)
	328	+	if errors.Is(err, sql.ErrNoRows) {
	329	+	replaced = nil
	330	+	} else if err != nil {
	331	+	return err
	332	+	}
	333	+
	334	+	next := newPost(toName, []byte(post.Text))
	335	+	metadata := PostMetaData{
	336	+	Post: &next,
	337	+	Cur: post,
	338	+	User: user,
	339	+	FileEntry: to,
	340	+	}
	341	+	if valid, err := h.Hooks.FileValidate(s, &metadata); !valid {
	342	+	return err
	343	+	}
	344	+	metadata.Data = post.Data
	345	+	metadata.PublishAt = post.PublishAt
	346	+	if err := h.Hooks.FileMeta(s, &metadata); err != nil {
	347	+	return err
	348	+	}
	349	+	if other, err := h.DBPool.FindPostWithSlug(metadata.Slug, user.ID, h.Cfg.Space); err == nil &&
	350	+	other.ID != post.ID && (replaced == nil \|\| other.ID != replaced.ID) {
	351	+	return fmt.Errorf("%s: %s already uses its slug (%s): %w", toName, other.Filename, metadata.Slug, os.ErrExist)
	352	+	}
	353	+
	354	+	metadata.UpdatedAt = post.UpdatedAt
	355	+	// The new name can parse the text differently, as .md and .lxt do.
	356	+	_, err = h.DBPool.RenamePost(post, metadata.Post, replaced, metadata.Tags, metadata.Aliases)
	357	+	if err != nil && !errors.Is(err, db.ErrPostChanged) {
	358	+	return fmt.Errorf("error for %s: %w", toName, err)
	359	+	}
	360	+	return err
	361	+	}
	362	+
272	363		func (h ScpUploadHandler) Delete(s pssh.SSHServerConnSession, entry *sendutils.FileEntry) error {
273	364		logger := pssh.GetLogger(s)
274	365		user := pssh.GetUser(s)

...	...		@@ -0,0 +1,71 @@
	1	+	package filehandlers
	2	+
	3	+	import (
	4	+	"context"
	5	+	"errors"
	6	+	"io"
	7	+	"log/slog"
	8	+	"strings"
	9	+	"testing"
	10	+	"testing/iotest"
	11	+
	12	+	"github.com/picosh/pico/pkg/db"
	13	+	"github.com/picosh/pico/pkg/db/stub"
	14	+	"github.com/picosh/pico/pkg/pssh"
	15	+	sendutils "github.com/picosh/pico/pkg/send/utils"
	16	+	"github.com/picosh/pico/pkg/shared"
	17	+	"golang.org/x/crypto/ssh"
	18	+	)
	19	+
	20	+	type uploadHooks struct {
	21	+	validated bool
	22	+	}
	23	+
	24	+	func (h uploadHooks) FileValidate(pssh.SSHServerConnSession, *PostMetaData) (bool, error) {
	25	+	h.validated = true
	26	+	return false, errors.New("unexpected validation after a failed read")
	27	+	}
	28	+
	29	+	func (h uploadHooks) FileMeta(pssh.SSHServerConnSession, *PostMetaData) error {
	30	+	return nil
	31	+	}
	32	+
	33	+	func TestPostUploadReadError(t *testing.T) {
	34	+	checksumErr := errors.New("rsync: file failed verification")
	35	+	tests := []struct {
	36	+	name string
	37	+	text string
	38	+	err error
	39	+	}{
	40	+	{name: "before contents", err: io.ErrClosedPipe},
	41	+	{name: "partial contents", text: "partial upload", err: io.ErrUnexpectedEOF},
	42	+	{name: "failed verification", text: "complete upload", err: checksumErr},
	43	+	}
	44	+	for _, tt := range tests {
	45	+	t.Run(tt.name, func(t *testing.T) {
	46	+	logger := slog.New(slog.DiscardHandler)
	47	+	session := &pssh.SSHServerConnSession{
	48	+	SSHServerConn: &pssh.SSHServerConn{
	49	+	Conn: &ssh.ServerConn{Permissions: &ssh.Permissions{Extensions: map[string]string{}}},
	50	+	Logger: logger,
	51	+	},
	52	+	Ctx: context.Background(),
	53	+	}
	54	+	pssh.SetUser(session, &db.User{ID: "user-1", Name: "tester"})
	55	+	hooks := &uploadHooks{}
	56	+	handler := NewScpPostHandler(stub.NewStubDB(logger), &shared.ConfigSite{Space: "pastes"}, hooks)
	57	+	reader := io.MultiReader(strings.NewReader(tt.text), iotest.ErrReader(tt.err))
	58	+
	59	+	message, err := handler.Write(session, &sendutils.FileEntry{Filepath: "/test.txt", Reader: reader})
	60	+	if !errors.Is(err, tt.err) {
	61	+	t.Fatalf("error = %v, want %v", err, tt.err)
	62	+	}
	63	+	if message != "" {
	64	+	t.Fatalf("failed upload returned a success message: %q", message)
	65	+	}
	66	+	if hooks.validated {
	67	+	t.Fatal("failed upload reached validation")
	68	+	}
	69	+	})
	70	+	}
	71	+	}

...	...		@@ -0,0 +1,286 @@
	1	+	package filehandlers
	2	+
	3	+	import (
	4	+	"context"
	5	+	"errors"
	6	+	"fmt"
	7	+	"log/slog"
	8	+	"os"
	9	+	"path/filepath"
	10	+	"slices"
	11	+	"testing"
	12	+	"time"
	13	+
	14	+	"github.com/picosh/pico/pkg/db"
	15	+	"github.com/picosh/pico/pkg/db/stub"
	16	+	"github.com/picosh/pico/pkg/pssh"
	17	+	sendutils "github.com/picosh/pico/pkg/send/utils"
	18	+	"github.com/picosh/pico/pkg/shared"
	19	+	"golang.org/x/crypto/ssh"
	20	+	)
	21	+
	22	+	// extHooks accepts .md and .lxt files, and tags each with its extension.
	23	+	type extHooks struct {
	24	+	// meta runs while a post's metadata is read, as a save from another
	25	+	// session could.
	26	+	meta func()
	27	+	}
	28	+
	29	+	func (extHooks) FileValidate(_ pssh.SSHServerConnSession, data PostMetaData) (bool, error) {
	30	+	if ext := filepath.Ext(data.Filename); ext != ".md" && ext != ".lxt" {
	31	+	return false, fmt.Errorf("%s: invalid extension", data.Filename)
	32	+	}
	33	+	return true, nil
	34	+	}
	35	+
	36	+	func (h extHooks) FileMeta(_ pssh.SSHServerConnSession, data PostMetaData) error {
	37	+	if h.meta != nil {
	38	+	h.meta()
	39	+	}
	40	+	ext := filepath.Ext(data.Filename)
	41	+	data.Tags = []string{ext}
	42	+	data.Aliases = []string{data.Slug + ext}
	43	+	data.Title = data.Slug
	44	+	if data.Cur != nil {
	45	+	data.Hidden = data.Cur.Hidden
	46	+	}
	47	+	return nil
	48	+	}
	49	+
	50	+	func newPostsSession(t testing.T) (pssh.SSHServerConnSession, *stub.PostsDB) {
	51	+	t.Helper()
	52	+	logger := slog.New(slog.DiscardHandler)
	53	+	user := &db.User{ID: "user-1", Name: "tester"}
	54	+	session := &pssh.SSHServerConnSession{
	55	+	SSHServerConn: &pssh.SSHServerConn{
	56	+	Conn: &ssh.ServerConn{Permissions: &ssh.Permissions{Extensions: map[string]string{}}},
	57	+	Logger: logger,
	58	+	},
	59	+	Ctx: context.Background(),
	60	+	}
	61	+	pssh.SetUser(session, user)
	62	+	return session, stub.NewPostsDB(logger, user)
	63	+	}
	64	+
	65	+	func insertPost(t testing.T, posts stub.PostsDB, filename, text string, hidden bool) *db.Post {
	66	+	t.Helper()
	67	+	p := newPost(filename, []byte(text))
	68	+	publish := time.Date(2020, 1, 2, 0, 0, 0, 0, time.UTC)
	69	+	p.UserID, p.Space, p.Hidden, p.PublishAt, p.UpdatedAt = "user-1", "prose", hidden, &publish, &publish
	70	+	created, err := posts.InsertPost(&p)
	71	+	if err != nil {
	72	+	t.Fatal(err)
	73	+	}
	74	+	return created
	75	+	}
	76	+
	77	+	func TestPostRename(t *testing.T) {
	78	+	session, posts := newPostsSession(t)
	79	+	h := NewScpPostHandler(posts, &shared.ConfigSite{Space: "prose"}, extHooks{})
	80	+	a := insertPost(t, posts, "a.md", "# a", true)
	81	+	b := insertPost(t, posts, "b.md", "# b", false)
	82	+	insertPost(t, posts, "c.md", "# c", false)
	83	+	rename := func(from, to string) error {
	84	+	return h.Rename(session, &sendutils.FileEntry{Filepath: "/" + from}, &sendutils.FileEntry{Filepath: "/" + to})
	85	+	}
	86	+
	87	+	if err := rename("a.md", "a.lxt"); err != nil {
	88	+	t.Fatal(err)
	89	+	}
	90	+	got, err := posts.FindPostWithFilename("a.lxt", "user-1", "prose")
	91	+	if err != nil {
	92	+	t.Fatal(err)
	93	+	}
	94	+	if got.ID != a.ID \|\| !got.PublishAt.Equal(*a.PublishAt) \|\| !got.Hidden \|\| got.Text != "# a" \|\| got.MimeType == a.MimeType {
	95	+	t.Errorf("renamed post = %+v, was %+v", got, a)
	96	+	}
	97	+
	98	+	// c.lxt has the slug c.md has.
	99	+	if err := rename("a.lxt", "c.lxt"); !errors.Is(err, os.ErrExist) {
	100	+	t.Errorf("rename onto a used slug: %v", err)
	101	+	}
	102	+	if err := rename("a.lxt", "a.txt"); err == nil {
	103	+	t.Error("rename to an invalid name succeeded")
	104	+	}
	105	+	if err := rename("missing.md", "m.md"); !errors.Is(err, os.ErrNotExist) {
	106	+	t.Errorf("rename of a missing post: %v", err)
	107	+	}
	108	+
	109	+	if err := rename("a.lxt", "b.md"); err != nil {
	110	+	t.Fatal(err)
	111	+	}
	112	+	got, err = posts.FindPostWithFilename("b.md", "user-1", "prose")
	113	+	if err != nil \|\| got.ID != a.ID \|\| got.Text != "# a" {
	114	+	t.Errorf("b.md = %+v, %v", got, err)
	115	+	}
	116	+	if _, err := posts.FindPost(b.ID); err == nil {
	117	+	t.Error("replaced post still exists")
	118	+	}
	119	+	all, _ := posts.FindAllPostsByUser("user-1", "prose")
	120	+	if len(all) != 2 {
	121	+	t.Errorf("posts = %d, want 2", len(all))
	122	+	}
	123	+	}
	124	+
	125	+	// otherHandler is a handler of a different kind, like prose's images.
	126	+	type otherHandler struct{ ReadWriteHandler }
	127	+
	128	+	func (otherHandler) List(*pssh.SSHServerConnSession, string, bool, bool) ([]os.FileInfo, error) {
	129	+	return []os.FileInfo{&sendutils.VirtualFile{FName: "/", FIsDir: true}, &sendutils.VirtualFile{FName: "x.png"}}, nil
	130	+	}
	131	+
	132	+	func TestRouterSharedHandler(t *testing.T) {
	133	+	session, posts := newPostsSession(t)
	134	+	cfg := &shared.ConfigSite{Space: "prose"}
	135	+	postHandler := NewScpPostHandler(posts, cfg, extHooks{})
	136	+	router := NewFileHandlerRouter(cfg, posts, map[string]ReadWriteHandler{
	137	+	".md": postHandler,
	138	+	".lxt": postHandler,
	139	+	".txt": NewScpPostHandler(posts, cfg, extHooks{}),
	140	+	"fallback": otherHandler{},
	141	+	})
	142	+	insertPost(t, posts, "a.md", "# a", false)
	143	+
	144	+	list, err := router.List(session, "/", true, false)
	145	+	if err != nil {
	146	+	t.Fatal(err)
	147	+	}
	148	+	var names []string
	149	+	for _, f := range list {
	150	+	names = append(names, f.Name())
	151	+	}
	152	+	if len(names) != 3 {
	153	+	t.Errorf("listed %q, want /, a.md and x.png once each", names)
	154	+	}
	155	+
	156	+	entry := func(name string) *sendutils.FileEntry { return &sendutils.FileEntry{Filepath: "/" + name} }
	157	+	if err := router.Rename(session, entry("a.md"), entry("a.lxt")); err != nil {
	158	+	t.Fatal(err)
	159	+	}
	160	+	for _, to := range []string{"a.txt", "a.png"} {
	161	+	if err := router.Rename(session, entry("a.lxt"), entry(to)); !errors.Is(err, errors.ErrUnsupported) {
	162	+	t.Errorf("rename to %s across handlers: %v", to, err)
	163	+	}
	164	+	}
	165	+	}
	166	+
	167	+	func TestPostRenameRelations(t *testing.T) {
	168	+	session, posts := newPostsSession(t)
	169	+	h := NewScpPostHandler(posts, &shared.ConfigSite{Space: "prose"}, extHooks{})
	170	+	a := insertPost(t, posts, "a.md", "# a", false)
	171	+	if err := h.Rename(session, &sendutils.FileEntry{Filepath: "/a.md"}, &sendutils.FileEntry{Filepath: "/a.lxt"}); err != nil {
	172	+	t.Fatal(err)
	173	+	}
	174	+	got, err := posts.FindPost(a.ID)
	175	+	if err != nil {
	176	+	t.Fatal(err)
	177	+	}
	178	+	if !slices.Equal(got.Tags, []string{".lxt"}) \|\| !slices.Equal(posts.Aliases(a.ID), []string{"a.lxt"}) {
	179	+	t.Errorf("tags %q and aliases %q, want the .lxt ones", got.Tags, posts.Aliases(a.ID))
	180	+	}
	181	+	}
	182	+
	183	+	func TestPostRenameConcurrentSave(t *testing.T) {
	184	+	for _, tt := range []struct {
	185	+	name string
	186	+	// saves is how many renames a save interrupts.
	187	+	saves int
	188	+	// post is the post that is saved.
	189	+	post string
	190	+	want error
	191	+	}{
	192	+	{name: "source saved once", saves: 1, post: "a.md"},
	193	+	{name: "replaced post saved once", saves: 1, post: "b.md"},
	194	+	{name: "source saved on every try", saves: 3, post: "a.md", want: db.ErrPostChanged},
	195	+	{name: "replaced post saved on every try", saves: 3, post: "b.md", want: db.ErrPostChanged},
	196	+	} {
	197	+	t.Run(tt.name, func(t *testing.T) {
	198	+	session, posts := newPostsSession(t)
	199	+	a := insertPost(t, posts, "a.md", "# a", true)
	200	+	b := insertPost(t, posts, "b.md", "# b", false)
	201	+	saves := 0
	202	+	save := func() {
	203	+	if saves == tt.saves {
	204	+	return
	205	+	}
	206	+	saves++
	207	+	p, err := posts.FindPostWithFilename(tt.post, "user-1", "prose")
	208	+	if err != nil {
	209	+	t.Fatal(err)
	210	+	}
	211	+	p.Text = fmt.Sprintf("save %d", saves)
	212	+	now := time.Now()
	213	+	p.UpdatedAt = &now
	214	+	if _, err := posts.UpdatePost(p); err != nil {
	215	+	t.Fatal(err)
	216	+	}
	217	+	}
	218	+	h := NewScpPostHandler(posts, &shared.ConfigSite{Space: "prose"}, extHooks{meta: save})
	219	+
	220	+	err := h.Rename(session, &sendutils.FileEntry{Filepath: "/a.md"}, &sendutils.FileEntry{Filepath: "/b.md"})
	221	+	if !errors.Is(err, tt.want) {
	222	+	t.Fatalf("rename: %v, want %v", err, tt.want)
	223	+	}
	224	+	lastSave := fmt.Sprintf("save %d", saves)
	225	+	if tt.want != nil {
	226	+	// Both posts are where they were, with the last save.
	227	+	for _, p := range []*db.Post{a, b} {
	228	+	got, err := posts.FindPost(p.ID)
	229	+	if err != nil {
	230	+	t.Fatalf("%s: %v", p.Filename, err)
	231	+	}
	232	+	want := p.Text
	233	+	if p.Filename == tt.post {
	234	+	want = lastSave
	235	+	}
	236	+	if got.Filename != p.Filename \|\| got.Text != want {
	237	+	t.Errorf("%s = %s %q, want %q", p.Filename, got.Filename, got.Text, want)
	238	+	}
	239	+	}
	240	+	return
	241	+	}
	242	+	got, err := posts.FindPostWithFilename("b.md", "user-1", "prose")
	243	+	if err != nil {
	244	+	t.Fatal(err)
	245	+	}
	246	+	want := "# a"
	247	+	if tt.post == "a.md" {
	248	+	want = lastSave
	249	+	}
	250	+	if got.ID != a.ID \|\| got.Text != want \|\| !got.Hidden {
	251	+	t.Errorf("b.md = %s %q hidden %v, want %s %q hidden", got.ID, got.Text, got.Hidden, a.ID, want)
	252	+	}
	253	+	})
	254	+	}
	255	+	}
	256	+
	257	+	func TestPostRenameMovedDestination(t *testing.T) {
	258	+	session, posts := newPostsSession(t)
	259	+	a := insertPost(t, posts, "a.md", "# a", false)
	260	+	b := insertPost(t, posts, "b.md", "# b", false)
	261	+	moved := false
	262	+	move := func() {
	263	+	if moved {
	264	+	return
	265	+	}
	266	+	moved = true
	267	+	cur, err := posts.FindPost(b.ID)
	268	+	if err != nil {
	269	+	t.Fatal(err)
	270	+	}
	271	+	next := *cur
	272	+	next.Filename, next.Slug = "f.md", "f"
	273	+	if _, err := posts.RenamePost(cur, &next, nil, nil, nil); err != nil {
	274	+	t.Fatal(err)
	275	+	}
	276	+	}
	277	+	h := NewScpPostHandler(posts, &shared.ConfigSite{Space: "prose"}, extHooks{meta: move})
	278	+	if err := h.Rename(session, &sendutils.FileEntry{Filepath: "/a.md"}, &sendutils.FileEntry{Filepath: "/b.md"}); err != nil {
	279	+	t.Fatal(err)
	280	+	}
	281	+	for name, id := range map[string]string{"b.md": a.ID, "f.md": b.ID} {
	282	+	if got, err := posts.FindPostWithFilename(name, "user-1", "prose"); err != nil \|\| got.ID != id {
	283	+	t.Errorf("%s = %+v, %v, want post %s", name, got, err, id)
	284	+	}
	285	+	}
	286	+	}

...	...		@@ -65,6 +65,17 @@ func (r FileHandlerRouter) Write(s pssh.SSHServerConnSession, entry *utils.Fil
65	65		return handler.Write(s, entry)
66	66		}
67	67
	68	+	func (r FileHandlerRouter) ValidateWrite(s pssh.SSHServerConnSession, entry *utils.FileEntry) error {
	69	+	handler, err := r.findHandler(entry.Filepath)
	70	+	if err != nil {
	71	+	return err
	72	+	}
	73	+	if v, ok := handler.(utils.WriteValidator); ok {
	74	+	return v.ValidateWrite(s, entry)
	75	+	}
	76	+	return nil
	77	+	}
	78	+
68	79		func (r FileHandlerRouter) Delete(s pssh.SSHServerConnSession, entry *utils.FileEntry) error {
69	80		handler, err := r.findHandler(entry.Filepath)
70	81		if err != nil {

...	...		@@ -73,6 +84,23 @@ func (r FileHandlerRouter) Delete(s pssh.SSHServerConnSession, entry *utils.Fi
73	84		return handler.Delete(s, entry)
74	85		}
75	86
	87	+	// Rename moves a file in place when one handler serves both names.
	88	+	func (r FileHandlerRouter) Rename(s pssh.SSHServerConnSession, from, to *utils.FileEntry) error {
	89	+	handler, err := r.findHandler(from.Filepath)
	90	+	if err != nil {
	91	+	return err
	92	+	}
	93	+	target, err := r.findHandler(to.Filepath)
	94	+	if err != nil {
	95	+	return err
	96	+	}
	97	+	renamer, ok := handler.(utils.FileRenamer)
	98	+	if !ok \|\| handler != target {
	99	+	return errors.ErrUnsupported
	100	+	}
	101	+	return renamer.Rename(s, from, to)
	102	+	}
	103	+
76	104		func (r FileHandlerRouter) Read(s pssh.SSHServerConnSession, entry *utils.FileEntry) (os.FileInfo, utils.ReadAndReaderAtCloser, error) {
77	105		handler, err := r.findHandler(entry.Filepath)
78	106		if err != nil {

...	...		@@ -83,18 +111,28 @@ func (r FileHandlerRouter) Read(s pssh.SSHServerConnSession, entry *utils.File
83	111
84	112		func (r FileHandlerRouter) List(s pssh.SSHServerConnSession, fpath string, isDir bool, recursive bool) ([]os.FileInfo, error) {
85	113		files := []os.FileInfo{}
86		-	for key, handler := range r.FileMap {
87		-	// TODO: hack because we have duplicate keys for .md and .css
88		-	if key == ".css" {
	114	+	// Several extensions can share a handler, and handlers can list the
	115	+	// same entries, such as the root directory.
	116	+	listed := map[ReadWriteHandler]bool{}
	117	+	seen := map[string]bool{}
	118	+	for _, handler := range r.FileMap {
	119	+	if listed[handler] {
89	120		continue
90	121		}
	122	+	listed[handler] = true
91	123
92	124		ff, err := handler.List(s, fpath, isDir, recursive)
93	125		if err != nil {
94	126		r.GetLogger(s).Error("handler list", "err", err)
95	127		continue
96	128		}
97		-	files = append(files, ff...)
	129	+	for _, f := range ff {
	130	+	key := fmt.Sprintf("%s:%v", f.Name(), f.IsDir())
	131	+	if !seen[key] {
	132	+	seen[key] = true
	133	+	files = append(files, f)
	134	+	}
	135	+	}
98	136		}
99	137		return files, nil
100	138		}

...	...		@@ -0,0 +1,282 @@
	1	+	package filehandlers_test
	2	+
	3	+	import (
	4	+	"context"
	5	+	"crypto/ed25519"
	6	+	"crypto/rand"
	7	+	"encoding/pem"
	8	+	"log/slog"
	9	+	"net"
	10	+	"os"
	11	+	"os/exec"
	12	+	"path/filepath"
	13	+	"slices"
	14	+	"strings"
	15	+	"testing"
	16	+	"time"
	17	+
	18	+	"github.com/picosh/pico/pkg/apps/pastes"
	19	+	"github.com/picosh/pico/pkg/apps/prose"
	20	+	"github.com/picosh/pico/pkg/db"
	21	+	"github.com/picosh/pico/pkg/db/stub"
	22	+	"github.com/picosh/pico/pkg/pssh"
	23	+	"github.com/picosh/pico/pkg/send/list"
	24	+	"github.com/picosh/pico/pkg/send/protocols/sftp"
	25	+	"github.com/picosh/pico/pkg/send/utils"
	26	+	"github.com/picosh/pico/pkg/shared"
	27	+	"github.com/picosh/pico/pkg/storage"
	28	+	"golang.org/x/crypto/ssh"
	29	+	)
	30	+
	31	+	type postsEnv struct {
	32	+	t *testing.T
	33	+	db *stub.PostsDB
	34	+	space string
	35	+	port string
	36	+	key string
	37	+	}
	38	+
	39	+	func newPostsEnv(t testing.T, app string) postsEnv {
	40	+	t.Helper()
	41	+	logger := slog.New(slog.DiscardHandler)
	42	+	_, userKey, err := ed25519.GenerateKey(rand.Reader)
	43	+	if err != nil {
	44	+	t.Fatal(err)
	45	+	}
	46	+	signer, err := ssh.NewSignerFromKey(userKey)
	47	+	if err != nil {
	48	+	t.Fatal(err)
	49	+	}
	50	+	posts := stub.NewPostsDB(logger, &db.User{ID: "user-1", Name: "tester"})
	51	+	posts.Pubkey = shared.KeyForKeyText(signer.PublicKey())
	52	+
	53	+	var cfg *shared.ConfigSite
	54	+	var handler interface {
	55	+	utils.CopyFromClientHandler
	56	+	pssh.GetLoggerInterface
	57	+	}
	58	+	switch app {
	59	+	case "prose":
	60	+	cfg = prose.NewConfigSite("prose-test")
	61	+	st, err := storage.NewStorageMemory(map[string]map[string]string{})
	62	+	if err != nil {
	63	+	t.Fatal(err)
	64	+	}
	65	+	handler = prose.NewFileHandler(cfg, posts, st)
	66	+	case "pastes":
	67	+	cfg = pastes.NewConfigSite("pastes-test")
	68	+	handler = pastes.NewFileHandler(cfg, posts)
	69	+	}
	70	+	cfg.Logger = logger
	71	+
	72	+	ctx, cancel := context.WithCancel(context.Background())
	73	+	t.Cleanup(cancel)
	74	+	auth := shared.NewSshAuthHandler(posts, logger, app)
	75	+	server, err := pssh.NewSSHServerWithConfig(ctx, logger, app, "127.0.0.1", "0", "", filepath.Join(t.TempDir(), "host_key"),
	76	+	auth.PubkeyAuthHandler,
	77	+	[]pssh.SSHServerMiddleware{list.Middleware(handler), pssh.LogMiddleware(handler, posts)},
	78	+	[]pssh.SSHServerMiddleware{sftp.Middleware(handler), pssh.LogMiddleware(handler, posts)},
	79	+	nil,
	80	+	)
	81	+	if err != nil {
	82	+	t.Fatal(err)
	83	+	}
	84	+	go func() { _ = server.ListenAndServe() }()
	85	+	var addr string
	86	+	for range 100 {
	87	+	server.Mu.Lock()
	88	+	if server.Listener != nil {
	89	+	addr = server.Listener.Addr().String()
	90	+	}
	91	+	server.Mu.Unlock()
	92	+	if addr != "" {
	93	+	break
	94	+	}
	95	+	time.Sleep(10 * time.Millisecond)
	96	+	}
	97	+	if addr == "" {
	98	+	t.Fatal("server listener not ready")
	99	+	}
	100	+	_, port, _ := net.SplitHostPort(addr)
	101	+
	102	+	b, err := ssh.MarshalPrivateKey(userKey, "")
	103	+	if err != nil {
	104	+	t.Fatal(err)
	105	+	}
	106	+	key := filepath.Join(t.TempDir(), "id_ed25519")
	107	+	if err := os.WriteFile(key, pem.EncodeToMemory(b), 0o600); err != nil {
	108	+	t.Fatal(err)
	109	+	}
	110	+	return &postsEnv{t: t, db: posts, space: cfg.Space, port: port, key: key}
	111	+	}
	112	+
	113	+	func (e *postsEnv) sshOpts() []string {
	114	+	return []string{"-o", "IdentityFile=" + e.key, "-o", "IdentitiesOnly=yes", "-o", "StrictHostKeyChecking=no",
	115	+	"-o", "UserKnownHostsFile=/dev/null", "-o", "LogLevel=ERROR"}
	116	+	}
	117	+
	118	+	func (e *postsEnv) run(name string, args ...string) string {
	119	+	e.t.Helper()
	120	+	out, err := exec.Command(name, args...).CombinedOutput()
	121	+	if err != nil {
	122	+	e.t.Fatalf("%s %v: %v\n%s", name, args, err, out)
	123	+	}
	124	+	return string(out)
	125	+	}
	126	+
	127	+	// seed stores a hidden post, made hidden the way each service does it.
	128	+	func (e postsEnv) seed(filename string) db.Post {
	129	+	e.t.Helper()
	130	+	published := time.Date(2020, 1, 2, 0, 0, 0, 0, time.UTC)
	131	+	post := &db.Post{
	132	+	UserID: "user-1", Space: e.space, Filename: filename, Slug: strings.TrimSuffix(filename, filepath.Ext(filename)),
	133	+	Text: seedText(e.space), FileSize: len(seedText(e.space)), Hidden: true, PublishAt: &published, UpdatedAt: &published,
	134	+	}
	135	+	if e.space == "pastes" {
	136	+	expires := time.Now().Add(time.Hour).Truncate(time.Second)
	137	+	post.ExpiresAt = &expires
	138	+	}
	139	+	p, err := e.db.InsertPost(post)
	140	+	if err != nil {
	141	+	e.t.Fatal(err)
	142	+	}
	143	+	return p
	144	+	}
	145	+
	146	+	func seedText(space string) string {
	147	+	if space == "prose" {
	148	+	return "---\ndraft: true\n---\nseeded\n"
	149	+	}
	150	+	return "seeded\n"
	151	+	}
	152	+
	153	+	// assertKept waits for the post at filename to have text and checks it is
	154	+	// the seeded one with its settings. sshfs sends a close after the command
	155	+	// that made it returns.
	156	+	func (e postsEnv) assertKept(filename, text string, seeded db.Post) {
	157	+	e.t.Helper()
	158	+	e.assertMoved(filename, text, seeded, seeded.Hidden)
	159	+	}
	160	+
	161	+	// assertMoved is assertKept for a post whose new name changes whether it
	162	+	// is hidden.
	163	+	func (e postsEnv) assertMoved(filename, text string, seeded db.Post, hidden bool) {
	164	+	e.t.Helper()
	165	+	var got *db.Post
	166	+	for range 100 {
	167	+	var err error
	168	+	if got, err = e.db.FindPostWithFilename(filename, "user-1", e.space); err == nil && got.Text == text {
	169	+	break
	170	+	}
	171	+	time.Sleep(20 * time.Millisecond)
	172	+	}
	173	+	if got == nil \|\| got.Text != text {
	174	+	e.t.Fatalf("%s = %+v, want text %q", filename, got, text)
	175	+	}
	176	+	if got.ID != seeded.ID \|\| !got.PublishAt.Equal(*seeded.PublishAt) \|\| got.Hidden != hidden \|\|
	177	+	(got.ExpiresAt == nil) != (seeded.ExpiresAt == nil) \|\| (got.ExpiresAt != nil && !got.ExpiresAt.Equal(*seeded.ExpiresAt)) {
	178	+	e.t.Errorf("%s = id %s published %v hidden %v expires %v, want %s %v %v %v", filename,
	179	+	got.ID, got.PublishAt, got.Hidden, got.ExpiresAt, seeded.ID, seeded.PublishAt, hidden, seeded.ExpiresAt)
	180	+	}
	181	+	}
	182	+
	183	+	func assertListed(t *testing.T, out string, want ...string) {
	184	+	t.Helper()
	185	+	var got []string
	186	+	for _, line := range strings.Split(out, "\n") {
	187	+	f := filepath.Base(strings.TrimSpace(line))
	188	+	if !strings.HasPrefix(line, "sftp>") && f != "." && f != ".." && f != "/" {
	189	+	got = append(got, f)
	190	+	}
	191	+	}
	192	+	slices.Sort(got)
	193	+	slices.Sort(want)
	194	+	if !slices.Equal(got, want) {
	195	+	t.Errorf("listed %q, want %q", got, want)
	196	+	}
	197	+	}
	198	+
	199	+	func TestPostsSftp(t *testing.T) {
	200	+	if _, err := exec.LookPath("sftp"); err != nil {
	201	+	t.Skip("sftp not installed")
	202	+	}
	203	+	for app, ext := range map[string]string{"prose": ".md", "pastes": ".txt"} {
	204	+	t.Run(app, func(t *testing.T) {
	205	+	e := newPostsEnv(t, app)
	206	+	a := e.seed("a" + ext)
	207	+	e.seed("c" + ext)
	208	+
	209	+	sftpCmd := func(cmds string) string {
	210	+	cmd := exec.Command("sftp", append(append([]string{"-b", "-", "-P", e.port}, e.sshOpts()...), "localhost")...)
	211	+	cmd.Stdin = strings.NewReader(cmds)
	212	+	out, err := cmd.CombinedOutput()
	213	+	if err != nil {
	214	+	t.Fatalf("sftp: %v\n%s", err, out)
	215	+	}
	216	+	return string(out)
	217	+	}
	218	+	assertListed(t, sftpCmd("ls -1\n"), "a"+ext, "c"+ext)
	219	+	assertListed(t, e.run("ssh", append(append([]string{"-p", e.port}, e.sshOpts()...), "localhost", "command", "ls")...), "a"+ext, "c"+ext)
	220	+
	221	+	sftpCmd("rename a" + ext + " b" + ext + "\n")
	222	+	e.assertKept("b"+ext, a.Text, a)
	223	+	sftpCmd("rename b" + ext + " c" + ext + "\n")
	224	+	e.assertKept("c"+ext, a.Text, a)
	225	+	assertListed(t, sftpCmd("ls -1\n"), "c"+ext)
	226	+	})
	227	+	}
	228	+	}
	229	+
	230	+	func TestPostsSshfs(t *testing.T) {
	231	+	if _, err := exec.LookPath("sshfs"); err != nil {
	232	+	t.Skip("sshfs not installed")
	233	+	}
	234	+	tests := map[string][]struct {
	235	+	cmd string
	236	+	name string
	237	+	text string
	238	+	// public is set when the new name drops the draft setting, which
	239	+	// .lxt files don't read from front matter.
	240	+	public bool
	241	+	}{
	242	+	"prose": {
	243	+	{cmd: "echo more >> a.md", name: "a.md", text: seedText("prose") + "more\n"},
	244	+	{cmd: "mv a.md b.md", name: "b.md", text: seedText("prose") + "more\n"},
	245	+	{cmd: "mv b.md b.lxt", name: "b.lxt", text: seedText("prose") + "more\n", public: true},
	246	+	{cmd: "mv b.lxt c.md", name: "c.md", text: seedText("prose") + "more\n"},
	247	+	},
	248	+	"pastes": {
	249	+	{cmd: "echo more >> a.txt", name: "a.txt", text: "seeded\nmore\n"},
	250	+	{cmd: "vim -u DEFAULTS -i NONE -N -es -c '%s/more/less/' -c wq a.txt", name: "a.txt", text: "seeded\nless\n"},
	251	+	{cmd: "mv a.txt b.txt", name: "b.txt", text: "seeded\nless\n"},
	252	+	{cmd: "mv b.txt c.txt", name: "c.txt", text: "seeded\nless\n"},
	253	+	},
	254	+	}
	255	+	for app, steps := range tests {
	256	+	t.Run(app, func(t *testing.T) {
	257	+	e := newPostsEnv(t, app)
	258	+	ext := filepath.Ext(steps[0].name)
	259	+	a := e.seed("a" + ext)
	260	+	e.seed("c" + ext)
	261	+
	262	+	mnt := t.TempDir()
	263	+	if out, err := exec.Command("sshfs", append(append([]string{"-p", e.port}, e.sshOpts()...), "localhost:/", mnt)...).CombinedOutput(); err != nil {
	264	+	t.Skipf("sshfs mount failed: %v\n%s", err, out)
	265	+	}
	266	+	t.Cleanup(func() { _ = exec.Command("fusermount3", "-u", "-z", mnt).Run() })
	267	+	assertListed(t, e.run("ls", "-1a", mnt), "a"+ext, "c"+ext)
	268	+
	269	+	home := t.TempDir()
	270	+	for _, step := range steps {
	271	+	cmd := exec.Command("bash", "-c", step.cmd)
	272	+	cmd.Dir = mnt
	273	+	cmd.Env = append(os.Environ(), "HOME="+home)
	274	+	if out, err := cmd.CombinedOutput(); err != nil {
	275	+	t.Fatalf("%s: %v\n%s", step.cmd, err, out)
	276	+	}
	277	+	e.assertMoved(step.name, step.text, a, !step.public)
	278	+	}
	279	+	assertListed(t, e.run("ls", "-1a", mnt), "c"+ext)
	280	+	})
	281	+	}
	282	+	}

...	...		@@ -1,7 +1,6 @@
1	1		package httpcache
2	2
3	3		import (
4		-	"encoding/json"
5	4		"log/slog"
6	5		"net/http"
7	6		"net/http/httptest"

...	...		@@ -241,7 +240,7 @@ func TestCacheVary(t *testing.T) {
241	240		// VaryRequestHeaders snapshots the request header values that were present
242	241		// when this entry was cached, keyed by the lowercase header name.
243	242		cv.VaryRequestHeaders = map[string]string{"accept-encoding": "gzip"}
244		-	cacheValue, _ := json.Marshal(cv)
	243	+	cacheValue := cv.encode()
245	244		handler.Cache.Add(cacheKey, cacheValue)
246	245
247	246		respMatch, _ := tc.DoWithHeaders(req, map[string][]string{

...	...		@@ -385,7 +384,7 @@ func TestCacheValidation(t *testing.T) {
385	384		cv := testCacheValue(250 * time.Second)
386	385		cv.Header["ETag"] = []string{"ccc"}
387	386		cv.Header["Last-Modified"] = []string{actualStr}
388		-	cacheValue, _ := json.Marshal(cv)
	387	+	cacheValue := cv.encode()
389	388		handler.Cache.Add(cacheKey, cacheValue)
390	389
391	390		t.Run(tt.name, func(t *testing.T) {

...	...		@@ -423,7 +422,7 @@ func TestCacheAge(t *testing.T) {
423	422
424	423		req, _ := http.NewRequest("GET", tc.cachedServer.URL+"/test", nil)
425	424		cacheKey := handler.GetCacheKey(req)
426		-	cacheValue, _ := json.Marshal(testCacheValue(250 * time.Second))
	425	+	cacheValue := testCacheValue(250 * time.Second).encode()
427	426		handler.Cache.Add(cacheKey, cacheValue)
428	427
429	428		resp, _ := tc.Do(req)

...	...		@@ -509,7 +508,7 @@ func TestCacheRequestDirectives(t *testing.T) {
509	508
510	509		req, _ := http.NewRequest("GET", tc.cachedServer.URL+"/test", nil)
511	510		cacheKey := handler.GetCacheKey(req)
512		-	cacheValue, _ := json.Marshal(testCacheValue(250 * time.Second))
	511	+	cacheValue := testCacheValue(250 * time.Second).encode()
513	512		handler.Cache.Add(cacheKey, cacheValue)
514	513
515	514		t.Run(tt.name, func(t *testing.T) {

...	...		@@ -762,7 +761,7 @@ func TestCache304NotModifiedMerge(t *testing.T) {
762	761		staleCv.Header["ETag"] = []string{"\"abc\""}
763	762		staleCv.Header["Cache-Control"] = []string{"max-age=60, must-revalidate"}
764	763		staleCv.Body = []byte("original body")
765		-	cacheData, _ := json.Marshal(staleCv)
	764	+	cacheData := staleCv.encode()
766	765		handler.Cache.Add(cacheKey, cacheData)
767	766
768	767		// First request with If-None-Match triggers validation; origin returns 304

...	...		@@ -890,7 +889,7 @@ func TestCache304NoBody(t *testing.T) {
890	889		cv.Header["ETag"] = []string{"\"abc\""}
891	890		cv.Header["Cache-Control"] = []string{"max-age=60, must-revalidate"}
892	891		cv.Body = []byte("original body")
893		-	cacheData, _ := json.Marshal(cv)
	892	+	cacheData := cv.encode()
894	893		handler.Cache.Add(cacheKey, cacheData)
895	894
896	895		// Trigger revalidation — upstream returns 304 with a spurious body.

...	...		@@ -944,7 +943,7 @@ func TestCache304IncludesCachedHeaders(t *testing.T) {
944	943		cv.Header["Content-Type"] = []string{"text/html; charset=utf-8"}
945	944		cv.Header["Cache-Control"] = []string{"max-age=300"}
946	945		cv.Body = []byte("<h1>hello</h1>")
947		-	cacheData, _ := json.Marshal(cv)
	946	+	cacheData := cv.encode()
948	947		handler.Cache.Add(cacheKey, cacheData)
949	948
950	949		// Send conditional request that triggers a 304 from the cache layer

...	...		@@ -1075,7 +1074,7 @@ func TestCacheMustRevalidateRevalidationHeaders(t *testing.T) {
1075	1074		}
1076	1075		cv.Header["Cache-Control"] = []string{"max-age=60, must-revalidate"}
1077	1076		cv.Body = []byte("cached body")
1078		-	cacheData, _ := json.Marshal(cv)
	1077	+	cacheData := cv.encode()
1079	1078		handler.Cache.Add(cacheKey, cacheData)
1080	1079
1081	1080		resp, _ := tc.Do(req)

...	...		@@ -1103,3 +1102,66 @@ func TestCacheMustRevalidateRevalidationHeaders(t *testing.T) {
1103	1102		})
1104	1103		}
1105	1104		}
	1105	+
	1106	+	// A range request that misses must not leave a partial response cached for
	1107	+	// requests of the whole resource.
	1108	+	func TestCacheRangeNotStored(t *testing.T) {
	1109	+	body := strings.Repeat("0123456789", 100)
	1110	+	upstream := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
	1111	+	http.ServeContent(w, r, "file.txt", time.Unix(1_700_000_000, 0), strings.NewReader(body))
	1112	+	})
	1113	+	handler := NewHttpCache(slog.New(slog.DiscardHandler), upstream)
	1114	+
	1115	+	req := httptest.NewRequest("GET", "http://example.com/file.txt", nil)
	1116	+	req.Header.Set("Range", "bytes=0-9")
	1117	+	rec := httptest.NewRecorder()
	1118	+	handler.ServeHTTP(rec, req)
	1119	+	if rec.Code != http.StatusPartialContent \|\| rec.Body.String() != body[:10] {
	1120	+	t.Fatalf("range request got %d %q", rec.Code, rec.Body.String())
	1121	+	}
	1122	+
	1123	+	for _, want := range []string{"miss", "hit"} {
	1124	+	rec = httptest.NewRecorder()
	1125	+	handler.ServeHTTP(rec, httptest.NewRequest("GET", "http://example.com/file.txt", nil))
	1126	+	if rec.Code != http.StatusOK \|\| rec.Body.String() != body {
	1127	+	t.Fatalf("full request got %d with %d bytes", rec.Code, rec.Body.Len())
	1128	+	}
	1129	+	if status := rec.Header().Get("cache-status"); !strings.Contains(status, want) {
	1130	+	t.Fatalf("full request cache-status %q, want %s", status, want)
	1131	+	}
	1132	+	}
	1133	+	}
	1134	+
	1135	+	// Bodies over MaxBodySize stream through and are not stored, whether or not
	1136	+	// upstream announces their length.
	1137	+	func TestCacheMaxBodySize(t *testing.T) {
	1138	+	body := strings.Repeat("x", 1000)
	1139	+	for name, setLength := range map[string]bool{"content-length": true, "chunked": false} {
	1140	+	t.Run(name, func(t *testing.T) {
	1141	+	upstream := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
	1142	+	if setLength {
	1143	+	w.Header().Set("content-length", strconv.Itoa(len(body)))
	1144	+	}
	1145	+	for i := 0; i < len(body); i += 100 {
	1146	+	_, _ = w.Write([]byte(body[i : i+100]))
	1147	+	}
	1148	+	})
	1149	+	handler := NewHttpCache(slog.New(slog.DiscardHandler), upstream)
	1150	+	handler.MaxBodySize = 250
	1151	+
	1152	+	for range 2 {
	1153	+	rec := httptest.NewRecorder()
	1154	+	handler.ServeHTTP(rec, httptest.NewRequest("GET", "http://example.com/big", nil))
	1155	+	if rec.Code != http.StatusOK \|\| rec.Body.String() != body {
	1156	+	t.Fatalf("got %d with %d bytes", rec.Code, rec.Body.Len())
	1157	+	}
	1158	+	if status := rec.Header().Get("cache-status"); !strings.Contains(status, "miss") {
	1159	+	t.Fatalf("cache-status %q, want a miss", status)
	1160	+	}
	1161	+	}
	1162	+	if handler.Cache.Len() != 0 {
	1163	+	t.Fatalf("stored %d entries", handler.Cache.Len())
	1164	+	}
	1165	+	})
	1166	+	}
	1167	+	}

...	...		@@ -1,26 +1,77 @@
1	1		package httpcache
2	2
3	3		import (
	4	+	"encoding/binary"
	5	+	"encoding/json"
	6	+	"errors"
4	7		"net/http"
	8	+	"strconv"
5	9		"strings"
6	10		"time"
7	11		)
8	12
	13	+	// responseWriter buffers an upstream response so it can be stored. A body
	14	+	// larger than maxBody is not stored: what was buffered is sent and the rest
	15	+	// streams straight through.
9	16		type responseWriter struct {
10	17		http.ResponseWriter
11	18		statusCode int
12	19		body []byte
	20	+	maxBody int64
	21	+	streaming bool
	22	+	// streamStatus is the cache-status sent with a streamed response.
	23	+	streamStatus string
13	24		}
14	25
15	26		func (rw *responseWriter) WriteHeader(code int) {
16	27		rw.statusCode = code
	28	+	size, err := strconv.ParseInt(rw.Header().Get("content-length"), 10, 64)
	29	+	if err == nil && rw.tooLarge(size) {
	30	+	_ = rw.stream()
	31	+	}
17	32		}
18	33
19	34		func (rw *responseWriter) Write(data []byte) (int, error) {
	35	+	if !rw.streaming && rw.tooLarge(int64(len(rw.body)+len(data))) {
	36	+	if err := rw.stream(); err != nil {
	37	+	return 0, err
	38	+	}
	39	+	}
	40	+	if rw.streaming {
	41	+	return rw.ResponseWriter.Write(data)
	42	+	}
20	43		rw.body = append(rw.body, data...)
21	44		return len(data), nil
22	45		}
23	46
	47	+	// Flush passes through once the response is streaming. Flushing a buffered
	48	+	// response would send its headers before it is known whether it is stored.
	49	+	func (rw *responseWriter) Flush() {
	50	+	if rw.streaming {
	51	+	_ = http.NewResponseController(rw.ResponseWriter).Flush()
	52	+	}
	53	+	}
	54	+
	55	+	func (rw *responseWriter) tooLarge(size int64) bool {
	56	+	return rw.maxBody > 0 && size > rw.maxBody
	57	+	}
	58	+
	59	+	func (rw *responseWriter) stream() error {
	60	+	if rw.streaming {
	61	+	return nil
	62	+	}
	63	+	rw.streaming = true
	64	+	rw.Header().Set("cache-status", rw.streamStatus)
	65	+	rw.ResponseWriter.WriteHeader(rw.StatusCode())
	66	+	body := rw.body
	67	+	rw.body = nil
	68	+	if len(body) == 0 {
	69	+	return nil
	70	+	}
	71	+	_, err := rw.ResponseWriter.Write(body)
	72	+	return err
	73	+	}
	74	+
24	75		// Body returns the captured response body.
25	76		func (rw *responseWriter) Body() []byte {
26	77		return rw.body

...	...		@@ -77,8 +128,33 @@ func (rw responseWriter) ToCacheValue(r http.Request) *CacheValue {
77	128
78	129		type CacheValue struct {
79	130		Header map[string][]string `json:"headers"`
80		-	Body []byte `json:"body"`
	131	+	Body []byte `json:"-"`
81	132		CreatedAt time.Time `json:"created_at"`
82	133		StatusCode int `json:"status_code"`
83	134		VaryRequestHeaders map[string]string `json:"vary_request_headers,omitempty"`
84	135		}
	136	+
	137	+	// encode stores the body after the JSON metadata so a cache hit decodes only
	138	+	// the metadata.
	139	+	func (cv *CacheValue) encode() []byte {
	140	+	meta, _ := json.Marshal(cv)
	141	+	out := make([]byte, 0, binary.MaxVarintLen64+len(meta)+len(cv.Body))
	142	+	out = binary.AppendUvarint(out, uint64(len(meta)))
	143	+	out = append(out, meta...)
	144	+	return append(out, cv.Body...)
	145	+	}
	146	+
	147	+	// decodeCacheValue reads an entry written by encode. The body shares memory
	148	+	// with data, which the cache never modifies.
	149	+	func decodeCacheValue(data []byte) (*CacheValue, error) {
	150	+	n, k := binary.Uvarint(data)
	151	+	if k <= 0 \|\| n > uint64(len(data)-k) {
	152	+	return nil, errors.New("corrupt cache entry")
	153	+	}
	154	+	cv := &CacheValue{}
	155	+	if err := json.Unmarshal(data[k:k+int(n)], cv); err != nil {
	156	+	return nil, err
	157	+	}
	158	+	cv.Body = data[k+int(n):]
	159	+	return cv, nil
	160	+	}

...	...		@@ -1,7 +1,6 @@
1	1		package httpcache
2	2
3	3		import (
4		-	"encoding/json"
5	4		"errors"
6	5		"fmt"
7	6		"log/slog"

...	...		@@ -52,6 +51,9 @@ type HttpCache struct {
52	51		Upstream http.Handler
53	52		Cache Cacher
54	53		Logger *slog.Logger
	54	+	// MaxBodySize is the largest body that is stored. Larger responses
	55	+	// stream through uncached. Zero means no limit.
	56	+	MaxBodySize int64
55	57		}
56	58
57	59		func NewHttpCache(log slog.Logger, upstream http.Handler) HttpCache {

...	...		@@ -78,6 +80,10 @@ func (c HttpCache) ServeHTTP(w http.ResponseWriter, r http.Request) {
78	80		cacheKey := c.GetCacheKey(r)
79	81		log := c.Logger.With("cache_key", cacheKey)
80	82
	83	+	// A partial response must not be stored under the key of the whole
	84	+	// resource, so a range request that misses goes straight upstream.
	85	+	hasRange := r.Header.Get("range") != ""
	86	+
81	87		err := c.maybeUseCache(cacheKey, w, r)
82	88		if err == nil {
83	89		log.Info("cache hit")

...	...		@@ -85,6 +91,15 @@ func (c HttpCache) ServeHTTP(w http.ResponseWriter, r http.Request) {
85	91		return
86	92		}
87	93
	94	+	if hasRange {
	95	+	log.Info("cache miss for range request, passing upstream", "err", err)
	96	+	c.AddCacheMiss()
	97	+	w.Header().Set("cache-status", cacheStatusMiss(cacheKey, false))
	98	+	c.Upstream.ServeHTTP(w, r)
	99	+	c.AddUpstreamRequest()
	100	+	return
	101	+	}
	102	+
88	103		// RFC 9111 5.2.1.7 only-if-cached - don't store new responses
89	104		cacheContState := parseCacheControl(r.Header.Get("cache-control"))
90	105		onlyIfCached := cacheContState.onlyIfCache

...	...		@@ -105,8 +120,7 @@ func (c HttpCache) ServeHTTP(w http.ResponseWriter, r http.Request) {
105	120
106	121		if errors.Is(err, ErrMustRevalidate) {
107	122		if cachedData, exists := c.Cache.Get(cacheKey); exists {
108		-	var cachedValue CacheValue
109		-	if json.Unmarshal(cachedData, &cachedValue) == nil {
	123	+	if cachedValue, err := decodeCacheValue(cachedData); err == nil {
110	124		if etag := getHeader(cachedValue.Header, "etag"); etag != "" {
111	125		r.Header.Set("if-none-match", etag)
112	126		}

...	...		@@ -119,9 +133,17 @@ func (c HttpCache) ServeHTTP(w http.ResponseWriter, r http.Request) {
119	133
120	134		log.Info("cache miss, requesting upstream", "err", err)
121	135		c.AddCacheMiss()
122		-	wrapped := &responseWriter{ResponseWriter: w}
	136	+	wrapped := &responseWriter{
	137	+	ResponseWriter: w,
	138	+	maxBody: c.MaxBodySize,
	139	+	streamStatus: cacheStatusMiss(cacheKey, false),
	140	+	}
123	141		c.Upstream.ServeHTTP(wrapped, r)
124	142		c.AddUpstreamRequest()
	143	+	if wrapped.streaming {
	144	+	log.Info("not cachable", "err", "response body too large")
	145	+	return
	146	+	}
125	147
126	148		// RFC 9111 4.3.4 304 Not Modified
127	149		// https://www.rfc-editor.org/rfc/rfc9111.html#section-4.3.4

...	...		@@ -135,10 +157,9 @@ func (c HttpCache) ServeHTTP(w http.ResponseWriter, r http.Request) {
135	157		return
136	158		}
137	159
138		-	var cacheValue CacheValue
139		-	err = json.Unmarshal(existingData, &cacheValue)
	160	+	cacheValue, err := decodeCacheValue(existingData)
140	161		if err != nil {
141		-	log.Error("json unmarshal", "err", err)
	162	+	log.Error("decode cache entry", "err", err)
142	163		wrapped.Send()
143	164		return
144	165		}

...	...		@@ -161,7 +182,7 @@ func (c HttpCache) ServeHTTP(w http.ResponseWriter, r http.Request) {
161	182		}
162	183		// Revalidation refreshes the entry -- reset CreatedAt so it's fresh again.
163	184		cacheValue.CreatedAt = time.Now()
164		-	enc, _ := json.Marshal(cacheValue)
	185	+	enc := cacheValue.encode()
165	186		log.Info("updating cached headers from 304 response")
166	187		c.Cache.Remove(cacheKey)
167	188		c.Cache.Add(cacheKey, enc)

...	...		@@ -172,9 +193,9 @@ func (c HttpCache) ServeHTTP(w http.ResponseWriter, r http.Request) {
172	193		// updated cached entry and return 304 if it still matches.
173	194		r.Header.Set("if-none-match", clientIfNoneMatch)
174	195		r.Header.Set("if-modified-since", clientIfModifiedSince)
175		-	valid := c.handleValidation(r, &cacheValue)
	196	+	valid := c.handleValidation(r, cacheValue)
176	197		if valid {
177		-	hdr := stripForbiddenHeaders(w, &cacheValue)
	198	+	hdr := stripForbiddenHeaders(w, cacheValue)
178	199		ageDur := calcAge(cacheValue.CreatedAt)
179	200		hdr.Set("age", strconv.Itoa(int(ageDur.Seconds())+1))
180	201		hdr.Set("cache-status", cacheStatusStale(cacheKey, wrapped.StatusCode()))

...	...		@@ -187,15 +208,14 @@ func (c HttpCache) ServeHTTP(w http.ResponseWriter, r http.Request) {
187	208		// Client request was unconditional (or conditional but no longer matches)
188	209		// serve the full cached response.
189	210		log.Info("serving full cached response to client")
190		-	serveCache(w, c.Ttl, cacheKey, &cacheValue)
	211	+	serveCache(w, c.Ttl, cacheKey, cacheValue)
191	212		return
192	213		}
193	214
194	215		err = isResponseCachable(r, wrapped)
195	216		if err == nil {
196	217		log.Info("storing cache")
197		-	nextValue := wrapped.ToCacheValue(r)
198		-	enc, _ := json.Marshal(nextValue)
	218	+	enc := wrapped.ToCacheValue(r).encode()
199	219		c.Cache.Remove(cacheKey)
200	220		c.Cache.Add(cacheKey, enc)
201	221		c.AddCacheItem(float64(len(enc)))

...	...		@@ -409,10 +429,11 @@ func isResponseCachable(r http.Request, resp responseWriter) error {
409	429		// 206, 300, 301,
410	430		// 308, 404, 405,
411	431		// 410, 414, 501.
	432	+	// 206 is left out: the cache key does not include the range.
412	433		func isCacheableStatusCode(code int) bool {
413	434		switch code {
414	435		case http.StatusOK, http.StatusNonAuthoritativeInfo, http.StatusNoContent,
415		-	http.StatusPartialContent, http.StatusMultipleChoices, http.StatusMovedPermanently,
	436	+	http.StatusMultipleChoices, http.StatusMovedPermanently,
416	437		http.StatusPermanentRedirect, http.StatusNotFound, http.StatusMethodNotAllowed,
417	438		http.StatusGone, http.StatusRequestURITooLong, http.StatusNotImplemented:
418	439		return true

...	...		@@ -559,14 +580,13 @@ func (c HttpCache) maybeUseCache(cacheKey string, w http.ResponseWriter, r htt
559	580		return fmt.Errorf("no cache stored")
560	581		}
561	582
562		-	var cacheValue CacheValue
563		-	err := json.Unmarshal(data, &cacheValue)
	583	+	cacheValue, err := decodeCacheValue(data)
564	584		if err != nil {
565		-	return fmt.Errorf("json unmarshal: %w", err)
	585	+	return fmt.Errorf("decode cache entry: %w", err)
566	586		}
567	587
568	588		// RFC 9111 4.1 Vary - check if request matches cached Vary values
569		-	if !matchVary(r, &cacheValue) {
	589	+	if !matchVary(r, cacheValue) {
570	590		return fmt.Errorf("vary mismatch")
571	591		}
572	592

...	...		@@ -624,9 +644,9 @@ func (c HttpCache) maybeUseCache(cacheKey string, w http.ResponseWriter, r htt
624	644		// RFC 9111 4.3 Validation - check validation headers first
625	645		// RFC 9110 13 Conditional Requests
626	646		// https://www.rfc-editor.org/rfc/rfc9110.html#section-13
627		-	valid := c.handleValidation(r, &cacheValue)
	647	+	valid := c.handleValidation(r, cacheValue)
628	648		if valid {
629		-	hdr := stripForbiddenHeaders(w, &cacheValue)
	649	+	hdr := stripForbiddenHeaders(w, cacheValue)
630	650		ageDur := calcAge(cacheValue.CreatedAt)
631	651		hdr.Set("age", strconv.Itoa(int(ageDur.Seconds())+1))
632	652		hdr.Set("cache-status", cacheStatusHit(cacheKey, freshness.Seconds()))

...	...		@@ -658,7 +678,7 @@ func (c HttpCache) maybeUseCache(cacheKey string, w http.ResponseWriter, r htt
658	678		return fmt.Errorf("response older than request max-age")
659	679		}
660	680
661		-	serveCache(w, freshness, cacheKey, &cacheValue)
	681	+	serveCache(w, freshness, cacheKey, cacheValue)
662	682		return nil
663	683		}
664	684

...	...		@@ -349,10 +349,12 @@ func (s *SSHServer) ListenAndServe() error {
349	349		},
350	350		}, []string{"command"})
351	351
352		-	go func() {
353		-	mux := http.NewServeMux()
354		-	mux.Handle("/metrics", promhttp.Handler())
	352	+	// The handler reads the default registerer, so build it before the
	353	+	// listener is published rather than in the goroutine.
	354	+	mux := http.NewServeMux()
	355	+	mux.Handle("/metrics", promhttp.Handler())
355	356
	357	+	go func() {
356	358		srv := &http.Server{Addr: s.Config.PromListenAddr, Handler: mux}
357	359
358	360		go func() {

...	...		@@ -1,5 +0,0 @@
1		-	package nofollow
2		-
3		-	// Maybe resolves to unix.O_NOFOLLOW on unix systems,
4		-	// 0 on other platforms. TODO(go1.24): use os.Root.
5		-	const Maybe = 0

...	...		@@ -1,36 +0,0 @@
1		-	package rsync
2		-
3		-	// rsync.h.
4		-	const (
5		-	XMIT_TOP_DIR = (1 << 0)
6		-	XMIT_SAME_MODE = (1 << 1)
7		-	XMIT_EXTENDED_FLAGS = (1 << 2)
8		-	XMIT_SAME_RDEV_pre28 = XMIT_EXTENDED_FLAGS /* Only in protocols < 28 */
9		-	XMIT_SAME_UID = (1 << 3)
10		-	XMIT_SAME_GID = (1 << 4)
11		-	XMIT_SAME_NAME = (1 << 5)
12		-	XMIT_LONG_NAME = (1 << 6)
13		-	XMIT_SAME_TIME = (1 << 7)
14		-	XMIT_SAME_RDEV_MAJOR = (1 << 8)
15		-	XMIT_HAS_IDEV_DATA = (1 << 9)
16		-	XMIT_SAME_DEV = (1 << 10)
17		-	XMIT_RDEV_MINOR_IS_SMALL = (1 << 11)
18		-	)
19		-
20		-	// as per /usr/include/bits/stat.h.
21		-	const (
22		-	S_IFMT = 0o0170000 // bits determining the file type
23		-	S_IFDIR = 0o0040000 // Directory
24		-	S_IFCHR = 0o0020000 // Character device
25		-	S_IFBLK = 0o0060000 // Block device
26		-	S_IFREG = 0o0100000 // Regular file
27		-	S_IFIFO = 0o0010000 // FIFO
28		-	S_IFLNK = 0o0120000 // Symbolic link
29		-	S_IFSOCK = 0o0140000 // Socket
30		-	)
31		-
32		-	// ProtocolVersion defines the currently implemented rsync protocol
33		-	// version. Protocol version 27 seems to be the safest bet for wide
34		-	// compatibility: version 27 was introduced by rsync 2.6.0 (released 2004), and
35		-	// is supported by openrsync and rsyn.
36		-	const ProtocolVersion = 27

...	...		@@ -1,6 +0,0 @@
1		-	// Package rsync contains a native Go rsync implementation.
2		-	//
3		-	// The only component currently is gokr-rsyncd, a read-only rsync daemon
4		-	// sender-only Go implementation of rsyncd. rsync daemon is a custom
5		-	// (un-standardized) network protocol, running on port 873 by default.
6		-	package rsync

...	...		@@ -1,7 +0,0 @@
1		-	package rsync
2		-
3		-	import "log/slog"
4		-
5		-	// Logger is an interface that allows specifying your own logger.
6		-	// By default, the Go log package is used, which prints to stderr.
7		-	type Logger = slog.Logger

...	...		@@ -1,86 +0,0 @@
1		-	package rsync
2		-
3		-	import (
4		-	"fmt"
5		-
6		-	"github.com/picosh/pico/pkg/rsync-receiver/rsyncwire"
7		-	)
8		-
9		-	// rsync/rsync.h:struct sum_buf.
10		-	type SumBuf struct {
11		-	Offset int64
12		-	Len int64
13		-	Index int32
14		-	Sum1 uint32
15		-	Sum2 [16]byte
16		-	}
17		-
18		-	// TODO: remove connection.go:sumHead in favor of this type.
19		-	type SumHead struct {
20		-	// “number of blocks” (openrsync)
21		-	// “how many chunks” (rsync)
22		-	ChecksumCount int32
23		-
24		-	// “block length in the file” (openrsync)
25		-	// maximum (1 << 29) for older rsync, (1 << 17) for newer
26		-	BlockLength int32
27		-
28		-	// “long checksum length” (openrsync)
29		-	ChecksumLength int32
30		-
31		-	// “terminal (remainder) block length” (openrsync)
32		-	// RemainderLength is flength % BlockLength
33		-	RemainderLength int32
34		-
35		-	Sums []SumBuf
36		-	}
37		-
38		-	func (sh SumHead) ReadFrom(c rsyncwire.Conn) error {
39		-	// TODO(protocol>=30): update maxBlockLen
40		-	const maxBlockLen = 1 << 29 // see rsync.h:OLD_MAX_BLOCK_SIZE
41		-
42		-	var err error
43		-	sh.ChecksumCount, err = c.ReadInt32()
44		-	if err != nil {
45		-	return err
46		-	}
47		-	if sh.ChecksumCount < 0 {
48		-	return fmt.Errorf("invalid checksum count %d", sh.ChecksumCount)
49		-	}
50		-
51		-	sh.BlockLength, err = c.ReadInt32()
52		-	if err != nil {
53		-	return err
54		-	}
55		-	if sh.BlockLength < 0 \|\| sh.BlockLength > maxBlockLen {
56		-	return fmt.Errorf("invalid block length %d", sh.BlockLength)
57		-	}
58		-
59		-	sh.ChecksumLength, err = c.ReadInt32()
60		-	if err != nil {
61		-	return err
62		-	}
63		-	// TODO(protocol>=27): update max sh.ChecksumLength check
64		-	if sh.ChecksumLength < 0 \|\| sh.ChecksumLength > 16 {
65		-	return fmt.Errorf("invalid checksum length %d", sh.ChecksumLength)
66		-	}
67		-
68		-	sh.RemainderLength, err = c.ReadInt32()
69		-	if err != nil {
70		-	return err
71		-	}
72		-	if sh.RemainderLength < 0 \|\| sh.RemainderLength > sh.BlockLength {
73		-	return fmt.Errorf("invalid remainder length %d", sh.RemainderLength)
74		-	}
75		-
76		-	return nil
77		-	}
78		-
79		-	func (sh SumHead) WriteTo(c rsyncwire.Conn) error {
80		-	var buf rsyncwire.Buffer
81		-	buf.WriteInt32(sh.ChecksumCount)
82		-	buf.WriteInt32(sh.BlockLength)
83		-	buf.WriteInt32(sh.ChecksumLength)
84		-	buf.WriteInt32(sh.RemainderLength)
85		-	return c.WriteString(buf.String())
86		-	}

...	...		@@ -1,73 +0,0 @@
1		-	package rsyncchecksum_test
2		-
3		-	import (
4		-	"bytes"
5		-	"os"
6		-	"path/filepath"
7		-	"testing"
8		-
9		-	"github.com/picosh/pico/pkg/rsync-receiver/rsyncchecksum"
10		-	)
11		-
12		-	func constructLargeDataFile(headPattern, bodyPattern, endPattern []byte) []byte {
13		-	// create large data file in source directory to be copied
14		-	head := bytes.Repeat(headPattern, 110241024)
15		-	body := bytes.Repeat(bodyPattern, 110241024)
16		-	end := bytes.Repeat(endPattern, 110241024)
17		-	return append(append(head, body...), end...)
18		-	}
19		-
20		-	func writeLargeDataFile(t *testing.T, source string, headPattern, bodyPattern, endPattern []byte) {
21		-	// create large data file in source directory to be copied
22		-	content := constructLargeDataFile(headPattern, bodyPattern, endPattern)
23		-	large := filepath.Join(source, "large-data-file")
24		-	if err := os.MkdirAll(filepath.Dir(large), 0755); err != nil {
25		-	t.Fatal(err)
26		-	}
27		-	if err := os.WriteFile(large, content, 0644); err != nil {
28		-	t.Fatal(err)
29		-	}
30		-	}
31		-
32		-	func TestSyncExtended(t *testing.T) {
33		-	tmp := t.TempDir()
34		-	source := filepath.Join(tmp, "source")
35		-
36		-	writeLargeDataFile(t, source, []byte{0x11}, []byte{0xbb}, []byte{0xee})
37		-
38		-	// These values are taken from the rsync debug output:
39		-	const k = 1768
40		-	want := make([]uint32, 1780)
41		-	for i := 0; i <= 592; i++ {
42		-	want[i] = 0xa5d47568
43		-	}
44		-	want[593] = 0x23645688
45		-	for i := 594; i <= 1185; i++ {
46		-	want[i] = 0x8c1c2378
47		-	}
48		-	want[1186] = 0x12504720
49		-	for i := 1187; i <= 1778; i++ {
50		-	want[i] = 0x7d9883b0
51		-	}
52		-	want[1779] = 0x61b8dff0
53		-
54		-	sourceLarge := filepath.Join(source, "large-data-file")
55		-	f, err := os.Open(sourceLarge)
56		-	if err != nil {
57		-	t.Fatal(err)
58		-	}
59		-	defer func() { _ = f.Close() }()
60		-	buf := make([]byte, k)
61		-	for idx, wantChecksum := range want {
62		-	n, err := f.Read(buf)
63		-	if err != nil {
64		-	t.Fatal(err)
65		-	}
66		-
67		-	chunk := buf[:n]
68		-	sum := rsyncchecksum.Checksum1(chunk)
69		-	if sum != wantChecksum {
70		-	t.Fatalf("checksum calculation error: got %08x, want %08x (idx %d), chunk: %#v", sum, wantChecksum, idx, chunk)
71		-	}
72		-	}
73		-	}

...	...		@@ -1,73 +0,0 @@
1		-	package rsyncchecksum
2		-
3		-	import (
4		-	"encoding/binary"
5		-	"io"
6		-	"os"
7		-
8		-	"github.com/mmcloughlin/md4"
9		-	)
10		-
11		-	func Tag2(s1, s2 uint16) uint16 {
12		-	return (((s1) + (s2)) & 0xFFFF)
13		-	}
14		-
15		-	func Tag(sum uint32) uint16 {
16		-	return Tag2(uint16(sum&0xFFFF), uint16(sum>>16))
17		-	}
18		-
19		-	// SignExtend mirrors how C converts from (signed char) to uint32, i.e. using
20		-	// sign extension. get_checksum1 treats the buffer as (signed char*) instead of
21		-	// (unsigned char*), which likely was not a conscious choice, but here we are.
22		-	//
23		-	// This function is exported for use in the rolling checksum in match.go.
24		-	func SignExtend(b byte) uint32 {
25		-	val := uint32(b)
26		-	return uint32(int32(val<<24) >> 24)
27		-	}
28		-
29		-	func Checksum1(buf []byte) uint32 {
30		-	bufLen := len(buf)
31		-	var s1, s2 uint32
32		-	var i int
33		-
34		-	if bufLen > 4 {
35		-	for i = 0; i < (bufLen - 4); i += 4 {
36		-	s2 += 4*(s1+SignExtend(buf[i])) +
37		-	3*SignExtend(buf[i+1]) +
38		-	2*SignExtend(buf[i+2]) +
39		-	SignExtend(buf[i+3])
40		-	s1 += SignExtend(buf[i+0]) +
41		-	SignExtend(buf[i+1]) +
42		-	SignExtend(buf[i+2]) +
43		-	SignExtend(buf[i+3])
44		-	}
45		-	}
46		-	for ; i < bufLen; i++ {
47		-	s1 += SignExtend(buf[i])
48		-	s2 += s1
49		-	}
50		-	return (s1 & 0xffff) + (s2 << 16)
51		-	}
52		-
53		-	func Checksum2(seed int32, buf []byte) []byte {
54		-	h := md4.New()
55		-	h.Write(buf)
56		-	_ = binary.Write(h, binary.LittleEndian, seed) // hash.Hash.Write never fails
57		-	return h.Sum(nil)
58		-	}
59		-
60		-	func FileChecksum(fn string) ([]byte, error) {
61		-	f, err := os.Open(fn)
62		-	if err != nil {
63		-	return nil, err
64		-	}
65		-	defer func() { _ = f.Close() }()
66		-	h := md4.New()
67		-	if _, err := io.Copy(h, f); err != nil {
68		-	return nil, err
69		-	}
70		-	return h.Sum(nil), nil
71		-	}
72		-
73		-	const Size = md4.Size

...	...		@@ -1,37 +0,0 @@
1		-	// Package rsynccommon contains functionality that both the sender and the
2		-	// receiver implementation need.
3		-	package rsynccommon
4		-
5		-	import (
6		-	"math"
7		-
8		-	"github.com/picosh/pico/pkg/rsync-receiver/rsync"
9		-	)
10		-
11		-	const blockSize = 700 // rsync/rsync.h
12		-
13		-	// Corresponds to rsync/generator.c:sum_sizes_sqroot.
14		-	func SumSizesSqroot(contentLen int64) rsync.SumHead {
15		-	// * The block size is a rounded square root of file length.
16		-
17		-	// The block size algorithm plays a crucial role in the protocol efficiency. In general, the block size is the rounded square root of the total file size. The minimum block size, however, is 700 B. Otherwise, the square root computation is simply sqrt(3) followed by ceil(3)
18		-
19		-	// For reasons unknown, the square root result is rounded up to the nearest multiple of eight.
20		-
21		-	// TODO: round this
22		-	blockLength := max(int32(math.Sqrt(float64(contentLen))), blockSize)
23		-
24		-	// * The checksum size is determined according to:
25		-	// * blocksum_bits = BLOCKSUM_EXP + 2*log2(file_len) - log2(block_len)
26		-	// * provided by Donovan Baarda which gives a probability of rsync
27		-	// * algorithm corrupting data and falling back using the whole md4
28		-	// * checksums.
29		-	const checksumLength = 16 // TODO?
30		-
31		-	return rsync.SumHead{
32		-	ChecksumCount: int32((contentLen + (int64(blockLength) - 1)) / int64(blockLength)),
33		-	RemainderLength: int32(contentLen % int64(blockLength)),
34		-	BlockLength: blockLength,
35		-	ChecksumLength: checksumLength,
36		-	}
37		-	}

...	...		@@ -1,109 +0,0 @@
1		-	package rsyncreceiver
2		-
3		-	import (
4		-	"fmt"
5		-	"io"
6		-	"log/slog"
7		-	"os"
8		-
9		-	"github.com/picosh/pico/pkg/rsync-receiver/rsync"
10		-	"github.com/picosh/pico/pkg/rsync-receiver/rsyncopts"
11		-	"github.com/picosh/pico/pkg/rsync-receiver/rsyncsender"
12		-	"github.com/picosh/pico/pkg/rsync-receiver/rsyncwire"
13		-	"github.com/picosh/pico/pkg/rsync-receiver/utils"
14		-	)
15		-
16		-	func ClientRun(logger slog.Logger, opts rsyncopts.Options, conn io.ReadWriter, filesystem utils.FS, paths []string, negotiate bool) error {
17		-	var err error
18		-
19		-	crd, cwr := rsyncwire.CounterPair(conn, conn)
20		-
21		-	const sessionChecksumSeed = 666
22		-
23		-	c := &rsyncwire.Conn{
24		-	Reader: crd,
25		-	Writer: cwr,
26		-	}
27		-
28		-	if negotiate {
29		-	remoteProtocol, err := c.ReadInt32()
30		-	if err != nil {
31		-	return err
32		-	}
33		-	logger.Debug("remote protocol", "protocol", remoteProtocol)
34		-	if err := c.WriteInt32(rsync.ProtocolVersion); err != nil {
35		-	return err
36		-	}
37		-	}
38		-
39		-	if err := c.WriteInt32(sessionChecksumSeed); err != nil {
40		-	return err
41		-	}
42		-
43		-	// Switch to multiplexing protocol, but only for server-side transmissions.
44		-	// Transmissions received from the client are not multiplexed.
45		-	mpx := &rsyncwire.MultiplexWriter{Writer: c.Writer}
46		-	c.Writer = mpx
47		-
48		-	defer func() {
49		-	if err != nil {
50		-	_, _ = mpx.WriteMsg(rsyncwire.MsgError, fmt.Appendf(nil, "gokr-rsync [receiver]: %v\n", err))
51		-	}
52		-	}()
53		-
54		-	rt := &Transfer{
55		-	Opts: &TransferOpts{
56		-	DryRun: opts.DryRun(),
57		-
58		-	DeleteMode: opts.DeleteMode(),
59		-	PreserveGid: opts.PreserveGid(),
60		-	PreserveUid: opts.PreserveUid(),
61		-	PreserveLinks: opts.PreserveLinks(),
62		-	PreservePerms: opts.PreservePerms(),
63		-	PreserveDevices: opts.PreserveDevices(),
64		-	PreserveSpecials: opts.PreserveSpecials(),
65		-	PreserveTimes: opts.PreserveMTimes(),
66		-	IgnoreTimes: opts.IgnoreTimes(),
67		-	SizeOnly: opts.SizeOnly(),
68		-	AlwaysChecksum: opts.AlwaysChecksum(),
69		-	// TODO: PreserveHardlinks: opts.PreserveHardlinks,
70		-	},
71		-	Dest: "/",
72		-	// TODO: what is Env used for and can we get rid of it?
73		-	Env: Osenv{
74		-	Stdout: os.Stdout,
75		-	Stderr: os.Stderr,
76		-	Stdin: os.Stdin,
77		-	},
78		-	Conn: c,
79		-	Seed: sessionChecksumSeed,
80		-
81		-	Files: filesystem,
82		-
83		-	Logger: logger,
84		-	}
85		-
86		-	if opts.DeleteMode() {
87		-	// receive the exclusion list (openrsync’s is always empty)
88		-	exclusionList, err := rsyncsender.RecvFilterList(c)
89		-	if err != nil {
90		-	return err
91		-	}
92		-	logger.Debug("exclusion list read", "filters", exclusionList.Filters)
93		-	}
94		-
95		-	// receive file list
96		-	logger.Debug("receiving file list")
97		-	fileList, err := rt.ReceiveFileList()
98		-	if err != nil {
99		-	return err
100		-	}
101		-	logger.Debug("received names", "files", fileList)
102		-	stats, err := rt.Do(c, fileList, true)
103		-	if err != nil {
104		-	return err
105		-	}
106		-
107		-	logger.Debug("stats", "stats", stats)
108		-	return nil
109		-	}

...	...		@@ -1,94 +0,0 @@
1		-	package rsyncreceiver
2		-
3		-	import (
4		-	"context"
5		-
6		-	"github.com/picosh/pico/pkg/rsync-receiver/rsyncstats"
7		-	"github.com/picosh/pico/pkg/rsync-receiver/rsyncwire"
8		-	"github.com/picosh/pico/pkg/rsync-receiver/utils"
9		-	"golang.org/x/sync/errgroup"
10		-	)
11		-
12		-	func (rt Transfer) deleteFiles(fileList []utils.ReceiverFile) error {
13		-	if rt.IOErrors > 0 {
14		-	rt.Logger.Debug("IO error encountered, skipping file deletion")
15		-	return nil
16		-	}
17		-
18		-	return rt.Files.Remove(fileList)
19		-	}
20		-
21		-	// rsync/main.c:do_recv.
22		-	func (rt Transfer) Do(c rsyncwire.Conn, fileList []utils.ReceiverFile, noReport bool) (rsyncstats.TransferStats, error) {
23		-	if rt.Opts.DeleteMode {
24		-	if err := rt.deleteFiles(fileList); err != nil {
25		-	return nil, err
26		-	}
27		-	}
28		-
29		-	ctx := context.Background()
30		-	eg, ctx := errgroup.WithContext(ctx)
31		-	eg.Go(func() error {
32		-	return rt.GenerateFiles(fileList)
33		-	})
34		-	eg.Go(func() error {
35		-	// Ensure we don’t block on the receiver when the generator returns an
36		-	// error.
37		-	errChan := make(chan error)
38		-	go func() {
39		-	errChan <- rt.RecvFiles(fileList)
40		-	}()
41		-	select {
42		-	case <-ctx.Done():
43		-	return ctx.Err()
44		-	case err := <-errChan:
45		-	return err
46		-	}
47		-	})
48		-	if err := eg.Wait(); err != nil {
49		-	return nil, err
50		-	}
51		-
52		-	var stats *rsyncstats.TransferStats
53		-	if !noReport {
54		-	var err error
55		-	stats, err = report(c)
56		-	rt.Logger.Debug("report", "stats", stats)
57		-	if err != nil {
58		-	return nil, err
59		-	}
60		-	}
61		-
62		-	// send final goodbye message
63		-	if err := c.WriteInt32(-1); err != nil {
64		-	return nil, err
65		-	}
66		-
67		-	return stats, nil
68		-	}
69		-
70		-	// rsync/main.c:report.
71		-	func report(c rsyncwire.Conn) (rsyncstats.TransferStats, error) {
72		-	// read statistics:
73		-	// total bytes read (from network connection)
74		-	read, err := c.ReadInt64()
75		-	if err != nil {
76		-	return nil, err
77		-	}
78		-	// total bytes written (to network connection)
79		-	written, err := c.ReadInt64()
80		-	if err != nil {
81		-	return nil, err
82		-	}
83		-	// total size of files
84		-	size, err := c.ReadInt64()
85		-	if err != nil {
86		-	return nil, err
87		-	}
88		-
89		-	return &rsyncstats.TransferStats{
90		-	Read: read,
91		-	Written: written,
92		-	Size: size,
93		-	}, nil
94		-	}

...	...		@@ -1,191 +0,0 @@
1		-	package rsyncreceiver
2		-
3		-	import (
4		-	"fmt"
5		-	"io"
6		-	"path/filepath"
7		-	"time"
8		-
9		-	"github.com/picosh/pico/pkg/rsync-receiver/rsync"
10		-	"github.com/picosh/pico/pkg/rsync-receiver/utils"
11		-	)
12		-
13		-	// rsync/flist.c:receive_file_entry.
14		-	func (rt Transfer) receiveFileEntry(flags uint16, last utils.ReceiverFile) (*utils.ReceiverFile, error) {
15		-	f := &utils.ReceiverFile{}
16		-
17		-	var l1 int
18		-	if flags&rsync.XMIT_SAME_NAME != 0 {
19		-	l, err := rt.Conn.ReadByte()
20		-	if err != nil {
21		-	return nil, err
22		-	}
23		-	l1 = int(l)
24		-	}
25		-
26		-	var l2 int
27		-	if flags&rsync.XMIT_LONG_NAME != 0 {
28		-	l, err := rt.Conn.ReadInt32()
29		-	if err != nil {
30		-	return nil, err
31		-	}
32		-	l2 = int(l)
33		-	} else {
34		-	l, err := rt.Conn.ReadByte()
35		-	if err != nil {
36		-	return nil, err
37		-	}
38		-	l2 = int(l)
39		-	}
40		-	// linux/limits.h
41		-	const PATH_MAX = 4096
42		-	if l2 >= PATH_MAX-l1 {
43		-	const lastname = ""
44		-	return nil, fmt.Errorf("overflow: flags=0x%x l1=%d l2=%d lastname=%s",
45		-	flags, l1, l2, lastname)
46		-	}
47		-	b := make([]byte, l1+l2)
48		-	readb := b
49		-	if l1 > 0 {
50		-	copy(b, []byte(last.Name))
51		-	readb = b[l1:]
52		-	}
53		-	if _, err := io.ReadFull(rt.Conn.Reader, readb); err != nil {
54		-	return nil, err
55		-	}
56		-	// TODO: does rsync’s clean_fname() and sanitize_path() combination do
57		-	// anything more than Go’s filepath.Clean()?
58		-	f.Name = filepath.Clean(string(b))
59		-
60		-	length, err := rt.Conn.ReadInt64()
61		-	if err != nil {
62		-	return nil, err
63		-	}
64		-	f.Length = length
65		-
66		-	if flags&rsync.XMIT_SAME_TIME != 0 {
67		-	f.ModTime = last.ModTime
68		-	} else {
69		-	modTime, err := rt.Conn.ReadInt32()
70		-	if err != nil {
71		-	return nil, err
72		-	}
73		-	f.ModTime = time.Unix(int64(modTime), 0)
74		-	}
75		-
76		-	if flags&rsync.XMIT_SAME_MODE != 0 {
77		-	f.Mode = last.Mode
78		-	} else {
79		-	mode, err := rt.Conn.ReadInt32()
80		-	if err != nil {
81		-	return nil, err
82		-	}
83		-	f.Mode = mode
84		-	}
85		-
86		-	if rt.Opts.PreserveUid {
87		-	if flags&rsync.XMIT_SAME_UID != 0 {
88		-	f.Uid = last.Uid
89		-	} else {
90		-	uid, err := rt.Conn.ReadInt32()
91		-	if err != nil {
92		-	return nil, err
93		-	}
94		-	f.Uid = uid
95		-	}
96		-	}
97		-
98		-	if rt.Opts.PreserveGid {
99		-	if flags&rsync.XMIT_SAME_GID != 0 {
100		-	f.Gid = last.Gid
101		-	} else {
102		-	gid, err := rt.Conn.ReadInt32()
103		-	if err != nil {
104		-	return nil, err
105		-	}
106		-	f.Gid = gid
107		-	}
108		-	}
109		-
110		-	mode := f.Mode & rsync.S_IFMT
111		-	isDev := mode == rsync.S_IFCHR \|\| mode == rsync.S_IFBLK
112		-	isSpecial := mode == rsync.S_IFIFO \|\| mode == rsync.S_IFSOCK
113		-	isLink := mode == rsync.S_IFLNK
114		-
115		-	if rt.Opts.PreserveDevices && (isDev \|\| isSpecial) {
116		-	// TODO(protocol >= 28): rdev/major/minor handling
117		-	if flags&rsync.XMIT_SAME_RDEV_pre28 != 0 {
118		-	f.Rdev = last.Rdev
119		-	} else {
120		-	rdev, err := rt.Conn.ReadInt32()
121		-	if err != nil {
122		-	return nil, err
123		-	}
124		-	f.Rdev = rdev
125		-	}
126		-	}
127		-
128		-	if rt.Opts.PreserveLinks && isLink {
129		-	length, err := rt.Conn.ReadInt32()
130		-	if err != nil {
131		-	return nil, err
132		-	}
133		-	b := make([]byte, length)
134		-	if _, err := io.ReadFull(rt.Conn.Reader, b); err != nil {
135		-	return nil, err
136		-	}
137		-	f.LinkTarget = string(b)
138		-	}
139		-
140		-	return f, nil
141		-	}
142		-
143		-	// rsync/flist.c:recv_file_list.
144		-	func (rt Transfer) ReceiveFileList() ([]utils.ReceiverFile, error) {
145		-	lastFileEntry := new(utils.ReceiverFile)
146		-	var fileList []*utils.ReceiverFile
147		-	for {
148		-	b, err := rt.Conn.ReadByte()
149		-	if err != nil {
150		-	return nil, err
151		-	}
152		-	if b == 0 {
153		-	break
154		-	}
155		-	flags := uint16(b)
156		-	// log.Printf("flags: %x", flags)
157		-	// TODO(protocol >= 28): extended flags
158		-
159		-	f, err := rt.receiveFileEntry(flags, lastFileEntry)
160		-	if err != nil {
161		-	return nil, err
162		-	}
163		-	lastFileEntry = f
164		-	// TODO: include depth in output?
165		-	rt.Logger.Debug("recv_file_list", "file", f.Name, "length", f.Length, "mode", f.Mode, "uid", f.Uid, "gid", f.Gid, "flags", flags)
166		-
167		-	fileList = append(fileList, f)
168		-	}
169		-
170		-	utils.SortFileList(fileList)
171		-
172		-	if rt.Opts.PreserveUid \|\| rt.Opts.PreserveGid {
173		-	// receive the uid/gid list
174		-	users, groups, err := rt.RecvIdList()
175		-	if err != nil {
176		-	return nil, err
177		-	}
178		-	_ = users
179		-	_ = groups
180		-	}
181		-
182		-	// read the i/o error flag
183		-	ioErrors, err := rt.Conn.ReadInt32()
184		-	if err != nil {
185		-	return nil, err
186		-	}
187		-	rt.Logger.Debug("ioErrors", "errs", ioErrors)
188		-	rt.IOErrors = ioErrors
189		-
190		-	return fileList, nil
191		-	}

...	...		@@ -1,156 +0,0 @@
1		-	package rsyncreceiver
2		-
3		-	import (
4		-	"fmt"
5		-	"io"
6		-	"os"
7		-
8		-	"github.com/picosh/pico/pkg/rsync-receiver/rsync"
9		-	"github.com/picosh/pico/pkg/rsync-receiver/rsyncchecksum"
10		-	"github.com/picosh/pico/pkg/rsync-receiver/rsynccommon"
11		-	"github.com/picosh/pico/pkg/rsync-receiver/utils"
12		-	)
13		-
14		-	// rsync/generator.c:generate_files().
15		-	func (rt Transfer) GenerateFiles(fileList []utils.ReceiverFile) error {
16		-	phase := 0
17		-	for idx, f := range fileList {
18		-	// TODO: use a copy of f with .Mode \|= S_IWUSR for directories, so
19		-	// that we can create files within all directories.
20		-	if err := rt.recvGenerator(idx, f); err != nil {
21		-	return err
22		-	}
23		-	}
24		-	phase++
25		-	rt.Logger.Debug("generateFiles", "phase", phase)
26		-	if err := rt.Conn.WriteInt32(-1); err != nil {
27		-	return err
28		-	}
29		-
30		-	// TODO: re-do any files that failed
31		-	phase++
32		-	rt.Logger.Debug("generateFiles", "phase", phase)
33		-	if err := rt.Conn.WriteInt32(-1); err != nil {
34		-	return err
35		-	}
36		-
37		-	rt.Logger.Debug("generateFiles finished")
38		-	return nil
39		-	}
40		-
41		-	// rsync/generator.c:skip_file.
42		-	func (rt Transfer) skipFile(f utils.ReceiverFile, st os.FileInfo) (bool, error) {
43		-	if rt.Opts.AlwaysChecksum \|\| rt.Opts.IgnoreTimes {
44		-	return false, nil
45		-	}
46		-
47		-	sizeMatch := st.Size() == f.Length
48		-	if rt.Opts.SizeOnly {
49		-	return sizeMatch, nil
50		-	}
51		-
52		-	timeMatch := st.ModTime().Equal(f.ModTime)
53		-	return sizeMatch && timeMatch, nil
54		-	}
55		-
56		-	// rsync/generator.c:recv_generator.
57		-	func (rt Transfer) recvGenerator(idx int, f utils.ReceiverFile) error {
58		-	if rt.listOnly() {
59		-	if _, err := fmt.Fprintf(rt.Env.Stdout, "%s %11.0f %s %s\n",
60		-	f.FileMode().String(),
61		-	float64(f.Length), // TODO: rsync prints decimal separators
62		-	f.ModTime.Format("2006/01/02 15:04:05"),
63		-	f.Name); err != nil {
64		-	return err
65		-	}
66		-	return nil
67		-	}
68		-	rt.Logger.Debug("recv_generator", "file", f)
69		-
70		-	if !f.FileMode().IsRegular() {
71		-	// None of the Preserve* options is enabled, so just skip over
72		-	// non-regular files.
73		-	return nil
74		-	}
75		-
76		-	requestFullFile := func() error {
77		-	rt.Logger.Debug("requesting", "file", f)
78		-	if err := rt.Conn.WriteInt32(int32(idx)); err != nil {
79		-	return err
80		-	}
81		-	if rt.Opts.DryRun {
82		-	return nil
83		-	}
84		-	var sh rsync.SumHead
85		-	if err := sh.WriteTo(rt.Conn); err != nil {
86		-	return err
87		-	}
88		-	return nil
89		-	}
90		-
91		-	st, in, err := rt.Files.Read(&utils.SenderFile{WPath: f.Name})
92		-	if err != nil {
93		-	rt.Logger.Error("failed to open file", "st", st, "file", f, "err", err)
94		-	return requestFullFile()
95		-	}
96		-
97		-	defer func() { _ = in.Close() }()
98		-
99		-	skip, err := rt.skipFile(f, st)
100		-	if err != nil {
101		-	return err
102		-	}
103		-
104		-	if skip {
105		-	rt.Logger.Debug("skipping", "file", f)
106		-	return nil
107		-	}
108		-
109		-	if rt.Opts.DryRun {
110		-	if err := rt.Conn.WriteInt32(int32(idx)); err != nil {
111		-	return err
112		-	}
113		-
114		-	return nil
115		-	}
116		-
117		-	rt.Logger.Debug("sending sums", "file", f, "st", st)
118		-	if err := rt.Conn.WriteInt32(int32(idx)); err != nil {
119		-	return err
120		-	}
121		-
122		-	err = rt.generateAndSendSums(in, st.Size())
123		-	if err != nil {
124		-	rt.Logger.Error("failed to send sums", "file", f, "err", err)
125		-	}
126		-
127		-	return err
128		-	}
129		-
130		-	// rsync/generator.c:generate_and_send_sums.
131		-	func (rt *Transfer) generateAndSendSums(in utils.ReaderAtCloser, fileLen int64) error {
132		-	sh := rsynccommon.SumSizesSqroot(fileLen)
133		-	if err := sh.WriteTo(rt.Conn); err != nil {
134		-	return err
135		-	}
136		-	buf := make([]byte, int(sh.BlockLength))
137		-	remaining := fileLen
138		-	for i := int32(0); i < sh.ChecksumCount; i++ {
139		-	n1 := min(int64(sh.BlockLength), remaining)
140		-	b := buf[:n1]
141		-	if _, err := io.ReadFull(in, b); err != nil {
142		-	return err
143		-	}
144		-
145		-	sum1 := rsyncchecksum.Checksum1(b)
146		-	sum2 := rsyncchecksum.Checksum2(rt.Seed, b)
147		-	if err := rt.Conn.WriteInt32(int32(sum1)); err != nil {
148		-	return err
149		-	}
150		-	if _, err := rt.Conn.Writer.Write(sum2); err != nil {
151		-	return err
152		-	}
153		-	remaining -= n1
154		-	}
155		-	return nil
156		-	}

...	...		@@ -1,170 +0,0 @@
1		-	package rsyncreceiver
2		-
3		-	import (
4		-	"bytes"
5		-	"encoding/binary"
6		-	"errors"
7		-	"fmt"
8		-	"io"
9		-	"sync"
10		-
11		-	"github.com/mmcloughlin/md4"
12		-	"github.com/picosh/pico/pkg/rsync-receiver/rsync"
13		-	"github.com/picosh/pico/pkg/rsync-receiver/utils"
14		-	)
15		-
16		-	// rsync/receiver.c:recv_files.
17		-	func (rt Transfer) RecvFiles(fileList []utils.ReceiverFile) error {
18		-	phase := 0
19		-	for {
20		-	idx, err := rt.Conn.ReadInt32()
21		-	if err != nil {
22		-	return err
23		-	}
24		-	if idx == -1 {
25		-	if phase == 0 {
26		-	phase++
27		-	rt.Logger.Debug("recvFiles phase", "phase", phase)
28		-	// TODO: send done message
29		-	continue
30		-	}
31		-	break
32		-	}
33		-	rt.Logger.Debug("receiving file", "idx", idx, "file", fileList[idx])
34		-	if err := rt.recvFile1(fileList[idx]); err != nil {
35		-	return err
36		-	}
37		-	}
38		-	rt.Logger.Debug("recvFiles finished")
39		-	return nil
40		-	}
41		-
42		-	func (rt Transfer) recvFile1(f utils.ReceiverFile) error {
43		-	if rt.Opts.DryRun {
44		-	fmt.Println(f.Name)
45		-	return nil
46		-	}
47		-
48		-	localFile, err := rt.openLocalFile(f)
49		-	if err != nil {
50		-	rt.Logger.Error("opening local file failed, continuing", "err", err, "file", f)
51		-	} else {
52		-	defer func() { _ = localFile.Close() }()
53		-	}
54		-
55		-	err = rt.receiveData(f, localFile)
56		-	if err != nil {
57		-	rt.Logger.Error("receiving data failed, continuing", "err", err, "file", f)
58		-	}
59		-	return err
60		-	}
61		-
62		-	func (rt Transfer) openLocalFile(f utils.ReceiverFile) (utils.ReaderAtCloser, error) {
63		-	_, r, err := rt.Files.Read(&utils.SenderFile{
64		-	WPath: f.Name,
65		-	Regular: true,
66		-	})
67		-
68		-	if err != nil {
69		-	return nil, err
70		-	}
71		-
72		-	return r, nil
73		-	}
74		-
75		-	// rsync/receiver.c:receive_data.
76		-	func (rt Transfer) receiveData(f utils.ReceiverFile, localFile utils.ReaderAtCloser) error {
77		-	var sh rsync.SumHead
78		-	if err := sh.ReadFrom(rt.Conn); err != nil {
79		-	return err
80		-	}
81		-
82		-	r, w := io.Pipe()
83		-
84		-	f.Reader = r
85		-
86		-	var wg sync.WaitGroup
87		-	wg.Add(1)
88		-
89		-	go func() {
90		-	defer func() {
91		-	wg.Done()
92		-	if err := r.Close(); err != nil {
93		-	return
94		-	}
95		-	}()
96		-
97		-	_, err := rt.Files.Put(f)
98		-	if err != nil {
99		-	return
100		-	}
101		-	}()
102		-
103		-	h := md4.New()
104		-	_ = binary.Write(h, binary.LittleEndian, rt.Seed) // hash.Hash.Write never fails
105		-
106		-	for {
107		-	token, data, err := rt.recvToken()
108		-	if err != nil {
109		-	return err
110		-	}
111		-	if token == 0 {
112		-	break
113		-	}
114		-	if token > 0 {
115		-	if _, err := h.Write(data); err != nil {
116		-	return err
117		-	}
118		-
119		-	if _, err := w.Write(data); err != nil {
120		-	if errors.Is(err, io.ErrClosedPipe) {
121		-	continue
122		-	}
123		-	return err
124		-	}
125		-	continue
126		-	}
127		-	if localFile == nil {
128		-	return fmt.Errorf("BUG: local file %s not open for copying chunk", localFile)
129		-	}
130		-	token = -(token + 1)
131		-	offset2 := int64(token) * int64(sh.BlockLength)
132		-	dataLen := sh.BlockLength
133		-	if token == sh.ChecksumCount-1 && sh.RemainderLength != 0 {
134		-	dataLen = sh.RemainderLength
135		-	}
136		-	data = make([]byte, dataLen)
137		-	if _, err := localFile.ReadAt(data, offset2); err != nil {
138		-	return err
139		-	}
140		-
141		-	if _, err := h.Write(data); err != nil {
142		-	return err
143		-	}
144		-
145		-	if _, err := w.Write(data); err != nil {
146		-	if errors.Is(err, io.ErrClosedPipe) {
147		-	continue
148		-	}
149		-	return err
150		-	}
151		-	}
152		-
153		-	if err := w.Close(); err != nil {
154		-	return err
155		-	}
156		-
157		-	wg.Wait()
158		-
159		-	localSum := h.Sum(nil)
160		-	remoteSum := make([]byte, len(localSum))
161		-	if _, err := io.ReadFull(rt.Conn.Reader, remoteSum); err != nil {
162		-	return err
163		-	}
164		-	if !bytes.Equal(localSum, remoteSum) {
165		-	return fmt.Errorf("file corruption in %s", f.Name)
166		-	}
167		-	rt.Logger.Debug("checksum matches!", "localSum", localSum)
168		-
169		-	return nil
170		-	}

...	...		@@ -1,20 +0,0 @@
1		-	package rsyncreceiver
2		-
3		-	import "io"
4		-
5		-	// rsync/token.c:recvToken.
6		-	func (rt *Transfer) recvToken() (token int32, data []byte, _ error) {
7		-	var err error
8		-	token, err = rt.Conn.ReadInt32()
9		-	if err != nil {
10		-	return 0, nil, err
11		-	}
12		-	if token <= 0 {
13		-	return token, nil, nil
14		-	}
15		-	data = make([]byte, int(token))
16		-	if _, err := io.ReadFull(rt.Conn.Reader, data); err != nil {
17		-	return 0, nil, err
18		-	}
19		-	return token, data, nil
20		-	}

...	...		@@ -1,53 +0,0 @@
1		-	package rsyncreceiver
2		-
3		-	import (
4		-	"io"
5		-	"log/slog"
6		-
7		-	"github.com/picosh/pico/pkg/rsync-receiver/rsyncwire"
8		-	"github.com/picosh/pico/pkg/rsync-receiver/utils"
9		-	)
10		-
11		-	type Osenv struct {
12		-	Stdin io.Reader
13		-	Stdout io.Writer
14		-	Stderr io.Writer
15		-	}
16		-
17		-	// TransferOpts is a subset of Opts which is required for implementing a receiver.
18		-	type TransferOpts struct {
19		-	Verbose bool
20		-	DryRun bool
21		-
22		-	DeleteMode bool
23		-	PreserveGid bool
24		-	PreserveUid bool
25		-	PreserveLinks bool
26		-	PreservePerms bool
27		-	PreserveDevices bool
28		-	PreserveSpecials bool
29		-	PreserveTimes bool
30		-	PreserveHardlinks bool
31		-	IgnoreTimes bool
32		-	SizeOnly bool
33		-	AlwaysChecksum bool
34		-	}
35		-
36		-	type Transfer struct {
37		-	// config
38		-	// Opts *Opts
39		-	Opts *TransferOpts
40		-	Dest string
41		-	Env Osenv
42		-
43		-	// state
44		-	Conn *rsyncwire.Conn
45		-	Seed int32
46		-	IOErrors int32
47		-
48		-	Files utils.FS
49		-
50		-	Logger *slog.Logger
51		-	}
52		-
53		-	func (rt *Transfer) listOnly() bool { return rt.Dest == "" }

...	...		@@ -1,69 +0,0 @@
1		-	package rsyncreceiver
2		-
3		-	import (
4		-	"io"
5		-	)
6		-
7		-	type mapping struct {
8		-	Name string
9		-	LocalId int32
10		-	}
11		-
12		-	func (rt *Transfer) recvIdMapping1(localId func(id int32, name string) int32) (map[int32]mapping, error) {
13		-	idMapping := make(map[int32]mapping)
14		-	for {
15		-	id, err := rt.Conn.ReadInt32()
16		-	if err != nil {
17		-	return nil, err
18		-	}
19		-	if id == 0 {
20		-	break
21		-	}
22		-	length, err := rt.Conn.ReadByte()
23		-	if err != nil {
24		-	return nil, err
25		-	}
26		-	name := make([]byte, length)
27		-	if _, err := io.ReadFull(rt.Conn.Reader, name); err != nil {
28		-	return nil, err
29		-	}
30		-	idMapping[id] = mapping{
31		-	Name: string(name),
32		-	LocalId: localId(id, string(name)),
33		-	}
34		-	}
35		-	return idMapping, nil
36		-	}
37		-
38		-	// rsync/uidlist.c:recv_id_list.
39		-	func (rt *Transfer) RecvIdList() (users map[int32]mapping, groups map[int32]mapping, _ error) {
40		-	if rt.Opts.PreserveUid {
41		-	var err error
42		-	users, err = rt.recvIdMapping1(func(remoteUid int32, remoteUsername string) int32 {
43		-	// TODO: look up local uid by username
44		-	return remoteUid
45		-	})
46		-	if err != nil {
47		-	return nil, nil, err
48		-	}
49		-	for remoteUid, mapping := range users {
50		-	rt.Logger.Debug("remote uid maps to local uid", "remoteUid", remoteUid, "name", mapping.Name, "localUid", mapping.LocalId)
51		-	}
52		-	}
53		-
54		-	if rt.Opts.PreserveGid {
55		-	var err error
56		-	groups, err = rt.recvIdMapping1(func(remoteGid int32, remoteGroupname string) int32 {
57		-	// TODO: look up local gid by groupname
58		-	return remoteGid
59		-	})
60		-	if err != nil {
61		-	return nil, nil, err
62		-	}
63		-	for remoteGid, mapping := range groups {
64		-	rt.Logger.Debug("remote gid maps to local gid", "remoteGid", remoteGid, "name", mapping.Name, "localGid", mapping.LocalId)
65		-	}
66		-	}
67		-
68		-	return users, groups, nil
69		-	}

...	...		@@ -1,75 +0,0 @@
1		-	package rsyncsender
2		-
3		-	import (
4		-	"fmt"
5		-	"io"
6		-	"log/slog"
7		-
8		-	"github.com/picosh/pico/pkg/rsync-receiver/rsync"
9		-	"github.com/picosh/pico/pkg/rsync-receiver/rsyncopts"
10		-	"github.com/picosh/pico/pkg/rsync-receiver/rsyncwire"
11		-	"github.com/picosh/pico/pkg/rsync-receiver/utils"
12		-	)
13		-
14		-	func ClientRun(logger slog.Logger, opts rsyncopts.Options, conn io.ReadWriter, filesystem utils.FS, paths []string, negotiate bool) error {
15		-	var err error
16		-
17		-	crd, cwr := rsyncwire.CounterPair(conn, conn)
18		-
19		-	const sessionChecksumSeed = 666
20		-
21		-	c := &rsyncwire.Conn{
22		-	Reader: crd,
23		-	Writer: cwr,
24		-	}
25		-
26		-	if negotiate {
27		-	remoteProtocol, err := c.ReadInt32()
28		-	if err != nil {
29		-	return err
30		-	}
31		-	logger.Debug("remote protocol", "remoteProtocol", remoteProtocol)
32		-	if err := c.WriteInt32(rsync.ProtocolVersion); err != nil {
33		-	return err
34		-	}
35		-	}
36		-
37		-	if err := c.WriteInt32(sessionChecksumSeed); err != nil {
38		-	return err
39		-	}
40		-
41		-	// Switch to multiplexing protocol, but only for server-side transmissions.
42		-	// Transmissions received from the client are not multiplexed.
43		-	mpx := &rsyncwire.MultiplexWriter{Writer: c.Writer}
44		-	c.Writer = mpx
45		-
46		-	defer func() {
47		-	if err != nil {
48		-	_, _ = mpx.WriteMsg(rsyncwire.MsgError, fmt.Appendf(nil, "gokr-rsync [sender]: %v\n", err))
49		-	}
50		-	}()
51		-
52		-	st := &Transfer{
53		-	Opts: opts,
54		-	Conn: c,
55		-	Seed: sessionChecksumSeed,
56		-	Files: filesystem,
57		-
58		-	Logger: logger,
59		-	}
60		-	// receive the exclusion list (openrsync’s is always empty)
61		-	exclusionList, err := RecvFilterList(st.Conn)
62		-	if err != nil {
63		-	return err
64		-	}
65		-	logger.Debug("exclusion list read", "filters", exclusionList.Filters)
66		-
67		-	stats, err := st.Do(crd, cwr, paths, exclusionList)
68		-	if err != nil {
69		-	return err
70		-	}
71		-
72		-	logger.Debug("handleConnSender done. stats", "stats", stats)
73		-
74		-	return err
75		-	}

...	...		@@ -1,68 +0,0 @@
1		-	package rsyncsender
2		-
3		-	import (
4		-	"fmt"
5		-	"sort"
6		-
7		-	"github.com/picosh/pico/pkg/rsync-receiver/rsyncstats"
8		-	"github.com/picosh/pico/pkg/rsync-receiver/rsyncwire"
9		-	)
10		-
11		-	// rsync/main.c:client_run am_sender.
12		-	func (st Transfer) Do(crd rsyncwire.CountingReader, cwr rsyncwire.CountingWriter, paths []string, exclusionList filterRuleList) (*rsyncstats.TransferStats, error) {
13		-	if exclusionList == nil {
14		-	exclusionList = &filterRuleList{}
15		-	}
16		-
17		-	// “Update exchange” as per
18		-	// https://github.com/kristapsdz/openrsync/blob/master/rsync.5
19		-
20		-	// send file list
21		-	fileList, err := st.SendFileList(st.Opts, paths, exclusionList)
22		-	if err != nil {
23		-	return nil, err
24		-	}
25		-
26		-	st.Logger.Debug("file list sent")
27		-
28		-	// Sort the file list. The client sorts, so we need to sort, too (in the
29		-	// same way!), otherwise our indices do not match what the client will
30		-	// request.
31		-	sort.Slice(fileList.Files, func(i, j int) bool {
32		-	return fileList.Files[i].WPath < fileList.Files[j].WPath
33		-	})
34		-
35		-	if err := st.SendFiles(fileList); err != nil {
36		-	return nil, err
37		-	}
38		-
39		-	// send statistics:
40		-	// total bytes read (from network connection)
41		-	if err := st.Conn.WriteInt64(crd.BytesRead); err != nil {
42		-	return nil, err
43		-	}
44		-	// total bytes written (to network connection)
45		-	if err := st.Conn.WriteInt64(cwr.BytesWritten); err != nil {
46		-	return nil, err
47		-	}
48		-	// total size of files
49		-	if err := st.Conn.WriteInt64(fileList.TotalSize); err != nil {
50		-	return nil, err
51		-	}
52		-
53		-	st.Logger.Debug("reading final int32")
54		-
55		-	finish, err := st.Conn.ReadInt32()
56		-	if err != nil {
57		-	return nil, err
58		-	}
59		-	if finish != -1 {
60		-	return nil, fmt.Errorf("protocol error: expected final -1, got %d", finish)
61		-	}
62		-
63		-	return &rsyncstats.TransferStats{
64		-	Read: crd.BytesRead,
65		-	Written: cwr.BytesWritten,
66		-	Size: fileList.TotalSize,
67		-	}, nil
68		-	}

...	...		@@ -1,109 +0,0 @@
1		-	package rsyncsender
2		-
3		-	import (
4		-	"io"
5		-	"path/filepath"
6		-	"strings"
7		-
8		-	"github.com/picosh/pico/pkg/rsync-receiver/rsyncwire"
9		-	)
10		-
11		-	type filterRuleList struct {
12		-	Filters []*filterRule
13		-	}
14		-
15		-	// exclude.c:add_rule.
16		-	func (l filterRuleList) addRule(fr filterRule) {
17		-	if strings.HasSuffix(fr.pattern, "/") {
18		-	fr.flag \|= filtruleDirectory
19		-	fr.pattern = strings.TrimSuffix(fr.pattern, "/")
20		-	}
21		-	if strings.ContainsFunc(fr.pattern, func(r rune) bool {
22		-	return r == '*' \|\| r == '[' \|\| r == '?'
23		-	}) {
24		-	fr.flag \|= filtruleWild
25		-	}
26		-	l.Filters = append(l.Filters, fr)
27		-	}
28		-
29		-	// exclude.c:check_filter.
30		-	func (l *filterRuleList) matches(name string) bool {
31		-	for _, fr := range l.Filters {
32		-	if fr.matches(name) {
33		-	return true
34		-	}
35		-	}
36		-	return false
37		-	}
38		-
39		-	// exclude.c:recv_filter_list.
40		-	func RecvFilterList(c rsyncwire.Conn) (filterRuleList, error) {
41		-	var l filterRuleList
42		-	const exclusionListEnd = 0
43		-	for {
44		-	length, err := c.ReadInt32()
45		-	if err != nil {
46		-	return nil, err
47		-	}
48		-	if length == exclusionListEnd {
49		-	break
50		-	}
51		-	line := make([]byte, length)
52		-	if _, err := io.ReadFull(c.Reader, line); err != nil {
53		-	return nil, err
54		-	}
55		-	fr, err := parseFilter(string(line))
56		-	if err != nil {
57		-	return nil, err
58		-	}
59		-	l.addRule(fr)
60		-	}
61		-	return &l, nil
62		-	}
63		-
64		-	const (
65		-	filtruleInclude = 1 << iota
66		-	filtruleClearList
67		-	filtruleDirectory
68		-	filtruleWild
69		-	)
70		-
71		-	type filterRule struct {
72		-	flag int
73		-	pattern string
74		-	}
75		-
76		-	// exclude.c:rule_matches.
77		-	func (fr *filterRule) matches(name string) bool {
78		-	if fr.flag&filtruleWild != 0 {
79		-	panic("wildcard filter rules not yet implemented")
80		-	}
81		-	if !strings.ContainsRune(fr.pattern, '/') &&
82		-	fr.flag&filtruleWild == 0 {
83		-	name = filepath.Base(name)
84		-	}
85		-	return fr.pattern == name
86		-	}
87		-
88		-	// exclude.c:parse_filter_str / exclude.c:parse_rule_tok.
89		-	func parseFilter(line string) (*filterRule, error) {
90		-	rule := new(filterRule)
91		-
92		-	// We only support what rsync calls XFLG_OLD_PREFIXES
93		-	if strings.HasPrefix(line, "- ") {
94		-	// clear include flag
95		-	rule.flag &= ^filtruleInclude
96		-	line = strings.TrimPrefix(line, "- ")
97		-	} else if strings.HasPrefix(line, "+ ") {
98		-	// set include flag
99		-	rule.flag \|= filtruleInclude
100		-	line = strings.TrimPrefix(line, "+ ")
101		-	} else if strings.HasPrefix(line, "!") {
102		-	// set clear_list flag
103		-	rule.flag \|= filtruleClearList
104		-	}
105		-
106		-	rule.pattern = line
107		-
108		-	return rule, nil
109		-	}

...	...		@@ -1,116 +0,0 @@
1		-	package rsyncsender
2		-
3		-	import (
4		-	"io"
5		-	"log/slog"
6		-	"os"
7		-	)
8		-
9		-	// rsync.h:map_struct.
10		-	type mapStruct struct {
11		-	fileSize int64 // file size (from stat)
12		-	pOffset int64 // window start
13		-	pFdOffset int64 // offset of cursor in fd ala lseek
14		-	window []byte // window pointer
15		-	pSize int64 // largest window we allocated
16		-	pLen int64 // latest (rounded) window size
17		-	defWindowSize int64 // default window size
18		-	f *os.File // file descriptor
19		-	err error // first read error
20		-	}
21		-
22		-	const alignBoundary = 1024
23		-
24		-	func alignedLength(l int64) int64 {
25		-	return ((l - 1) \| (alignBoundary - 1)) + 1
26		-	}
27		-
28		-	func alignedOvershoot(off int64) int64 {
29		-	return off & (alignBoundary - 1)
30		-	}
31		-
32		-	func mapFile(f os.File, len int64, readSize int32, blkSize int32) mapStruct {
33		-	if blkSize > 0 && readSize%blkSize != 0 {
34		-	readSize += blkSize - (readSize % blkSize)
35		-	}
36		-	return &mapStruct{
37		-	fileSize: len,
38		-	defWindowSize: alignedLength(int64(readSize)),
39		-	f: f,
40		-	}
41		-	}
42		-
43		-	func (ms *mapStruct) ptr(offset int64, l int32) []byte {
44		-	//log.Printf("ptr(offset=%d, l=%d)", offset, l)
45		-	len := int64(l)
46		-	if len == 0 {
47		-	return nil
48		-	}
49		-	if len < 0 {
50		-	slog.Debug("BUG: invalid len", "len", len)
51		-	return nil
52		-	}
53		-
54		-	if offset >= ms.pOffset && offset+int64(len) <= ms.pOffset+int64(ms.pLen) {
55		-	//log.Printf("-> already available")
56		-	// region already available
57		-	off := offset - ms.pOffset
58		-	return ms.window[off : off+int64(len)]
59		-	}
60		-
61		-	alignFudge := alignedOvershoot(offset)
62		-	windowStart := offset - alignFudge
63		-	windowSize := int64(ms.defWindowSize)
64		-	if windowStart+windowSize > ms.fileSize {
65		-	windowSize = ms.fileSize - windowStart
66		-	}
67		-	if windowSize < len+alignFudge {
68		-	windowSize = alignedLength(len + alignFudge)
69		-	}
70		-	if windowSize > ms.pSize {
71		-	win := make([]byte, windowSize)
72		-	copy(win, ms.window)
73		-	ms.window = win
74		-	ms.pSize = windowSize
75		-	}
76		-	readStart := windowStart
77		-	readSize := windowSize
78		-	readOffset := int64(0)
79		-
80		-	//log.Printf("windowSize: %d, ms=%+v", windowSize, ms)
81		-	if windowStart >= ms.pOffset && windowStart < ms.pOffset+ms.pLen &&
82		-	windowStart+windowSize >= ms.pOffset+ms.pLen {
83		-	readStart = ms.pOffset + ms.pLen
84		-	readOffset = readStart - windowStart
85		-	readSize = windowSize - readOffset
86		-	off := ms.pLen - readOffset
87		-	copy(ms.window[:], ms.window[off:off+readOffset])
88		-	}
89		-	if readSize <= 0 {
90		-	slog.Debug("BUG: invalid readSize", "readSize", readSize)
91		-	return nil
92		-	}
93		-	if ms.pFdOffset != readStart {
94		-	if _, err := ms.f.Seek(readStart, io.SeekStart); err != nil {
95		-	slog.Error("seek error", "err", err)
96		-	return nil
97		-	}
98		-	ms.pFdOffset = readStart
99		-	}
100		-	ms.pOffset = windowStart
101		-	ms.pLen = windowSize
102		-	//log.Printf("-> reading %d bytes from %d into buffer at offset=%d", readSize, readStart, readOffset)
103		-	for readSize > 0 {
104		-	n, err := ms.f.Read(ms.window[readOffset : readOffset+readSize])
105		-	if err != nil {
106		-	ms.err = err
107		-	// TODO: zero the buffer, file has changed mid-transfer
108		-	slog.Debug("file has changed mid-transfer")
109		-	return nil
110		-	}
111		-	ms.pFdOffset += int64(n)
112		-	readOffset += int64(n)
113		-	readSize -= int64(n)
114		-	}
115		-	return ms.window[alignFudge : alignFudge+len]
116		-	}

...	...		@@ -1,236 +0,0 @@
1		-	package rsyncsender
2		-
3		-	import (
4		-	"os"
5		-	"os/user"
6		-	"strconv"
7		-	"sync"
8		-
9		-	"github.com/picosh/pico/pkg/rsync-receiver/rsync"
10		-	"github.com/picosh/pico/pkg/rsync-receiver/rsyncchecksum"
11		-	"github.com/picosh/pico/pkg/rsync-receiver/rsyncopts"
12		-	"github.com/picosh/pico/pkg/rsync-receiver/rsyncwire"
13		-	"github.com/picosh/pico/pkg/rsync-receiver/utils"
14		-	)
15		-
16		-	type fileList struct {
17		-	TotalSize int64
18		-	Files []utils.SenderFile
19		-	}
20		-
21		-	// rsync/rsync.h defines chunkSize as 32 * 1024, but increasing it to 256K
22		-	// increases throughput with “tridge” rsync as client by 50 Mbit/s.
23		-	const chunkSize = 256 * 1024
24		-
25		-	var (
26		-	lookupOnce sync.Once
27		-	lookupGroupOnce sync.Once
28		-	)
29		-
30		-	// rsync/flist.c:send_file_list.
31		-	func (st Transfer) SendFileList(opts rsyncopts.Options, paths []string, excl filterRuleList) (fileList, error) {
32		-	var fileList fileList
33		-	fec := &rsyncwire.Buffer{}
34		-
35		-	uidMap := make(map[int32]string)
36		-	gidMap := make(map[int32]string)
37		-
38		-	// TODO: flush in between to keep the pipes filled when traversal takes long
39		-
40		-	// TODO: handle info == nil case (permission denied?): should set an i/o
41		-	// error flag, but traversal should continue
42		-
43		-	st.Logger.Debug("sendFileList()")
44		-	// TODO: handle \|root\| referring to an individual file, symlink or special (skip)
45		-	for _, requested := range paths {
46		-	files, err := st.Files.List(requested)
47		-	if err != nil {
48		-	return nil, err
49		-	}
50		-
51		-	for _, info := range files {
52		-	// Only ever transmit long names, like openrsync
53		-	flags := byte(rsync.XMIT_LONG_NAME)
54		-
55		-	// log.Printf("Trim(path=%q, %q) = %q", path, strip, name)
56		-	name := info.Name()
57		-	path := name
58		-	if name == "/" {
59		-	name = "."
60		-	flags \|= rsync.XMIT_TOP_DIR
61		-	}
62		-	// log.Printf("flags for %q: %v", name, flags)
63		-
64		-	if excl.matches(name) {
65		-	continue
66		-	}
67		-
68		-	fileList.Files = append(fileList.Files, utils.SenderFile{
69		-	Path: "/",
70		-	Regular: info.Mode().IsRegular(),
71		-	WPath: name,
72		-	})
73		-
74		-	// 1. status byte (integer)
75		-	_ = fec.WriteByte(flags)
76		-
77		-	// 2. inherited filename length (optional, byte)
78		-	// 3. filename length (integer or byte)
79		-	fec.WriteInt32(int32(len(name)))
80		-
81		-	// 4. file (byte array)
82		-	fec.WriteString(name)
83		-
84		-	// 5. file length (long)
85		-	size := info.Size()
86		-	if info.Mode().IsDir() {
87		-	// tmpfs returns non-4K sizes for directories. Override with
88		-	// 4096 to make the tests succeed regardless of the /tmp file
89		-	// system type.
90		-	size = 4096
91		-	}
92		-	fec.WriteInt64(size)
93		-
94		-	fileList.TotalSize += size
95		-
96		-	// 6. file modification time (optional, integer)
97		-	// TODO: this will overflow in 2038! :(
98		-	fec.WriteInt32(int32(info.ModTime().Unix()))
99		-
100		-	// 7. file mode (optional, mode_t, integer)
101		-	mode := int32(info.Mode() & os.ModePerm)
102		-	isDev := false
103		-	isSpecial := false
104		-	if info.Mode().IsDir() {
105		-	mode \|= rsync.S_IFDIR
106		-	} else if info.Mode().IsRegular() {
107		-	mode \|= rsync.S_IFREG
108		-	} else if info.Mode().Type()&os.ModeSymlink != 0 {
109		-	mode \|= rsync.S_IFLNK
110		-	// TODO: skip symlink if PreserveSymlinks is not set
111		-	}
112		-
113		-	if info.Mode().Type()&os.ModeCharDevice != 0 {
114		-	mode \|= rsync.S_IFCHR
115		-	isDev = true
116		-	} else if info.Mode().Type()&os.ModeDevice != 0 {
117		-	mode \|= rsync.S_IFBLK
118		-	isDev = true
119		-	}
120		-
121		-	if info.Mode().Type()&os.ModeNamedPipe != 0 {
122		-	mode \|= rsync.S_IFIFO
123		-	isSpecial = true
124		-	}
125		-
126		-	if info.Mode().Type()&os.ModeSocket != 0 {
127		-	mode \|= rsync.S_IFSOCK
128		-	isSpecial = true
129		-	}
130		-
131		-	fec.WriteInt32(mode)
132		-
133		-	if opts.PreserveUid() {
134		-	uid, ok := uidFromFileInfo(info)
135		-	if ok {
136		-	if _, ok := uidMap[uid]; !ok && uid != 0 {
137		-	u, err := user.LookupId(strconv.Itoa(int(uid)))
138		-	if err != nil {
139		-	lookupOnce.Do(func() {
140		-	st.Logger.Error("lookup", "uid", uid, "err", err)
141		-	})
142		-	} else {
143		-	uidMap[uid] = u.Username
144		-	}
145		-	}
146		-	}
147		-	// 8. if -o, the user id (integer)
148		-	fec.WriteInt32(uid)
149		-	}
150		-
151		-	if opts.PreserveGid() {
152		-	gid, ok := gidFromFileInfo(info)
153		-	if ok {
154		-	if _, ok := gidMap[gid]; !ok && gid != 0 {
155		-	g, err := user.LookupGroupId(strconv.Itoa(int(gid)))
156		-	if err != nil {
157		-	lookupGroupOnce.Do(func() {
158		-	st.Logger.Error("lookupgroup", "gid", gid, "err", err)
159		-	})
160		-	} else {
161		-	gidMap[gid] = g.Name
162		-	}
163		-	}
164		-	}
165		-	// 9. if -g, the group id (integer)
166		-	fec.WriteInt32(gid)
167		-	}
168		-
169		-	if (opts.PreserveDevices() && isDev) \|\|
170		-	(opts.PreserveSpecials() && isSpecial) {
171		-	// 10. if a special file and -D, the device “rdev” type (integer)
172		-	rdev, _ := rdevFromFileInfo(info)
173		-	fec.WriteInt32(rdev)
174		-	}
175		-
176		-	if opts.PreserveLinks() && info.Mode().Type()&os.ModeSymlink != 0 {
177		-	// 11. if a symbolic link and -l, the link target's length (integer)
178		-	// 12. if a symbolic link and -l, the link target (byte array)
179		-	target, err := os.Readlink(path)
180		-	if err != nil {
181		-	continue
182		-	}
183		-	fec.WriteInt32(int32(len(target)))
184		-	fec.WriteString(target)
185		-	}
186		-
187		-	if opts.AlwaysChecksum() {
188		-	var emptyChecksum [rsyncchecksum.Size]byte
189		-	checksum := emptyChecksum[:]
190		-	if info.Mode().IsRegular() {
191		-	// TODO: send md4 checksum of this file
192		-	checksum, err = rsyncchecksum.FileChecksum(path)
193		-	if err != nil {
194		-	continue
195		-	}
196		-	}
197		-	// For non-regular files, send empty md4 checksum
198		-	fec.WriteString(string(checksum))
199		-	}
200		-	}
201		-	if err != nil {
202		-	return nil, err
203		-	}
204		-	}
205		-
206		-	const endOfFileList = 0
207		-	_ = fec.WriteByte(endOfFileList)
208		-
209		-	const endOfSet = 0
210		-	if opts.PreserveUid() {
211		-	for uid, name := range uidMap {
212		-	fec.WriteInt32(uid)
213		-	_ = fec.WriteByte(byte(len(name)))
214		-	fec.WriteString(name)
215		-	}
216		-	fec.WriteInt32(endOfSet)
217		-	}
218		-
219		-	if opts.PreserveGid() {
220		-	for gid, name := range gidMap {
221		-	fec.WriteInt32(gid)
222		-	_ = fec.WriteByte(byte(len(name)))
223		-	fec.WriteString(name)
224		-	}
225		-	fec.WriteInt32(endOfSet)
226		-	}
227		-
228		-	const ioErrors = 0
229		-	fec.WriteInt32(ioErrors)
230		-
231		-	if err := st.Conn.WriteString(fec.String()); err != nil {
232		-	return nil, err
233		-	}
234		-
235		-	return &fileList, nil
236		-	}

...	...		@@ -1,257 +0,0 @@
1		-	package rsyncsender
2		-
3		-	import (
4		-	"bytes"
5		-	"encoding/binary"
6		-	"fmt"
7		-	"hash"
8		-	"os"
9		-
10		-	"github.com/mmcloughlin/md4"
11		-	"github.com/picosh/pico/pkg/rsync-receiver/nofollow"
12		-	"github.com/picosh/pico/pkg/rsync-receiver/rsync"
13		-	"github.com/picosh/pico/pkg/rsync-receiver/rsyncchecksum"
14		-	"github.com/picosh/pico/pkg/rsync-receiver/utils"
15		-	)
16		-
17		-	type target struct {
18		-	index int32
19		-	tag uint16
20		-	}
21		-
22		-	// rsync/match.c:hash_search.
23		-	func (st *Transfer) hashSearch(targets []target, tagTable map[uint16]int, head rsync.SumHead, fileIndex int32, fl utils.SenderFile) error {
24		-	st.Logger.Debug("hashSearch", "file", fl, "head", head)
25		-	f, err := os.OpenFile(fl.Path, os.O_RDONLY\|nofollow.Maybe, 0)
26		-	if err != nil {
27		-	return err
28		-	}
29		-	defer func() { _ = f.Close() }()
30		-
31		-	fi, err := f.Stat()
32		-	if err != nil {
33		-	return err
34		-	}
35		-
36		-	readSize := max(3head.BlockLength, 2561024)
37		-	ms := mapFile(f, fi.Size(), readSize, head.BlockLength)
38		-
39		-	if err := st.Conn.WriteInt32(fileIndex); err != nil {
40		-	return err
41		-	}
42		-
43		-	if err := head.WriteTo(st.Conn); err != nil {
44		-	return err
45		-	}
46		-
47		-	// sum_init()
48		-	h := md4.New()
49		-	_ = binary.Write(h, binary.LittleEndian, st.Seed) // hash.Hash.Write never fails
50		-
51		-	// The following quotes are citations from
52		-	// https://www.samba.org/~tridge/phd_thesis.pdf, section 3.2.6 The
53		-	// signature search algorithm (PDF page 64).
54		-
55		-	// “Once the sorted signature table and the index table have been formed the
56		-	// signature search process can begin. For each byte offset in a_i the fast
57		-	// signature is computed, along with the 16 bit hash of the fast
58		-	// signature. The 16 bit hash is then used to lookup the signature index,
59		-	// giving the index in the signature table of the first fast signature with
60		-	// that hash.”
61		-
62		-	var k int
63		-	var sum uint32
64		-	var s1, s2 uint32
65		-	var offset int64
66		-	end := fi.Size() + 1 - head.Sums[len(head.Sums)-1].Len
67		-	st.Logger.Debug("last block", "len", head.Sums[len(head.Sums)-1].Len, "end", end)
68		-
69		-	readChunk := func() error {
70		-	k = int(head.BlockLength)
71		-	if remaining := int(fi.Size() - offset); remaining < k {
72		-	k = remaining
73		-	}
74		-
75		-	chunk := ms.ptr(offset, int32(k))
76		-	sum = rsyncchecksum.Checksum1(chunk)
77		-	s1 = uint32(sum & 0xFFFF)
78		-	s2 = uint32(sum >> 16)
79		-	return nil
80		-	}
81		-	if err := readChunk(); err != nil {
82		-	return err
83		-	}
84		-
85		-	tagHits := 0
86		-	Outer:
87		-	for {
88		-	tag := rsyncchecksum.Tag2(uint16(s1), uint16(s2))
89		-	var sum2 []byte
90		-	doneCsum2 := false
91		-	j, ok := tagTable[tag]
92		-	if ok {
93		-	// “A linear search is then performed through the signature table, stopping
94		-	// when an entry is found with a 16 bit hash which doesn’t match. For each
95		-	// entry the current 32 bit fast signature is compared to the entry in the
96		-	// signature table, and if that matches then the full 128 bit strong
97		-	// signature is computed at the current byte offset and compared to the
98		-	// strong signature in the signature table”
99		-	sum = (uint32(s1) & 0xFFFF) \| (uint32(s2) << 16)
100		-	tagHits++
101		-	for ; j < int(head.ChecksumCount) && targets[j].tag == tag; j++ {
102		-	i := targets[j].index
103		-	if sum != head.Sums[i].Sum1 {
104		-	continue
105		-	}
106		-
107		-	l := int64(head.BlockLength)
108		-	if v := fi.Size() - offset; v < l {
109		-	l = v
110		-	}
111		-	if l != head.Sums[i].Len {
112		-	continue
113		-	}
114		-
115		-	// log.Printf("potential match at %d target=%d %d sum=%08x", offset, j, i, sum)
116		-
117		-	if !doneCsum2 {
118		-	buf := ms.ptr(offset, int32(l))
119		-	sum2 = rsyncchecksum.Checksum2(st.Seed, buf[:])
120		-	doneCsum2 = true
121		-	}
122		-
123		-	if local, remote := sum2[:head.ChecksumLength], head.Sums[i].Sum2[:head.ChecksumLength]; !bytes.Equal(local, remote) {
124		-	st.Logger.Debug("false alarm", "local", local, "remote", remote)
125		-	//falseAlarms++
126		-	continue
127		-	}
128		-
129		-	// TODO(optimization): tridge rsync locates adjacent matches
130		-	// here for better run-length encoding, but I’m not sure where
131		-	// (if at all) we currently use run-length encoding:
132		-	// https://github.com/WayneD/rsync/commit/923fa978088f4c044eec528d9472962d9c9d13c3
133		-
134		-	// “If the strong signature is found to match then A emits a
135		-	// token telling B that a match was found and which block in bi
136		-	// was matched12. The search then continues at the byte after
137		-	// the matching block.”
138		-
139		-	if err := st.matched(h, ms, head, offset, i); err != nil {
140		-	return err
141		-	}
142		-
143		-	// rsync doesn’t read the next chunk (offset+sums[i].len),
144		-	// rsync starts reading one byte before the next chunk
145		-	// (offset+sums[i].len-1), because the code path starting at
146		-	// “null_tag” removes the chunk’s first byte and adds the
147		-	// next byte after the chunk.
148		-	offset += head.Sums[i].Len - 1
149		-	if err := readChunk(); err != nil {
150		-	return fmt.Errorf("readChunk: %v", err)
151		-	}
152		-
153		-	if offset >= end {
154		-	break Outer
155		-	}
156		-
157		-	break
158		-	}
159		-	}
160		-
161		-	// Update the rolling checksum by removing the oldest byte (update[0])
162		-	// and adding the newest byte (update[k]).
163		-	backup := max(offset-st.lastMatch, 0)
164		-
165		-	more := offset+int64(k) < fi.Size()
166		-	mmore := int64(0)
167		-	if more {
168		-	mmore = 1
169		-	}
170		-	update := ms.ptr(offset-backup, int32(int64(k)+mmore+backup))
171		-	update = update[backup:]
172		-
173		-	s1 -= rsyncchecksum.SignExtend(update[0])
174		-	s2 -= uint32(k) * rsyncchecksum.SignExtend(update[0])
175		-
176		-	if more {
177		-	s1 += rsyncchecksum.SignExtend(update[k])
178		-	s2 += s1
179		-	} else {
180		-	k--
181		-	}
182		-	s1 = uint32(uint16(s1))
183		-	s2 = uint32(uint16(s2))
184		-
185		-	if backup >= int64(head.BlockLength)+chunkSize && end-offset > chunkSize {
186		-	// Prevent offset-st.lastMatch from growing too large by flushing
187		-	// intermediate chunks.
188		-	if err := st.matched(h, ms, head, offset-int64(head.BlockLength), -2); err != nil {
189		-	return err
190		-	}
191		-	}
192		-
193		-	offset++
194		-	if offset >= end {
195		-	break
196		-	}
197		-	}
198		-
199		-	if err := st.matched(h, ms, head, fi.Size(), -1); err != nil {
200		-	return err
201		-	}
202		-
203		-	{
204		-	sum := h.Sum(nil)
205		-	st.Logger.Debug("sum info", "sum", sum, "len", len(sum))
206		-	if _, err := st.Conn.Writer.Write(sum); err != nil {
207		-	return err
208		-	}
209		-	}
210		-
211		-	return nil
212		-
213		-	}
214		-
215		-	// rsync/match.c:matched.
216		-	func (st Transfer) matched(h hash.Hash, ms mapStruct, head rsync.SumHead, offset int64, i int32) error {
217		-	n := offset - st.lastMatch
218		-
219		-	transmitAccumulated := i < 0
220		-
221		-	// if !transmitAccumulated {
222		-	// log.Printf("match at offset=%d last_match=%d i=%d len=%d n=%d",
223		-	// offset, st.lastMatch, i, head.Sums[i].Len, n)
224		-	// } else {
225		-	// log.Printf("transmit accumulated at offset=%d", offset)
226		-	// }
227		-
228		-	/* FIXME: this is not used
229		-	l := int64(0)
230		-	if !transmitAccumulated {
231		-	l = head.Sums[i].Len
232		-	}
233		-	*/
234		-
235		-	if err := st.sendToken(ms, i, st.lastMatch, n); err != nil {
236		-	return fmt.Errorf("sendToken: %v", err)
237		-	}
238		-	// TODO: data_transfer += n;
239		-
240		-	if !transmitAccumulated {
241		-	// stats.matched_data += s->sums[i].len;
242		-	n += head.Sums[i].Len
243		-	}
244		-
245		-	for j := int64(0); j < n; j += chunkSize {
246		-	n1 := min(int64(chunkSize), n-j)
247		-	chunk := ms.ptr(st.lastMatch+j, int32(n1))
248		-	h.Write(chunk)
249		-	}
250		-
251		-	if !transmitAccumulated {
252		-	st.lastMatch = offset + head.Sums[i].Len
253		-	} else {
254		-	st.lastMatch = offset
255		-	}
256		-	return nil
257		-	}

...	...		@@ -1,216 +0,0 @@
1		-	package rsyncsender
2		-
3		-	import (
4		-	"encoding/binary"
5		-	"io"
6		-	"os"
7		-	"sort"
8		-
9		-	"github.com/mmcloughlin/md4"
10		-	"github.com/picosh/pico/pkg/rsync-receiver/rsync"
11		-	"github.com/picosh/pico/pkg/rsync-receiver/rsyncchecksum"
12		-	"github.com/picosh/pico/pkg/rsync-receiver/rsynccommon"
13		-	"github.com/picosh/pico/pkg/rsync-receiver/utils"
14		-	)
15		-
16		-	// rsync/sender.c:send_files().
17		-	func (st Transfer) SendFiles(fileList fileList) error {
18		-	phase := 0
19		-	for {
20		-	// receive data about receiver’s copy of the file list contents (not
21		-	// ordered)
22		-	// see (*rsync.Receiver).Generator()
23		-	fileIndex, err := st.Conn.ReadInt32()
24		-	if err != nil {
25		-	return err
26		-	}
27		-	if fileIndex == -1 {
28		-	if phase == 0 {
29		-	phase++
30		-	// acknowledge phase change by sending -1
31		-	if err := st.Conn.WriteInt32(-1); err != nil {
32		-	return err
33		-	}
34		-	continue
35		-	}
36		-	break
37		-	}
38		-
39		-	if st.Opts.DryRun() {
40		-	if err := st.Conn.WriteInt32(fileIndex); err != nil {
41		-	return err
42		-	}
43		-	continue
44		-	}
45		-
46		-	head, err := st.receiveSums()
47		-	if err != nil {
48		-	return err
49		-	}
50		-
51		-	// The following quotes are citations from
52		-	// https://www.samba.org/~tridge/phd_thesis.pdf, section 3.2.6 The
53		-	// signature search algorithm (PDF page 64).
54		-
55		-	// rsync/match.c:build_hash_table
56		-	targets := make([]target, len(head.Sums))
57		-	tagTable := make(map[uint16]int) // TODO: or int32 more specifically?
58		-	{
59		-	// “The first step in the algorithm is to sort the received
60		-	// signatures by a 16 bit hash of the fast signature.”
61		-	for idx, sum := range head.Sums {
62		-	targets[idx] = target{
63		-	index: int32(idx),
64		-	tag: rsyncchecksum.Tag(sum.Sum1),
65		-	}
66		-	}
67		-	sort.Slice(targets, func(i, j int) bool {
68		-	return targets[i].tag < targets[j].tag
69		-	})
70		-
71		-	// “A 16 bit index table is then formed which takes a 16 bit hash
72		-	// value and gives an index into the sorted signature table which
73		-	// points to the first entry in the table which has a matching
74		-	// hash.”
75		-	for idx := len(head.Sums) - 1; idx >= 0; idx-- {
76		-	tagTable[targets[idx].tag] = idx
77		-	}
78		-	}
79		-
80		-	st.lastMatch = 0
81		-	if len(head.Sums) == 0 {
82		-	// fast path: send the whole file
83		-	err = st.sendFile(fileIndex, fileList.Files[fileIndex])
84		-	} else {
85		-	err = st.hashSearch(targets, tagTable, head, fileIndex, fileList.Files[fileIndex])
86		-	}
87		-	if err != nil {
88		-	if _, ok := err.(*os.PathError); ok {
89		-	// OpenFile() failed. Log the error (server side only) and
90		-	// proceed. Only starting with protocol 30, an I/O error flag is
91		-	// sent after the file transfer phase.
92		-	if os.IsNotExist(err) {
93		-	st.Logger.Debug("file has vanished", "file", fileList.Files[fileIndex])
94		-	} else {
95		-	st.Logger.Error("sendFiles", "err", err)
96		-	}
97		-	continue
98		-	} else {
99		-	return err
100		-	}
101		-	}
102		-	}
103		-
104		-	// phase done
105		-	if err := st.Conn.WriteInt32(-1); err != nil {
106		-	return err
107		-	}
108		-
109		-	return nil
110		-	}
111		-
112		-	// rsync/sender.c:receive_sums().
113		-	func (st *Transfer) receiveSums() (rsync.SumHead, error) {
114		-	var head rsync.SumHead
115		-	if err := head.ReadFrom(st.Conn); err != nil {
116		-	return head, err
117		-	}
118		-	var offset int64
119		-	head.Sums = make([]rsync.SumBuf, int(head.ChecksumCount))
120		-	for i := int32(0); i < head.ChecksumCount; i++ {
121		-	shortChecksum, err := st.Conn.ReadInt32()
122		-	if err != nil {
123		-	return head, err
124		-	}
125		-	sb := rsync.SumBuf{
126		-	Index: i,
127		-	Offset: offset,
128		-	Sum1: uint32(shortChecksum),
129		-	}
130		-	if i == head.ChecksumCount-1 && head.RemainderLength != 0 {
131		-	sb.Len = int64(head.RemainderLength)
132		-	} else {
133		-	sb.Len = int64(head.BlockLength)
134		-	}
135		-	offset += sb.Len
136		-	n, err := io.ReadFull(st.Conn.Reader, sb.Sum2[:head.ChecksumLength])
137		-	if err != nil {
138		-	return head, err
139		-	}
140		-	_ = n
141		-	// log.Printf("chunk[%d] len=%d offset=%.0f sum1=%08x, sum2=%x",
142		-	// i, sb.len, float64(sb.offset), sb.sum1, sb.sum2[:n])
143		-	head.Sums[i] = sb
144		-	}
145		-	return head, nil
146		-	}
147		-
148		-	func (st *Transfer) sendFile(fileIndex int32, fl utils.SenderFile) error {
149		-	// rsync/rsync.h defines CHUNK_SIZE as 32 * 1024. openrsync (tridge)
150		-	// uses 256K, but standard rsync rejects tokens larger than 32K.
151		-	const chunkSize = 32 * 1024
152		-
153		-	fi, r, err := st.Files.Read(&fl)
154		-	if err != nil {
155		-	return err
156		-	}
157		-	defer func() { _ = r.Close() }()
158		-
159		-	if err := st.Conn.WriteInt32(fileIndex); err != nil {
160		-	return err
161		-	}
162		-
163		-	sh := rsynccommon.SumSizesSqroot(fi.Size())
164		-	// log.Printf("sh = %+v", sh)
165		-	if err := sh.WriteTo(st.Conn); err != nil {
166		-	return err
167		-	}
168		-
169		-	h := md4.New()
170		-	_ = binary.Write(h, binary.LittleEndian, st.Seed) // hash.Hash.Write never fails
171		-
172		-	buf := make([]byte, chunkSize)
173		-	for {
174		-	shouldBreak := false
175		-	n, err := r.Read(buf)
176		-	if err != nil {
177		-	if err == io.EOF {
178		-	shouldBreak = true
179		-	} else {
180		-	return err
181		-	}
182		-	}
183		-	chunk := buf[:n]
184		-
185		-	if len(chunk) == 0 {
186		-	break
187		-	}
188		-
189		-	_, err = h.Write(chunk)
190		-	if err != nil {
191		-	return err
192		-	}
193		-	// chunk size (“rawtok” variable in openrsync)
194		-	if err := st.Conn.WriteInt32(int32(len(chunk))); err != nil {
195		-	return err
196		-	}
197		-	if _, err := st.Conn.Writer.Write(chunk); err != nil {
198		-	return err
199		-	}
200		-
201		-	if shouldBreak {
202		-	break
203		-	}
204		-	}
205		-	// transfer finished:
206		-	if err := st.Conn.WriteInt32(0); err != nil {
207		-	return err
208		-	}
209		-
210		-	sum := h.Sum(nil)
211		-	// log.Printf("sum: %x (len = %d)", sum, len(sum))
212		-	if _, err := st.Conn.Writer.Write(sum); err != nil {
213		-	return err
214		-	}
215		-	return nil
216		-	}

...	...		@@ -1,15 +0,0 @@
1		-	package rsyncsender
2		-
3		-	import "io/fs"
4		-
5		-	func uidFromFileInfo(fs.FileInfo) (int32, bool) {
6		-	return 1000, false
7		-	}
8		-
9		-	func gidFromFileInfo(fs.FileInfo) (int32, bool) {
10		-	return 1000, false
11		-	}
12		-
13		-	func rdevFromFileInfo(fs.FileInfo) (int32, bool) {
14		-	return 1000, false
15		-	}

...	...		@@ -1,34 +0,0 @@
1		-	package rsyncsender
2		-
3		-	// rsync/token.c:simple_send_token.
4		-	func (st Transfer) simpleSendToken(ms mapStruct, token int32, offset int64, n int64) error {
5		-	if n > 0 {
6		-	st.Logger.Debug("sending unmatched chunks", "offset", offset, "n", n)
7		-	l := int64(0)
8		-	for l < n {
9		-	n1 := min(int64(chunkSize), n-l)
10		-
11		-	chunk := ms.ptr(offset+l, int32(n1))
12		-
13		-	if err := st.Conn.WriteInt32(int32(n1)); err != nil {
14		-	return err
15		-	}
16		-
17		-	if _, err := st.Conn.Writer.Write(chunk); err != nil {
18		-	return err
19		-	}
20		-
21		-	l += n1
22		-	}
23		-	}
24		-	if token != -2 {
25		-	return st.Conn.WriteInt32(-(token + 1))
26		-	}
27		-	return nil
28		-	}
29		-
30		-	// rsync/token.c:send_token.
31		-	func (st Transfer) sendToken(ms mapStruct, i int32, offset int64, n int64) error {
32		-	// TODO(compression): send deflated token
33		-	return st.simpleSendToken(ms, i, offset, n)
34		-	}

...	...		@@ -1,49 +0,0 @@
1		-	package rsyncsender
2		-
3		-	import (
4		-	"io"
5		-	"log/slog"
6		-
7		-	"github.com/picosh/pico/pkg/rsync-receiver/rsyncopts"
8		-	"github.com/picosh/pico/pkg/rsync-receiver/rsyncwire"
9		-	"github.com/picosh/pico/pkg/rsync-receiver/utils"
10		-	)
11		-
12		-	type Osenv struct {
13		-	Stdin io.Reader
14		-	Stdout io.Writer
15		-	Stderr io.Writer
16		-	}
17		-
18		-	// TransferOpts is a subset of Opts which is required for implementing a receiver.
19		-	type TransferOpts struct {
20		-	Verbose bool
21		-	DryRun bool
22		-
23		-	DeleteMode bool
24		-	PreserveGid bool
25		-	PreserveUid bool
26		-	PreserveLinks bool
27		-	PreservePerms bool
28		-	PreserveDevices bool
29		-	PreserveSpecials bool
30		-	PreserveTimes bool
31		-	PreserveHardlinks bool
32		-	}
33		-
34		-	type Transfer struct {
35		-	// config
36		-	// Opts *Opts
37		-	Opts *rsyncopts.Options
38		-
39		-	// state
40		-	Conn *rsyncwire.Conn
41		-	Seed int32
42		-	lastMatch int64
43		-
44		-	Files utils.FS
45		-
46		-	Logger *slog.Logger
47		-	}
48		-
49		-	//func (rt *Transfer) listOnly() bool { return rt.Dest == "" }

...	...		@@ -1,7 +0,0 @@
1		-	package rsyncstats
2		-
3		-	type TransferStats struct {
4		-	Read int64 // total bytes read (from network connection)
5		-	Written int64 // total bytes written (to network connection)
6		-	Size int64 // total size of files
7		-	}

...	...		@@ -1,159 +0,0 @@
1		-	package rsyncwire
2		-
3		-	import (
4		-	"bytes"
5		-	"encoding/binary"
6		-	"io"
7		-	)
8		-
9		-	const (
10		-	MsgData uint8 = 0
11		-	MsgInfo uint8 = 2
12		-	MsgError uint8 = 1
13		-	)
14		-
15		-	const mplexBase = 7
16		-
17		-	type MultiplexWriter struct {
18		-	Writer io.Writer
19		-	}
20		-
21		-	func (w *MultiplexWriter) Write(p []byte) (n int, err error) {
22		-	return w.WriteMsg(MsgData, p)
23		-	}
24		-
25		-	func (w *MultiplexWriter) WriteMsg(tag uint8, p []byte) (n int, err error) {
26		-	header := uint32(mplexBase+tag)<<24 \| uint32(len(p))
27		-	// log.Printf("len %d (hex %x)", len(p), uint32(len(p)))
28		-	// log.Printf("header=%v (%x)", header, header)
29		-	if err := binary.Write(w.Writer, binary.LittleEndian, header); err != nil {
30		-	return 0, err
31		-	}
32		-	return w.Writer.Write(p)
33		-	}
34		-
35		-	type Buffer struct {
36		-	// buf.Write() never fails, making for a convenient API.
37		-	buf bytes.Buffer
38		-	}
39		-
40		-	func (b *Buffer) WriteByte(data byte) error {
41		-	return binary.Write(&b.buf, binary.LittleEndian, data)
42		-	}
43		-
44		-	func (b *Buffer) WriteInt32(data int32) {
45		-	_ = binary.Write(&b.buf, binary.LittleEndian, data)
46		-	}
47		-
48		-	func (b *Buffer) WriteInt64(data int64) {
49		-	// send as a 32-bit integer if possible
50		-	if data <= 0x7FFFFFFF && data >= 0 {
51		-	b.WriteInt32(int32(data))
52		-	return
53		-	}
54		-	// otherwise, send -1 followed by the 64-bit integer
55		-	b.WriteInt32(-1)
56		-	_ = binary.Write(&b.buf, binary.LittleEndian, data)
57		-	}
58		-
59		-	func (b *Buffer) WriteString(data string) {
60		-	_, _ = io.WriteString(&b.buf, data)
61		-	}
62		-
63		-	func (b *Buffer) String() string {
64		-	return b.buf.String()
65		-	}
66		-
67		-	type Conn struct {
68		-	Writer io.Writer
69		-	Reader io.Reader
70		-	}
71		-
72		-	func (c *Conn) WriteByte(data byte) error {
73		-	return binary.Write(c.Writer, binary.LittleEndian, data)
74		-	}
75		-
76		-	func (c *Conn) WriteInt32(data int32) error {
77		-	return binary.Write(c.Writer, binary.LittleEndian, data)
78		-	}
79		-
80		-	func (c *Conn) WriteInt64(data int64) error {
81		-	// send as a 32-bit integer if possible
82		-	if data <= 0x7FFFFFFF && data >= 0 {
83		-	return c.WriteInt32(int32(data))
84		-	}
85		-	// otherwise, send -1 followed by the 64-bit integer
86		-	if err := c.WriteInt32(-1); err != nil {
87		-	return err
88		-	}
89		-	return binary.Write(c.Writer, binary.LittleEndian, data)
90		-	}
91		-
92		-	func (c *Conn) WriteString(data string) error {
93		-	_, err := io.WriteString(c.Writer, data)
94		-	return err
95		-	}
96		-
97		-	func (c *Conn) ReadByte() (byte, error) {
98		-	var buf [1]byte
99		-	if _, err := io.ReadFull(c.Reader, buf[:]); err != nil {
100		-	return 0, err
101		-	}
102		-	return buf[0], nil
103		-	}
104		-
105		-	func (c *Conn) ReadInt32() (int32, error) {
106		-	var buf [4]byte
107		-	if _, err := io.ReadFull(c.Reader, buf[:]); err != nil {
108		-	return 0, err
109		-	}
110		-	return int32(binary.LittleEndian.Uint32(buf[:])), nil
111		-	}
112		-
113		-	func (c *Conn) ReadInt64() (int64, error) {
114		-	{
115		-	data, err := c.ReadInt32()
116		-	if err != nil {
117		-	return 0, err
118		-	}
119		-	if data != -1 {
120		-	// The value was small enough to fit into a 32 bit int, so it was
121		-	// transferred directly.
122		-	return int64(data), nil
123		-	}
124		-	// Otherwise, -1 was transmitted, followed by the int64.
125		-	}
126		-	var data int64
127		-	if err := binary.Read(c.Reader, binary.LittleEndian, &data); err != nil {
128		-	return 0, err
129		-	}
130		-	return data, nil
131		-	}
132		-
133		-	type CountingReader struct {
134		-	R io.Reader
135		-	BytesRead int64
136		-	}
137		-
138		-	func (r *CountingReader) Read(p []byte) (n int, err error) {
139		-	n, err = r.R.Read(p)
140		-	r.BytesRead += int64(n)
141		-	return n, err
142		-	}
143		-
144		-	type CountingWriter struct {
145		-	W io.Writer
146		-	BytesWritten int64
147		-	}
148		-
149		-	func (w *CountingWriter) Write(p []byte) (n int, err error) {
150		-	n, err = w.W.Write(p)
151		-	w.BytesWritten += int64(n)
152		-	return n, err
153		-	}
154		-
155		-	func CounterPair(r io.Reader, w io.Writer) (CountingReader, CountingWriter) {
156		-	crd := &CountingReader{R: r}
157		-	cwr := &CountingWriter{W: w}
158		-	return crd, cwr
159		-	}

...	...		@@ -1,59 +0,0 @@
1		-	package utils
2		-
3		-	import (
4		-	"io"
5		-	"io/fs"
6		-	"sort"
7		-	"time"
8		-
9		-	"github.com/picosh/pico/pkg/rsync-receiver/rsync"
10		-	)
11		-
12		-	type SenderFile struct {
13		-	// TODO: store relative to the root to conserve RAM
14		-	Path string
15		-	WPath string
16		-	Regular bool
17		-	}
18		-
19		-	type ReceiverFile struct {
20		-	Name string
21		-	Length int64
22		-	ModTime time.Time
23		-	Mode int32
24		-	Uid int32
25		-	Gid int32
26		-	LinkTarget string
27		-	Rdev int32
28		-	Reader io.Reader
29		-	}
30		-
31		-	// FileMode converts from the Linux permission bits to Go’s permission bits.
32		-	func (f *ReceiverFile) FileMode() fs.FileMode {
33		-	ret := fs.FileMode(f.Mode) & fs.ModePerm
34		-
35		-	mode := f.Mode & rsync.S_IFMT
36		-	switch mode {
37		-	case rsync.S_IFCHR:
38		-	ret \|= fs.ModeCharDevice
39		-	case rsync.S_IFBLK:
40		-	ret \|= fs.ModeDevice
41		-	case rsync.S_IFIFO:
42		-	ret \|= fs.ModeNamedPipe
43		-	case rsync.S_IFSOCK:
44		-	ret \|= fs.ModeSocket
45		-	case rsync.S_IFLNK:
46		-	ret \|= fs.ModeSymlink
47		-	case rsync.S_IFDIR:
48		-	ret \|= fs.ModeDir
49		-	}
50		-
51		-	return ret
52		-	}
53		-
54		-	// rsync/flist.c:flist_sort_and_clean.
55		-	func SortFileList(fileList []*ReceiverFile) {
56		-	sort.Slice(fileList, func(i, j int) bool {
57		-	return fileList[i].Name < fileList[j].Name
58		-	})
59		-	}

...	...		@@ -1,20 +0,0 @@
1		-	package utils
2		-
3		-	import (
4		-	"io"
5		-	"os"
6		-	)
7		-
8		-	type ReaderAtCloser interface {
9		-	io.Reader
10		-	io.ReaderAt
11		-	io.Closer
12		-	}
13		-
14		-	// File System: need to handle all type of files: regular, folder, symlink, etc.
15		-	type FS interface {
16		-	Put(*ReceiverFile) (int64, error)
17		-	List(string) ([]os.FileInfo, error)
18		-	Read(*SenderFile) (os.FileInfo, ReaderAtCloser, error)
19		-	Remove([]*ReceiverFile) error
20		-	}

...	...		@@ -0,0 +1,172 @@
	1	+	package rsync
	2	+
	3	+	import (
	4	+	"crypto/md5"
	5	+	"encoding/binary"
	6	+	"fmt"
	7	+	"hash"
	8	+
	9	+	"github.com/cespare/xxhash/v2"
	10	+	"github.com/mmcloughlin/md4"
	11	+	)
	12	+
	13	+	// csumType identifies a strong checksum. md4Old is the implied checksum of
	14	+	// protocols 27-29, which seeds the whole-file sum; md4 is the same digest
	15	+	// when negotiated by name in protocol 30+.
	16	+	type csumType int
	17	+
	18	+	const (
	19	+	csumNone csumType = iota
	20	+	csumMD4Old
	21	+	csumMD4
	22	+	csumMD5
	23	+	csumXXH64
	24	+	)
	25	+
	26	+	// checksumNames is our preference order for negotiation. It matches rsync's
	27	+	// table order so both sides converge on the same choice.
	28	+	var checksumNames = []struct {
	29	+	name string
	30	+	typ csumType
	31	+	}{
	32	+	{"xxh64", csumXXH64},
	33	+	{"xxhash", csumXXH64},
	34	+	{"md5", csumMD5},
	35	+	{"md4", csumMD4},
	36	+	{"none", csumNone},
	37	+	}
	38	+
	39	+	func parseChecksumName(name string) (csumType, error) {
	40	+	for _, c := range checksumNames {
	41	+	if c.name == name {
	42	+	return c.typ, nil
	43	+	}
	44	+	}
	45	+	return 0, fmt.Errorf("unknown checksum name: %s", name)
	46	+	}
	47	+
	48	+	func (t csumType) size() int {
	49	+	switch t {
	50	+	case csumNone:
	51	+	return 1
	52	+	case csumXXH64:
	53	+	return 8
	54	+	default:
	55	+	return 16
	56	+	}
	57	+	}
	58	+
	59	+	// checksum1 is rsync's rolling weak checksum. The bytes are treated as
	60	+	// signed chars, as rsync's C implementation does.
	61	+	func checksum1(buf []byte) uint32 {
	62	+	var s1, s2 uint32
	63	+	i := 0
	64	+	for ; i < len(buf)-4; i += 4 {
	65	+	b0 := uint32(int8(buf[i]))
	66	+	b1 := uint32(int8(buf[i+1]))
	67	+	b2 := uint32(int8(buf[i+2]))
	68	+	b3 := uint32(int8(buf[i+3]))
	69	+	s2 += 4(s1+b0) + 3b1 + 2*b2 + b3
	70	+	s1 += b0 + b1 + b2 + b3
	71	+	}
	72	+	for ; i < len(buf); i++ {
	73	+	s1 += uint32(int8(buf[i]))
	74	+	s2 += s1
	75	+	}
	76	+	return (s1 & 0xffff) + (s2 << 16)
	77	+	}
	78	+
	79	+	// checksums holds the negotiated digests and the session seed.
	80	+	type checksums struct {
	81	+	xfer csumType
	82	+	file csumType
	83	+	seed int32
	84	+	properSeedOrder bool
	85	+	}
	86	+
	87	+	func (c *checksums) seedBytes() []byte {
	88	+	var b [4]byte
	89	+	binary.LittleEndian.PutUint32(b[:], uint32(c.seed))
	90	+	return b[:]
	91	+	}
	92	+
	93	+	// blockSum is rsync's get_checksum2: the strong checksum of one block.
	94	+	func (c *checksums) blockSum(buf []byte, out []byte) []byte {
	95	+	switch c.xfer {
	96	+	case csumXXH64:
	97	+	d := xxhash.NewWithSeed(uint64(int64(c.seed)))
	98	+	_, _ = d.Write(buf)
	99	+	return binary.LittleEndian.AppendUint64(out, d.Sum64())
	100	+	case csumMD5:
	101	+	h := md5.New()
	102	+	if c.properSeedOrder {
	103	+	if c.seed != 0 {
	104	+	h.Write(c.seedBytes())
	105	+	}
	106	+	h.Write(buf)
	107	+	} else {
	108	+	h.Write(buf)
	109	+	if c.seed != 0 {
	110	+	h.Write(c.seedBytes())
	111	+	}
	112	+	}
	113	+	return h.Sum(out)
	114	+	case csumMD4, csumMD4Old:
	115	+	h := md4.New()
	116	+	h.Write(buf)
	117	+	if c.seed != 0 {
	118	+	h.Write(c.seedBytes())
	119	+	}
	120	+	return h.Sum(out)
	121	+	default:
	122	+	return append(out, 0)
	123	+	}
	124	+	}
	125	+
	126	+	// newXferSum starts the whole-file checksum that follows each transferred
	127	+	// file (rsync's sum_init with the transfer digest).
	128	+	func (c *checksums) newXferSum() hash.Hash {
	129	+	switch c.xfer {
	130	+	case csumMD4Old:
	131	+	h := md4.New()
	132	+	h.Write(c.seedBytes())
	133	+	return h
	134	+	default:
	135	+	return newPlainHash(c.xfer)
	136	+	}
	137	+	}
	138	+
	139	+	// newFileSum starts the --checksum digest of a whole file.
	140	+	func (c *checksums) newFileSum() hash.Hash {
	141	+	return newPlainHash(c.file)
	142	+	}
	143	+
	144	+	func newPlainHash(t csumType) hash.Hash {
	145	+	switch t {
	146	+	case csumMD4, csumMD4Old:
	147	+	return md4.New()
	148	+	case csumMD5:
	149	+	return md5.New()
	150	+	case csumXXH64:
	151	+	return &xxh64LE{d: xxhash.NewWithSeed(0)}
	152	+	default:
	153	+	return &noneHash{}
	154	+	}
	155	+	}
	156	+
	157	+	// xxh64LE emits the digest little-endian, as rsync's SIVAL64 does.
	158	+	type xxh64LE struct{ d *xxhash.Digest }
	159	+
	160	+	func (x *xxh64LE) Write(p []byte) (int, error) { return x.d.Write(p) }
	161	+	func (x *xxh64LE) Sum(b []byte) []byte { return binary.LittleEndian.AppendUint64(b, x.d.Sum64()) }
	162	+	func (x *xxh64LE) Reset() { x.d.Reset() }
	163	+	func (x *xxh64LE) Size() int { return 8 }
	164	+	func (x *xxh64LE) BlockSize() int { return 32 }
	165	+
	166	+	type noneHash struct{}
	167	+
	168	+	func (noneHash) Write(p []byte) (int, error) { return len(p), nil }
	169	+	func (noneHash) Sum(b []byte) []byte { return append(b, 0) }
	170	+	func (noneHash) Reset() {}
	171	+	func (noneHash) Size() int { return 1 }
	172	+	func (noneHash) BlockSize() int { return 1 }

...	...		@@ -0,0 +1,681 @@
	1	+	package rsync
	2	+
	3	+	import (
	4	+	"bufio"
	5	+	"encoding/binary"
	6	+	"errors"
	7	+	"fmt"
	8	+	"io"
	9	+	"sync"
	10	+	)
	11	+
	12	+	// remoteExitError is returned when the peer aborts with MSG_ERROR_EXIT.
	13	+	type remoteExitError struct{ code int }
	14	+
	15	+	func (e *remoteExitError) Error() string {
	16	+	return fmt.Sprintf("remote side exited with code %d", e.code)
	17	+	}
	18	+
	19	+	// muxWriter queues protocol data and messages in order and writes them from
	20	+	// its own goroutine. Once multiplexing is on, data goes out as MSG_DATA
	21	+	// frames. Writers of data block when too much is queued; messages never
	22	+	// block, so the receiver can report progress while the generator waits on
	23	+	// the peer.
	24	+	type muxWriter struct {
	25	+	mu sync.Mutex
	26	+	cond *sync.Cond
	27	+	w io.Writer
	28	+	queue []muxItem
	29	+	queued int
	30	+	kick bool
	31	+	busy bool
	32	+	multiplex bool
	33	+	written int64
	34	+	err error
	35	+	}
	36	+
	37	+	type muxItem struct {
	38	+	tag byte
	39	+	data []byte
	40	+	}
	41	+
	42	+	const (
	43	+	muxBatchSize = 32 * 1024
	44	+	muxMaxQueued = 256 * 1024
	45	+	)
	46	+
	47	+	func newMuxWriter(w io.Writer) *muxWriter {
	48	+	m := &muxWriter{w: w}
	49	+	m.cond = sync.NewCond(&m.mu)
	50	+	go m.pump()
	51	+	return m
	52	+	}
	53	+
	54	+	func (m *muxWriter) Write(p []byte) (int, error) {
	55	+	m.mu.Lock()
	56	+	defer m.mu.Unlock()
	57	+	for m.err == nil && m.queued >= muxMaxQueued {
	58	+	m.kick = true
	59	+	m.cond.Broadcast()
	60	+	m.cond.Wait()
	61	+	}
	62	+	if m.err != nil {
	63	+	return 0, m.err
	64	+	}
	65	+	if n := len(m.queue); n > 0 && m.queue[n-1].tag == msgData && len(m.queue[n-1].data) < muxBatchSize {
	66	+	m.queue[n-1].data = append(m.queue[n-1].data, p...)
	67	+	} else {
	68	+	m.queue = append(m.queue, muxItem{tag: msgData, data: append([]byte(nil), p...)})
	69	+	}
	70	+	m.queued += len(p)
	71	+	if m.queued >= muxBatchSize {
	72	+	m.kick = true
	73	+	m.cond.Broadcast()
	74	+	}
	75	+	return len(p), nil
	76	+	}
	77	+
	78	+	// WriteMsg queues an out-of-band message behind any pending data.
	79	+	func (m *muxWriter) WriteMsg(tag byte, p []byte) error {
	80	+	m.mu.Lock()
	81	+	defer m.mu.Unlock()
	82	+	if m.err != nil {
	83	+	return m.err
	84	+	}
	85	+	if !m.multiplex {
	86	+	return errors.New("rsync: message sent before multiplexing started")
	87	+	}
	88	+	m.queue = append(m.queue, muxItem{tag: tag, data: append([]byte(nil), p...)})
	89	+	m.kick = true
	90	+	m.cond.Broadcast()
	91	+	return nil
	92	+	}
	93	+
	94	+	// Kick asks for queued output to go out without waiting for it.
	95	+	func (m *muxWriter) Kick() error {
	96	+	m.mu.Lock()
	97	+	defer m.mu.Unlock()
	98	+	if len(m.queue) > 0 {
	99	+	m.kick = true
	100	+	m.cond.Broadcast()
	101	+	}
	102	+	return m.err
	103	+	}
	104	+
	105	+	// Flush waits until everything queued has been written.
	106	+	func (m *muxWriter) Flush() error {
	107	+	m.mu.Lock()
	108	+	defer m.mu.Unlock()
	109	+	for m.err == nil && (len(m.queue) > 0 \|\| m.busy) {
	110	+	m.kick = true
	111	+	m.cond.Broadcast()
	112	+	m.cond.Wait()
	113	+	}
	114	+	return m.err
	115	+	}
	116	+
	117	+	// errMuxClosed is returned to writers that outlive Close.
	118	+	var errMuxClosed = errors.New("rsync: write after close")
	119	+
	120	+	// Close flushes and stops the writer goroutine. Later writes fail.
	121	+	func (m *muxWriter) Close() error {
	122	+	err := m.Flush()
	123	+	m.mu.Lock()
	124	+	if m.err == nil {
	125	+	m.err = errMuxClosed
	126	+	}
	127	+	m.cond.Broadcast()
	128	+	m.mu.Unlock()
	129	+	return err
	130	+	}
	131	+
	132	+	func (m *muxWriter) setMultiplex(on bool) error {
	133	+	if err := m.Flush(); err != nil {
	134	+	return err
	135	+	}
	136	+	m.mu.Lock()
	137	+	m.multiplex = on
	138	+	m.mu.Unlock()
	139	+	return nil
	140	+	}
	141	+
	142	+	func (m *muxWriter) pump() {
	143	+	m.mu.Lock()
	144	+	defer m.mu.Unlock()
	145	+	for {
	146	+	for m.err == nil && (!m.kick \|\| len(m.queue) == 0) {
	147	+	m.cond.Wait()
	148	+	}
	149	+	if m.err != nil {
	150	+	m.queue = nil
	151	+	m.cond.Broadcast()
	152	+	return
	153	+	}
	154	+	items := m.queue
	155	+	m.queue = nil
	156	+	m.queued = 0
	157	+	m.kick = false
	158	+	m.busy = true
	159	+	multiplex := m.multiplex
	160	+	m.cond.Broadcast()
	161	+	m.mu.Unlock()
	162	+
	163	+	var n int64
	164	+	var err error
	165	+	for _, it := range items {
	166	+	var k int64
	167	+	k, err = writeItem(m.w, it, multiplex)
	168	+	n += k
	169	+	if err != nil {
	170	+	break
	171	+	}
	172	+	}
	173	+
	174	+	m.mu.Lock()
	175	+	m.busy = false
	176	+	m.written += n
	177	+	if err != nil {
	178	+	m.err = err
	179	+	}
	180	+	m.cond.Broadcast()
	181	+	}
	182	+	}
	183	+
	184	+	func writeItem(w io.Writer, it muxItem, multiplex bool) (int64, error) {
	185	+	if !multiplex {
	186	+	n, err := w.Write(it.data)
	187	+	return int64(n), err
	188	+	}
	189	+	var total int64
	190	+	data := it.data
	191	+	for {
	192	+	chunk := data[:min(len(data), 0xFFFFFF)]
	193	+	var hdr [4]byte
	194	+	binary.LittleEndian.PutUint32(hdr[:], uint32(mplexBase+int(it.tag))<<24\|uint32(len(chunk)))
	195	+	n, err := w.Write(append(hdr[:], chunk...))
	196	+	total += int64(n)
	197	+	if err != nil {
	198	+	return total, err
	199	+	}
	200	+	data = data[len(chunk):]
	201	+	if len(data) == 0 {
	202	+	return total, nil
	203	+	}
	204	+	}
	205	+	}
	206	+
	207	+	func (m *muxWriter) bytesWritten() int64 {
	208	+	m.mu.Lock()
	209	+	defer m.mu.Unlock()
	210	+	n := m.written + int64(m.queued)
	211	+	for _, it := range m.queue {
	212	+	if it.tag != msgData {
	213	+	n += int64(len(it.data)) + 4
	214	+	}
	215	+	}
	216	+	return n
	217	+	}
	218	+
	219	+	// flushingReader sends our pending output before it blocks on the peer,
	220	+	// which is what keeps request/response exchanges from stalling.
	221	+	type flushingReader struct {
	222	+	r io.Reader
	223	+	flush func() error
	224	+	read int64
	225	+	}
	226	+
	227	+	func (f *flushingReader) Read(p []byte) (int, error) {
	228	+	if err := f.flush(); err != nil {
	229	+	return 0, err
	230	+	}
	231	+	n, err := f.r.Read(p)
	232	+	f.read += int64(n)
	233	+	return n, err
	234	+	}
	235	+
	236	+	// muxReader returns the data stream of a possibly multiplexed input and
	237	+	// hands every other message to onMsg.
	238	+	type muxReader struct {
	239	+	r *bufio.Reader
	240	+	src *flushingReader
	241	+	multiplex bool
	242	+	remaining int
	243	+	onMsg func(tag byte, data []byte) error
	244	+	}
	245	+
	246	+	func (m *muxReader) fill() error {
	247	+	for m.remaining == 0 {
	248	+	var hdr [4]byte
	249	+	if _, err := io.ReadFull(m.r, hdr[:]); err != nil {
	250	+	return err
	251	+	}
	252	+	v := binary.LittleEndian.Uint32(hdr[:])
	253	+	tag := int(v>>24) - mplexBase
	254	+	size := int(v & 0xFFFFFF)
	255	+	if tag == msgData {
	256	+	m.remaining = size
	257	+	continue
	258	+	}
	259	+	if tag < 0 {
	260	+	return fmt.Errorf("rsync: unexpected tag %d", tag)
	261	+	}
	262	+	data := make([]byte, size)
	263	+	if _, err := io.ReadFull(m.r, data); err != nil {
	264	+	return err
	265	+	}
	266	+	if err := m.onMsg(byte(tag), data); err != nil {
	267	+	return err
	268	+	}
	269	+	}
	270	+	return nil
	271	+	}
	272	+
	273	+	func (m *muxReader) Read(p []byte) (int, error) {
	274	+	if len(p) == 0 {
	275	+	return 0, nil
	276	+	}
	277	+	if !m.multiplex {
	278	+	return m.r.Read(p)
	279	+	}
	280	+	if err := m.fill(); err != nil {
	281	+	return 0, err
	282	+	}
	283	+	n, err := m.r.Read(p[:min(len(p), m.remaining)])
	284	+	m.remaining -= n
	285	+	return n, err
	286	+	}
	287	+
	288	+	func (m *muxReader) ReadByte() (byte, error) {
	289	+	if !m.multiplex {
	290	+	return m.r.ReadByte()
	291	+	}
	292	+	if err := m.fill(); err != nil {
	293	+	return 0, err
	294	+	}
	295	+	b, err := m.r.ReadByte()
	296	+	if err == nil {
	297	+	m.remaining--
	298	+	}
	299	+	return b, err
	300	+	}
	301	+
	302	+	// ndxState holds the delta-encoding state of one direction of write_ndx.
	303	+	type ndxState struct {
	304	+	prevPositive int32
	305	+	prevNegative int32
	306	+	}
	307	+
	308	+	func newNdxState() ndxState { return ndxState{prevPositive: -1, prevNegative: 1} }
	309	+
	310	+	// conn carries the rsync protocol over the session's stdin and stdout.
	311	+	type conn struct {
	312	+	protocol int
	313	+	r *muxReader
	314	+	w *muxWriter
	315	+	ndxIn ndxState
	316	+	ndxOut ndxState
	317	+	rbuf [16]byte
	318	+	wbuf [16]byte
	319	+	}
	320	+
	321	+	func newConn(rw io.ReadWriter) *conn {
	322	+	w := newMuxWriter(rw)
	323	+	src := &flushingReader{r: rw, flush: w.Kick}
	324	+	c := &conn{
	325	+	r: &muxReader{r: bufio.NewReaderSize(src, 64*1024), src: src},
	326	+	w: w,
	327	+	ndxIn: newNdxState(),
	328	+	ndxOut: newNdxState(),
	329	+	}
	330	+	c.r.onMsg = func(tag byte, data []byte) error {
	331	+	return fmt.Errorf("rsync: unexpected message %d", tag)
	332	+	}
	333	+	return c
	334	+	}
	335	+
	336	+	func (c *conn) bytesRead() int64 { return c.r.src.read }
	337	+	func (c *conn) bytesWritten() int64 { return c.w.bytesWritten() }
	338	+	func (c *conn) flush() error { return c.w.Flush() }
	339	+	func (c *conn) close() error { return c.w.Close() }
	340	+
	341	+	func (c *conn) readFull(p []byte) error {
	342	+	_, err := io.ReadFull(c.r, p)
	343	+	if errors.Is(err, io.EOF) {
	344	+	return io.ErrUnexpectedEOF
	345	+	}
	346	+	return err
	347	+	}
	348	+
	349	+	func (c *conn) readByte() (byte, error) {
	350	+	b, err := c.r.ReadByte()
	351	+	if errors.Is(err, io.EOF) {
	352	+	return 0, io.ErrUnexpectedEOF
	353	+	}
	354	+	return b, err
	355	+	}
	356	+
	357	+	func (c *conn) readInt32() (int32, error) {
	358	+	if err := c.readFull(c.rbuf[:4]); err != nil {
	359	+	return 0, err
	360	+	}
	361	+	return int32(binary.LittleEndian.Uint32(c.rbuf[:4])), nil
	362	+	}
	363	+
	364	+	func (c *conn) readShortint() (uint16, error) {
	365	+	if err := c.readFull(c.rbuf[:2]); err != nil {
	366	+	return 0, err
	367	+	}
	368	+	return binary.LittleEndian.Uint16(c.rbuf[:2]), nil
	369	+	}
	370	+
	371	+	// readLongint reads the pre-30 64-bit encoding: an int32, or -1 followed by
	372	+	// the full 64-bit value.
	373	+	func (c *conn) readLongint() (int64, error) {
	374	+	v, err := c.readInt32()
	375	+	if err != nil \|\| v != -1 {
	376	+	return int64(v), err
	377	+	}
	378	+	if err := c.readFull(c.rbuf[:8]); err != nil {
	379	+	return 0, err
	380	+	}
	381	+	return int64(binary.LittleEndian.Uint64(c.rbuf[:8])), nil
	382	+	}
	383	+
	384	+	// intByteExtra maps the high bits of a varint's first byte to the number of
	385	+	// bytes that follow it.
	386	+	var intByteExtra = [64]int{
	387	+	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
	388	+	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
	389	+	1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
	390	+	2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 4, 4, 5, 6,
	391	+	}
	392	+
	393	+	func (c *conn) readVarint() (int32, error) {
	394	+	ch, err := c.readByte()
	395	+	if err != nil {
	396	+	return 0, err
	397	+	}
	398	+	var b [5]byte
	399	+	extra := intByteExtra[ch/4]
	400	+	if extra == 0 {
	401	+	return int32(ch), nil
	402	+	}
	403	+	if extra >= len(b) {
	404	+	return 0, errors.New("rsync: overflow in varint")
	405	+	}
	406	+	if err := c.readFull(b[:extra]); err != nil {
	407	+	return 0, err
	408	+	}
	409	+	bit := byte(1) << (8 - extra)
	410	+	b[extra] = ch & (bit - 1)
	411	+	return int32(binary.LittleEndian.Uint32(b[:4])), nil
	412	+	}
	413	+
	414	+	func (c *conn) readVarlong(minBytes int) (int64, error) {
	415	+	var b2 [8]byte
	416	+	if err := c.readFull(b2[:minBytes]); err != nil {
	417	+	return 0, err
	418	+	}
	419	+	var b [9]byte
	420	+	copy(b[:], b2[1:minBytes])
	421	+	extra := intByteExtra[b2[0]/4]
	422	+	if extra == 0 {
	423	+	b[minBytes-1] = b2[0]
	424	+	return int64(binary.LittleEndian.Uint64(b[:8])), nil
	425	+	}
	426	+	if minBytes+extra > len(b) {
	427	+	return 0, errors.New("rsync: overflow in varlong")
	428	+	}
	429	+	if err := c.readFull(b[minBytes-1 : minBytes-1+extra]); err != nil {
	430	+	return 0, err
	431	+	}
	432	+	bit := byte(1) << (8 - extra)
	433	+	b[minBytes+extra-1] = b2[0] & (bit - 1)
	434	+	return int64(binary.LittleEndian.Uint64(b[:8])), nil
	435	+	}
	436	+
	437	+	func (c *conn) readVarint30() (int32, error) {
	438	+	if c.protocol < 30 {
	439	+	return c.readInt32()
	440	+	}
	441	+	return c.readVarint()
	442	+	}
	443	+
	444	+	func (c *conn) readVarlong30(minBytes int) (int64, error) {
	445	+	if c.protocol < 30 {
	446	+	return c.readLongint()
	447	+	}
	448	+	return c.readVarlong(minBytes)
	449	+	}
	450	+
	451	+	func (c *conn) readVstring(max int) (string, error) {
	452	+	b, err := c.readByte()
	453	+	if err != nil {
	454	+	return "", err
	455	+	}
	456	+	n := int(b)
	457	+	if n&0x80 != 0 {
	458	+	lo, err := c.readByte()
	459	+	if err != nil {
	460	+	return "", err
	461	+	}
	462	+	n = (n&^0x80)*0x100 + int(lo)
	463	+	}
	464	+	if n >= max {
	465	+	return "", fmt.Errorf("rsync: over-long vstring received (%d > %d)", n, max-1)
	466	+	}
	467	+	buf := make([]byte, n)
	468	+	if err := c.readFull(buf); err != nil {
	469	+	return "", err
	470	+	}
	471	+	return string(buf), nil
	472	+	}
	473	+
	474	+	// readNdx reads a file-list index, which protocol 30 delta-encodes.
	475	+	func (c *conn) readNdx() (int32, error) {
	476	+	if c.protocol < 30 {
	477	+	return c.readInt32()
	478	+	}
	479	+	b0, err := c.readByte()
	480	+	if err != nil {
	481	+	return 0, err
	482	+	}
	483	+	prev := &c.ndxIn.prevPositive
	484	+	switch b0 {
	485	+	case 0:
	486	+	return ndxDone, nil
	487	+	case 0xFF:
	488	+	prev = &c.ndxIn.prevNegative
	489	+	if b0, err = c.readByte(); err != nil {
	490	+	return 0, err
	491	+	}
	492	+	}
	493	+	var unum uint32
	494	+	if b0 == 0xFE {
	495	+	var b [4]byte
	496	+	if err := c.readFull(b[:2]); err != nil {
	497	+	return 0, err
	498	+	}
	499	+	if b[0]&0x80 != 0 {
	500	+	b[3] = b[0] &^ 0x80
	501	+	b[0] = b[1]
	502	+	if err := c.readFull(b[1:3]); err != nil {
	503	+	return 0, err
	504	+	}
	505	+	unum = binary.LittleEndian.Uint32(b[:])
	506	+	} else {
	507	+	unum = uint32(b[0])<<8 + uint32(b[1]) + uint32(*prev)
	508	+	}
	509	+	} else {
	510	+	unum = uint32(b0) + uint32(*prev)
	511	+	}
	512	+	if unum > 0x7FFFFFFF {
	513	+	return 0, fmt.Errorf("rsync: invalid file index %d", unum)
	514	+	}
	515	+	num := int32(unum)
	516	+	*prev = num
	517	+	if prev == &c.ndxIn.prevNegative {
	518	+	num = -num
	519	+	}
	520	+	return num, nil
	521	+	}
	522	+
	523	+	func (c *conn) write(p []byte) error {
	524	+	_, err := c.w.Write(p)
	525	+	return err
	526	+	}
	527	+
	528	+	func (c *conn) writeByte(b byte) error {
	529	+	return c.write([]byte{b})
	530	+	}
	531	+
	532	+	func (c *conn) writeInt32(v int32) error {
	533	+	var b [4]byte
	534	+	binary.LittleEndian.PutUint32(b[:], uint32(v))
	535	+	return c.write(b[:])
	536	+	}
	537	+
	538	+	func (c *conn) writeShortint(v uint16) error {
	539	+	var b [2]byte
	540	+	binary.LittleEndian.PutUint16(b[:], v)
	541	+	return c.write(b[:])
	542	+	}
	543	+
	544	+	func (c *conn) writeLongint(v int64) error {
	545	+	if v >= 0 && v <= 0x7FFFFFFF {
	546	+	return c.writeInt32(int32(v))
	547	+	}
	548	+	var b [12]byte
	549	+	binary.LittleEndian.PutUint32(b[:4], 0xFFFFFFFF)
	550	+	binary.LittleEndian.PutUint64(b[4:], uint64(v))
	551	+	return c.write(b[:])
	552	+	}
	553	+
	554	+	func appendVarint(dst []byte, x int32) []byte {
	555	+	var b [5]byte
	556	+	binary.LittleEndian.PutUint32(b[1:], uint32(x))
	557	+	cnt := 4
	558	+	for cnt > 1 && b[cnt] == 0 {
	559	+	cnt--
	560	+	}
	561	+	bit := byte(1) << (7 - cnt + 1)
	562	+	switch {
	563	+	case b[cnt] >= bit:
	564	+	cnt++
	565	+	b[0] = ^(bit - 1)
	566	+	case cnt > 1:
	567	+	b[0] = b[cnt] \| ^(bit*2 - 1)
	568	+	default:
	569	+	b[0] = b[1]
	570	+	}
	571	+	return append(dst, b[:cnt]...)
	572	+	}
	573	+
	574	+	func appendVarlong(dst []byte, x int64, minBytes int) []byte {
	575	+	var b [9]byte
	576	+	binary.LittleEndian.PutUint64(b[1:], uint64(x))
	577	+	cnt := 8
	578	+	for cnt > minBytes && b[cnt] == 0 {
	579	+	cnt--
	580	+	}
	581	+	bit := byte(1) << (7 - cnt + minBytes)
	582	+	switch {
	583	+	case b[cnt] >= bit:
	584	+	cnt++
	585	+	b[0] = ^(bit - 1)
	586	+	case cnt > minBytes:
	587	+	b[0] = b[cnt] \| ^(bit*2 - 1)
	588	+	default:
	589	+	b[0] = b[cnt]
	590	+	}
	591	+	return append(dst, b[:cnt]...)
	592	+	}
	593	+
	594	+	func (c *conn) writeVarint(v int32) error {
	595	+	return c.write(appendVarint(c.wbuf[:0], v))
	596	+	}
	597	+
	598	+	func (c *conn) writeVarlong(v int64, minBytes int) error {
	599	+	return c.write(appendVarlong(c.wbuf[:0], v, minBytes))
	600	+	}
	601	+
	602	+	func (c *conn) writeVarint30(v int32) error {
	603	+	if c.protocol < 30 {
	604	+	return c.writeInt32(v)
	605	+	}
	606	+	return c.writeVarint(v)
	607	+	}
	608	+
	609	+	func (c *conn) writeVarlong30(v int64, minBytes int) error {
	610	+	if c.protocol < 30 {
	611	+	return c.writeLongint(v)
	612	+	}
	613	+	return c.writeVarlong(v, minBytes)
	614	+	}
	615	+
	616	+	func (c *conn) writeVstring(s string) error {
	617	+	n := len(s)
	618	+	if n > 0x7FFF {
	619	+	return fmt.Errorf("rsync: attempting to send over-long vstring (%d > %d)", n, 0x7FFF)
	620	+	}
	621	+	var hdr []byte
	622	+	if n > 0x7F {
	623	+	hdr = []byte{byte(n/0x100 + 0x80), byte(n)}
	624	+	} else {
	625	+	hdr = []byte{byte(n)}
	626	+	}
	627	+	if err := c.write(hdr); err != nil {
	628	+	return err
	629	+	}
	630	+	return c.write([]byte(s))
	631	+	}
	632	+
	633	+	func (c *conn) writeNdx(ndx int32) error {
	634	+	if c.protocol < 30 {
	635	+	return c.writeInt32(ndx)
	636	+	}
	637	+	var b [6]byte
	638	+	cnt := 0
	639	+	var diff int32
	640	+	switch {
	641	+	case ndx >= 0:
	642	+	diff = ndx - c.ndxOut.prevPositive
	643	+	c.ndxOut.prevPositive = ndx
	644	+	case ndx == ndxDone:
	645	+	return c.writeByte(0)
	646	+	default:
	647	+	b[cnt] = 0xFF
	648	+	cnt++
	649	+	ndx = -ndx
	650	+	diff = ndx - c.ndxOut.prevNegative
	651	+	c.ndxOut.prevNegative = ndx
	652	+	}
	653	+	switch {
	654	+	case diff > 0 && diff < 0xFE:
	655	+	b[cnt] = byte(diff)
	656	+	cnt++
	657	+	case diff < 0 \|\| diff > 0x7FFF:
	658	+	b[cnt] = 0xFE
	659	+	b[cnt+1] = byte(ndx>>24) \| 0x80
	660	+	b[cnt+2] = byte(ndx)
	661	+	b[cnt+3] = byte(ndx >> 8)
	662	+	b[cnt+4] = byte(ndx >> 16)
	663	+	cnt += 5
	664	+	default:
	665	+	b[cnt] = 0xFE
	666	+	b[cnt+1] = byte(diff >> 8)
	667	+	b[cnt+2] = byte(diff)
	668	+	cnt += 3
	669	+	}
	670	+	return c.write(b[:cnt])
	671	+	}
	672	+
	673	+	func (c *conn) writeMsg(tag byte, data []byte) error {
	674	+	return c.w.WriteMsg(tag, data)
	675	+	}
	676	+
	677	+	func (c *conn) writeMsgInt(tag byte, v int32) error {
	678	+	var b [4]byte
	679	+	binary.LittleEndian.PutUint32(b[:], uint32(v))
	680	+	return c.w.WriteMsg(tag, b[:])
	681	+	}

...	...		@@ -0,0 +1,133 @@
	1	+	package rsync
	2	+
	3	+	// Protocol versions this implementation speaks. 27 is rsync 2.6.0 and 31 is
	4	+	// what every rsync since 3.1.0 negotiates down to.
	5	+	const (
	6	+	minProtocol = 27
	7	+	maxProtocol = 31
	8	+	)
	9	+
	10	+	// File-list transmission flags (rsync.h XMIT_*).
	11	+	const (
	12	+	xmitTopDir = 1 << 0
	13	+	xmitSameMode = 1 << 1
	14	+	xmitSameRdevPre28 = 1 << 2
	15	+	xmitExtendedFlags = 1 << 2
	16	+	xmitSameUID = 1 << 3
	17	+	xmitSameGID = 1 << 4
	18	+	xmitSameName = 1 << 5
	19	+	xmitLongName = 1 << 6
	20	+	xmitSameTime = 1 << 7
	21	+	xmitSameRdevMajor = 1 << 8
	22	+	xmitNoContentDir = 1 << 8
	23	+	xmitHlinked = 1 << 9
	24	+	xmitUserNameFollows = 1 << 10
	25	+	xmitRdevMinor8Pre30 = 1 << 11
	26	+	xmitGroupNameFollows = 1 << 11
	27	+	xmitHlinkFirst = 1 << 12
	28	+	xmitIOErrorEndList = 1 << 12
	29	+	xmitModNsec = 1 << 13
	30	+	xmitSameAtime = 1 << 14
	31	+	)
	32	+
	33	+	// Sizes and limits (rsync.h).
	34	+	const (
	35	+	maxPathLen = 4096
	36	+	maxWireNsec = 999999999
	37	+	blockSize = 700
	38	+	maxBlockSize = 1 << 17
	39	+	oldMaxBlockSize = 1 << 29
	40	+	chunkSize = 32 * 1024
	41	+	maxMapSize = 256 * 1024
	42	+	sumLength = 16
	43	+	shortSumLength = 2
	44	+	blocksumBias = 10
	45	+	maxDataCount = 16383
	46	+	mplexBase = 7
	47	+	maxChainLen = 1024
	48	+	traditionalTableSize = 1 << 16
	49	+	)
	50	+
	51	+	// Unix file type bits as sent on the wire.
	52	+	const (
	53	+	sIFMT = 0o170000
	54	+	sIFSOCK = 0o140000
	55	+	sIFLNK = 0o120000
	56	+	sIFREG = 0o100000
	57	+	sIFBLK = 0o060000
	58	+	sIFDIR = 0o040000
	59	+	sIFCHR = 0o020000
	60	+	sIFIFO = 0o010000
	61	+	)
	62	+
	63	+	// Item flags sent alongside a file index (rsync.h ITEM_*).
	64	+	const (
	65	+	itemReportChange = 1 << 1
	66	+	itemReportSize = 1 << 2
	67	+	itemReportTime = 1 << 3
	68	+	itemBasisTypeFollows = 1 << 11
	69	+	itemXnameFollows = 1 << 12
	70	+	itemIsNew = 1 << 13
	71	+	itemTransfer = 1 << 15
	72	+	itemMissingData = 1 << 16
	73	+	)
	74	+
	75	+	// Basis file types sent with itemBasisTypeFollows.
	76	+	const (
	77	+	fnamecmpFname = 0x80
	78	+	fnamecmpPartialDir = 0x81
	79	+	)
	80	+
	81	+	// Multiplexed message tags (rsync.h enum msgcode).
	82	+	const (
	83	+	msgData = 0
	84	+	msgErrorXfer = 1
	85	+	msgInfo = 2
	86	+	msgError = 3
	87	+	msgWarning = 4
	88	+	msgIOError = 22
	89	+	msgNoop = 42
	90	+	msgErrorExit = 86
	91	+	msgSuccess = 100
	92	+	msgDeleted = 101
	93	+	msgNoSend = 102
	94	+	)
	95	+
	96	+	// Special file-list index values.
	97	+	const (
	98	+	ndxDone = -1
	99	+	ndxFlistEOF = -2
	100	+	ndxDelStats = -3
	101	+	)
	102	+
	103	+	// I/O error bits reported at the end of the file list.
	104	+	const (
	105	+	ioErrGeneral = 1 << 0
	106	+	ioErrVanished = 1 << 1
	107	+	ioErrDelLimit = 1 << 2
	108	+	ioErrMask = ioErrGeneral \| ioErrVanished \| ioErrDelLimit
	109	+	)
	110	+
	111	+	// Compat flags a protocol 30+ server sends. We never offer incremental
	112	+	// recursion, symlink times or iconv.
	113	+	const (
	114	+	cfSafeFlist = 1 << 3
	115	+	cfAvoidXattrOptim = 1 << 4
	116	+	cfChksumSeedFix = 1 << 5
	117	+	cfInplacePartialDir = 1 << 6
	118	+	cfVarintFlistFlags = 1 << 7
	119	+	cfID0Names = 1 << 8
	120	+	)
	121	+
	122	+	// Exit codes (errcode.h).
	123	+	const (
	124	+	exitSyntax = 1
	125	+	exitProtocol = 2
	126	+	exitFileSelect = 3
	127	+	exitUnsupported = 4
	128	+	exitFileIO = 11
	129	+	exitStreamIO = 12
	130	+	exitPartial = 23
	131	+	exitVanished = 24
	132	+	exitDelLimit = 25
	133	+	)

...	...		@@ -0,0 +1,8 @@
	1	+	// Package rsync implements the server side of the rsync protocol, the end
	2	+	// an rsync client talks to over SSH ("rsync --server ..."). It speaks
	3	+	// protocol versions 27 through 31, negotiates xxh64, md5 or md4 block
	4	+	// checksums and zlib or zlibx compression, sends and receives deltas
	5	+	// against existing files, and supports --checksum and the --delete modes.
	6	+	//
	7	+	// Files live in an FS, which the pico services back with their storage.
	8	+	package rsync

...	...		@@ -0,0 +1,930 @@
	1	+	package rsync
	2	+
	3	+	import (
	4	+	"errors"
	5	+	"fmt"
	6	+	"io/fs"
	7	+	"path"
	8	+	"slices"
	9	+	"strings"
	10	+	)
	11	+
	12	+	// Filter rule flags (rsync.h FILTRULE_*).
	13	+	const (
	14	+	ruleWild = 1 << iota
	15	+	ruleWild2
	16	+	ruleWild2Prefix
	17	+	ruleWild3Suffix
	18	+	ruleAbsPath
	19	+	ruleInclude
	20	+	ruleDirectory
	21	+	ruleWordSplit
	22	+	ruleNoInherit
	23	+	ruleNoPrefixes
	24	+	ruleMergeFile
	25	+	rulePerdirMerge
	26	+	ruleExcludeSelf
	27	+	ruleFinishSetup
	28	+	ruleNegate
	29	+	ruleCVSIgnore
	30	+	ruleSenderSide
	31	+	ruleReceiverSide
	32	+	ruleClearList
	33	+	rulePerishable
	34	+	ruleXattr
	35	+	)
	36	+
	37	+	// containerFlags are the flags rules in a merge file take from the merge
	38	+	// rule (FILTRULES_FROM_CONTAINER).
	39	+	const containerFlags = ruleAbsPath \| ruleInclude \| ruleDirectory \| ruleNegate \| rulePerishable
	40	+
	41	+	const sideFlags = ruleSenderSide \| ruleReceiverSide
	42	+
	43	+	// maxMergeFileSize bounds a per-directory merge file read from storage.
	44	+	const maxMergeFileSize = 1 << 20
	45	+
	46	+	type filterRule struct {
	47	+	pattern string
	48	+	flags int
	49	+	slashCnt int
	50	+	// merge indexes filterList.merges for a per-directory merge rule.
	51	+	merge int
	52	+	}
	53	+
	54	+	// filterList holds the rules the client sent. The client only sends the
	55	+	// rules that apply to our side of the transfer. A per-directory merge rule
	56	+	// stands for the rules read from that file in each directory, which a
	57	+	// filterScope holds.
	58	+	type filterList struct {
	59	+	rules []*filterRule
	60	+	merges []*filterRule
	61	+	// otherSide marks merge-file rules that only apply to the client.
	62	+	otherSide int
	63	+	deleteExcluded bool
	64	+	cvsPerishable bool
	65	+	}
	66	+
	67	+	// filterScope is the per-directory merge-file rules in effect in one
	68	+	// directory, each list holding the directory's own rules before the
	69	+	// inherited ones.
	70	+	type filterScope struct {
	71	+	// root is the storage path of the transfer root, which absolute
	72	+	// patterns are matched from.
	73	+	root string
	74	+	lists [][]*filterRule
	75	+	}
	76	+
	77	+	// mergeFileError is a per-directory merge file that could not be used.
	78	+	// Skipping its rules could delete or send files they cover, so callers
	79	+	// stop instead.
	80	+	type mergeFileError struct {
	81	+	name string
	82	+	err error
	83	+	}
	84	+
	85	+	func (e *mergeFileError) Error() string {
	86	+	return fmt.Sprintf("per-directory filter file %q: %v", "/"+e.name, e.err)
	87	+	}
	88	+
	89	+	func (e *mergeFileError) Unwrap() error { return e.err }
	90	+
	91	+	// defaultCVSIgnore is the list rsync's -C adds.
	92	+	const defaultCVSIgnore = "RCS SCCS CVS CVS.adm RCSLOG cvslog.* tags TAGS" +
	93	+	" .make.state .nse_depinfo ~ # .#* ,* _$* *$" +
	94	+	" .old .bak .BAK .orig .rej .del-" +
	95	+	" .a .olb .o .obj .so .exe" +
	96	+	" .Z .elc *.ln core" +
	97	+	" .svn/ .git/ .hg/ .bzr/"
	98	+
	99	+	var longRuleNames = map[byte]struct {
	100	+	word string
	101	+	ch byte
	102	+	}{
	103	+	'c': {"clear", '!'},
	104	+	'd': {"dir-merge", ':'},
	105	+	'e': {"exclude", '-'},
	106	+	'h': {"hide", 'H'},
	107	+	'i': {"include", '+'},
	108	+	'm': {"merge", '.'},
	109	+	'p': {"protect", 'P'},
	110	+	'r': {"risk", 'R'},
	111	+	's': {"show", 'S'},
	112	+	}
	113	+
	114	+	func isSpace(c byte) bool { return c == ' ' \|\| (c >= '\t' && c <= '\r') }
	115	+
	116	+	// parseRuleTok parses one rule (parse_rule_tok). tmpl is the flags of the
	117	+	// merge rule whose file the rule was read from, or 0 for the client's
	118	+	// rules. It returns a nil rule for an empty one.
	119	+	func parseRuleTok(s string, tmpl int, oldPrefixes bool) (*filterRule, error) {
	120	+	if tmpl&ruleWordSplit != 0 {
	121	+	s = strings.TrimLeftFunc(s, func(r rune) bool { return r < 0x80 && isSpace(byte(r)) })
	122	+	}
	123	+	if s == "" {
	124	+	return nil, nil
	125	+	}
	126	+	rule := &filterRule{flags: tmpl & containerFlags}
	127	+	switch {
	128	+	case tmpl&ruleNoPrefixes != 0:
	129	+	if s[0] == '!' && tmpl&ruleCVSIgnore != 0 {
	130	+	rule.flags \|= ruleClearList
	131	+	}
	132	+	case oldPrefixes:
	133	+	switch {
	134	+	case strings.HasPrefix(s, "- "):
	135	+	rule.flags &^= ruleInclude
	136	+	s = s[2:]
	137	+	case strings.HasPrefix(s, "+ "):
	138	+	rule.flags \|= ruleInclude
	139	+	s = s[2:]
	140	+	case s[0] == '!':
	141	+	rule.flags \|= ruleClearList
	142	+	}
	143	+	default:
	144	+	// p ends on the last character of the rule name, as the modifier
	145	+	// loop starts by stepping past it.
	146	+	ch, p := s[0], 0
	147	+	if long, ok := longRuleNames[s[0]]; ok {
	148	+	rest, found := strings.CutPrefix(s, long.word)
	149	+	switch {
	150	+	case !found:
	151	+	return nil, errors.New("unknown filter rule")
	152	+	case rest == "" \|\| rest[0] == '_' \|\| isSpace(rest[0]):
	153	+	ch, p = long.ch, len(long.word)-1
	154	+	case rest[0] == ',':
	155	+	ch, p = long.ch, len(long.word)
	156	+	default:
	157	+	return nil, errors.New("unknown filter rule")
	158	+	}
	159	+	} else if len(s) > 1 && s[1] == ',' {
	160	+	p = 1
	161	+	}
	162	+	sideFromPrefix := false
	163	+	switch ch {
	164	+	case ':':
	165	+	rule.flags \|= rulePerdirMerge \| ruleFinishSetup \| ruleMergeFile
	166	+	case '.':
	167	+	rule.flags \|= ruleMergeFile
	168	+	case '+':
	169	+	rule.flags \|= ruleInclude
	170	+	case '-':
	171	+	case 'S':
	172	+	rule.flags \|= ruleInclude \| ruleSenderSide
	173	+	sideFromPrefix = true
	174	+	case 'H':
	175	+	rule.flags \|= ruleSenderSide
	176	+	sideFromPrefix = true
	177	+	case 'R':
	178	+	rule.flags \|= ruleInclude \| ruleReceiverSide
	179	+	sideFromPrefix = true
	180	+	case 'P':
	181	+	rule.flags \|= ruleReceiverSide
	182	+	sideFromPrefix = true
	183	+	case '!':
	184	+	rule.flags \|= ruleClearList
	185	+	default:
	186	+	return nil, errors.New("unknown filter rule")
	187	+	}
	188	+	merge := rule.flags&ruleMergeFile != 0
	189	+	for ch != '!' {
	190	+	p++
	191	+	if p >= len(s) \|\| s[p] == ' ' \|\| s[p] == '_' {
	192	+	break
	193	+	}
	194	+	if tmpl&ruleWordSplit != 0 && isSpace(s[p]) {
	195	+	p--
	196	+	break
	197	+	}
	198	+	valid := true
	199	+	switch s[p] {
	200	+	case '-':
	201	+	valid = merge && rule.flags&ruleNoPrefixes == 0
	202	+	rule.flags \|= ruleNoPrefixes
	203	+	case '+':
	204	+	valid = merge && rule.flags&ruleNoPrefixes == 0
	205	+	rule.flags \|= ruleNoPrefixes \| ruleInclude
	206	+	case '/':
	207	+	rule.flags \|= ruleAbsPath
	208	+	case '!':
	209	+	valid = !merge
	210	+	rule.flags \|= ruleNegate
	211	+	case 'C':
	212	+	valid = rule.flags&ruleNoPrefixes == 0 && !sideFromPrefix
	213	+	rule.flags \|= ruleNoPrefixes \| ruleWordSplit \| ruleNoInherit \| ruleCVSIgnore
	214	+	case 'e':
	215	+	valid = merge
	216	+	rule.flags \|= ruleExcludeSelf
	217	+	case 'n':
	218	+	valid = merge
	219	+	rule.flags \|= ruleNoInherit
	220	+	case 'p':
	221	+	rule.flags \|= rulePerishable
	222	+	case 'r':
	223	+	valid = !sideFromPrefix
	224	+	rule.flags \|= ruleReceiverSide
	225	+	case 's':
	226	+	valid = !sideFromPrefix
	227	+	rule.flags \|= ruleSenderSide
	228	+	case 'w':
	229	+	valid = merge
	230	+	rule.flags \|= ruleWordSplit
	231	+	case 'x':
	232	+	rule.flags \|= ruleXattr
	233	+	default:
	234	+	valid = false
	235	+	}
	236	+	if !valid {
	237	+	return nil, fmt.Errorf("invalid modifier %q in filter rule", s[p])
	238	+	}
	239	+	}
	240	+	if p < len(s) {
	241	+	p++
	242	+	}
	243	+	s = s[min(p, len(s)):]
	244	+	}
	245	+	if tmpl&sideFlags != 0 {
	246	+	if rule.flags&sideFlags != 0 {
	247	+	return nil, errors.New("specified-side merge file contains specified-side filter")
	248	+	}
	249	+	rule.flags \|= tmpl & sideFlags
	250	+	}
	251	+
	252	+	if tmpl&ruleWordSplit != 0 {
	253	+	if i := strings.IndexFunc(s, func(r rune) bool { return r < 0x80 && isSpace(byte(r)) }); i >= 0 {
	254	+	s = s[:i]
	255	+	}
	256	+	}
	257	+	if rule.flags&ruleClearList != 0 {
	258	+	if tmpl&ruleNoPrefixes == 0 && !oldPrefixes && s != "" {
	259	+	return nil, errors.New("'!' rule has trailing characters")
	260	+	}
	261	+	if len(s) > 1 {
	262	+	rule.flags &^= ruleClearList
	263	+	}
	264	+	} else if s == "" && rule.flags&ruleCVSIgnore == 0 {
	265	+	return nil, errors.New("unexpected end of filter rule")
	266	+	}
	267	+	if len(s) >= maxPathLen {
	268	+	return nil, errors.New("filter rule too long")
	269	+	}
	270	+	rule.pattern = s
	271	+	return rule, nil
	272	+	}
	273	+
	274	+	// parse adds the rules in one line received from the client. Before
	275	+	// protocol 29 only the "- " and "+ " prefixes exist.
	276	+	func (l *filterList) parse(line string, oldPrefixes bool) error {
	277	+	rule, err := parseRuleTok(line, 0, oldPrefixes)
	278	+	switch {
	279	+	case err != nil:
	280	+	return fmt.Errorf("%w: %q", err, line)
	281	+	case rule == nil:
	282	+	return nil
	283	+	case rule.flags&ruleClearList != 0:
	284	+	l.rules, l.merges = nil, nil
	285	+	return nil
	286	+	case rule.flags&ruleMergeFile != 0:
	287	+	if rule.flags&rulePerdirMerge == 0 {
	288	+	// Clients read merge files themselves and send their rules.
	289	+	return fmt.Errorf("merge-file filter rules are not supported: %q", line)
	290	+	}
	291	+	return l.addMerge(rule)
	292	+	case rule.flags&ruleCVSIgnore != 0:
	293	+	l.rules = append(l.rules, l.cvsIgnoreRules()...)
	294	+	return nil
	295	+	}
	296	+	l.add(rule.pattern, rule.flags)
	297	+	return nil
	298	+	}
	299	+
	300	+	// addMerge adds a per-directory merge rule. Only the file's base name is
	301	+	// read in each directory; a directory part names where the scan of parent
	302	+	// directories starts (setup_merge_file).
	303	+	func (l filterList) addMerge(rule filterRule) error {
	304	+	if rule.pattern == "" {
	305	+	rule.pattern = ".cvsignore"
	306	+	}
	307	+	base := path.Base(rule.pattern)
	308	+	if strings.HasSuffix(rule.pattern, "/") \|\| base == "." \|\| base == ".." {
	309	+	return fmt.Errorf("invalid merge-file name: %q", rule.pattern)
	310	+	}
	311	+	if rule.flags&ruleExcludeSelf != 0 {
	312	+	l.add(base, 0)
	313	+	rule.flags &^= ruleExcludeSelf
	314	+	}
	315	+	for _, m := range l.merges {
	316	+	if path.Base(m.pattern) == base {
	317	+	return nil
	318	+	}
	319	+	}
	320	+	rule.merge = len(l.merges)
	321	+	l.rules = append(l.rules, rule)
	322	+	l.merges = append(l.merges, rule)
	323	+	return nil
	324	+	}
	325	+
	326	+	// cvsIgnoreRules returns rsync's default CVS excludes, which are
	327	+	// perishable from protocol 30 (get_cvs_excludes).
	328	+	func (l filterList) cvsIgnoreRules() []filterRule {
	329	+	flags := ruleNoPrefixes
	330	+	if l.cvsPerishable {
	331	+	flags \|= rulePerishable
	332	+	}
	333	+	var rules []*filterRule
	334	+	for _, pat := range strings.Fields(defaultCVSIgnore) {
	335	+	rules = append(rules, newRule(pat, flags))
	336	+	}
	337	+	return rules
	338	+	}
	339	+
	340	+	func newRule(pat string, flags int) *filterRule {
	341	+	rule := &filterRule{flags: flags}
	342	+	if len(pat) > 1 && strings.HasSuffix(pat, "/") {
	343	+	pat = pat[:len(pat)-1]
	344	+	rule.flags \|= ruleDirectory
	345	+	}
	346	+	rule.slashCnt = strings.Count(pat, "/")
	347	+	rule.pattern = pat
	348	+	if strings.ContainsAny(pat, "*[?") {
	349	+	rule.flags \|= ruleWild
	350	+	if i := strings.Index(pat, "**"); i >= 0 {
	351	+	rule.flags \|= ruleWild2
	352	+	if i == 0 {
	353	+	rule.flags \|= ruleWild2Prefix
	354	+	}
	355	+	if strings.HasSuffix(pat, "***") {
	356	+	rule.flags \|= ruleWild3Suffix
	357	+	}
	358	+	}
	359	+	}
	360	+	return rule
	361	+	}
	362	+
	363	+	func (l *filterList) add(pat string, flags int) {
	364	+	l.rules = append(l.rules, newRule(pat, flags))
	365	+	}
	366	+
	367	+	// scopeTree builds the filter scope of each directory below a traversal
	368	+	// root the first time it is needed.
	369	+	type scopeTree struct {
	370	+	l *filterList
	371	+	root string
	372	+	dir string
	373	+	read func(string) ([]byte, error)
	374	+	scopes map[string]*filterScope
	375	+	}
	376	+
	377	+	// newScopeTree starts at dir, the storage path of the first directory
	378	+	// traversed. root is the storage path of the transfer root.
	379	+	func (l filterList) newScopeTree(root, dir string, read func(string) ([]byte, error)) scopeTree {
	380	+	return &scopeTree{l: l, root: root, dir: dir, read: read, scopes: map[string]*filterScope{}}
	381	+	}
	382	+
	383	+	// scope returns the scope for rel, a directory relative to the tree's
	384	+	// first directory, which is ".".
	385	+	func (t scopeTree) scope(rel string) (filterScope, error) {
	386	+	if sc, ok := t.scopes[rel]; ok {
	387	+	return sc, nil
	388	+	}
	389	+	var sc *filterScope
	390	+	if rel == "." {
	391	+	var err error
	392	+	if sc, err = t.l.newScope(t.root, t.dir, t.read); err != nil {
	393	+	return nil, err
	394	+	}
	395	+	} else {
	396	+	parent, err := t.scope(path.Dir(rel))
	397	+	if err != nil {
	398	+	return nil, err
	399	+	}
	400	+	if sc, err = t.l.enter(parent, path.Join(t.dir, rel), t.read); err != nil {
	401	+	return nil, err
	402	+	}
	403	+	}
	404	+	t.scopes[rel] = sc
	405	+	return sc, nil
	406	+	}
	407	+
	408	+	// mergeReader reads per-directory merge files from storage, returning nil
	409	+	// for a missing one. files lists the regular files below tree, relative to
	410	+	// it; other files are looked up in their directory, as not every backend
	411	+	// reports a missing file from Open.
	412	+	func (s *session) mergeReader(tree string, files map[string]bool) func(string) ([]byte, error) {
	413	+	return func(name string) ([]byte, error) {
	414	+	rel, under := name, tree == ""
	415	+	if !under {
	416	+	rel, under = strings.CutPrefix(name, tree+"/")
	417	+	}
	418	+	found := false
	419	+	if under && files != nil {
	420	+	found = files[rel]
	421	+	} else {
	422	+	dir := path.Dir(name)
	423	+	if dir == "." {
	424	+	dir = ""
	425	+	}
	426	+	entries, err := s.fs.ReadDir(dir, false)
	427	+	if err != nil && !errors.Is(err, fs.ErrNotExist) {
	428	+	return nil, err
	429	+	}
	430	+	base := path.Base(name)
	431	+	found = slices.ContainsFunc(entries, func(e FileInfo) bool {
	432	+	return !e.IsDir && strings.Trim(e.Name, "/") == base
	433	+	})
	434	+	}
	435	+	if !found {
	436	+	return nil, nil
	437	+	}
	438	+	f, info, err := s.fs.Open(name)
	439	+	if err != nil {
	440	+	return nil, err
	441	+	}
	442	+	defer func() { _ = f.Close() }()
	443	+	if info.Size > maxMergeFileSize {
	444	+	return nil, fmt.Errorf("larger than %d bytes", maxMergeFileSize)
	445	+	}
	446	+	buf := make([]byte, info.Size)
	447	+	if err := readFullAt(f, buf, 0); err != nil {
	448	+	return nil, err
	449	+	}
	450	+	return buf, nil
	451	+	}
	452	+	}
	453	+
	454	+	// newScope returns the scope for dir, the storage path of the first
	455	+	// directory traversed. A merge rule whose name has a directory part that
	456	+	// is an ancestor of dir first reads the file in that directory and each
	457	+	// one below it down to dir's parent (setup_merge_file).
	458	+	func (l filterList) newScope(root, dir string, read func(string) ([]byte, error)) (filterScope, error) {
	459	+	parent := &filterScope{root: root, lists: make([][]*filterRule, len(l.merges))}
	460	+	for i, m := range l.merges {
	461	+	if !strings.Contains(m.pattern, "/") {
	462	+	continue
	463	+	}
	464	+	from := path.Dir(m.pattern)
	465	+	if !strings.HasPrefix(from, "/") {
	466	+	from = path.Join("/"+dir, from)
	467	+	}
	468	+	from = strings.TrimPrefix(path.Clean(from), "/")
	469	+	if from == dir \|\| (from != "" && !strings.HasPrefix(dir, from+"/")) {
	470	+	continue
	471	+	}
	472	+	rest := strings.Split(strings.TrimPrefix(dir, from), "/")
	473	+	d := from
	474	+	for _, elem := range rest {
	475	+	if elem == "" {
	476	+	continue
	477	+	}
	478	+	rules, cleared, err := l.readMerge(m, d, read)
	479	+	if err != nil {
	480	+	return nil, err
	481	+	}
	482	+	if cleared {
	483	+	parent.lists[i] = nil
	484	+	}
	485	+	if m.flags&ruleNoInherit == 0 {
	486	+	parent.lists[i] = append(rules, parent.lists[i]...)
	487	+	}
	488	+	d = path.Join(d, elem)
	489	+	}
	490	+	}
	491	+	return l.enter(parent, dir, read)
	492	+	}
	493	+
	494	+	// enter returns the scope for dir, a storage path, below parent's
	495	+	// directory (push_local_filters).
	496	+	func (l filterList) enter(parent filterScope, dir string, read func(string) ([]byte, error)) (*filterScope, error) {
	497	+	if len(l.merges) == 0 {
	498	+	return parent, nil
	499	+	}
	500	+	sc := &filterScope{root: parent.root, lists: make([][]*filterRule, len(l.merges))}
	501	+	for i, m := range l.merges {
	502	+	var inherited []*filterRule
	503	+	if m.flags&ruleNoInherit == 0 {
	504	+	inherited = parent.lists[i]
	505	+	}
	506	+	rules, cleared, err := l.readMerge(m, dir, read)
	507	+	if err != nil {
	508	+	return nil, err
	509	+	}
	510	+	if cleared {
	511	+	inherited = nil
	512	+	}
	513	+	sc.lists[i] = append(rules, inherited...)
	514	+	}
	515	+	return sc, nil
	516	+	}
	517	+
	518	+	// readMerge parses the merge file for m in dir (parse_filter_file). A "!"
	519	+	// clears the rules before it, inherited ones included.
	520	+	func (l filterList) readMerge(m filterRule, dir string, read func(string) ([]byte, error)) (rules []*filterRule, cleared bool, err error) {
	521	+	name := path.Join(dir, path.Base(m.pattern))
	522	+	data, err := read(name)
	523	+	if err != nil {
	524	+	return nil, false, &mergeFileError{name: name, err: err}
	525	+	}
	526	+	var toks []string
	527	+	if m.flags&ruleWordSplit != 0 {
	528	+	toks = strings.FieldsFunc(string(data), func(r rune) bool { return r < 0x80 && isSpace(byte(r)) })
	529	+	} else {
	530	+	text := strings.ReplaceAll(string(data), "\r\n", "\n")
	531	+	for line := range strings.FieldsFuncSeq(text, func(r rune) bool { return r == '\n' \|\| r == '\r' }) {
	532	+	if line[0] != '#' && line[0] != ';' {
	533	+	toks = append(toks, line)
	534	+	}
	535	+	}
	536	+	}
	537	+	for _, tok := range toks {
	538	+	rule, err := parseRuleTok(tok, m.flags, false)
	539	+	if err != nil {
	540	+	return nil, false, &mergeFileError{name: name, err: err}
	541	+	}
	542	+	if rule == nil {
	543	+	continue
	544	+	}
	545	+	switch {
	546	+	case rule.flags&ruleClearList != 0:
	547	+	rules, cleared = nil, true
	548	+	continue
	549	+	case rule.flags&ruleMergeFile != 0:
	550	+	return nil, false, &mergeFileError{name: name, err: errors.New("merge rules inside a merge file are not supported")}
	551	+	case rule.flags&ruleCVSIgnore != 0:
	552	+	rules = append(rules, l.cvsIgnoreRules()...)
	553	+	continue
	554	+	}
	555	+	if l.deleteExcluded && rule.flags&sideFlags == 0 {
	556	+	rule.flags \|= ruleSenderSide
	557	+	}
	558	+	if rule.flags&sideFlags == l.otherSide {
	559	+	continue
	560	+	}
	561	+	pat := rule.pattern
	562	+	if rule.flags&ruleAbsPath == 0 && strings.HasPrefix(pat, "/") {
	563	+	// Anchored patterns are relative to the file's directory.
	564	+	rule.flags \|= ruleAbsPath
	565	+	if dir != "" {
	566	+	pat = "/" + dir + pat
	567	+	}
	568	+	}
	569	+	r := newRule(pat, rule.flags)
	570	+	rules = append(rules, r)
	571	+	}
	572	+	return rules, cleared, nil
	573	+	}
	574	+
	575	+	// check returns -1 if name is excluded, 1 if it is included and 0 when no
	576	+	// rule matched. name is relative to the transfer root. sc holds the
	577	+	// per-directory rules for name's directory, if any have been read.
	578	+	func (l filterList) check(name string, isDir bool, sc filterScope) int {
	579	+	return l.checkPerishable(name, isDir, sc, false)
	580	+	}
	581	+
	582	+	// checkPerishable is check, skipping perishable rules when
	583	+	// ignorePerishable is set (rsync's ignore_perishable).
	584	+	func (l filterList) checkPerishable(name string, isDir bool, sc filterScope, ignorePerishable bool) int {
	585	+	root := ""
	586	+	if sc != nil {
	587	+	root = sc.root
	588	+	}
	589	+	for _, r := range l.rules {
	590	+	if ignorePerishable && r.flags&rulePerishable != 0 {
	591	+	continue
	592	+	}
	593	+	if r.flags&rulePerdirMerge != 0 {
	594	+	if sc != nil {
	595	+	if rc := checkRules(sc.lists[r.merge], name, isDir, root, ignorePerishable); rc != 0 {
	596	+	return rc
	597	+	}
	598	+	}
	599	+	continue
	600	+	}
	601	+	if rc := checkRules([]*filterRule{r}, name, isDir, root, ignorePerishable); rc != 0 {
	602	+	return rc
	603	+	}
	604	+	}
	605	+	return 0
	606	+	}
	607	+
	608	+	func checkRules(rules []*filterRule, name string, isDir bool, root string, ignorePerishable bool) int {
	609	+	for _, r := range rules {
	610	+	if ignorePerishable && r.flags&rulePerishable != 0 {
	611	+	continue
	612	+	}
	613	+	if r.matches(name, isDir, root) {
	614	+	if r.flags&ruleInclude != 0 {
	615	+	return 1
	616	+	}
	617	+	return -1
	618	+	}
	619	+	}
	620	+	return 0
	621	+	}
	622	+
	623	+	func (l filterList) excluded(name string, isDir bool, sc filterScope) bool {
	624	+	return l != nil && l.check(name, isDir, sc) < 0
	625	+	}
	626	+
	627	+	// protectsInDeletedDir reports whether a rule keeps name, which is inside a
	628	+	// directory being deleted. Perishable rules don't, so the directory can go.
	629	+	func (l filterList) protectsInDeletedDir(name string, isDir bool, sc filterScope) bool {
	630	+	return l != nil && l.checkPerishable(name, isDir, sc, true) < 0
	631	+	}
	632	+
	633	+	// matches is rsync's rule_matches. Absolute patterns match the storage
	634	+	// path, which is name below root.
	635	+	func (r *filterRule) matches(fname string, isDir bool, root string) bool {
	636	+	ret := r.flags&ruleNegate == 0
	637	+	name := strings.TrimPrefix(fname, "/")
	638	+	if name == "" \|\| r.flags&ruleXattr != 0 {
	639	+	return false
	640	+	}
	641	+
	642	+	var strs []string
	643	+	switch {
	644	+	case r.slashCnt == 0 && r.flags&ruleWild2 == 0:
	645	+	if i := strings.LastIndexByte(name, '/'); i >= 0 {
	646	+	name = name[i+1:]
	647	+	}
	648	+	case r.flags&ruleAbsPath != 0 && !strings.HasPrefix(fname, "/") && root != "":
	649	+	strs = append(strs, root, "/")
	650	+	case r.flags&ruleWild2Prefix != 0 && !strings.HasPrefix(fname, "/"):
	651	+	strs = append(strs, "/")
	652	+	}
	653	+	strs = append(strs, name)
	654	+	if isDir {
	655	+	if r.flags&ruleWild3Suffix != 0 {
	656	+	strs = append(strs, "/")
	657	+	}
	658	+	} else if r.flags&ruleDirectory != 0 {
	659	+	return !ret
	660	+	}
	661	+
	662	+	pattern := r.pattern
	663	+	anchored := false
	664	+	if strings.HasPrefix(pattern, "/") {
	665	+	anchored = true
	666	+	pattern = pattern[1:]
	667	+	}
	668	+
	669	+	var where int
	670	+	switch {
	671	+	case !anchored && r.slashCnt > 0 && r.flags&ruleWild2 == 0:
	672	+	where = r.slashCnt + 1
	673	+	case !anchored && r.flags&ruleWild2Prefix == 0 && r.flags&ruleWild2 != 0:
	674	+	where = -1
	675	+	}
	676	+
	677	+	text := strings.Join(strs, "")
	678	+	switch {
	679	+	case r.flags&ruleWild != 0:
	680	+	if wildmatchWhere(pattern, text, where) {
	681	+	return ret
	682	+	}
	683	+	case len(strs) > 1:
	684	+	if t, ok := trailingElements(text, where); ok && t == pattern {
	685	+	return ret
	686	+	}
	687	+	case anchored:
	688	+	if name == pattern {
	689	+	return ret
	690	+	}
	691	+	default:
	692	+	if strings.HasSuffix(name, pattern) &&
	693	+	(len(name) == len(pattern) \|\| name[len(name)-len(pattern)-1] == '/') {
	694	+	return ret
	695	+	}
	696	+	}
	697	+	return !ret
	698	+	}
	699	+
	700	+	// trailingElements returns the last count slash-separated elements of s.
	701	+	// A count of zero or less returns s unchanged.
	702	+	func trailingElements(s string, count int) (string, bool) {
	703	+	if count <= 0 {
	704	+	return s, true
	705	+	}
	706	+	for i := len(s) - 1; i >= 0; i-- {
	707	+	if s[i] == '/' {
	708	+	count--
	709	+	if count == 0 {
	710	+	return s[i+1:], true
	711	+	}
	712	+	}
	713	+	}
	714	+	if count == 1 {
	715	+	return s, true
	716	+	}
	717	+	return "", false
	718	+	}
	719	+
	720	+	// wildmatchWhere matches pattern against the trailing where elements of
	721	+	// text (where > 0), at the start of text (where == 0), or at the start or
	722	+	// after any slash (where < 0).
	723	+	func wildmatchWhere(pattern, text string, where int) bool {
	724	+	t, ok := trailingElements(text, where)
	725	+	if !ok {
	726	+	return false
	727	+	}
	728	+	m := dowild(pattern, t)
	729	+	if m == wildMatch \|\| where >= 0 \|\| m == wildAbortAll {
	730	+	return m == wildMatch
	731	+	}
	732	+	for i := 0; i < len(t); i++ {
	733	+	if t[i] != '/' {
	734	+	continue
	735	+	}
	736	+	m = dowild(pattern, t[i+1:])
	737	+	if m != wildNoMatch && m != wildAbortToStarStar {
	738	+	return m == wildMatch
	739	+	}
	740	+	}
	741	+	return false
	742	+	}
	743	+
	744	+	// wildmatch reports whether text matches the rsync wildcard pattern. "*"
	745	+	// and "?" don't match "/", "**" does.
	746	+	func wildmatch(pattern, text string) bool {
	747	+	return dowild(pattern, text) == wildMatch
	748	+	}
	749	+
	750	+	const (
	751	+	wildNoMatch = iota
	752	+	wildMatch
	753	+	wildAbortAll
	754	+	wildAbortToStarStar
	755	+	)
	756	+
	757	+	func dowild(p, text string) int {
	758	+	pi, ti := 0, 0
	759	+	for ; pi < len(p); pi, ti = pi+1, ti+1 {
	760	+	pch := p[pi]
	761	+	if ti >= len(text) && pch != '*' {
	762	+	return wildAbortAll
	763	+	}
	764	+	var tch byte
	765	+	if ti < len(text) {
	766	+	tch = text[ti]
	767	+	}
	768	+	switch pch {
	769	+	case '\\':
	770	+	pi++
	771	+	if pi >= len(p) \|\| tch != p[pi] {
	772	+	return wildNoMatch
	773	+	}
	774	+	case '?':
	775	+	if tch == '/' {
	776	+	return wildNoMatch
	777	+	}
	778	+	case '*':
	779	+	special := false
	780	+	pi++
	781	+	if pi < len(p) && p[pi] == '*' {
	782	+	for pi < len(p) && p[pi] == '*' {
	783	+	pi++
	784	+	}
	785	+	special = true
	786	+	}
	787	+	if pi >= len(p) {
	788	+	// A trailing "*" matches everything, "" only if
	789	+	// there are no more slashes.
	790	+	if !special && strings.IndexByte(text[ti:], '/') >= 0 {
	791	+	return wildNoMatch
	792	+	}
	793	+	return wildMatch
	794	+	}
	795	+	for ; ; ti++ {
	796	+	if ti >= len(text) {
	797	+	return wildAbortAll
	798	+	}
	799	+	m := dowild(p[pi:], text[ti:])
	800	+	if m != wildNoMatch {
	801	+	if !special \|\| m != wildAbortToStarStar {
	802	+	return m
	803	+	}
	804	+	} else if !special && text[ti] == '/' {
	805	+	return wildAbortToStarStar
	806	+	}
	807	+	}
	808	+	case '[':
	809	+	pi++
	810	+	if pi >= len(p) {
	811	+	return wildAbortAll
	812	+	}
	813	+	negate := false
	814	+	if p[pi] == '!' \|\| p[pi] == '^' {
	815	+	negate = true
	816	+	pi++
	817	+	}
	818	+	matched := false
	819	+	var prev byte
	820	+	for first := true; ; first = false {
	821	+	if pi >= len(p) {
	822	+	return wildAbortAll
	823	+	}
	824	+	c := p[pi]
	825	+	if c == ']' && !first {
	826	+	break
	827	+	}
	828	+	switch {
	829	+	case c == '\\':
	830	+	pi++
	831	+	if pi >= len(p) {
	832	+	return wildAbortAll
	833	+	}
	834	+	c = p[pi]
	835	+	if tch == c {
	836	+	matched = true
	837	+	}
	838	+	case c == '-' && prev != 0 && pi+1 < len(p) && p[pi+1] != ']':
	839	+	pi++
	840	+	c = p[pi]
	841	+	if c == '\\' {
	842	+	pi++
	843	+	if pi >= len(p) {
	844	+	return wildAbortAll
	845	+	}
	846	+	c = p[pi]
	847	+	}
	848	+	if tch <= c && tch >= prev {
	849	+	matched = true
	850	+	}
	851	+	c = 0
	852	+	case c == '[' && pi+1 < len(p) && p[pi+1] == ':':
	853	+	k := strings.IndexByte(p[pi+2:], ']')
	854	+	if k < 0 {
	855	+	return wildAbortAll
	856	+	}
	857	+	if k == 0 \|\| p[pi+2+k-1] != ':' {
	858	+	// No ":]", so the '[' is literal.
	859	+	if tch == '[' {
	860	+	matched = true
	861	+	}
	862	+	break
	863	+	}
	864	+	ok, valid := charClass(p[pi+2:pi+2+k-1], tch)
	865	+	if !valid {
	866	+	return wildAbortAll
	867	+	}
	868	+	if ok {
	869	+	matched = true
	870	+	}
	871	+	pi += 2 + k
	872	+	c = 0
	873	+	default:
	874	+	if tch == c {
	875	+	matched = true
	876	+	}
	877	+	}
	878	+	prev = c
	879	+	pi++
	880	+	}
	881	+	if matched == negate \|\| tch == '/' {
	882	+	return wildNoMatch
	883	+	}
	884	+	default:
	885	+	if tch != pch {
	886	+	return wildNoMatch
	887	+	}
	888	+	}
	889	+	}
	890	+	if ti < len(text) {
	891	+	return wildNoMatch
	892	+	}
	893	+	return wildMatch
	894	+	}
	895	+
	896	+	func charClass(class string, c byte) (match, valid bool) {
	897	+	isUpper := c >= 'A' && c <= 'Z'
	898	+	isLower := c >= 'a' && c <= 'z'
	899	+	isDigit := c >= '0' && c <= '9'
	900	+	isAlpha := isUpper \|\| isLower
	901	+	isSpace := c == ' ' \|\| (c >= '\t' && c <= '\r')
	902	+	isPrint := c >= 0x20 && c < 0x7f
	903	+	switch class {
	904	+	case "alnum":
	905	+	return isAlpha \|\| isDigit, true
	906	+	case "alpha":
	907	+	return isAlpha, true
	908	+	case "blank":
	909	+	return c == ' ' \|\| c == '\t', true
	910	+	case "cntrl":
	911	+	return c < 0x20 \|\| c == 0x7f, true
	912	+	case "digit":
	913	+	return isDigit, true
	914	+	case "graph":
	915	+	return isPrint && c != ' ', true
	916	+	case "lower":
	917	+	return isLower, true
	918	+	case "print":
	919	+	return isPrint, true
	920	+	case "punct":
	921	+	return isPrint && c != ' ' && !isAlpha && !isDigit, true
	922	+	case "space":
	923	+	return isSpace, true
	924	+	case "upper":
	925	+	return isUpper, true
	926	+	case "xdigit":
	927	+	return isDigit \|\| (c >= 'a' && c <= 'f') \|\| (c >= 'A' && c <= 'F'), true
	928	+	}
	929	+	return false, false
	930	+	}

...	...		@@ -0,0 +1,752 @@
	1	+	package rsync
	2	+
	3	+	import (
	4	+	"errors"
	5	+	"fmt"
	6	+	"path"
	7	+	"slices"
	8	+	"strings"
	9	+	)
	10	+
	11	+	// Live flags of a file-list entry.
	12	+	const (
	13	+	flagTopDir = 1 << 0
	14	+	flagContentDir = 1 << 2
	15	+	flagImpliedDir = 1 << 6
	16	+	)
	17	+
	18	+	// fileEntry is one file-list entry.
	19	+	type fileEntry struct {
	20	+	dirname string // "" for entries at the top
	21	+	basename string
	22	+	mode uint32 // unix mode including type bits
	23	+	size int64
	24	+	mtime int64
	25	+	nsec uint32
	26	+	uid uint32
	27	+	gid uint32
	28	+	link string
	29	+	sum []byte // whole-file checksum with --checksum
	30	+	flags int
	31	+	inactive bool // removed as a duplicate
	32	+
	33	+	// sender only: where to read the file from
	34	+	src string
	35	+	}
	36	+
	37	+	func (f *fileEntry) name() string {
	38	+	if f.dirname == "" {
	39	+	return f.basename
	40	+	}
	41	+	return f.dirname + "/" + f.basename
	42	+	}
	43	+
	44	+	func (f *fileEntry) isDir() bool { return f.mode&sIFMT == sIFDIR }
	45	+	func (f *fileEntry) isReg() bool { return f.mode&sIFMT == sIFREG }
	46	+	func (f *fileEntry) isLink() bool { return f.mode&sIFMT == sIFLNK }
	47	+	func (f *fileEntry) isDevice() bool {
	48	+	t := f.mode & sIFMT
	49	+	return t == sIFCHR \|\| t == sIFBLK
	50	+	}
	51	+	func (f *fileEntry) isSpecial() bool {
	52	+	t := f.mode & sIFMT
	53	+	return t == sIFIFO \|\| t == sIFSOCK
	54	+	}
	55	+
	56	+	func newFileEntry(name string) *fileEntry {
	57	+	dir, base := path.Split(name)
	58	+	return &fileEntry{dirname: strings.TrimSuffix(dir, "/"), basename: base}
	59	+	}
	60	+
	61	+	// flistCodec holds the state and options shared by the encoder and decoder.
	62	+	// Both compress each entry against the previous one.
	63	+	type flistCodec struct {
	64	+	protocol int
	65	+	varintFlags bool
	66	+	safeFlist bool
	67	+	preserveUID bool
	68	+	preserveGID bool
	69	+	preserveLinks bool
	70	+	preserveDevs bool
	71	+	preserveSpecs bool
	72	+	preserveAtime bool
	73	+	alwaysSum bool
	74	+	sumLen int
	75	+
	76	+	last fileEntry
	77	+	lastName string
	78	+	lastAtime int64
	79	+	rdevMaj uint32
	80	+	}
	81	+
	82	+	func (fc flistCodec) sendEntry(c conn, f *fileEntry) error {
	83	+	name := f.name()
	84	+	var xflags int
	85	+	switch {
	86	+	case f.isDir():
	87	+	if fc.protocol >= 30 {
	88	+	switch {
	89	+	case f.flags&flagContentDir != 0:
	90	+	xflags = f.flags & flagTopDir
	91	+	case f.flags&flagImpliedDir != 0:
	92	+	xflags = xmitTopDir \| xmitNoContentDir
	93	+	default:
	94	+	xflags = xmitNoContentDir
	95	+	}
	96	+	} else {
	97	+	xflags = f.flags & flagTopDir
	98	+	}
	99	+	}
	100	+	if f.mode == fc.last.mode && fc.lastName != "" {
	101	+	xflags \|= xmitSameMode
	102	+	}
	103	+	if !fc.preserveUID \|\| (f.uid == fc.last.uid && fc.lastName != "") {
	104	+	xflags \|= xmitSameUID
	105	+	}
	106	+	if !fc.preserveGID \|\| (f.gid == fc.last.gid && fc.lastName != "") {
	107	+	xflags \|= xmitSameGID
	108	+	}
	109	+	if f.mtime == fc.last.mtime && fc.lastName != "" {
	110	+	xflags \|= xmitSameTime
	111	+	}
	112	+	if f.nsec != 0 && fc.protocol >= 31 {
	113	+	xflags \|= xmitModNsec
	114	+	}
	115	+	// We don't track access times, so the modification time stands in.
	116	+	atime := f.mtime
	117	+	if fc.preserveAtime && !f.isDir() {
	118	+	if atime == fc.lastAtime {
	119	+	xflags \|= xmitSameAtime
	120	+	} else {
	121	+	fc.lastAtime = atime
	122	+	}
	123	+	}
	124	+	if fc.preserveSpecs && f.isSpecial() && fc.protocol < 31 {
	125	+	if fc.protocol < 28 {
	126	+	xflags \|= xmitSameRdevPre28
	127	+	} else {
	128	+	xflags \|= xmitSameRdevMajor
	129	+	if fc.protocol < 30 {
	130	+	xflags \|= xmitRdevMinor8Pre30
	131	+	}
	132	+	}
	133	+	}
	134	+
	135	+	l1 := 0
	136	+	for l1 < len(fc.lastName) && l1 < len(name) && l1 < 255 && name[l1] == fc.lastName[l1] {
	137	+	l1++
	138	+	}
	139	+	l2 := len(name) - l1
	140	+	if l1 > 0 {
	141	+	xflags \|= xmitSameName
	142	+	}
	143	+	if l2 > 255 {
	144	+	xflags \|= xmitLongName
	145	+	}
	146	+
	147	+	switch {
	148	+	case fc.varintFlags:
	149	+	if xflags == 0 {
	150	+	xflags = xmitExtendedFlags
	151	+	}
	152	+	if err := c.writeVarint(int32(xflags)); err != nil {
	153	+	return err
	154	+	}
	155	+	case fc.protocol >= 28:
	156	+	if xflags == 0 && !f.isDir() {
	157	+	xflags \|= xmitTopDir
	158	+	}
	159	+	if xflags&0xFF00 != 0 \|\| xflags == 0 {
	160	+	xflags \|= xmitExtendedFlags
	161	+	if err := c.writeShortint(uint16(xflags)); err != nil {
	162	+	return err
	163	+	}
	164	+	} else if err := c.writeByte(byte(xflags)); err != nil {
	165	+	return err
	166	+	}
	167	+	default:
	168	+	if xflags&0xFF == 0 {
	169	+	if f.isDir() {
	170	+	xflags \|= xmitLongName
	171	+	} else {
	172	+	xflags \|= xmitTopDir
	173	+	}
	174	+	}
	175	+	if err := c.writeByte(byte(xflags)); err != nil {
	176	+	return err
	177	+	}
	178	+	}
	179	+	if xflags&xmitSameName != 0 {
	180	+	if err := c.writeByte(byte(l1)); err != nil {
	181	+	return err
	182	+	}
	183	+	}
	184	+	if xflags&xmitLongName != 0 {
	185	+	if err := c.writeVarint30(int32(l2)); err != nil {
	186	+	return err
	187	+	}
	188	+	} else if err := c.writeByte(byte(l2)); err != nil {
	189	+	return err
	190	+	}
	191	+	if err := c.write([]byte(name[l1:])); err != nil {
	192	+	return err
	193	+	}
	194	+
	195	+	if err := c.writeVarlong30(f.size, 3); err != nil {
	196	+	return err
	197	+	}
	198	+	if xflags&xmitSameTime == 0 {
	199	+	var err error
	200	+	if fc.protocol >= 30 {
	201	+	err = c.writeVarlong(f.mtime, 4)
	202	+	} else {
	203	+	err = c.writeInt32(int32(f.mtime))
	204	+	}
	205	+	if err != nil {
	206	+	return err
	207	+	}
	208	+	}
	209	+	if xflags&xmitModNsec != 0 {
	210	+	if err := c.writeVarint(int32(f.nsec)); err != nil {
	211	+	return err
	212	+	}
	213	+	}
	214	+	if xflags&xmitSameMode == 0 {
	215	+	if err := c.writeInt32(int32(f.mode)); err != nil {
	216	+	return err
	217	+	}
	218	+	}
	219	+	if fc.preserveAtime && !f.isDir() && xflags&xmitSameAtime == 0 {
	220	+	if err := c.writeVarlong(atime, 4); err != nil {
	221	+	return err
	222	+	}
	223	+	}
	224	+	if fc.preserveUID && xflags&xmitSameUID == 0 {
	225	+	if err := c.writeVarint30(int32(f.uid)); err != nil {
	226	+	return err
	227	+	}
	228	+	}
	229	+	if fc.preserveGID && xflags&xmitSameGID == 0 {
	230	+	if err := c.writeVarint30(int32(f.gid)); err != nil {
	231	+	return err
	232	+	}
	233	+	}
	234	+	if fc.preserveSpecs && f.isSpecial() && fc.protocol < 31 {
	235	+	// The rdev of a special file is meaningless, so it is always
	236	+	// sent as "same major" with a zero minor.
	237	+	switch {
	238	+	case fc.protocol >= 30:
	239	+	if err := c.writeVarint(0); err != nil {
	240	+	return err
	241	+	}
	242	+	case fc.protocol >= 28:
	243	+	if err := c.writeByte(0); err != nil {
	244	+	return err
	245	+	}
	246	+	}
	247	+	}
	248	+	if fc.preserveLinks && f.isLink() {
	249	+	if err := c.writeVarint30(int32(len(f.link))); err != nil {
	250	+	return err
	251	+	}
	252	+	if err := c.write([]byte(f.link)); err != nil {
	253	+	return err
	254	+	}
	255	+	}
	256	+	if fc.alwaysSum && (f.isReg() \|\| fc.protocol < 28) {
	257	+	sum := f.sum
	258	+	if !f.isReg() \|\| len(sum) != fc.sumLen {
	259	+	sum = make([]byte, fc.sumLen)
	260	+	}
	261	+	if err := c.write(sum); err != nil {
	262	+	return err
	263	+	}
	264	+	}
	265	+
	266	+	fc.last = *f
	267	+	fc.lastName = name
	268	+	return nil
	269	+	}
	270	+
	271	+	// sendEnd terminates the file list.
	272	+	func (fc flistCodec) sendEnd(c conn, ioError int32) error {
	273	+	switch {
	274	+	case fc.varintFlags:
	275	+	if err := c.writeVarint(0); err != nil {
	276	+	return err
	277	+	}
	278	+	return c.writeVarint(ioError)
	279	+	case ioError != 0 && fc.safeFlist:
	280	+	if err := c.writeShortint(xmitExtendedFlags \| xmitIOErrorEndList); err != nil {
	281	+	return err
	282	+	}
	283	+	return c.writeVarint(ioError)
	284	+	default:
	285	+	return c.writeByte(0)
	286	+	}
	287	+	}
	288	+
	289	+	// recvFlags reads the flags of the next entry. It returns ok=false at the end
	290	+	// of the list, along with any I/O error the sender reported there.
	291	+	func (fc flistCodec) recvFlags(c conn) (xflags int, ok bool, ioError int32, err error) {
	292	+	if fc.varintFlags {
	293	+	v, err := c.readVarint()
	294	+	if err != nil {
	295	+	return 0, false, 0, err
	296	+	}
	297	+	if v == 0 {
	298	+	e, err := c.readVarint()
	299	+	return 0, false, e, err
	300	+	}
	301	+	return int(v), true, 0, nil
	302	+	}
	303	+	b, err := c.readByte()
	304	+	if err != nil {
	305	+	return 0, false, 0, err
	306	+	}
	307	+	if b == 0 {
	308	+	return 0, false, 0, nil
	309	+	}
	310	+	xflags = int(b)
	311	+	if fc.protocol >= 28 && xflags&xmitExtendedFlags != 0 {
	312	+	hi, err := c.readByte()
	313	+	if err != nil {
	314	+	return 0, false, 0, err
	315	+	}
	316	+	xflags \|= int(hi) << 8
	317	+	}
	318	+	if xflags == xmitExtendedFlags\|xmitIOErrorEndList {
	319	+	if !fc.safeFlist {
	320	+	return 0, false, 0, fmt.Errorf("invalid flist flag: %x", xflags)
	321	+	}
	322	+	e, err := c.readVarint()
	323	+	return 0, false, e, err
	324	+	}
	325	+	return xflags, true, 0, nil
	326	+	}
	327	+
	328	+	func (fc flistCodec) recvEntry(c conn, xflags int) (*fileEntry, error) {
	329	+	var l1, l2 int
	330	+	if xflags&xmitSameName != 0 {
	331	+	b, err := c.readByte()
	332	+	if err != nil {
	333	+	return nil, err
	334	+	}
	335	+	l1 = int(b)
	336	+	}
	337	+	if xflags&xmitLongName != 0 {
	338	+	v, err := c.readVarint30()
	339	+	if err != nil {
	340	+	return nil, err
	341	+	}
	342	+	l2 = int(v)
	343	+	} else {
	344	+	b, err := c.readByte()
	345	+	if err != nil {
	346	+	return nil, err
	347	+	}
	348	+	l2 = int(b)
	349	+	}
	350	+	if l1 > len(fc.lastName) \|\| l2 < 0 \|\| l2 >= maxPathLen-l1 {
	351	+	return nil, fmt.Errorf("overflow: xflags=0x%x l1=%d l2=%d lastname=%s", xflags, l1, l2, fc.lastName)
	352	+	}
	353	+	nameBuf := make([]byte, l1+l2)
	354	+	copy(nameBuf, fc.lastName[:l1])
	355	+	if err := c.readFull(nameBuf[l1:]); err != nil {
	356	+	return nil, err
	357	+	}
	358	+	rawName := string(nameBuf)
	359	+	fc.lastName = rawName
	360	+
	361	+	name, err := cleanName(rawName)
	362	+	if err != nil {
	363	+	return nil, err
	364	+	}
	365	+	f := newFileEntry(name)
	366	+	if name == "." {
	367	+	f.dirname, f.basename = "", "."
	368	+	}
	369	+
	370	+	if xflags&(xmitHlinked) != 0 && fc.protocol >= 30 && xflags&xmitHlinkFirst == 0 {
	371	+	return nil, errors.New("hard links are not supported")
	372	+	}
	373	+
	374	+	if f.size, err = c.readVarlong30(3); err != nil {
	375	+	return nil, err
	376	+	}
	377	+	if f.size < 0 {
	378	+	return nil, errors.New("offset underflow: file-length is negative")
	379	+	}
	380	+	if xflags&xmitSameTime != 0 {
	381	+	f.mtime = fc.last.mtime
	382	+	} else if fc.protocol >= 30 {
	383	+	if f.mtime, err = c.readVarlong(4); err != nil {
	384	+	return nil, err
	385	+	}
	386	+	} else {
	387	+	v, err := c.readInt32()
	388	+	if err != nil {
	389	+	return nil, err
	390	+	}
	391	+	f.mtime = int64(uint32(v))
	392	+	}
	393	+	if xflags&xmitModNsec != 0 {
	394	+	v, err := c.readVarint()
	395	+	if err != nil {
	396	+	return nil, err
	397	+	}
	398	+	if v < 0 \|\| v > maxWireNsec {
	399	+	return nil, fmt.Errorf("invalid modtime nsec %d", v)
	400	+	}
	401	+	f.nsec = uint32(v)
	402	+	}
	403	+	if xflags&xmitSameMode != 0 {
	404	+	f.mode = fc.last.mode
	405	+	} else {
	406	+	v, err := c.readInt32()
	407	+	if err != nil {
	408	+	return nil, err
	409	+	}
	410	+	f.mode = uint32(v)
	411	+	switch f.mode & sIFMT {
	412	+	case sIFREG, sIFDIR, sIFLNK, sIFCHR, sIFBLK, sIFIFO, sIFSOCK:
	413	+	default:
	414	+	return nil, fmt.Errorf("invalid file mode 0%o for %s", f.mode, name)
	415	+	}
	416	+	}
	417	+	if fc.preserveAtime && !f.isDir() && xflags&xmitSameAtime == 0 {
	418	+	if fc.lastAtime, err = c.readVarlong(4); err != nil {
	419	+	return nil, err
	420	+	}
	421	+	}
	422	+	f.uid, f.gid = fc.last.uid, fc.last.gid
	423	+	if fc.preserveUID && xflags&xmitSameUID == 0 {
	424	+	v, err := c.readVarint30()
	425	+	if err != nil {
	426	+	return nil, err
	427	+	}
	428	+	f.uid = uint32(v)
	429	+	if fc.protocol >= 30 && xflags&xmitUserNameFollows != 0 {
	430	+	if _, err := fc.readIDName(c); err != nil {
	431	+	return nil, err
	432	+	}
	433	+	}
	434	+	}
	435	+	if fc.preserveGID && xflags&xmitSameGID == 0 {
	436	+	v, err := c.readVarint30()
	437	+	if err != nil {
	438	+	return nil, err
	439	+	}
	440	+	f.gid = uint32(v)
	441	+	if fc.protocol >= 30 && xflags&xmitGroupNameFollows != 0 {
	442	+	if _, err := fc.readIDName(c); err != nil {
	443	+	return nil, err
	444	+	}
	445	+	}
	446	+	}
	447	+	if (fc.preserveDevs && f.isDevice()) \|\| (fc.preserveSpecs && f.isSpecial() && fc.protocol < 31) {
	448	+	if err := fc.skipRdev(c, xflags); err != nil {
	449	+	return nil, err
	450	+	}
	451	+	f.size = 0
	452	+	} else if f.isDevice() {
	453	+	f.size = 0
	454	+	}
	455	+	if fc.preserveLinks && f.isLink() {
	456	+	n, err := c.readVarint30()
	457	+	if err != nil {
	458	+	return nil, err
	459	+	}
	460	+	if n < 0 \|\| n >= maxPathLen {
	461	+	return nil, fmt.Errorf("overflow: linkname_len=%d", n)
	462	+	}
	463	+	link := make([]byte, n)
	464	+	if err := c.readFull(link); err != nil {
	465	+	return nil, err
	466	+	}
	467	+	f.link = string(link)
	468	+	}
	469	+	if xflags&xmitHlinked != 0 && fc.protocol < 30 {
	470	+	return nil, errors.New("hard links are not supported")
	471	+	}
	472	+	if fc.alwaysSum && (f.isReg() \|\| fc.protocol < 28) {
	473	+	sum := make([]byte, fc.sumLen)
	474	+	if err := c.readFull(sum); err != nil {
	475	+	return nil, err
	476	+	}
	477	+	if f.isReg() {
	478	+	f.sum = sum
	479	+	}
	480	+	}
	481	+
	482	+	if name == "." && !f.isDir() {
	483	+	return nil, fmt.Errorf("rejecting non-directory transfer-root entry: %s", name)
	484	+	}
	485	+	if f.isDir() {
	486	+	if fc.protocol >= 30 {
	487	+	if xflags&xmitNoContentDir == 0 {
	488	+	if xflags&xmitTopDir != 0 {
	489	+	f.flags \|= flagTopDir
	490	+	}
	491	+	f.flags \|= flagContentDir
	492	+	} else if xflags&xmitTopDir != 0 {
	493	+	f.flags \|= flagImpliedDir
	494	+	}
	495	+	} else if xflags&xmitTopDir != 0 {
	496	+	f.flags \|= flagTopDir \| flagContentDir
	497	+	}
	498	+	}
	499	+
	500	+	fc.last = *f
	501	+	return f, nil
	502	+	}
	503	+
	504	+	func (fc flistCodec) readIDName(c conn) (string, error) {
	505	+	n, err := c.readByte()
	506	+	if err != nil {
	507	+	return "", err
	508	+	}
	509	+	buf := make([]byte, n)
	510	+	if err := c.readFull(buf); err != nil {
	511	+	return "", err
	512	+	}
	513	+	return string(buf), nil
	514	+	}
	515	+
	516	+	func (fc flistCodec) skipRdev(c conn, xflags int) error {
	517	+	if fc.protocol < 28 {
	518	+	if xflags&xmitSameRdevPre28 == 0 {
	519	+	_, err := c.readInt32()
	520	+	return err
	521	+	}
	522	+	return nil
	523	+	}
	524	+	if xflags&xmitSameRdevMajor == 0 {
	525	+	v, err := c.readVarint30()
	526	+	if err != nil {
	527	+	return err
	528	+	}
	529	+	fc.rdevMaj = uint32(v)
	530	+	}
	531	+	var err error
	532	+	switch {
	533	+	case fc.protocol >= 30:
	534	+	_, err = c.readVarint()
	535	+	case xflags&xmitRdevMinor8Pre30 != 0:
	536	+	_, err = c.readByte()
	537	+	default:
	538	+	_, err = c.readInt32()
	539	+	}
	540	+	return err
	541	+	}
	542	+
	543	+	// cleanName validates a name received from the peer and normalizes it the
	544	+	// way clean_fname does: collapse slashes, drop "." components, refuse "..".
	545	+	func cleanName(name string) (string, error) {
	546	+	if name == "" {
	547	+	return "", errors.New("empty file name from sender")
	548	+	}
	549	+	if strings.HasPrefix(name, "/") {
	550	+	return "", fmt.Errorf("ABORTING due to unsafe pathname from sender: %s", name)
	551	+	}
	552	+	var parts []string
	553	+	for _, p := range strings.Split(name, "/") {
	554	+	switch p {
	555	+	case "", ".":
	556	+	continue
	557	+	case "..":
	558	+	return "", fmt.Errorf("ABORTING due to unsafe pathname from sender: %s", name)
	559	+	}
	560	+	parts = append(parts, p)
	561	+	}
	562	+	if len(parts) == 0 {
	563	+	return ".", nil
	564	+	}
	565	+	return strings.Join(parts, "/"), nil
	566	+	}
	567	+
	568	+	// Name-comparison states and types for fileNameCmp.
	569	+	const (
	570	+	sDir = iota
	571	+	sSlash
	572	+	sBase
	573	+	sTrailing
	574	+	)
	575	+
	576	+	const (
	577	+	tPath = iota
	578	+	tItem
	579	+	)
	580	+
	581	+	// fileNameCmp orders entries the way rsync's f_name_cmp does. Both sides
	582	+	// sort the list independently and address files by index, so this has to
	583	+	// match byte for byte. From protocol 29 on, a directory sorts after the
	584	+	// files beside it, as if its name ended in "/".
	585	+	func fileNameCmp(f1, f2 *fileEntry, protocol int) int {
	586	+	if f1.inactive {
	587	+	if f2.inactive {
	588	+	return 0
	589	+	}
	590	+	return -1
	591	+	}
	592	+	if f2.inactive {
	593	+	return 1
	594	+	}
	595	+	tp := tItem
	596	+	if protocol >= 29 {
	597	+	tp = tPath
	598	+	}
	599	+
	600	+	a := newNameCursor(f1, tp)
	601	+	b := newNameCursor(f2, tp)
	602	+	if f1.dirname == f2.dirname {
	603	+	a.atBase()
	604	+	b.atBase()
	605	+	}
	606	+	if a.typ != b.typ {
	607	+	if a.typ == tPath {
	608	+	return 1
	609	+	}
	610	+	return -1
	611	+	}
	612	+
	613	+	for {
	614	+	if a.empty() {
	615	+	a.advance()
	616	+	if !b.empty() && a.typ != b.typ {
	617	+	if a.typ == tPath {
	618	+	return 1
	619	+	}
	620	+	return -1
	621	+	}
	622	+	}
	623	+	if b.empty() {
	624	+	if b.state == sTrailing {
	625	+	if a.empty() {
	626	+	return 0
	627	+	}
	628	+	b.typ = tItem
	629	+	} else {
	630	+	b.advance()
	631	+	}
	632	+	if a.typ != b.typ {
	633	+	if a.typ == tPath {
	634	+	return 1
	635	+	}
	636	+	return -1
	637	+	}
	638	+	}
	639	+	ca, cb := a.next(), b.next()
	640	+	if ca != cb {
	641	+	return int(ca) - int(cb)
	642	+	}
	643	+	if ca == 0 {
	644	+	// Both strings ended in the trailing state.
	645	+	return 0
	646	+	}
	647	+	}
	648	+	}
	649	+
	650	+	type nameCursor struct {
	651	+	f *fileEntry
	652	+	tp int
	653	+	s string
	654	+	i int
	655	+	state int
	656	+	typ int
	657	+	}
	658	+
	659	+	func newNameCursor(f fileEntry, tp int) nameCursor {
	660	+	c := &nameCursor{f: f, tp: tp}
	661	+	if f.dirname == "" {
	662	+	c.atBase()
	663	+	} else {
	664	+	c.s = f.dirname
	665	+	c.typ = tp
	666	+	c.state = sDir
	667	+	}
	668	+	return c
	669	+	}
	670	+
	671	+	func (c *nameCursor) atBase() {
	672	+	c.typ = tItem
	673	+	if c.f.isDir() {
	674	+	c.typ = c.tp
	675	+	}
	676	+	c.s, c.i = c.f.basename, 0
	677	+	if c.typ == tPath && c.s == "." {
	678	+	c.typ = tItem
	679	+	c.state = sTrailing
	680	+	c.s = ""
	681	+	} else {
	682	+	c.state = sBase
	683	+	}
	684	+	}
	685	+
	686	+	func (c *nameCursor) empty() bool { return c.i >= len(c.s) }
	687	+
	688	+	// advance moves past the end of the current string the way the C state
	689	+	// machine does when it hits a NUL.
	690	+	func (c *nameCursor) advance() {
	691	+	switch c.state {
	692	+	case sDir:
	693	+	c.state = sSlash
	694	+	c.s, c.i = "/", 0
	695	+	case sSlash:
	696	+	c.atBase()
	697	+	case sBase:
	698	+	c.state = sTrailing
	699	+	if c.typ == tPath {
	700	+	c.s, c.i = "/", 0
	701	+	return
	702	+	}
	703	+	c.typ = tItem
	704	+	case sTrailing:
	705	+	c.typ = tItem
	706	+	}
	707	+	}
	708	+
	709	+	func (c *nameCursor) next() byte {
	710	+	if c.i >= len(c.s) {
	711	+	return 0
	712	+	}
	713	+	b := c.s[c.i]
	714	+	c.i++
	715	+	return b
	716	+	}
	717	+
	718	+	// sortFileList sorts the list and, on the receiving side, marks duplicate
	719	+	// names inactive without disturbing the indexes (flist_sort_and_clean).
	720	+	func sortFileList(files []*fileEntry, protocol int, receiver bool) {
	721	+	slices.SortStableFunc(files, func(a, b *fileEntry) int {
	722	+	return fileNameCmp(a, b, protocol)
	723	+	})
	724	+	if !receiver {
	725	+	return
	726	+	}
	727	+	seen := make(map[string]int, len(files))
	728	+	for i, f := range files {
	729	+	if f.inactive {
	730	+	continue
	731	+	}
	732	+	name := f.name()
	733	+	j, dup := seen[name]
	734	+	if !dup {
	735	+	seen[name] = i
	736	+	continue
	737	+	}
	738	+	// Keep a directory over a non-directory, otherwise the first.
	739	+	prev := files[j]
	740	+	switch {
	741	+	case f.isDir() && !prev.isDir():
	742	+	prev.inactive = true
	743	+	seen[name] = i
	744	+	case f.isDir():
	745	+	prev.flags \|= f.flags & (flagTopDir \| flagContentDir)
	746	+	prev.flags &= f.flags \| ^flagImpliedDir
	747	+	f.inactive = true
	748	+	default:
	749	+	f.inactive = true
	750	+	}
	751	+	}
	752	+	}

...	...		@@ -0,0 +1,104 @@
	1	+	package rsync
	2	+
	3	+	import (
	4	+	"errors"
	5	+	"io"
	6	+	"time"
	7	+	)
	8	+
	9	+	// ErrFileIgnored lets Put skip a file without failing the transfer or
	10	+	// acknowledging it for --remove-source-files.
	11	+	var ErrFileIgnored = errors.New("file ignored")
	12	+
	13	+	// FileInfo describes an entry in an FS.
	14	+	type FileInfo struct {
	15	+	// Name is slash-separated and relative to the directory being listed.
	16	+	Name string
	17	+	Size int64
	18	+	ModTime time.Time
	19	+	IsDir bool
	20	+	}
	21	+
	22	+	// File is a file opened for reading.
	23	+	type File interface {
	24	+	io.ReaderAt
	25	+	io.Closer
	26	+	}
	27	+
	28	+	// FS is the storage the server reads from and writes to. Paths are
	29	+	// slash-separated, relative to the root of the storage, and never start
	30	+	// with a slash. The root itself is "".
	31	+	type FS interface {
	32	+	// Stat returns the entry at name, or an error wrapping fs.ErrNotExist.
	33	+	Stat(name string) (FileInfo, error)
	34	+	// ReadDir returns the entries below dir with names relative to dir,
	35	+	// descending into subdirectories when recursive is set.
	36	+	ReadDir(dir string, recursive bool) ([]FileInfo, error)
	37	+	// Open opens a regular file for reading.
	38	+	Open(name string) (File, FileInfo, error)
	39	+	// Put stores a regular file with the data read from r. If the transfer
	40	+	// fails, r returns an error instead of io.EOF and Put must not keep
	41	+	// what it read. The returned message, if any, is shown to the user.
	42	+	// Return an error wrapping ErrFileIgnored for an intentional skip.
	43	+	Put(name string, info FileInfo, r io.Reader) (string, error)
	44	+	// Remove deletes a file or an empty directory.
	45	+	Remove(name string, isDir bool) error
	46	+	}
	47	+
	48	+	// mapFile gives windowed access to a file being sent, like rsync's
	49	+	// map_ptr. Returned slices are valid until the next call. After a read error
	50	+	// it returns zeroes and records the error, which the caller checks once the
	51	+	// file is done.
	52	+	type mapFile struct {
	53	+	r io.ReaderAt
	54	+	size int64
	55	+	window []byte
	56	+	start int64
	57	+	defSize int64
	58	+	err error
	59	+	}
	60	+
	61	+	func newMapFile(r io.ReaderAt, size int64, readSize int64, blength int32) *mapFile {
	62	+	if blength > 0 && readSize%int64(blength) != 0 {
	63	+	readSize += int64(blength) - readSize%int64(blength)
	64	+	}
	65	+	return &mapFile{r: r, size: size, defSize: alignUp(readSize)}
	66	+	}
	67	+
	68	+	func alignUp(n int64) int64 { return (n + 1023) &^ 1023 }
	69	+
	70	+	func (m *mapFile) ptr(offset int64, n int) []byte {
	71	+	if n == 0 {
	72	+	return nil
	73	+	}
	74	+	if m.err != nil {
	75	+	return make([]byte, n)
	76	+	}
	77	+	end := offset + int64(n)
	78	+	if offset >= m.start && end <= m.start+int64(len(m.window)) {
	79	+	return m.window[offset-m.start : end-m.start]
	80	+	}
	81	+	start := offset &^ 1023
	82	+	size := max(m.defSize, alignUp(end-start))
	83	+	if start+size > m.size {
	84	+	size = m.size - start
	85	+	}
	86	+	buf := make([]byte, size)
	87	+	// Reuse the overlap with the current window.
	88	+	read := buf
	89	+	readAt := start
	90	+	if start >= m.start && start < m.start+int64(len(m.window)) {
	91	+	k := copy(buf, m.window[start-m.start:])
	92	+	read = buf[k:]
	93	+	readAt = start + int64(k)
	94	+	}
	95	+	if len(read) > 0 {
	96	+	if err := readFullAt(m.r, read, readAt); err != nil {
	97	+	m.err = err
	98	+	return make([]byte, n)
	99	+	}
	100	+	}
	101	+	m.window = buf
	102	+	m.start = start
	103	+	return m.window[offset-m.start : end-m.start]
	104	+	}

...	...		@@ -0,0 +1,933 @@
	1	+	package rsync
	2	+
	3	+	import (
	4	+	"bytes"
	5	+	"crypto/rand"
	6	+	"errors"
	7	+	"fmt"
	8	+	"io"
	9	+	"io/fs"
	10	+	"os"
	11	+	"os/exec"
	12	+	"path/filepath"
	13	+	"slices"
	14	+	"strings"
	15	+	"testing"
	16	+	"time"
	17	+	)
	18	+
	19	+	// dirFS serves a local directory.
	20	+	type dirFS struct{ root string }
	21	+
	22	+	func (d dirFS) full(name string) string { return filepath.Join(d.root, filepath.FromSlash(name)) }
	23	+
	24	+	func (d dirFS) Stat(name string) (FileInfo, error) {
	25	+	st, err := os.Stat(d.full(name))
	26	+	if err != nil {
	27	+	return FileInfo{}, err
	28	+	}
	29	+	return FileInfo{Name: st.Name(), Size: st.Size(), ModTime: st.ModTime(), IsDir: st.IsDir()}, nil
	30	+	}
	31	+
	32	+	func (d dirFS) ReadDir(dir string, recursive bool) ([]FileInfo, error) {
	33	+	base := d.full(dir)
	34	+	var out []FileInfo
	35	+	err := filepath.WalkDir(base, func(p string, e fs.DirEntry, err error) error {
	36	+	if err != nil {
	37	+	return err
	38	+	}
	39	+	if p == base {
	40	+	return nil
	41	+	}
	42	+	st, err := e.Info()
	43	+	if err != nil {
	44	+	return err
	45	+	}
	46	+	rel, _ := filepath.Rel(base, p)
	47	+	out = append(out, FileInfo{Name: filepath.ToSlash(rel), Size: st.Size(), ModTime: st.ModTime(), IsDir: st.IsDir()})
	48	+	if e.IsDir() && !recursive {
	49	+	return filepath.SkipDir
	50	+	}
	51	+	return nil
	52	+	})
	53	+	return out, err
	54	+	}
	55	+
	56	+	func (d dirFS) Open(name string) (File, FileInfo, error) {
	57	+	f, err := os.Open(d.full(name))
	58	+	if err != nil {
	59	+	return nil, FileInfo{}, err
	60	+	}
	61	+	st, err := f.Stat()
	62	+	if err != nil {
	63	+	_ = f.Close()
	64	+	return nil, FileInfo{}, err
	65	+	}
	66	+	return f, FileInfo{Name: st.Name(), Size: st.Size(), ModTime: st.ModTime()}, nil
	67	+	}
	68	+
	69	+	func (d dirFS) Put(name string, info FileInfo, r io.Reader) (string, error) {
	70	+	p := d.full(name)
	71	+	if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil {
	72	+	return "", err
	73	+	}
	74	+	tmp, err := os.CreateTemp(filepath.Dir(p), ".tmp-")
	75	+	if err != nil {
	76	+	return "", err
	77	+	}
	78	+	if _, err := io.Copy(tmp, r); err != nil {
	79	+	_ = tmp.Close()
	80	+	_ = os.Remove(tmp.Name())
	81	+	return "", err
	82	+	}
	83	+	if err := tmp.Close(); err != nil {
	84	+	return "", err
	85	+	}
	86	+	if err := os.Rename(tmp.Name(), p); err != nil {
	87	+	return "", err
	88	+	}
	89	+	return "", os.Chtimes(p, info.ModTime, info.ModTime)
	90	+	}
	91	+
	92	+	func (d dirFS) Remove(name string, isDir bool) error { return os.Remove(d.full(name)) }
	93	+
	94	+	type rejectingFS struct {
	95	+	dirFS
	96	+	err error
	97	+	}
	98	+
	99	+	func (d rejectingFS) Put(name string, info FileInfo, r io.Reader) (string, error) {
	100	+	if filepath.Base(name) == "skip.txt" {
	101	+	return "", d.err
	102	+	}
	103	+	return d.dirFS.Put(name, info, r)
	104	+	}
	105	+
	106	+	// TestHelperProcess is the remote shell rsync runs in these tests. It serves
	107	+	// the directory in RSYNC_TEST_ROOT on stdin and stdout.
	108	+	func TestHelperProcess(t *testing.T) {
	109	+	root := os.Getenv("RSYNC_TEST_ROOT")
	110	+	if root == "" {
	111	+	t.Skip("helper process")
	112	+	}
	113	+	args := os.Args
	114	+	for len(args) > 0 && args[0] != "--" {
	115	+	args = args[1:]
	116	+	}
	117	+	// Skip "--", the host name and "rsync".
	118	+	if len(args) < 3 {
	119	+	os.Exit(2)
	120	+	}
	121	+	args = args[3:]
	122	+	rw := struct {
	123	+	io.Reader
	124	+	io.Writer
	125	+	}{os.Stdin, os.Stdout}
	126	+	var fsys FS = dirFS{root: root}
	127	+	switch os.Getenv("RSYNC_TEST_PUT_RESULT") {
	128	+	case "ignored":
	129	+	fsys = rejectingFS{dirFS{root: root}, fmt.Errorf("policy: %w", ErrFileIgnored)}
	130	+	case "failed":
	131	+	fsys = rejectingFS{dirFS{root: root}, errors.New("storage backend failed")}
	132	+	}
	133	+	err := Serve(Config{FS: fsys, Stderr: os.Stderr}, rw, args)
	134	+	code := 0
	135	+	var exit *ExitError
	136	+	if errors.As(err, &exit) {
	137	+	code = exit.Code
	138	+	} else if err != nil {
	139	+	code = 1
	140	+	}
	141	+	os.Exit(code)
	142	+	}
	143	+
	144	+	type rsyncEnv struct {
	145	+	t *testing.T
	146	+	remote string // directory our server serves
	147	+	local string
	148	+	}
	149	+
	150	+	func newRsyncEnv(t testing.T) rsyncEnv {
	151	+	t.Helper()
	152	+	if _, err := exec.LookPath("rsync"); err != nil {
	153	+	t.Skip("rsync binary not found")
	154	+	}
	155	+	return &rsyncEnv{t: t, remote: t.TempDir(), local: t.TempDir()}
	156	+	}
	157	+
	158	+	// run executes the rsync client with our server on the other end.
	159	+	func (e *rsyncEnv) run(args ...string) (string, error) {
	160	+	e.t.Helper()
	161	+	rsh := fmt.Sprintf("%s -test.run=^TestHelperProcess$ --", os.Args[0])
	162	+	cmd := exec.Command("rsync", append([]string{"-e", rsh}, args...)...)
	163	+	cmd.Env = append(os.Environ(), "RSYNC_TEST_ROOT="+e.remote)
	164	+	out, err := cmd.CombinedOutput()
	165	+	return string(out), err
	166	+	}
	167	+
	168	+	func (e *rsyncEnv) mustRun(args ...string) string {
	169	+	e.t.Helper()
	170	+	out, err := e.run(args...)
	171	+	if err != nil {
	172	+	e.t.Fatalf("rsync %s: %v\n%s", strings.Join(args, " "), err, out)
	173	+	}
	174	+	return out
	175	+	}
	176	+
	177	+	func writeFile(t *testing.T, path string, data []byte) {
	178	+	t.Helper()
	179	+	if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
	180	+	t.Fatal(err)
	181	+	}
	182	+	if err := os.WriteFile(path, data, 0o644); err != nil {
	183	+	t.Fatal(err)
	184	+	}
	185	+	}
	186	+
	187	+	func randomBytes(t *testing.T, n int) []byte {
	188	+	t.Helper()
	189	+	b := make([]byte, n)
	190	+	if _, err := rand.Read(b); err != nil {
	191	+	t.Fatal(err)
	192	+	}
	193	+	return b
	194	+	}
	195	+
	196	+	// textBytes returns compressible data.
	197	+	func textBytes(n int, seed string) []byte {
	198	+	var b bytes.Buffer
	199	+	for i := 0; b.Len() < n; i++ {
	200	+	fmt.Fprintf(&b, "%s line %d: the quick brown fox jumps over the lazy dog\n", seed, i)
	201	+	}
	202	+	return b.Bytes()[:n]
	203	+	}
	204	+
	205	+	// tree returns the regular files below root with their contents.
	206	+	func tree(t *testing.T, root string) map[string]string {
	207	+	t.Helper()
	208	+	out := map[string]string{}
	209	+	err := filepath.WalkDir(root, func(p string, e fs.DirEntry, err error) error {
	210	+	if err != nil {
	211	+	return err
	212	+	}
	213	+	if e.Type().IsRegular() {
	214	+	b, err := os.ReadFile(p)
	215	+	if err != nil {
	216	+	return err
	217	+	}
	218	+	rel, _ := filepath.Rel(root, p)
	219	+	out[filepath.ToSlash(rel)] = string(b)
	220	+	}
	221	+	return nil
	222	+	})
	223	+	if err != nil {
	224	+	t.Fatal(err)
	225	+	}
	226	+	return out
	227	+	}
	228	+
	229	+	func assertSameTree(t *testing.T, want, got string) {
	230	+	t.Helper()
	231	+	a, b := tree(t, want), tree(t, got)
	232	+	var names []string
	233	+	for k := range a {
	234	+	names = append(names, k)
	235	+	}
	236	+	for k := range b {
	237	+	if _, ok := a[k]; !ok {
	238	+	names = append(names, k)
	239	+	}
	240	+	}
	241	+	slices.Sort(names)
	242	+	for _, n := range slices.Compact(names) {
	243	+	av, aok := a[n]
	244	+	bv, bok := b[n]
	245	+	switch {
	246	+	case !aok:
	247	+	t.Errorf("unexpected file %s", n)
	248	+	case !bok:
	249	+	t.Errorf("missing file %s", n)
	250	+	case av != bv:
	251	+	t.Errorf("file %s differs (%d bytes, want %d)", n, len(bv), len(av))
	252	+	}
	253	+	}
	254	+	}
	255	+
	256	+	func populate(t *testing.T, root string) {
	257	+	t.Helper()
	258	+	writeFile(t, filepath.Join(root, "index.html"), textBytes(3000, "index"))
	259	+	writeFile(t, filepath.Join(root, "empty.txt"), nil)
	260	+	writeFile(t, filepath.Join(root, "css/site.css"), textBytes(800, "css"))
	261	+	writeFile(t, filepath.Join(root, "img/logo.bin"), randomBytes(t, 200_000))
	262	+	writeFile(t, filepath.Join(root, "a/b/c/deep.txt"), textBytes(50, "deep"))
	263	+	writeFile(t, filepath.Join(root, "a.b"), []byte("dot file next to dir a"))
	264	+	writeFile(t, filepath.Join(root, "a-b"), []byte("dash file next to dir a"))
	265	+	writeFile(t, filepath.Join(root, "big.dat"), randomBytes(t, 3_000_000))
	266	+	}
	267	+
	268	+	var protocols = []string{"27", "28", "29", "30", "31", ""}
	269	+
	270	+	func protoArgs(p string) []string {
	271	+	if p == "" {
	272	+	return nil
	273	+	}
	274	+	return []string{"--protocol=" + p}
	275	+	}
	276	+
	277	+	func TestRsyncUpload(t *testing.T) {
	278	+	for _, proto := range protocols {
	279	+	for _, compress := range [][]string{nil, {"-z"}, {"--compress-choice=zlib"}, {"--compress-choice=zlibx"}} {
	280	+	name := fmt.Sprintf("proto=%s/%s", proto, strings.Join(compress, ""))
	281	+	t.Run(name, func(t *testing.T) {
	282	+	if proto != "" && proto < "30" && len(compress) > 0 && compress[0] != "-z" {
	283	+	t.Skip("compression choice needs protocol 30")
	284	+	}
	285	+	e := newRsyncEnv(t)
	286	+	populate(t, e.local)
	287	+	args := append(protoArgs(proto), compress...)
	288	+	e.mustRun(append(args, "-rt", e.local+"/", "host:site")...)
	289	+	assertSameTree(t, e.local, filepath.Join(e.remote, "site"))
	290	+
	291	+	// Change part of a big file and upload again: only the
	292	+	// changed region should cross the wire.
	293	+	big := filepath.Join(e.local, "big.dat")
	294	+	data, _ := os.ReadFile(big)
	295	+	copy(data[1_000_000:], randomBytes(t, 5000))
	296	+	data = append(data[:2_000_000], append(randomBytes(t, 1234), data[2_000_000:]...)...)
	297	+	writeFile(t, big, data)
	298	+	writeFile(t, filepath.Join(e.local, "css/site.css"), textBytes(900, "css2"))
	299	+	out := e.mustRun(append(args, "-rt", "--stats", e.local+"/", "host:site")...)
	300	+	assertSameTree(t, e.local, filepath.Join(e.remote, "site"))
	301	+	literal := statValue(t, out, "Literal data")
	302	+	if literal > 100_000 {
	303	+	t.Errorf("literal data %d, want a delta transfer\n%s", literal, out)
	304	+	}
	305	+	if matched := statValue(t, out, "Matched data"); matched < 2_000_000 {
	306	+	t.Errorf("matched data %d, want most of big.dat\n%s", matched, out)
	307	+	}
	308	+	})
	309	+	}
	310	+	}
	311	+	}
	312	+
	313	+	func TestRsyncDownload(t *testing.T) {
	314	+	for _, proto := range protocols {
	315	+	for _, compress := range [][]string{nil, {"-z"}, {"--compress-choice=zlib"}} {
	316	+	name := fmt.Sprintf("proto=%s/%s", proto, strings.Join(compress, ""))
	317	+	t.Run(name, func(t *testing.T) {
	318	+	if proto != "" && proto < "30" && len(compress) > 0 && compress[0] != "-z" {
	319	+	t.Skip("compression choice needs protocol 30")
	320	+	}
	321	+	e := newRsyncEnv(t)
	322	+	site := filepath.Join(e.remote, "site")
	323	+	populate(t, site)
	324	+	args := append(protoArgs(proto), compress...)
	325	+	e.mustRun(append(args, "-rt", "host:site/", e.local+"/")...)
	326	+	assertSameTree(t, site, e.local)
	327	+
	328	+	// Change the server's copy and download again.
	329	+	big := filepath.Join(site, "big.dat")
	330	+	data, _ := os.ReadFile(big)
	331	+	copy(data[500_000:], randomBytes(t, 3000))
	332	+	writeFile(t, big, data)
	333	+	// rsync compares whole seconds, so make the change visible.
	334	+	later := time.Now().Add(time.Minute)
	335	+	if err := os.Chtimes(big, later, later); err != nil {
	336	+	t.Fatal(err)
	337	+	}
	338	+	out := e.mustRun(append(args, "-rt", "--stats", "host:site/", e.local+"/")...)
	339	+	assertSameTree(t, site, e.local)
	340	+	if literal := statValue(t, out, "Literal data"); literal > 100_000 {
	341	+	t.Errorf("literal data %d, want a delta transfer\n%s", literal, out)
	342	+	}
	343	+	})
	344	+	}
	345	+	}
	346	+	}
	347	+
	348	+	func TestRsyncChecksumChoices(t *testing.T) {
	349	+	for _, choice := range []string{"md4", "md5", "xxh64", "md5,md4"} {
	350	+	t.Run(choice, func(t *testing.T) {
	351	+	e := newRsyncEnv(t)
	352	+	populate(t, e.local)
	353	+	e.mustRun("-r", "--checksum-choice="+choice, e.local+"/", "host:site")
	354	+	assertSameTree(t, e.local, filepath.Join(e.remote, "site"))
	355	+	e.mustRun("-r", "-c", "--checksum-choice="+choice, "host:site/", t.TempDir())
	356	+	})
	357	+	}
	358	+	}
	359	+
	360	+	func TestRsyncAlwaysChecksum(t *testing.T) {
	361	+	for _, proto := range protocols {
	362	+	t.Run("proto="+proto, func(t *testing.T) {
	363	+	e := newRsyncEnv(t)
	364	+	populate(t, e.local)
	365	+	args := protoArgs(proto)
	366	+	e.mustRun(append(args, "-r", e.local+"/", "host:site")...)
	367	+
	368	+	// Same size and time, different content: only -c notices.
	369	+	p := filepath.Join(e.local, "css/site.css")
	370	+	st, _ := os.Stat(filepath.Join(e.remote, "site/css/site.css"))
	371	+	data, _ := os.ReadFile(p)
	372	+	data[10] ^= 0xff
	373	+	writeFile(t, p, data)
	374	+	_ = os.Chtimes(p, st.ModTime(), st.ModTime())
	375	+
	376	+	out := e.mustRun(append(args, "-rv", e.local+"/", "host:site")...)
	377	+	if strings.Contains(out, "css/site.css") {
	378	+	t.Fatalf("quick check should have skipped the file\n%s", out)
	379	+	}
	380	+	out = e.mustRun(append(args, "-rvc", e.local+"/", "host:site")...)
	381	+	if !strings.Contains(out, "css/site.css") {
	382	+	t.Fatalf("checksum run did not transfer the file\n%s", out)
	383	+	}
	384	+	assertSameTree(t, e.local, filepath.Join(e.remote, "site"))
	385	+
	386	+	// And the other direction.
	387	+	dl := t.TempDir()
	388	+	e.mustRun(append(args, "-rt", "host:site/", dl+"/")...)
	389	+	data, _ = os.ReadFile(filepath.Join(dl, "index.html"))
	390	+	st, _ = os.Stat(filepath.Join(dl, "index.html"))
	391	+	data[0] ^= 0xff
	392	+	writeFile(t, filepath.Join(dl, "index.html"), data)
	393	+	_ = os.Chtimes(filepath.Join(dl, "index.html"), st.ModTime(), st.ModTime())
	394	+	e.mustRun(append(args, "-rtc", "host:site/", dl+"/")...)
	395	+	assertSameTree(t, filepath.Join(e.remote, "site"), dl)
	396	+	})
	397	+	}
	398	+	}
	399	+
	400	+	func TestRsyncDelete(t *testing.T) {
	401	+	for _, proto := range protocols {
	402	+	for _, when := range []string{"--delete", "--delete-before", "--delete-during", "--delete-after", "--delete-delay"} {
	403	+	t.Run(fmt.Sprintf("proto=%s/%s", proto, when), func(t *testing.T) {
	404	+	if proto != "" && proto < "30" && when == "--delete-delay" {
	405	+	t.Skip("needs protocol 30")
	406	+	}
	407	+	e := newRsyncEnv(t)
	408	+	populate(t, e.local)
	409	+	args := protoArgs(proto)
	410	+	e.mustRun(append(args, "-r", e.local+"/", "host:site")...)
	411	+
	412	+	writeFile(t, filepath.Join(e.remote, "site/stale.txt"), []byte("old"))
	413	+	writeFile(t, filepath.Join(e.remote, "site/old/dir/file.txt"), []byte("old"))
	414	+	writeFile(t, filepath.Join(e.remote, "site/keep.log"), []byte("protected"))
	415	+	if err := os.RemoveAll(filepath.Join(e.local, "a")); err != nil {
	416	+	t.Fatal(err)
	417	+	}
	418	+
	419	+	out := e.mustRun(append(args, "-rv", when, "--exclude=*.log", e.local+"/", "host:site")...)
	420	+	if !strings.Contains(out, "deleting stale.txt") {
	421	+	t.Errorf("missing deletion message\n%s", out)
	422	+	}
	423	+	remote := tree(t, filepath.Join(e.remote, "site"))
	424	+	if _, ok := remote["keep.log"]; !ok {
	425	+	t.Error("excluded file was deleted")
	426	+	}
	427	+	delete(remote, "keep.log")
	428	+	if err := os.Remove(filepath.Join(e.remote, "site/keep.log")); err != nil {
	429	+	t.Fatal(err)
	430	+	}
	431	+	assertSameTree(t, e.local, filepath.Join(e.remote, "site"))
	432	+	if _, err := os.Stat(filepath.Join(e.remote, "site/old")); !os.IsNotExist(err) {
	433	+	t.Errorf("stale directory survived: %v", err)
	434	+	}
	435	+	})
	436	+	}
	437	+	}
	438	+	}
	439	+
	440	+	func TestRsyncDownloadFilters(t *testing.T) {
	441	+	e := newRsyncEnv(t)
	442	+	site := filepath.Join(e.remote, "site")
	443	+	populate(t, site)
	444	+	e.mustRun("-r", "--exclude=*.bin", "--exclude=/a/", "host:site/", e.local+"/")
	445	+	got := tree(t, e.local)
	446	+	if _, ok := got["img/logo.bin"]; ok {
	447	+	t.Error("excluded file was sent")
	448	+	}
	449	+	if _, ok := got["a/b/c/deep.txt"]; ok {
	450	+	t.Error("excluded directory was sent")
	451	+	}
	452	+	if _, ok := got["index.html"]; !ok {
	453	+	t.Error("index.html missing")
	454	+	}
	455	+	}
	456	+
	457	+	func TestRsyncSingleFile(t *testing.T) {
	458	+	e := newRsyncEnv(t)
	459	+	writeFile(t, filepath.Join(e.local, "one.txt"), []byte("hello"))
	460	+
	461	+	e.mustRun(filepath.Join(e.local, "one.txt"), "host:site/renamed.txt")
	462	+	if b, err := os.ReadFile(filepath.Join(e.remote, "site/renamed.txt")); err != nil \|\| string(b) != "hello" {
	463	+	t.Fatalf("single file upload: %q %v", b, err)
	464	+	}
	465	+	e.mustRun(filepath.Join(e.local, "one.txt"), "host:site/")
	466	+	if b, err := os.ReadFile(filepath.Join(e.remote, "site/one.txt")); err != nil \|\| string(b) != "hello" {
	467	+	t.Fatalf("upload into dir: %q %v", b, err)
	468	+	}
	469	+
	470	+	dl := t.TempDir()
	471	+	e.mustRun("host:site/renamed.txt", dl+"/")
	472	+	if b, err := os.ReadFile(filepath.Join(dl, "renamed.txt")); err != nil \|\| string(b) != "hello" {
	473	+	t.Fatalf("single file download: %q %v", b, err)
	474	+	}
	475	+	}
	476	+
	477	+	func TestRsyncListOnly(t *testing.T) {
	478	+	e := newRsyncEnv(t)
	479	+	populate(t, filepath.Join(e.remote, "site"))
	480	+	out := e.mustRun("host:site/")
	481	+	for _, want := range []string{"index.html", "css", "big.dat"} {
	482	+	if !strings.Contains(out, want) {
	483	+	t.Errorf("listing lacks %s\n%s", want, out)
	484	+	}
	485	+	}
	486	+	if strings.Contains(out, "site.css") {
	487	+	t.Errorf("non-recursive listing descended\n%s", out)
	488	+	}
	489	+	out = e.mustRun("-r", "host:site/")
	490	+	if !strings.Contains(out, "css/site.css") {
	491	+	t.Errorf("recursive listing lacks css/site.css\n%s", out)
	492	+	}
	493	+	}
	494	+
	495	+	func TestRsyncDryRun(t *testing.T) {
	496	+	e := newRsyncEnv(t)
	497	+	populate(t, e.local)
	498	+	out := e.mustRun("-rvn", e.local+"/", "host:site")
	499	+	if !strings.Contains(out, "big.dat") {
	500	+	t.Errorf("dry run did not list files\n%s", out)
	501	+	}
	502	+	if _, err := os.Stat(filepath.Join(e.remote, "site")); !os.IsNotExist(err) {
	503	+	t.Errorf("dry run wrote files: %v", err)
	504	+	}
	505	+	}
	506	+
	507	+	func TestRsyncMissingSource(t *testing.T) {
	508	+	e := newRsyncEnv(t)
	509	+	out, err := e.run("-r", "host:nope/", e.local+"/")
	510	+	var exit *exec.ExitError
	511	+	if !errors.As(err, &exit) \|\| exit.ExitCode() != exitPartial {
	512	+	t.Fatalf("want exit code %d, got %v\n%s", exitPartial, err, out)
	513	+	}
	514	+	if !strings.Contains(out, "No such file or directory") {
	515	+	t.Errorf("missing error message\n%s", out)
	516	+	}
	517	+	}
	518	+
	519	+	func TestRsyncUnchangedSkipped(t *testing.T) {
	520	+	e := newRsyncEnv(t)
	521	+	populate(t, e.local)
	522	+	e.mustRun("-rt", e.local+"/", "host:site")
	523	+	out := e.mustRun("-rt", "--stats", e.local+"/", "host:site")
	524	+	if n := statValue(t, out, "Number of regular files transferred"); n != 0 {
	525	+	t.Errorf("transferred %d files, want 0\n%s", n, out)
	526	+	}
	527	+	}
	528	+
	529	+	func statValue(t *testing.T, out, label string) int64 {
	530	+	t.Helper()
	531	+	for _, line := range strings.Split(out, "\n") {
	532	+	rest, ok := strings.CutPrefix(line, label+": ")
	533	+	if !ok {
	534	+	continue
	535	+	}
	536	+	rest = strings.TrimSuffix(rest, " bytes")
	537	+	rest = strings.ReplaceAll(rest, ",", "")
	538	+	var v int64
	539	+	if _, err := fmt.Sscan(rest, &v); err == nil {
	540	+	return v
	541	+	}
	542	+	}
	543	+	t.Fatalf("no %q in output\n%s", label, out)
	544	+	return 0
	545	+	}
	546	+
	547	+	func TestRsyncCompression(t *testing.T) {
	548	+	type tc struct {
	549	+	proto string
	550	+	args []string
	551	+	}
	552	+	cases := []tc{
	553	+	{"27", []string{"-z"}},
	554	+	{"29", []string{"-z"}},
	555	+	{"30", []string{"-z"}},
	556	+	{"30", []string{"--compress-choice=zlib"}},
	557	+	{"31", []string{"--compress-choice=zlib"}},
	558	+	{"", []string{"-z"}},
	559	+	{"", []string{"--compress-choice=zlibx"}},
	560	+	{"", []string{"-z", "--compress-level=1"}},
	561	+	{"", []string{"-z", "--compress-level=9"}},
	562	+	}
	563	+	for _, c := range cases {
	564	+	for _, block := range []string{"", "--block-size=100000"} {
	565	+	t.Run(fmt.Sprintf("proto=%s/%s/%s", c.proto, strings.Join(c.args, ""), block), func(t *testing.T) {
	566	+	e := newRsyncEnv(t)
	567	+	text := textBytes(2_000_000, "compressible")
	568	+	writeFile(t, filepath.Join(e.local, "text.txt"), text)
	569	+	args := append(protoArgs(c.proto), c.args...)
	570	+	if block != "" {
	571	+	args = append(args, block)
	572	+	}
	573	+
	574	+	out := e.mustRun(append(args, "-rt", "--stats", e.local+"/", "host:site")...)
	575	+	assertSameTree(t, e.local, filepath.Join(e.remote, "site"))
	576	+	if sent := statValue(t, out, "Total bytes sent"); sent > 500_000 {
	577	+	t.Errorf("upload sent %d bytes, compression is not in effect\n%s", sent, out)
	578	+	}
	579	+
	580	+	// A delta with matched blocks exercises the deflate history.
	581	+	text = append(textBytes(150_000, "prefix"), text...)
	582	+	text = append(text, textBytes(70_000, "suffix")...)
	583	+	writeFile(t, filepath.Join(e.local, "text.txt"), text)
	584	+	out = e.mustRun(append(args, "-rt", "--stats", e.local+"/", "host:site")...)
	585	+	assertSameTree(t, e.local, filepath.Join(e.remote, "site"))
	586	+	if matched := statValue(t, out, "Matched data"); matched < 1_000_000 {
	587	+	t.Errorf("matched %d bytes, want a delta\n%s", matched, out)
	588	+	}
	589	+
	590	+	dl := t.TempDir()
	591	+	writeFile(t, filepath.Join(dl, "text.txt"), textBytes(2_000_000, "compressible"))
	592	+	old := time.Now().Add(-time.Hour)
	593	+	_ = os.Chtimes(filepath.Join(dl, "text.txt"), old, old)
	594	+	out = e.mustRun(append(args, "-rt", "--stats", "host:site/", dl+"/")...)
	595	+	assertSameTree(t, filepath.Join(e.remote, "site"), dl)
	596	+	if recv := statValue(t, out, "Total bytes received"); recv > 300_000 {
	597	+	t.Errorf("download received %d bytes, want a compressed delta\n%s", recv, out)
	598	+	}
	599	+	if matched := statValue(t, out, "Matched data"); matched < 1_000_000 {
	600	+	t.Errorf("download matched %d bytes, want a delta\n%s", matched, out)
	601	+	}
	602	+	})
	603	+	}
	604	+	}
	605	+	}
	606	+
	607	+	func TestRsyncArchiveMode(t *testing.T) {
	608	+	for _, proto := range protocols {
	609	+	t.Run("proto="+proto, func(t *testing.T) {
	610	+	e := newRsyncEnv(t)
	611	+	populate(t, e.local)
	612	+	if err := os.Symlink("index.html", filepath.Join(e.local, "link.html")); err != nil {
	613	+	t.Fatal(err)
	614	+	}
	615	+	args := protoArgs(proto)
	616	+	out := e.mustRun(append(args, "-av", e.local+"/", "host:site")...)
	617	+	if !strings.Contains(out, `skipping non-regular file "link.html"`) {
	618	+	t.Errorf("symlink not reported as skipped\n%s", out)
	619	+	}
	620	+	_ = os.Remove(filepath.Join(e.local, "link.html"))
	621	+	assertSameTree(t, e.local, filepath.Join(e.remote, "site"))
	622	+
	623	+	dl := t.TempDir()
	624	+	e.mustRun(append(args, "-a", "host:site/", dl+"/")...)
	625	+	assertSameTree(t, e.local, dl)
	626	+	})
	627	+	}
	628	+	}
	629	+
	630	+	func TestRsyncTransferOptions(t *testing.T) {
	631	+	cases := [][]string{
	632	+	{"-s"},
	633	+	{"-W"},
	634	+	{"--inplace"},
	635	+	{"-u"},
	636	+	{"--size-only"},
	637	+	{"-I"},
	638	+	{"--block-size=2048"},
	639	+	{"--checksum-seed=12345"},
	640	+	{"--numeric-ids", "-og"},
	641	+	{"-U"},
	642	+	}
	643	+	for _, opts := range cases {
	644	+	t.Run(strings.Join(opts, " "), func(t *testing.T) {
	645	+	e := newRsyncEnv(t)
	646	+	populate(t, e.local)
	647	+	e.mustRun(append(slices.Clone(opts), "-rt", e.local+"/", "host:site")...)
	648	+	assertSameTree(t, e.local, filepath.Join(e.remote, "site"))
	649	+
	650	+	big := filepath.Join(e.remote, "site/big.dat")
	651	+	data, _ := os.ReadFile(big)
	652	+	copy(data[100_000:], randomBytes(t, 4000))
	653	+	copy(data[2_500_000:], data[10_000:20_000])
	654	+	writeFile(t, big, data)
	655	+	later := time.Now().Add(time.Minute)
	656	+	_ = os.Chtimes(big, later, later)
	657	+
	658	+	dl := t.TempDir()
	659	+	e.mustRun(append(slices.Clone(opts), "-rt", e.local+"/", dl+"/")...)
	660	+	e.mustRun(append(slices.Clone(opts), "-rt", "host:site/", dl+"/")...)
	661	+	if opts[0] != "--size-only" {
	662	+	assertSameTree(t, filepath.Join(e.remote, "site"), dl)
	663	+	}
	664	+	})
	665	+	}
	666	+	}
	667	+
	668	+	func TestRsyncSkipOptions(t *testing.T) {
	669	+	e := newRsyncEnv(t)
	670	+	writeFile(t, filepath.Join(e.local, "new.txt"), []byte("new"))
	671	+	writeFile(t, filepath.Join(e.local, "old.txt"), []byte("changed locally"))
	672	+	writeFile(t, filepath.Join(e.local, "huge.txt"), textBytes(50_000, "huge"))
	673	+	writeFile(t, filepath.Join(e.remote, "site/old.txt"), []byte("remote"))
	674	+
	675	+	e.mustRun("-r", "--existing", e.local+"/", "host:site")
	676	+	got := tree(t, filepath.Join(e.remote, "site"))
	677	+	if _, ok := got["new.txt"]; ok \|\| got["old.txt"] != "changed locally" {
	678	+	t.Errorf("--existing: %v", got)
	679	+	}
	680	+
	681	+	writeFile(t, filepath.Join(e.remote, "site/old.txt"), []byte("remote"))
	682	+	e.mustRun("-r", "--ignore-existing", "--max-size=10k", e.local+"/", "host:site")
	683	+	got = tree(t, filepath.Join(e.remote, "site"))
	684	+	if got["old.txt"] != "remote" \|\| got["new.txt"] != "new" {
	685	+	t.Errorf("--ignore-existing: %v", got)
	686	+	}
	687	+	if _, ok := got["huge.txt"]; ok {
	688	+	t.Error("--max-size let a large file through")
	689	+	}
	690	+	}
	691	+
	692	+	func TestRsyncPutResults(t *testing.T) {
	693	+	for _, proto := range protocols {
	694	+	for _, result := range []string{"ignored", "failed"} {
	695	+	t.Run("protocol_"+proto+"/"+result, func(t *testing.T) {
	696	+	t.Setenv("RSYNC_TEST_PUT_RESULT", result)
	697	+	e := newRsyncEnv(t)
	698	+	for _, name := range []string{"a.txt", "skip.txt", "z.txt"} {
	699	+	writeFile(t, filepath.Join(e.local, name), textBytes(100_000, name))
	700	+	}
	701	+	writeFile(t, filepath.Join(e.remote, "site/skip.txt"), []byte("original"))
	702	+	out, err := e.run(append(protoArgs(proto), "-r", "--remove-source-files", e.local+"/", "host:site")...)
	703	+	if result == "ignored" {
	704	+	if err != nil {
	705	+	t.Fatalf("ignored file failed the batch: %v\n%s", err, out)
	706	+	}
	707	+	} else {
	708	+	var exit *exec.ExitError
	709	+	if !errors.As(err, &exit) \|\| exit.ExitCode() != 23 {
	710	+	t.Fatalf("want exit 23, got %v\n%s", err, out)
	711	+	}
	712	+	}
	713	+	got := tree(t, filepath.Join(e.remote, "site"))
	714	+	if len(got) != 3 \|\| got["skip.txt"] != "original" {
	715	+	t.Fatalf("rejected file changed: %v", got)
	716	+	}
	717	+	for _, name := range []string{"a.txt", "z.txt"} {
	718	+	if got[name] != string(textBytes(100_000, name)) {
	719	+	t.Errorf("accepted file %s not stored", name)
	720	+	}
	721	+	if _, err := os.Stat(filepath.Join(e.local, name)); !errors.Is(err, os.ErrNotExist) {
	722	+	t.Errorf("accepted file %s not removed from source: %v", name, err)
	723	+	}
	724	+	}
	725	+	if data, err := os.ReadFile(filepath.Join(e.local, "skip.txt")); err != nil \|\| !bytes.Equal(data, textBytes(100_000, "skip.txt")) {
	726	+	t.Fatalf("rejected source file changed: %v", err)
	727	+	}
	728	+	})
	729	+	}
	730	+	}
	731	+	}
	732	+
	733	+	// writeTree writes files, named relative to root.
	734	+	func writeTree(t *testing.T, root string, files map[string]string) {
	735	+	t.Helper()
	736	+	for name, contents := range files {
	737	+	writeFile(t, filepath.Join(root, name), []byte(contents))
	738	+	}
	739	+	}
	740	+
	741	+	// mergeFilterTree has per-directory merge files covering inheritance,
	742	+	// overrides, anchoring, clearing and protect rules.
	743	+	var mergeFilterTree = map[string]string{
	744	+	".rsync-filter": "- keep.log\nP precious/\n- .tmp\n-p .perish\n",
	745	+	"keep.log": "keep",
	746	+	"stale.txt": "stale",
	747	+	"x.tmp": "tmp",
	748	+	"precious/a.txt": "precious",
	749	+	"sub/.rsync-filter": "+ sub-only.tmp\n- /anchored.txt\n",
	750	+	"sub/a.tmp": "tmp",
	751	+	"sub/sub-only.tmp": "kept by include",
	752	+	"sub/anchored.txt": "anchored",
	753	+	"sub/deeper/anchored.txt": "not anchored here",
	754	+	"sub/deeper/keep.log": "inherited",
	755	+	"cleared/.rsync-filter": "!\n- only.txt\n",
	756	+	"cleared/keep.log": "no longer excluded",
	757	+	"cleared/only.txt": "only",
	758	+	"gone/.rsync-filter": "- inside.log\n",
	759	+	"gone/inside.log": "protects its directory",
	760	+	"gone/other.txt": "other",
	761	+	".cvsignore": "precious.dat\n",
	762	+	"precious.dat": "cvs",
	763	+	"obj.o": "default cvs pattern",
	764	+	"nested/.cvsignore": "nested.dat",
	765	+	"nested/nested.dat": "nested",
	766	+	"nested/deeper/nested.dat": "cvsignore does not inherit",
	767	+	"nested/deeper/precious.dat": "nor does the parent's",
	768	+	"top.perish": "perishable rules protect a kept directory's entries",
	769	+	"perish/keep.perish": "but not a deleted directory's",
	770	+	"perish/deeper/keep.perish": "at any depth",
	771	+	"objdir/x.o": "default CVS patterns are perishable from protocol 30",
	772	+	}
	773	+
	774	+	// TestRsyncMergeFiltersMatchRsync runs the same transfer against real rsync
	775	+	// and against our server, each from a copy of mergeFilterTree, and expects
	776	+	// the same result.
	777	+	func TestRsyncMergeFiltersMatchRsync(t *testing.T) {
	778	+	local := map[string]string{"index.html": "new", "sub/new.txt": "new"}
	779	+	pushes := [][]string{
	780	+	{"-F", "--delete"},
	781	+	{"-F", "--delete-before"},
	782	+	{"-F", "--delete-during"},
	783	+	{"-F", "--delete-after"},
	784	+	{"-F", "--delete-delay"},
	785	+	{"-FF", "--delete"},
	786	+	{"-F", "--delete", "--delete-excluded"},
	787	+	{"-C", "--delete"},
	788	+	{"-CF", "--delete"},
	789	+	{"--filter=:e .rsync-filter", "--delete"},
	790	+	{"--filter=:n- .rsync-filter", "--delete"},
	791	+	{"--filter=dir-merge,- .rsync-filter", "--delete"},
	792	+	}
	793	+	for _, proto := range []string{"29", "30", "31", ""} {
	794	+	for _, flags := range pushes {
	795	+	t.Run(fmt.Sprintf("push/proto=%s/%s", proto, strings.Join(flags, " ")), func(t *testing.T) {
	796	+	if proto == "29" && slices.Contains(flags, "--delete-delay") {
	797	+	t.Skip("needs protocol 30")
	798	+	}
	799	+	e := newRsyncEnv(t)
	800	+	writeTree(t, e.local, local)
	801	+	ref := filepath.Join(t.TempDir(), "site")
	802	+	writeTree(t, ref, mergeFilterTree)
	803	+	writeTree(t, filepath.Join(e.remote, "site"), mergeFilterTree)
	804	+	args := append(protoArgs(proto), "-r")
	805	+	args = append(args, flags...)
	806	+
	807	+	refCmd := exec.Command("rsync", append(slices.Clone(args), e.local+"/", ref)...)
	808	+	if out, err := refCmd.CombinedOutput(); err != nil {
	809	+	t.Fatalf("reference rsync: %v\n%s", err, out)
	810	+	}
	811	+	e.mustRun(append(args, e.local+"/", "host:site")...)
	812	+	if slices.Contains(flags, "--delete-delay") {
	813	+	// rsync drops inherited per-directory rules before its
	814	+	// delayed deletions, so it removes this file. We keep it.
	815	+	inherited := filepath.Join(e.remote, "site/sub/deeper/keep.log")
	816	+	if err := os.Remove(inherited); err != nil {
	817	+	t.Fatalf("file protected by an inherited rule was deleted: %v", err)
	818	+	}
	819	+	}
	820	+	assertSameTree(t, ref, filepath.Join(e.remote, "site"))
	821	+	})
	822	+	}
	823	+	}
	824	+
	825	+	pulls := [][]string{{"-F"}, {"-FF"}, {"-C"}, {"--filter=:n .rsync-filter"}}
	826	+	for _, proto := range []string{"29", "30", "31", ""} {
	827	+	for _, flags := range pulls {
	828	+	for _, src := range []string{"site/", "site"} {
	829	+	t.Run(fmt.Sprintf("pull/proto=%s/%s/%s", proto, strings.Join(flags, " "), src), func(t *testing.T) {
	830	+	e := newRsyncEnv(t)
	831	+	writeTree(t, filepath.Join(e.remote, "site"), mergeFilterTree)
	832	+	refSrc := t.TempDir()
	833	+	writeTree(t, filepath.Join(refSrc, "site"), mergeFilterTree)
	834	+	ref := t.TempDir()
	835	+	args := append(protoArgs(proto), "-r")
	836	+	args = append(args, flags...)
	837	+
	838	+	refCmd := exec.Command("rsync", append(slices.Clone(args), filepath.Join(refSrc, src), ref+"/")...)
	839	+	if strings.HasSuffix(src, "/") {
	840	+	refCmd.Args[len(refCmd.Args)-2] += "/"
	841	+	}
	842	+	if out, err := refCmd.CombinedOutput(); err != nil {
	843	+	t.Fatalf("reference rsync: %v\n%s", err, out)
	844	+	}
	845	+	e.mustRun(append(args, "host:"+src, e.local+"/")...)
	846	+	assertSameTree(t, ref, e.local)
	847	+	})
	848	+	}
	849	+	}
	850	+	}
	851	+	}
	852	+
	853	+	// TestRsyncMergeFilterParentScan checks that "-F" reads .rsync-filter in
	854	+	// the directories above the transfer, as rsync does up to "/".
	855	+	func TestRsyncMergeFilterParentScan(t *testing.T) {
	856	+	e := newRsyncEnv(t)
	857	+	writeFile(t, filepath.Join(e.local, "index.html"), []byte("new"))
	858	+	writeTree(t, e.remote, map[string]string{
	859	+	".rsync-filter": "- /site/scanned.txt\n",
	860	+	"site/scanned.txt": "protected from above",
	861	+	"site/stale.txt": "stale",
	862	+	"other/scanned.txt": "outside the transfer",
	863	+	})
	864	+	e.mustRun("-r", "-F", "--delete", e.local+"/", "host:site")
	865	+	got := tree(t, filepath.Join(e.remote, "site"))
	866	+	if _, ok := got["scanned.txt"]; !ok {
	867	+	t.Error("rule from the parent directory's merge file was ignored")
	868	+	}
	869	+	if _, ok := got["stale.txt"]; ok {
	870	+	t.Error("unprotected file survived --delete")
	871	+	}
	872	+	}
	873	+
	874	+	// TestRsyncMergeFilterErrors checks that a merge file we can't use stops
	875	+	// deletions on upload and the whole transfer on download.
	876	+	func TestRsyncMergeFilterErrors(t *testing.T) {
	877	+	for _, contents := range []string{"merge other\n", "bogus rule\n"} {
	878	+	t.Run("push/"+contents, func(t *testing.T) {
	879	+	e := newRsyncEnv(t)
	880	+	writeFile(t, filepath.Join(e.local, "index.html"), []byte("new"))
	881	+	writeTree(t, filepath.Join(e.remote, "site"), map[string]string{
	882	+	"sub/.rsync-filter": contents,
	883	+	"sub/stale.txt": "stale",
	884	+	"stale.txt": "stale",
	885	+	})
	886	+	out, err := e.run("-r", "-F", "--delete-after", e.local+"/", "host:site")
	887	+	var exit *exec.ExitError
	888	+	if !errors.As(err, &exit) \|\| exit.ExitCode() != exitPartial {
	889	+	t.Fatalf("want exit %d, got %v\n%s", exitPartial, err, out)
	890	+	}
	891	+	if !strings.Contains(out, "skipping file deletion") {
	892	+	t.Errorf("no explanation for skipped deletion\n%s", out)
	893	+	}
	894	+	got := tree(t, filepath.Join(e.remote, "site"))
	895	+	if got["index.html"] != "new" {
	896	+	t.Error("file was not transferred")
	897	+	}
	898	+	for _, name := range []string{"stale.txt", "sub/stale.txt"} {
	899	+	if _, ok := got[name]; !ok {
	900	+	t.Errorf("%s was deleted despite the unusable merge file", name)
	901	+	}
	902	+	}
	903	+	})
	904	+	t.Run("pull/"+contents, func(t *testing.T) {
	905	+	e := newRsyncEnv(t)
	906	+	writeTree(t, filepath.Join(e.remote, "site"), map[string]string{
	907	+	"sub/.rsync-filter": contents,
	908	+	"index.html": "index",
	909	+	})
	910	+	out, err := e.run("-r", "-F", "host:site/", e.local+"/")
	911	+	if err == nil \|\| !strings.Contains(out, "per-directory filter file") {
	912	+	t.Fatalf("want a merge-file error, got %v\n%s", err, out)
	913	+	}
	914	+	if got := tree(t, e.local); len(got) != 0 {
	915	+	t.Errorf("files transferred despite the error: %v", got)
	916	+	}
	917	+	})
	918	+	}
	919	+	}
	920	+
	921	+	func TestRsyncUnsupportedOptions(t *testing.T) {
	922	+	e := newRsyncEnv(t)
	923	+	populate(t, e.local)
	924	+	for _, opt := range []string{"-H", "-A", "-X", "-R"} {
	925	+	out, err := e.run("-r", opt, e.local+"/", "host:site")
	926	+	if err == nil {
	927	+	t.Errorf("%s: want an error\n%s", opt, out)
	928	+	}
	929	+	if !strings.Contains(out, "not supported") {
	930	+	t.Errorf("%s: missing explanation\n%s", opt, out)
	931	+	}
	932	+	}
	933	+	}

...	...		@@ -0,0 +1,233 @@
	1	+	package rsync
	2	+
	3	+	import (
	4	+	"bytes"
	5	+	"hash"
	6	+	)
	7	+
	8	+	// matcher finds the receiver's blocks in the file being sent and emits the
	9	+	// delta (rsync's match.c).
	10	+	type matcher struct {
	11	+	head sumHead
	12	+	sums []sumBuf
	13	+	cs *checksums
	14	+	tok tokenSender
	15	+	buf *mapFile
	16	+	sum hash.Hash
	17	+	lastMatch int64
	18	+	inplace bool // the receiver updates its basis file in place
	19	+
	20	+	tablesize uint32
	21	+	table []int32
	22	+
	23	+	literal int64
	24	+	matched int64
	25	+	}
	26	+
	27	+	// matched sends the literal data before offset and, for i >= 0, the token of
	28	+	// block i. i == -1 ends the file and i == -2 only flushes literal data.
	29	+	func (m *matcher) emit(offset int64, i int32) error {
	30	+	n := offset - m.lastMatch
	31	+	var block []byte
	32	+	if i >= 0 {
	33	+	block = m.buf.ptr(offset, int(m.sums[i].len))
	34	+	// Copy, the literal ptr below may move the window.
	35	+	block = append([]byte(nil), block...)
	36	+	}
	37	+	literal := m.buf.ptr(m.lastMatch, int(n))
	38	+	if err := m.tok.send(i, literal, block); err != nil {
	39	+	return err
	40	+	}
	41	+	m.literal += n
	42	+	total := n
	43	+	if i >= 0 {
	44	+	m.matched += int64(m.sums[i].len)
	45	+	total += int64(m.sums[i].len)
	46	+	}
	47	+	for j := int64(0); j < total; j += chunkSize {
	48	+	n1 := min(chunkSize, total-j)
	49	+	m.sum.Write(m.buf.ptr(m.lastMatch+j, int(n1)))
	50	+	}
	51	+	if i >= 0 {
	52	+	m.lastMatch = offset + int64(m.sums[i].len)
	53	+	} else {
	54	+	m.lastMatch = offset
	55	+	}
	56	+	return nil
	57	+	}
	58	+
	59	+	func (m *matcher) buildTable() {
	60	+	m.tablesize = uint32(m.head.count/8)*10 + 11
	61	+	if m.tablesize < traditionalTableSize {
	62	+	m.tablesize = traditionalTableSize
	63	+	}
	64	+	m.table = make([]int32, m.tablesize)
	65	+	for i := range m.table {
	66	+	m.table[i] = -1
	67	+	}
	68	+	for i := range m.sums {
	69	+	var t uint32
	70	+	if m.tablesize == traditionalTableSize {
	71	+	s := m.sums[i].sum1
	72	+	t = (s&0xFFFF + s>>16) & 0xFFFF
	73	+	} else {
	74	+	t = m.sums[i].sum1 % m.tablesize
	75	+	}
	76	+	m.sums[i].chain = m.table[t]
	77	+	m.table[t] = int32(i)
	78	+	}
	79	+	}
	80	+
	81	+	// run sends a file of length size.
	82	+	func (m *matcher) run(size int64) error {
	83	+	if size > 0 && m.head.count > 0 {
	84	+	m.buildTable()
	85	+	if err := m.hashSearch(size); err != nil {
	86	+	return err
	87	+	}
	88	+	} else {
	89	+	for j := m.lastMatch + chunkSize; j < size; j += chunkSize {
	90	+	if err := m.emit(j, -2); err != nil {
	91	+	return err
	92	+	}
	93	+	}
	94	+	if err := m.emit(size, -1); err != nil {
	95	+	return err
	96	+	}
	97	+	}
	98	+	return nil
	99	+	}
	100	+
	101	+	func (m *matcher) hashSearch(size int64) error {
	102	+	var sum2 []byte
	103	+	wantI := int32(0)
	104	+	blength := int64(m.head.blength)
	105	+	k := min(size, blength)
	106	+	sum := checksum1(m.buf.ptr(0, int(k)))
	107	+	s1, s2 := sum&0xFFFF, sum>>16
	108	+	var offset, alignedOffset int64
	109	+	alignedI := int32(0)
	110	+	end := size + 1 - int64(m.sums[len(m.sums)-1].len)
	111	+	s2len := int(m.head.s2length)
	112	+
	113	+	for {
	114	+	doneCsum2 := false
	115	+	var hashEntry uint32
	116	+	if m.tablesize == traditionalTableSize {
	117	+	hashEntry = (s1 + s2) & 0xFFFF
	118	+	sum = s1&0xffff \| s2<<16
	119	+	} else {
	120	+	sum = s1&0xffff \| s2<<16
	121	+	hashEntry = sum % m.tablesize
	122	+	}
	123	+	i := m.table[hashEntry]
	124	+	prev := &m.table[hashEntry]
	125	+	chainLen := 0
	126	+	for ; i >= 0; i = m.sums[i].chain {
	127	+	// An in-place receiver can only use blocks at or after the
	128	+	// current offset, or ones it has left untouched.
	129	+	if m.inplace && m.sums[i].offset < offset && !m.sums[i].same {
	130	+	*prev = m.sums[i].chain
	131	+	continue
	132	+	}
	133	+	prev = &m.sums[i].chain
	134	+	if sum != m.sums[i].sum1 {
	135	+	continue
	136	+	}
	137	+	chainLen++
	138	+	if chainLen > maxChainLen {
	139	+	break
	140	+	}
	141	+	l := int32(min(blength, size-offset))
	142	+	if l != m.sums[i].len {
	143	+	continue
	144	+	}
	145	+	if !doneCsum2 {
	146	+	sum2 = m.cs.blockSum(m.buf.ptr(offset, int(l)), sum2[:0])
	147	+	doneCsum2 = true
	148	+	}
	149	+	if !bytes.Equal(sum2[:s2len], m.sums[i].sum2) {
	150	+	continue
	151	+	}
	152	+
	153	+	if m.inplace {
	154	+	for alignedOffset < offset {
	155	+	alignedOffset += blength
	156	+	alignedI++
	157	+	}
	158	+	if (offset == alignedOffset \|\| (sum == 0 && int64(l) == blength && alignedOffset+int64(l) <= size)) && alignedI < m.head.count {
	159	+	ok := true
	160	+	if i != alignedI {
	161	+	a := &m.sums[alignedI]
	162	+	if sum != a.sum1 \|\| l != a.len \|\| !bytes.Equal(sum2[:s2len], a.sum2) {
	163	+	ok = false
	164	+	} else {
	165	+	i = alignedI
	166	+	}
	167	+	}
	168	+	if ok && offset != alignedOffset {
	169	+	backup := max(alignedOffset-m.lastMatch, 0)
	170	+	block := m.buf.ptr(alignedOffset-backup, int(int64(l)+backup))[backup:]
	171	+	if checksum1(block) != m.sums[i].sum1 \|\|
	172	+	!bytes.Equal(m.cs.blockSum(block, nil)[:s2len], m.sums[i].sum2) {
	173	+	ok = false
	174	+	} else {
	175	+	offset = alignedOffset
	176	+	}
	177	+	}
	178	+	if ok {
	179	+	m.sums[i].same = true
	180	+	wantI = i
	181	+	}
	182	+	}
	183	+	}
	184	+
	185	+	if i != wantI && wantI < m.head.count &&
	186	+	(!m.inplace \|\| m.sums[wantI].offset >= offset \|\| m.sums[wantI].same) &&
	187	+	sum == m.sums[wantI].sum1 && l == m.sums[wantI].len &&
	188	+	bytes.Equal(sum2[:s2len], m.sums[wantI].sum2) {
	189	+	// Prefer the adjacent block so the token runs compress.
	190	+	i = wantI
	191	+	}
	192	+	wantI = i + 1
	193	+
	194	+	if err := m.emit(offset, i); err != nil {
	195	+	return err
	196	+	}
	197	+	offset += int64(m.sums[i].len) - 1
	198	+	k = min(blength, size-offset)
	199	+	sum = checksum1(m.buf.ptr(offset, int(k)))
	200	+	s1, s2 = sum&0xFFFF, sum>>16
	201	+	break
	202	+	}
	203	+
	204	+	backup := max(offset-m.lastMatch, 0)
	205	+	more := offset+k < size
	206	+	n := k + backup
	207	+	if more {
	208	+	n++
	209	+	}
	210	+	window := m.buf.ptr(offset-backup, int(n))[backup:]
	211	+	first := uint32(int8(window[0]))
	212	+	s1 -= first
	213	+	s2 -= uint32(k) * first
	214	+	if more {
	215	+	s1 += uint32(int8(window[k]))
	216	+	s2 += s1
	217	+	} else {
	218	+	k--
	219	+	}
	220	+
	221	+	if backup >= blength+chunkSize && end-offset > chunkSize {
	222	+	if err := m.emit(offset-blength, -2); err != nil {
	223	+	return err
	224	+	}
	225	+	}
	226	+
	227	+	offset++
	228	+	if offset >= end {
	229	+	break
	230	+	}
	231	+	}
	232	+	return m.emit(size, -1)
	233	+	}

...	...		@@ -0,0 +1,871 @@
	1	+	package rsync
	2	+
	3	+	import (
	4	+	"bytes"
	5	+	"context"
	6	+	"errors"
	7	+	"fmt"
	8	+	"io"
	9	+	"io/fs"
	10	+	"math"
	11	+	"path"
	12	+	"slices"
	13	+	"strings"
	14	+	"time"
	15	+	)
	16	+
	17	+	var errChecksumMismatch = errors.New("rsync: file failed verification")
	18	+
	19	+	// recvEvent is what the receiver tells the generator, standing in for the
	20	+	// pipe between rsync's two receiving processes.
	21	+	type recvEvent struct {
	22	+	done bool
	23	+	redo int32
	24	+	}
	25	+
	26	+	// destination maps file-list names to storage paths (get_local_name).
	27	+	type destination struct {
	28	+	dir string
	29	+	single bool // the transfer is one file written to dir itself
	30	+	}
	31	+
	32	+	func (d destination) path(f *fileEntry) string {
	33	+	if d.single {
	34	+	return d.dir
	35	+	}
	36	+	name := f.name()
	37	+	if name == "." {
	38	+	return d.dir
	39	+	}
	40	+	return path.Join(d.dir, name)
	41	+	}
	42	+
	43	+	// runReceiver is the server side of an upload (do_server_recv).
	44	+	func (s *session) runReceiver(args []string) error {
	45	+	if err := s.recvFilterList(); err != nil {
	46	+	return err
	47	+	}
	48	+	files, err := s.recvFileList()
	49	+	if err != nil {
	50	+	return err
	51	+	}
	52	+	if len(files) == 0 {
	53	+	return s.fatal(exitFileSelect, errors.New("server_recv: recv_file_list error"))
	54	+	}
	55	+
	56	+	base := ""
	57	+	if len(args) > 0 {
	58	+	base = args[0]
	59	+	}
	60	+	destArg := ""
	61	+	if len(args) > 1 {
	62	+	destArg = args[1]
	63	+	}
	64	+	dest, err := s.resolveDestination(files, base, destArg)
	65	+	if err != nil {
	66	+	return s.fatal(exitFileSelect, err)
	67	+	}
	68	+
	69	+	ctx, cancel := context.WithCancel(context.Background())
	70	+	defer cancel()
	71	+	events := make(chan recvEvent, 64)
	72	+	g := &generator{s: s, files: files, dest: dest, events: events, ctx: ctx}
	73	+	r := &receiver{s: s, files: files, dest: dest, events: events, ctx: ctx, sent: make([]bool, len(files))}
	74	+
	75	+	errc := make(chan error, 2)
	76	+	go func() { errc <- g.run() }()
	77	+	go func() { errc <- r.run() }()
	78	+	for range 2 {
	79	+	if err := <-errc; err != nil {
	80	+	cancel()
	81	+	return err
	82	+	}
	83	+	}
	84	+	return s.finish()
	85	+	}
	86	+
	87	+	func (s session) recvFileList() ([]fileEntry, error) {
	88	+	codec := s.newFlistCodec()
	89	+	var files []*fileEntry
	90	+	for {
	91	+	xflags, ok, ioErr, err := codec.recvFlags(s.c)
	92	+	if err != nil {
	93	+	return nil, err
	94	+	}
	95	+	if !ok {
	96	+	if !s.opts.ignoreErrors {
	97	+	s.addIOError(ioErr & ioErrMask)
	98	+	}
	99	+	break
	100	+	}
	101	+	f, err := codec.recvEntry(s.c, xflags)
	102	+	if err != nil {
	103	+	return nil, s.fatal(exitProtocol, err)
	104	+	}
	105	+	files = append(files, f)
	106	+	}
	107	+	if !s.opts.numericIDs {
	108	+	for _, want := range []bool{s.opts.preserveUID, s.opts.preserveGID} {
	109	+	if !want {
	110	+	continue
	111	+	}
	112	+	for {
	113	+	id, err := s.c.readVarint30()
	114	+	if err != nil {
	115	+	return nil, err
	116	+	}
	117	+	if id == 0 {
	118	+	break
	119	+	}
	120	+	if _, err := codec.readIDName(s.c); err != nil {
	121	+	return nil, err
	122	+	}
	123	+	}
	124	+	if s.xmitID0Names {
	125	+	if _, err := codec.readIDName(s.c); err != nil {
	126	+	return nil, err
	127	+	}
	128	+	}
	129	+	}
	130	+	}
	131	+	sortFileList(files, s.c.protocol, true)
	132	+	if s.c.protocol < 30 {
	133	+	v, err := s.c.readInt32()
	134	+	if err != nil {
	135	+	return nil, err
	136	+	}
	137	+	if !s.opts.ignoreErrors {
	138	+	s.addIOError(v & ioErrMask)
	139	+	}
	140	+	}
	141	+	return files, nil
	142	+	}
	143	+
	144	+	func (s session) resolveDestination(files []fileEntry, base, destArg string) (destination, error) {
	145	+	dir, err := cleanSourcePath(strings.TrimLeft(path.Join(base, destArg), "/"))
	146	+	if err != nil {
	147	+	return destination{}, err
	148	+	}
	149	+	trailingSlash := strings.HasSuffix(destArg, "/")
	150	+	if dir == "" {
	151	+	return destination{dir: dir}, nil
	152	+	}
	153	+	info, err := s.fs.Stat(dir)
	154	+	exists := err == nil
	155	+	if err != nil && !errors.Is(err, fs.ErrNotExist) {
	156	+	return destination{}, fmt.Errorf("ERROR: cannot stat destination %q: %w", "/"+dir, err)
	157	+	}
	158	+	if exists {
	159	+	if info.IsDir {
	160	+	return destination{dir: dir}, nil
	161	+	}
	162	+	if len(files) > 1 {
	163	+	return destination{}, errors.New("ERROR: destination must be a directory when copying more than 1 file")
	164	+	}
	165	+	if files[0].isDir() {
	166	+	return destination{}, errors.New("ERROR: cannot overwrite non-directory with a directory")
	167	+	}
	168	+	}
	169	+	if len(files) > 1 \|\| trailingSlash {
	170	+	if exists {
	171	+	return destination{}, errors.New("ERROR: destination path is not a directory")
	172	+	}
	173	+	return destination{dir: dir}, nil
	174	+	}
	175	+	return destination{dir: dir, single: true}, nil
	176	+	}
	177	+
	178	+	// generator decides what the client has to send and sends the signatures
	179	+	// of our copies (generator.c).
	180	+	type generator struct {
	181	+	s *session
	182	+	files []*fileEntry
	183	+	dest destination
	184	+	events chan recvEvent
	185	+	ctx context.Context
	186	+	done int
	187	+
	188	+	deletedFiles int32
	189	+	deletedDirs int32
	190	+	}
	191	+
	192	+	func (g *generator) run() error {
	193	+	s := g.s
	194	+	c := s.c
	195	+	earlyDelayDone := !s.opts.delayUpdates
	196	+	earlyDeleteDone := s.opts.deleteDuring != 2 && !s.opts.deleteAfter
	197	+
	198	+	if s.opts.deleteMode && !g.dest.single && (s.opts.deleteBefore \|\| s.opts.deleteDuring == 1) {
	199	+	if err := g.deletePass(); err != nil {
	200	+	return err
	201	+	}
	202	+	}
	203	+
	204	+	for i, f := range g.files {
	205	+	if f.inactive {
	206	+	continue
	207	+	}
	208	+	if err := g.ctx.Err(); err != nil {
	209	+	return err
	210	+	}
	211	+	if err := g.recvGenerator(int32(i), f, false); err != nil {
	212	+	return err
	213	+	}
	214	+	}
	215	+	if err := c.writeNdx(ndxDone); err != nil {
	216	+	return err
	217	+	}
	218	+
	219	+	// Redo files that failed verification until the receiver is done
	220	+	// with the first pass.
	221	+	if err := g.wait(1, true); err != nil {
	222	+	return err
	223	+	}
	224	+
	225	+	if err := c.writeNdx(ndxDone); err != nil {
	226	+	return err
	227	+	}
	228	+	if c.protocol >= 29 && earlyDelayDone {
	229	+	if err := c.writeNdx(ndxDone); err != nil {
	230	+	return err
	231	+	}
	232	+	}
	233	+	if c.protocol >= 31 && earlyDeleteDone {
	234	+	if s.opts.deleteMode {
	235	+	if err := g.writeDelStats(); err != nil {
	236	+	return err
	237	+	}
	238	+	}
	239	+	if earlyDelayDone {
	240	+	if err := c.writeNdx(ndxDone); err != nil {
	241	+	return err
	242	+	}
	243	+	}
	244	+	}
	245	+	if err := g.wait(2, false); err != nil {
	246	+	return err
	247	+	}
	248	+
	249	+	if c.protocol >= 29 {
	250	+	if !earlyDelayDone {
	251	+	if err := c.writeNdx(ndxDone); err != nil {
	252	+	return err
	253	+	}
	254	+	if c.protocol >= 31 && earlyDeleteDone {
	255	+	if err := c.writeNdx(ndxDone); err != nil {
	256	+	return err
	257	+	}
	258	+	}
	259	+	}
	260	+	if err := g.wait(3, false); err != nil {
	261	+	return err
	262	+	}
	263	+	}
	264	+
	265	+	if s.opts.deleteMode && !g.dest.single && (s.opts.deleteDuring == 2 \|\| s.opts.deleteAfter) {
	266	+	if err := g.deletePass(); err != nil {
	267	+	return err
	268	+	}
	269	+	}
	270	+
	271	+	if c.protocol >= 31 {
	272	+	if !earlyDeleteDone {
	273	+	if s.opts.deleteMode {
	274	+	if err := g.writeDelStats(); err != nil {
	275	+	return err
	276	+	}
	277	+	}
	278	+	if err := c.writeNdx(ndxDone); err != nil {
	279	+	return err
	280	+	}
	281	+	}
	282	+	if err := g.wait(4, false); err != nil {
	283	+	return err
	284	+	}
	285	+	}
	286	+
	287	+	// The final goodbye.
	288	+	if err := c.writeNdx(ndxDone); err != nil {
	289	+	return err
	290	+	}
	291	+	return c.flush()
	292	+	}
	293	+
	294	+	// wait handles receiver events until it has finished n phases.
	295	+	func (g *generator) wait(n int, redo bool) error {
	296	+	if err := g.s.c.w.Kick(); err != nil {
	297	+	return err
	298	+	}
	299	+	for g.done < n {
	300	+	select {
	301	+	case <-g.ctx.Done():
	302	+	return g.ctx.Err()
	303	+	case ev := <-g.events:
	304	+	if ev.done {
	305	+	g.done++
	306	+	continue
	307	+	}
	308	+	if redo {
	309	+	if err := g.recvGenerator(ev.redo, g.files[ev.redo], true); err != nil {
	310	+	return err
	311	+	}
	312	+	if err := g.s.c.w.Kick(); err != nil {
	313	+	return err
	314	+	}
	315	+	}
	316	+	}
	317	+	}
	318	+	return nil
	319	+	}
	320	+
	321	+	// recvGenerator handles one file-list entry (generator.c:recv_generator).
	322	+	// A redo asks again for a file that failed verification, with full-length
	323	+	// block checksums.
	324	+	func (g generator) recvGenerator(ndx int32, f fileEntry, redo bool) error {
	325	+	s := g.s
	326	+	o := &s.opts
	327	+	if !f.isReg() {
	328	+	if !f.isDir() && o.infoNonreg > 0 {
	329	+	s.info("skipping non-regular file %q", f.name())
	330	+	}
	331	+	return nil
	332	+	}
	333	+	if !redo {
	334	+	if o.maxSize >= 0 && f.size > o.maxSize {
	335	+	return nil
	336	+	}
	337	+	if o.minSize >= 0 && f.size < o.minSize {
	338	+	return nil
	339	+	}
	340	+	}
	341	+
	342	+	target := g.dest.path(f)
	343	+	st, err := s.fs.Stat(target)
	344	+	exists := err == nil
	345	+	if err != nil && !errors.Is(err, fs.ErrNotExist) {
	346	+	s.xferError("recv_generator: failed to stat %q: %v", "/"+target, err)
	347	+	return nil
	348	+	}
	349	+	if !redo {
	350	+	if o.ignoreNonExist && !exists {
	351	+	return nil
	352	+	}
	353	+	if o.ignoreExisting && exists {
	354	+	return nil
	355	+	}
	356	+	if o.updateOnly && exists && !st.IsDir && f.mtime-st.ModTime.Unix() < 0 {
	357	+	return nil
	358	+	}
	359	+	}
	360	+	if exists && st.IsDir {
	361	+	s.xferError("cannot replace directory %q with a file", "/"+target)
	362	+	return nil
	363	+	}
	364	+
	365	+	if exists && !redo {
	366	+	same, err := g.quickCheck(f, target, st)
	367	+	if err != nil {
	368	+	s.xferError("failed to checksum %q: %v", "/"+target, err)
	369	+	} else if same {
	370	+	if o.removeSource && !o.dryRun {
	371	+	return s.c.writeMsgInt(msgSuccess, ndx)
	372	+	}
	373	+	return nil
	374	+	}
	375	+	}
	376	+
	377	+	// Build the signature of our copy before announcing the file, so a
	378	+	// failure to read it can fall back to sending the whole file.
	379	+	var head sumHead
	380	+	var sums []byte
	381	+	if exists && !o.wholeFile && !o.dryRun && st.Size > 0 {
	382	+	csumLength := shortSumLength
	383	+	if redo {
	384	+	csumLength = sumLength
	385	+	}
	386	+	head, sums, err = g.signature(target, csumLength)
	387	+	if err != nil {
	388	+	s.send(msgError, fmt.Sprintf("rsync: [generator] failed to read %q, continuing: %v\n", "/"+target, err))
	389	+	exists = false
	390	+	head, sums = sumHead{}, nil
	391	+	}
	392	+	}
	393	+
	394	+	if err := s.c.writeNdx(ndx); err != nil {
	395	+	return err
	396	+	}
	397	+	if s.c.protocol >= 29 {
	398	+	iflags := itemTransfer
	399	+	if o.alwaysChecksum {
	400	+	iflags \|= itemReportChange
	401	+	}
	402	+	if exists {
	403	+	if st.Size != f.size {
	404	+	iflags \|= itemReportSize
	405	+	}
	406	+	if !o.preserveTimes \|\| st.ModTime.Unix() != f.mtime {
	407	+	iflags \|= itemReportTime
	408	+	}
	409	+	} else {
	410	+	iflags \|= itemIsNew
	411	+	}
	412	+	if err := s.c.writeShortint(uint16(iflags)); err != nil {
	413	+	return err
	414	+	}
	415	+	}
	416	+	if o.dryRun {
	417	+	return nil
	418	+	}
	419	+	if err := s.c.writeSumHead(head); err != nil {
	420	+	return err
	421	+	}
	422	+	return s.c.write(sums)
	423	+	}
	424	+
	425	+	// signature computes the block checksums of our copy of a file.
	426	+	func (g *generator) signature(target string, csumLength int) (sumHead, []byte, error) {
	427	+	s := g.s
	428	+	basis, info, err := s.fs.Open(target)
	429	+	if err != nil {
	430	+	return sumHead{}, nil, err
	431	+	}
	432	+	defer func() { _ = basis.Close() }()
	433	+	if info.Size <= 0 {
	434	+	return sumHead{}, nil, nil
	435	+	}
	436	+	head, err := sumSizesSqroot(info.Size, s.c.protocol, s.opts.blockSize, csumLength, s.cs.xfer.size())
	437	+	if err != nil {
	438	+	return sumHead{}, nil, err
	439	+	}
	440	+	sums, err := encodeSums(basis, info.Size, head, &s.cs)
	441	+	if err != nil {
	442	+	return sumHead{}, nil, err
	443	+	}
	444	+	return head, sums, nil
	445	+	}
	446	+
	447	+	// quickCheck reports whether our copy already matches (quick_check_ok).
	448	+	func (g generator) quickCheck(f fileEntry, target string, st FileInfo) (bool, error) {
	449	+	o := &g.s.opts
	450	+	if st.Size != f.size {
	451	+	return false, nil
	452	+	}
	453	+	if o.alwaysChecksum {
	454	+	sum, err := g.s.fileChecksum(target)
	455	+	if err != nil {
	456	+	return false, err
	457	+	}
	458	+	return bytes.Equal(sum, f.sum), nil
	459	+	}
	460	+	if o.sizeOnly {
	461	+	return true, nil
	462	+	}
	463	+	if o.ignoreTimes {
	464	+	return false, nil
	465	+	}
	466	+	// Storage keeps whole seconds, so nanoseconds are not compared.
	467	+	return st.ModTime.Unix() == f.mtime, nil
	468	+	}
	469	+
	470	+	func (g *generator) writeDelStats() error {
	471	+	return g.s.writeDelStats([5]int32{g.deletedFiles, g.deletedDirs, 0, 0, 0})
	472	+	}
	473	+
	474	+	// deletePass removes what the destination has beyond the file list.
	475	+	func (g *generator) deletePass() error {
	476	+	s := g.s
	477	+	s.mu.Lock()
	478	+	ioError := s.ioError
	479	+	s.mu.Unlock()
	480	+	if ioError != 0 && !s.opts.ignoreErrors {
	481	+	s.info("IO error encountered -- skipping file deletion")
	482	+	return nil
	483	+	}
	484	+
	485	+	existing, err := s.fs.ReadDir(g.dest.dir, true)
	486	+	if errors.Is(err, fs.ErrNotExist) {
	487	+	return nil
	488	+	}
	489	+	if err != nil {
	490	+	s.xferError("delete: failed to list %q: %v", "/"+g.dest.dir, err)
	491	+	return nil
	492	+	}
	493	+
	494	+	listed := make(map[string]*fileEntry, len(g.files))
	495	+	for _, f := range g.files {
	496	+	if !f.inactive {
	497	+	listed[f.name()] = f
	498	+	}
	499	+	}
	500	+	contentDir := func(name string) bool {
	501	+	f, ok := listed[name]
	502	+	if !ok \|\| !f.isDir() {
	503	+	return false
	504	+	}
	505	+	if g.s.c.protocol < 30 && s.opts.recurse {
	506	+	return true
	507	+	}
	508	+	return f.flags&flagContentDir != 0
	509	+	}
	510	+
	511	+	files := make(map[string]bool, len(existing))
	512	+	for _, e := range existing {
	513	+	if name, err := cleanSourcePath(e.Name); err == nil && !e.IsDir {
	514	+	files[name] = true
	515	+	}
	516	+	}
	517	+	scopes := s.filters.newScopeTree(g.dest.dir, g.dest.dir, s.mergeReader(g.dest.dir, files))
	518	+
	519	+	type victim struct {
	520	+	name string
	521	+	isDir bool
	522	+	}
	523	+	var victims []victim
	524	+	var protectedNames []string
	525	+	for _, e := range existing {
	526	+	name, err := cleanSourcePath(e.Name)
	527	+	if err != nil \|\| name == "" {
	528	+	continue
	529	+	}
	530	+	if f, ok := listed[name]; ok && f.isDir() == e.IsDir {
	531	+	continue
	532	+	}
	533	+	// Find the closest listed ancestor. Everything between it and
	534	+	// the entry is unlisted and goes too, unless a filter protects
	535	+	// part of the way.
	536	+	anc := path.Dir(name)
	537	+	var chain []string
	538	+	chain = append(chain, name)
	539	+	for anc != "." {
	540	+	if _, ok := listed[anc]; ok {
	541	+	break
	542	+	}
	543	+	chain = append(chain, anc)
	544	+	anc = path.Dir(anc)
	545	+	}
	546	+	if !contentDir(anc) {
	547	+	continue
	548	+	}
	549	+	// Walk down from the ancestor so merge files are only read in
	550	+	// directories rsync would enter. Below it everything is inside a
	551	+	// directory being deleted.
	552	+	protected := false
	553	+	for i := len(chain) - 1; i >= 0 && !protected; i-- {
	554	+	sc, err := scopes.scope(path.Dir(chain[i]))
	555	+	if err != nil {
	556	+	s.xferError("%v -- skipping file deletion", err)
	557	+	return nil
	558	+	}
	559	+	isDir := i > 0 \|\| e.IsDir
	560	+	if i == len(chain)-1 {
	561	+	protected = s.filters.excluded(chain[i], isDir, sc)
	562	+	} else {
	563	+	protected = s.filters.protectsInDeletedDir(chain[i], isDir, sc)
	564	+	}
	565	+	}
	566	+	if protected {
	567	+	protectedNames = append(protectedNames, name)
	568	+	} else {
	569	+	victims = append(victims, victim{name: name, isDir: e.IsDir})
	570	+	}
	571	+	}
	572	+
	573	+	// Children before parents.
	574	+	slices.SortStableFunc(victims, func(a, b victim) int {
	575	+	return strings.Count(b.name, "/") - strings.Count(a.name, "/")
	576	+	})
	577	+
	578	+	maxDelete := s.opts.maxDelete
	579	+	if maxDelete < 0 && maxDelete != math.MinInt32 {
	580	+	maxDelete = 0
	581	+	}
	582	+	skipped := 0
	583	+	for _, v := range victims {
	584	+	if maxDelete >= 0 && int(g.deletedFiles+g.deletedDirs) >= maxDelete {
	585	+	skipped++
	586	+	continue
	587	+	}
	588	+	if v.isDir && slices.ContainsFunc(protectedNames, func(p string) bool { return strings.HasPrefix(p, v.name+"/") }) {
	589	+	s.info("cannot delete non-empty directory: %s", v.name)
	590	+	continue
	591	+	}
	592	+	full := path.Join(g.dest.dir, v.name)
	593	+	if !s.opts.dryRun {
	594	+	if err := s.fs.Remove(full, v.isDir); err != nil {
	595	+	op := "unlink"
	596	+	if v.isDir {
	597	+	op = "rmdir"
	598	+	}
	599	+	s.xferError("delete_file: %s(%s) failed: %v", op, v.name, err)
	600	+	continue
	601	+	}
	602	+	}
	603	+	if v.isDir {
	604	+	g.deletedDirs++
	605	+	} else {
	606	+	g.deletedFiles++
	607	+	}
	608	+	g.logDelete(v.name, v.isDir)
	609	+	}
	610	+	if skipped > 0 {
	611	+	s.warning("Deletions stopped due to --max-delete limit (%d skipped)", skipped)
	612	+	s.addIOError(ioErrDelLimit)
	613	+	}
	614	+	return nil
	615	+	}
	616	+
	617	+	func (g *generator) logDelete(name string, isDir bool) {
	618	+	s := g.s
	619	+	if s.c.protocol >= 29 {
	620	+	msg := []byte(name)
	621	+	if isDir {
	622	+	msg = append(msg, 0)
	623	+	}
	624	+	if err := s.c.writeMsg(msgDeleted, msg); err != nil {
	625	+	s.logger.Debug("rsync message not delivered", "err", err)
	626	+	}
	627	+	return
	628	+	}
	629	+	if s.opts.infoDel > 0 {
	630	+	if isDir {
	631	+	name += "/"
	632	+	}
	633	+	s.info("deleting %s", name)
	634	+	}
	635	+	}
	636	+
	637	+	// receiver applies the deltas the client sends (receiver.c).
	638	+	type receiver struct {
	639	+	s *session
	640	+	files []*fileEntry
	641	+	dest destination
	642	+	events chan recvEvent
	643	+	ctx context.Context
	644	+	sent []bool
	645	+	}
	646	+
	647	+	func (r *receiver) notify(ev recvEvent) error {
	648	+	select {
	649	+	case r.events <- ev:
	650	+	return nil
	651	+	case <-r.ctx.Done():
	652	+	return r.ctx.Err()
	653	+	}
	654	+	}
	655	+
	656	+	func (r *receiver) run() error {
	657	+	s := r.s
	658	+	c := s.c
	659	+	phase := 0
	660	+	maxPhase := 1
	661	+	if c.protocol >= 29 {
	662	+	maxPhase = 2
	663	+	}
	664	+	tok := newTokenReceiver(c, s.compression)
	665	+	for {
	666	+	req, err := s.readNdxAndAttrs(len(r.files))
	667	+	if err != nil {
	668	+	return err
	669	+	}
	670	+	if req.ndx == ndxDone {
	671	+	phase++
	672	+	if phase > maxPhase {
	673	+	break
	674	+	}
	675	+	if err := r.notify(recvEvent{done: true}); err != nil {
	676	+	return err
	677	+	}
	678	+	continue
	679	+	}
	680	+	f := r.files[req.ndx]
	681	+	if f.inactive {
	682	+	return s.fatal(exitProtocol, fmt.Errorf("refusing transfer of cleared file index %d", req.ndx))
	683	+	}
	684	+	if req.iflags&itemTransfer == 0 {
	685	+	continue
	686	+	}
	687	+	if !f.isReg() {
	688	+	return s.fatal(exitProtocol, fmt.Errorf("received request to transfer non-regular file: %d", req.ndx))
	689	+	}
	690	+	if phase == 2 {
	691	+	return s.fatal(exitProtocol, errors.New("got transfer request in phase 2"))
	692	+	}
	693	+	redoing := r.sent[req.ndx]
	694	+	if s.opts.dryRun {
	695	+	continue
	696	+	}
	697	+
	698	+	err = r.receiveFile(f, tok)
	699	+	switch {
	700	+	case err == nil:
	701	+	if s.opts.removeSource {
	702	+	if err := c.writeMsgInt(msgSuccess, req.ndx); err != nil {
	703	+	return err
	704	+	}
	705	+	}
	706	+	case errors.Is(err, errChecksumMismatch):
	707	+	if redoing {
	708	+	s.xferError("ERROR: %s failed verification -- update discarded.", f.name())
	709	+	continue
	710	+	}
	711	+	if s.opts.infoName > 0 {
	712	+	s.warning("WARNING: %s failed verification -- update discarded (will try again).", f.name())
	713	+	}
	714	+	r.sent[req.ndx] = true
	715	+	if err := r.notify(recvEvent{redo: req.ndx}); err != nil {
	716	+	return err
	717	+	}
	718	+	default:
	719	+	var putErr *putError
	720	+	if !errors.As(err, &putErr) {
	721	+	return err
	722	+	}
	723	+	if errors.Is(putErr.err, ErrFileIgnored) {
	724	+	s.warning("rsync: [receiver] %s skipped: %v", f.name(), putErr.err)
	725	+	} else {
	726	+	s.xferError("%s not stored: %v", f.name(), putErr.err)
	727	+	}
	728	+	}
	729	+	}
	730	+	if err := r.notify(recvEvent{done: true}); err != nil {
	731	+	return err
	732	+	}
	733	+
	734	+	if c.protocol >= 31 {
	735	+	// The sender confirms our final NDX_DONE with one of its own.
	736	+	req, err := s.readNdxAndAttrs(0)
	737	+	if err != nil {
	738	+	return err
	739	+	}
	740	+	if req.ndx != ndxDone {
	741	+	return s.fatal(exitProtocol, fmt.Errorf("invalid packet at end of run (%d)", req.ndx))
	742	+	}
	743	+	return r.notify(recvEvent{done: true})
	744	+	}
	745	+	return nil
	746	+	}
	747	+
	748	+	// putError is a failure of the storage backend to keep a file.
	749	+	type putError struct{ err error }
	750	+
	751	+	func (e *putError) Error() string { return e.err.Error() }
	752	+
	753	+	// receiveFile reads one file's delta, rebuilds the file from it and our old
	754	+	// copy, and streams the result to storage. The stream only ends cleanly
	755	+	// once the whole-file checksum matched (receiver.c:receive_data).
	756	+	func (r receiver) receiveFile(f fileEntry, tok tokenReceiver) error {
	757	+	s := r.s
	758	+	target := r.dest.path(f)
	759	+
	760	+	head, err := s.c.readSumHead(s.cs.xfer.size())
	761	+	if err != nil {
	762	+	return s.fatal(exitProtocol, err)
	763	+	}
	764	+
	765	+	var basis File
	766	+	var basisSize int64
	767	+	if head.count > 0 {
	768	+	b, info, err := s.fs.Open(target)
	769	+	if err == nil {
	770	+	basis, basisSize = b, info.Size
	771	+	defer func() { _ = basis.Close() }()
	772	+	}
	773	+	}
	774	+
	775	+	pr, pw := io.Pipe()
	776	+	type result struct {
	777	+	msg string
	778	+	err error
	779	+	}
	780	+	resc := make(chan result, 1)
	781	+	go func() {
	782	+	info := FileInfo{Name: target, Size: f.size, ModTime: time.Unix(f.mtime, int64(f.nsec))}
	783	+	msg, err := s.fs.Put(target, info, pr)
	784	+	if err == nil {
	785	+	err = errPutReturned
	786	+	}
	787	+	_ = pr.CloseWithError(err)
	788	+	if errors.Is(err, errPutReturned) {
	789	+	err = nil
	790	+	}
	791	+	resc <- result{msg, err}
	792	+	}()
	793	+
	794	+	sum := s.cs.newXferSum()
	795	+	write := func(p []byte) {
	796	+	sum.Write(p)
	797	+	// After a failed Put the pipe returns its error; keep reading the
	798	+	// protocol stream regardless.
	799	+	_, _ = pw.Write(p)
	800	+	}
	801	+
	802	+	var block []byte
	803	+	var tokErr error
	804	+	for {
	805	+	i, data, err := tok.recv()
	806	+	if err != nil {
	807	+	tokErr = err
	808	+	break
	809	+	}
	810	+	if i == 0 {
	811	+	break
	812	+	}
	813	+	if i > 0 {
	814	+	write(data)
	815	+	continue
	816	+	}
	817	+	idx := -(i + 1)
	818	+	if idx >= head.count {
	819	+	tokErr = s.fatal(exitProtocol, fmt.Errorf("invalid block index %d (count=%d)", idx, head.count))
	820	+	break
	821	+	}
	822	+	l := head.blength
	823	+	if idx == head.count-1 && head.remainder != 0 {
	824	+	l = head.remainder
	825	+	}
	826	+	offset := int64(idx) * int64(head.blength)
	827	+	if basis == nil \|\| offset+int64(l) > basisSize {
	828	+	tokErr = s.fatal(exitProtocol, fmt.Errorf("got a block match with no basis file for %q", "/"+target))
	829	+	break
	830	+	}
	831	+	if cap(block) < int(l) {
	832	+	block = make([]byte, l)
	833	+	}
	834	+	block = block[:l]
	835	+	if err := readFullAt(basis, block, offset); err != nil {
	836	+	tokErr = s.fatal(exitFileIO, fmt.Errorf("failed to read basis %q: %w", "/"+target, err))
	837	+	break
	838	+	}
	839	+	tok.see(block)
	840	+	write(block)
	841	+	}
	842	+	if tokErr != nil {
	843	+	_ = pw.CloseWithError(tokErr)
	844	+	<-resc
	845	+	return tokErr
	846	+	}
	847	+
	848	+	remote := make([]byte, s.cs.xfer.size())
	849	+	if err := s.c.readFull(remote); err != nil {
	850	+	_ = pw.CloseWithError(err)
	851	+	<-resc
	852	+	return err
	853	+	}
	854	+	if !bytes.Equal(sum.Sum(nil), remote) {
	855	+	_ = pw.CloseWithError(errChecksumMismatch)
	856	+	<-resc
	857	+	return errChecksumMismatch
	858	+	}
	859	+	_ = pw.Close()
	860	+	res := <-resc
	861	+	if res.err != nil {
	862	+	return &putError{err: res.err}
	863	+	}
	864	+	if res.msg != "" {
	865	+	s.info("%s", res.msg)
	866	+	}
	867	+	return nil
	868	+	}
	869	+
	870	+	// errPutReturned unblocks writers to the pipe once Put is done reading.
	871	+	var errPutReturned = errors.New("rsync: storage finished reading")

...	...		@@ -1,18 +1,15 @@
1		-	// Package rsyncopts implements a parser for command-line options that
2		-	// implements a subset of popt(3) semantics; just enough to parse typical
3		-	// rsync(1) invocations without the advanced popt features like aliases
4		-	// or option prefix matching (not --del, only --delete).
	1	+	// Package rsyncopts parses the command line an rsync client sends to the
	2	+	// server. It implements just enough of popt(3) for typical rsync(1)
	3	+	// invocations, without aliases or option prefix matching (not --del, only
	4	+	// --delete).
5	5		//
6	6		// If we encounter arguments that rsync(1) parses differently compared to this
7	7		// package, then this package should be adjusted to match rsync(1).
8	8		package rsyncopts
9	9
10	10		import (
11		-	"errors"
12	11		"fmt"
13		-	"log/slog"
14	12		"math"
15		-	"slices"
16	13		"strconv"
17	14		"strings"
18	15		"syscall"

...	...		@@ -91,42 +88,27 @@ func NewOptions() *Options {
91	88		relative_paths: -1,
92	89		implied_dirs: 1,
93	90		max_delete: math.MinInt32,
	91	+	max_size: -1,
	92	+	min_size: -1,
94	93		whole_file: -1,
95	94		do_compression_level: math.MinInt32,
96	95		rsync_path: "rsync",
97	96		default_af_hint: syscall.AF_INET6,
98	97		blocking_io: -1,
99		-	protocol_version: 27,
100		-	}
101		-	}
102		-
103		-	// GokrazyOptions contains additional command-line flags, prefixed with
104		-	// gokr. (like --gokr.modulemap) to not clash with rsync flag names.
105		-	type GokrazyOptions struct {
106		-	Config string
107		-	Listen string
108		-	MonitoringListen string
109		-	AnonSSHListen string
110		-	ModuleMap string
111		-	}
112		-
113		-	func (o *GokrazyOptions) table() []poptOption {
114		-	return []poptOption{
115		-	/* longName, shortName, argInfo, arg, val */
116		-	{"gokr.config", "", POPT_ARG_STRING, &o.Config, 0},
117		-	{"gokr.listen", "", POPT_ARG_STRING, &o.Listen, 0},
118		-	{"gokr.monitoring_listen", "", POPT_ARG_STRING, &o.MonitoringListen, 0},
119		-	{"gokr.anonssh_listen", "", POPT_ARG_STRING, &o.AnonSSHListen, 0},
120		-	{"gokr.modulemap", "", POPT_ARG_STRING, &o.ModuleMap, 0},
121	98		}
122	99		}
123	100
124	101		type Options struct {
125		-	Gokrazy GokrazyOptions
126		-
127	102		// not directly referenced in the table, but used in the special case code.
128		-	do_compression int
129		-	info [COUNT_INFO]uint16
	103	+	do_compression int
	104	+	info [COUNT_INFO]uint16
	105	+	version_opt_cnt int
	106	+	block_size int64
	107	+	max_size int64
	108	+	min_size int64
	109	+	batch_mode int
	110	+	basis_dirs []string
	111	+	modify_window_set int
130	112
131	113		// order matches long_options order
132	114		verbose int

...	...		@@ -247,12 +229,6 @@ type Options struct {
247	229		checksum_seed int
248	230		am_server int
249	231		am_sender int
250		-	am_daemon int
251		-
252		-	daemon_bwlimit int
253		-	config_file string
254		-	daemon_opt int
255		-	no_detach int
256	232		}
257	233
258	234		type priority int

...	...		@@ -354,48 +330,6 @@ func (o *Options) setOutputVerbosity(prio priority) {
354	330		}
355	331		}
356	332
357		-	func (o *Options) DryRun() bool { return o.dry_run != 0 }
358		-	func (o *Options) PreserveLinks() bool { return o.preserve_links != 0 }
359		-	func (o *Options) PreserveUid() bool { return o.preserve_uid != 0 }
360		-	func (o *Options) PreserveGid() bool { return o.preserve_gid != 0 }
361		-	func (o *Options) PreserveDevices() bool { return o.preserve_devices != 0 }
362		-	func (o *Options) PreserveMTimes() bool { return o.preserve_mtimes != 0 }
363		-	func (o *Options) PreservePerms() bool { return o.preserve_perms != 0 }
364		-	func (o *Options) PreserveSpecials() bool { return o.preserve_specials != 0 }
365		-	func (o *Options) Recurse() bool { return o.recurse != 0 }
366		-	func (o *Options) DeleteMode() bool { return o.delete_mode != 0 }
367		-	func (o *Options) AlwaysChecksum() bool { return o.always_checksum != 0 }
368		-	func (o *Options) Compress() bool { return o.do_compression != 0 }
369		-	func (o *Options) IgnoreTimes() bool { return o.ignore_times == 1 }
370		-	func (o *Options) SizeOnly() bool { return o.size_only == 1 }
371		-
372		-	func (o *Options) daemonTable() []poptOption {
373		-	return []poptOption{
374		-	/* longName, shortName, argInfo, arg, val */
375		-	{"help", "", POPT_ARG_NONE, nil, OPT_HELP},
376		-	{"address", "", POPT_ARG_STRING, &o.bind_address, 0},
377		-	{"bwlimit", "", POPT_ARG_INT, &o.daemon_bwlimit, 0},
378		-	{"config", "", POPT_ARG_STRING, &o.config_file, 0},
379		-	{"daemon", "", POPT_ARG_NONE, &o.daemon_opt, 0},
380		-	{"dparam", "M", POPT_ARG_STRING, nil, 'M'},
381		-	{"ipv4", "4", POPT_ARG_VAL, &o.default_af_hint, syscall.AF_INET},
382		-	{"ipv6", "6", POPT_ARG_VAL, &o.default_af_hint, syscall.AF_INET6},
383		-	{"detach", "", POPT_ARG_VAL, &o.no_detach, 0},
384		-	{"no-detach", "", POPT_ARG_VAL, &o.no_detach, 1},
385		-	{"log-file", "", POPT_ARG_STRING, &o.logfile_name, 0},
386		-	{"log-file-format", "", POPT_ARG_STRING, &o.logfile_format, 0},
387		-	{"port", "", POPT_ARG_INT, &o.rsync_port, 0},
388		-	{"sockopts", "", POPT_ARG_STRING, &o.sockopts, 0},
389		-	{"protocol", "", POPT_ARG_INT, &o.protocol_version, 0},
390		-	{"server", "", POPT_ARG_NONE, &o.am_server, 0},
391		-	{"temp-dir", "T", POPT_ARG_STRING, &o.tmpdir, 0},
392		-	{"verbose", "v", POPT_ARG_NONE, 0, 'v'},
393		-	{"no-verbose", "", POPT_ARG_VAL, &o.verbose, 0},
394		-	{"no-v", "", POPT_ARG_VAL, &o.verbose, 0},
395		-	{"help", "h", POPT_ARG_NONE, 0, 'h'},
396		-	}
397		-	}
398		-
399	333		func (o *Options) table() []poptOption {
400	334		return []poptOption{
401	335		/* longName, shortName, argInfo, arg, val */

...	...		@@ -631,6 +565,7 @@ func (o *Options) table() []poptOption {
631	565		{"mkpath", "", POPT_ARG_VAL, &o.mkpath_dest_arg, 1},
632	566		{"no-mkpath", "", POPT_ARG_VAL, &o.mkpath_dest_arg, 0},
633	567		{"qsort", "", POPT_ARG_NONE, &o.use_qsort, 0},
	568	+	{"use-qsort", "", POPT_ARG_NONE, &o.use_qsort, 0},
634	569		{"copy-as", "", POPT_ARG_STRING, &o.copy_as, 0},
635	570		{"address", "", POPT_ARG_STRING, &o.bind_address, 0},
636	571		{"port", "", POPT_ARG_INT, &o.rsync_port, 0},

...	...		@@ -654,25 +589,13 @@ func (o *Options) table() []poptOption {
654	589		}
655	590		}
656	591
657		-	var errNotYetImplemented = errors.New("option not yet implemented in gokrazy/rsync")
658		-
659		-	// rsync/options.c:parse_arguments.
660		-	func ParseArguments(args []string, gokrazyTable bool) (*Context, error) {
661		-	// NOTE: We do not implement support for refusing options per rsyncd.conf
662		-	// here, as we have our own configuration file.
663		-
664		-	version_opt_cnt := 0
665		-
	592	+	// ParseArguments parses the arguments that follow "rsync" on the command
	593	+	// line (rsync/options.c:parse_arguments).
	594	+	func ParseArguments(args []string) (*Context, error) {
666	595		opts := NewOptions()
667		-	table := opts.table()
668		-	if gokrazyTable {
669		-	// We need to make the --gokr.* flags known, otherwise the first parsing
670		-	// attempt fails and the daemon mode parsing is never run.
671		-	table = slices.Concat(opts.Gokrazy.table(), table)
672		-	}
673	596		pc := Context{
674	597		Options: opts,
675		-	table: table,
	598	+	table: opts.table(),
676	599		args: args,
677	600		}
678	601

...	...		@@ -688,7 +611,7 @@ func ParseArguments(args []string, gokrazyTable bool) (*Context, error) {
688	611		// are returned and handled here.
689	612		switch opt {
690	613		case 'V':
691		-	version_opt_cnt++
	614	+	opts.version_opt_cnt++
692	615
693	616		case OPT_SERVER:
694	617		opts.am_server = 1

...	...		@@ -700,47 +623,11 @@ func ParseArguments(args []string, gokrazyTable bool) (*Context, error) {
700	623		opts.am_sender = 1
701	624
702	625		case OPT_DAEMON:
703		-	// Parse the whole command-line using the daemon options table.
704		-	table := opts.daemonTable()
705		-	if gokrazyTable {
706		-	table = slices.Concat(opts.Gokrazy.table(), table)
707		-	}
708		-	pc := Context{
709		-	Options: opts,
710		-	table: table,
711		-	args: args,
712		-	}
	626	+	return nil, fmt.Errorf("daemon mode is not supported")
713	627
714		-	for {
715		-	opt, err := pc.poptGetNextOpt()
716		-	if err != nil {
717		-	return nil, err
718		-	}
719		-	if opt == -1 {
720		-	break // done
721		-	}
722		-	// Most options are handled by poptGetNextOpt, only special cases
723		-	// are returned and handled here.
724		-	switch opt {
725		-	case 'M':
726		-	return nil, errNotYetImplemented
727		-	case 'v':
728		-	opts.verbose++
729		-	default:
730		-	return nil, fmt.Errorf("unhandled special case opt: %v", opt)
731		-	}
732		-	}
733		-
734		-	opts.am_daemon = 1
735		-
736		-	return &pc, nil
737		-
738		-	case OPT_FILTER,
739		-	OPT_EXCLUDE,
740		-	OPT_INCLUDE,
741		-	OPT_INCLUDE_FROM,
742		-	OPT_EXCLUDE_FROM:
743		-	return nil, errNotYetImplemented
	628	+	case OPT_FILTER, OPT_EXCLUDE, OPT_INCLUDE, OPT_INCLUDE_FROM, OPT_EXCLUDE_FROM, 'F':
	629	+	// Clients send their filter rules over the protocol instead.
	630	+	return nil, fmt.Errorf("filter options are not accepted on the server side")
744	631
745	632		case 'a':
746	633		if opts.recurse == 0 {

...	...		@@ -766,7 +653,7 @@ func ParseArguments(args []string, gokrazyTable bool) (*Context, error) {
766	653		opts.human_readable++
767	654
768	655		case 'H':
769		-	opts.preserve_hard_links = 1
	656	+	opts.preserve_hard_links++
770	657
771	658		case 'i':
772	659		opts.itemize_changes++

...	...		@@ -781,7 +668,7 @@ func ParseArguments(args []string, gokrazyTable bool) (*Context, error) {
781	668		opts.verbose++
782	669
783	670		case 'y':
784		-	return nil, errNotYetImplemented
	671	+	opts.fuzzy_basis++
785	672
786	673		case 'q':
787	674		opts.quiet++

...	...		@@ -789,9 +676,6 @@ func ParseArguments(args []string, gokrazyTable bool) (*Context, error) {
789	676		case 'x':
790	677		opts.one_file_system++
791	678
792		-	case 'F':
793		-	return nil, errNotYetImplemented
794		-
795	679		case 'P':
796	680		opts.do_progress = 1
797	681		opts.keep_partial = 1

...	...		@@ -810,57 +694,57 @@ func ParseArguments(args []string, gokrazyTable bool) (*Context, error) {
810	694		opts.compress_choice = ""
811	695
812	696		case OPT_OLD_ARGS:
813		-	return nil, errNotYetImplemented
	697	+	opts.old_style_args++
814	698
815		-	case 'M': // --remote-option
816		-	return nil, errNotYetImplemented
	699	+	case 'M':
	700	+	return nil, fmt.Errorf("--remote-option is not accepted on the server side")
817	701
818		-	case OPT_WRITE_BATCH,
819		-	OPT_ONLY_WRITE_BATCH,
820		-	OPT_READ_BATCH:
821		-	return nil, errNotYetImplemented
	702	+	case OPT_WRITE_BATCH, OPT_ONLY_WRITE_BATCH, OPT_READ_BATCH:
	703	+	opts.batch_mode = 1
822	704
823	705		case OPT_BLOCK_SIZE:
824		-	return nil, errNotYetImplemented
	706	+	size, err := parseSizeArg(pc.poptGetOptArg(), 'b')
	707	+	if err != nil \|\| size < 0 {
	708	+	return nil, fmt.Errorf("invalid --block-size value")
	709	+	}
	710	+	opts.block_size = size
825	711
826		-	case OPT_MAX_SIZE, // (needs parse_size_arg)
827		-	OPT_MIN_SIZE,
828		-	OPT_BWLIMIT:
829		-	return nil, errNotYetImplemented
	712	+	case OPT_MAX_SIZE:
	713	+	size, err := parseSizeArg(opts.max_size_arg, 'b')
	714	+	if err != nil {
	715	+	return nil, fmt.Errorf("invalid --max-size value: %s", opts.max_size_arg)
	716	+	}
	717	+	opts.max_size = size
830	718
831		-	case OPT_APPEND:
832		-	return nil, errNotYetImplemented
	719	+	case OPT_MIN_SIZE:
	720	+	size, err := parseSizeArg(opts.min_size_arg, 'b')
	721	+	if err != nil {
	722	+	return nil, fmt.Errorf("invalid --min-size value: %s", opts.min_size_arg)
	723	+	}
	724	+	opts.min_size = size
833	725
834		-	case OPT_LINK_DEST,
835		-	OPT_COPY_DEST,
836		-	OPT_COMPARE_DEST:
837		-	return nil, errNotYetImplemented
	726	+	case OPT_APPEND:
	727	+	opts.append_mode++
838	728
839		-	case OPT_CHMOD: // (needs parse_chmod):
840		-	return nil, errNotYetImplemented
	729	+	case OPT_LINK_DEST, OPT_COPY_DEST, OPT_COMPARE_DEST:
	730	+	opts.basis_dirs = append(opts.basis_dirs, pc.poptGetOptArg())
841	731
842	732		case OPT_INFO:
843	733		parseOutputWords(infoWords[:], opts.info[:], pc.poptGetOptArg(), USER_PRIORITY)
844	734
845		-	case OPT_DEBUG:
846		-	// TODO: plumb the debug level that make sense for our implementation
847		-	slog.Info("TODO: set debug level", "to", pc.poptGetOptArg())
848		-
849		-	case OPT_USERMAP,
850		-	OPT_GROUPMAP,
851		-	OPT_CHOWN:
852		-	return nil, errNotYetImplemented
853		-
854	735		case 'A':
855		-	return nil, fmt.Errorf("ACLs are not supported by gokrazy/rsync")
	736	+	opts.preserve_acls++
856	737
857	738		case 'X':
858	739		opts.preserve_xattrs++
859	740
860		-	case OPT_STOP_AFTER,
861		-	OPT_STOP_AT,
862		-	OPT_STDERR:
863		-	return nil, errNotYetImplemented
	741	+	case OPT_MODIFY_WINDOW:
	742	+	opts.modify_window_set = 1
	743	+
	744	+	case OPT_DEBUG, OPT_CHMOD, OPT_USERMAP, OPT_GROUPMAP, OPT_CHOWN,
	745	+	OPT_BWLIMIT, OPT_STOP_AFTER, OPT_STOP_AT, OPT_STDERR, OPT_NO_ICONV:
	746	+	// These don't change what the server does.
	747	+	_ = pc.poptGetOptArg()
864	748
865	749		default:
866	750		return nil, fmt.Errorf("unhandled special case opt: %v", opt)

...	...		@@ -872,6 +756,15 @@ func ParseArguments(args []string, gokrazyTable bool) (*Context, error) {
872	756
873	757		opts.setOutputVerbosity(DEFAULT_PRIORITY)
874	758
	759	+	if opts.delete_before+min(opts.delete_during, 1)+opts.delete_after > 1 {
	760	+	return nil, fmt.Errorf("you may not combine multiple --delete-WHEN options")
	761	+	}
	762	+	if opts.delete_before != 0 \|\| opts.delete_during != 0 \|\| opts.delete_after != 0 {
	763	+	opts.delete_mode = 1
	764	+	} else if opts.delete_mode != 0 \|\| opts.delete_excluded != 0 {
	765	+	opts.delete_mode = 1
	766	+	}
	767	+
875	768		if opts.recurse != 0 {
876	769		opts.xfer_dirs = 1
877	770		}

...	...		@@ -911,7 +804,7 @@ func ParseArguments(args []string, gokrazyTable bool) (*Context, error) {
911	804		opts.make_backups = 1 // --backup-dir implies --backup
912	805		}
913	806
914		-	if opts.do_progress != 0 /* && !opts.am_server */ {
	807	+	if opts.do_progress != 0 {
915	808		if opts.info[INFO_NAME] == 0 {
916	809		opts.info[INFO_NAME] = 1
917	810		}

...	...		@@ -923,3 +816,122 @@ func ParseArguments(args []string, gokrazyTable bool) (*Context, error) {
923	816
924	817		return &pc, nil
925	818		}
	819	+
	820	+	// parseSizeArg parses a size like 100K or 1.5MB (rsync/options.c
	821	+	// parse_size_arg). A bare number is in units of defSuffix.
	822	+	func parseSizeArg(arg string, defSuffix byte) (int64, error) {
	823	+	s := arg
	824	+	i := 0
	825	+	for i < len(s) && (s[i] >= '0' && s[i] <= '9' \|\| s[i] == '.') {
	826	+	i++
	827	+	}
	828	+	if i == 0 {
	829	+	return 0, fmt.Errorf("invalid size %q", arg)
	830	+	}
	831	+	num, err := strconv.ParseFloat(s[:i], 64)
	832	+	if err != nil {
	833	+	return 0, err
	834	+	}
	835	+	s = s[i:]
	836	+	suffix := defSuffix
	837	+	if s != "" {
	838	+	suffix = s[0]
	839	+	s = s[1:]
	840	+	}
	841	+	mult := float64(1024)
	842	+	switch {
	843	+	case strings.HasPrefix(strings.ToLower(s), "ib"):
	844	+	s = s[2:]
	845	+	case strings.HasPrefix(strings.ToLower(s), "b"):
	846	+	mult = 1000
	847	+	s = s[1:]
	848	+	}
	849	+	var size float64
	850	+	switch suffix \| 0x20 {
	851	+	case 'b':
	852	+	size = num
	853	+	case 'k':
	854	+	size = num * mult
	855	+	case 'm':
	856	+	size = num * mult * mult
	857	+	case 'g':
	858	+	size = num * mult * mult * mult
	859	+	case 't':
	860	+	size = num * mult * mult * mult * mult
	861	+	case 'p':
	862	+	size = num * mult * mult * mult * mult * mult
	863	+	default:
	864	+	return 0, fmt.Errorf("invalid size %q", arg)
	865	+	}
	866	+	switch s {
	867	+	case "":
	868	+	case "+1":
	869	+	size++
	870	+	case "-1":
	871	+	size--
	872	+	default:
	873	+	return 0, fmt.Errorf("invalid size %q", arg)
	874	+	}
	875	+	return int64(size), nil
	876	+	}
	877	+
	878	+	func (o *Options) Verbose() int { return o.verbose }
	879	+	func (o *Options) Info(level infoLevel) int { return int(o.info[level]) }
	880	+	func (o *Options) DryRun() bool { return o.dry_run != 0 }
	881	+	func (o *Options) Server() bool { return o.am_server != 0 }
	882	+	func (o *Options) Sender() bool { return o.am_sender != 0 }
	883	+	func (o *Options) Recurse() bool { return o.recurse != 0 }
	884	+	func (o *Options) XferDirs() bool { return o.xfer_dirs != 0 }
	885	+	func (o *Options) ListOnly() bool { return o.list_only != 0 }
	886	+	func (o *Options) RelativePaths() bool { return o.relative_paths != 0 }
	887	+	func (o *Options) PreserveLinks() bool { return o.preserve_links != 0 }
	888	+	func (o *Options) PreserveHardLinks() bool { return o.preserve_hard_links != 0 }
	889	+	func (o *Options) PreserveUid() bool { return o.preserve_uid != 0 }
	890	+	func (o *Options) PreserveGid() bool { return o.preserve_gid != 0 }
	891	+	func (o *Options) PreserveDevices() bool { return o.preserve_devices != 0 }
	892	+	func (o *Options) PreserveSpecials() bool { return o.preserve_specials != 0 }
	893	+	func (o *Options) PreserveMTimes() bool { return o.preserve_mtimes != 0 }
	894	+	func (o *Options) PreservePerms() bool { return o.preserve_perms != 0 }
	895	+	func (o *Options) PreserveACLs() bool { return o.preserve_acls != 0 }
	896	+	func (o *Options) PreserveXattrs() bool { return o.preserve_xattrs != 0 }
	897	+	func (o *Options) PreserveAtimes() bool { return o.preserve_atimes != 0 }
	898	+	func (o *Options) PreserveCrtimes() bool { return o.preserve_crtimes != 0 }
	899	+	func (o *Options) NumericIDs() bool { return o.numeric_ids != 0 }
	900	+	func (o *Options) DeleteMode() bool { return o.delete_mode != 0 }
	901	+	func (o *Options) DeleteBefore() bool { return o.delete_before != 0 }
	902	+	func (o *Options) DeleteDuring() int { return o.delete_during }
	903	+	func (o *Options) DeleteAfter() bool { return o.delete_after != 0 }
	904	+	func (o *Options) DeleteExcluded() bool { return o.delete_excluded != 0 }
	905	+	func (o *Options) ForceDelete() bool { return o.force_delete != 0 }
	906	+	func (o *Options) MaxDelete() int { return o.max_delete }
	907	+	func (o *Options) IgnoreErrors() bool { return o.ignore_errors != 0 }
	908	+	func (o *Options) PruneEmptyDirs() bool { return o.prune_empty_dirs != 0 }
	909	+	func (o *Options) AlwaysChecksum() bool { return o.always_checksum != 0 }
	910	+	func (o *Options) ChecksumChoice() string { return o.checksum_choice }
	911	+	func (o *Options) ChecksumSeed() int { return o.checksum_seed }
	912	+	func (o *Options) Compress() bool { return o.do_compression != 0 }
	913	+	func (o *Options) CompressChoice() string { return o.compress_choice }
	914	+	func (o *Options) CompressLevel() int { return o.do_compression_level }
	915	+	func (o *Options) IgnoreTimes() bool { return o.ignore_times != 0 }
	916	+	func (o *Options) SizeOnly() bool { return o.size_only != 0 }
	917	+	func (o *Options) UpdateOnly() bool { return o.update_only != 0 }
	918	+	func (o *Options) IgnoreExisting() bool { return o.ignore_existing != 0 }
	919	+	func (o *Options) IgnoreNonExisting() bool { return o.ignore_non_existing != 0 }
	920	+	func (o *Options) WholeFile() bool { return o.whole_file > 0 }
	921	+	func (o *Options) BlockSize() int64 { return o.block_size }
	922	+	func (o *Options) MaxSize() int64 { return o.max_size }
	923	+	func (o *Options) MinSize() int64 { return o.min_size }
	924	+	func (o *Options) Inplace() bool { return o.inplace != 0 }
	925	+	func (o *Options) AppendMode() int { return o.append_mode }
	926	+	func (o *Options) DelayUpdates() bool { return o.delay_updates != 0 }
	927	+	func (o *Options) RemoveSourceFiles() int { return o.remove_source_files }
	928	+	func (o *Options) CvsExclude() bool { return o.cvs_exclude != 0 }
	929	+	func (o *Options) ProtocolVersion() int { return o.protocol_version }
	930	+	func (o *Options) ProtectArgs() bool { return o.protect_args != 0 }
	931	+	func (o *Options) ShellCmd() string { return o.shell_cmd }
	932	+	func (o *Options) FilesFrom() string { return o.files_from }
	933	+	func (o *Options) MissingArgs() int { return o.missing_args }
	934	+	func (o *Options) BatchMode() bool { return o.batch_mode != 0 }
	935	+	func (o *Options) BasisDirs() []string { return o.basis_dirs }
	936	+	func (o *Options) Iconv() string { return o.iconv_opt }
	937	+	func (o *Options) ModifyWindowSet() bool { return o.modify_window_set != 0 }

...	...		@@ -0,0 +1,492 @@
	1	+	package rsync
	2	+
	3	+	import (
	4	+	"errors"
	5	+	"fmt"
	6	+	"io"
	7	+	"io/fs"
	8	+	"path"
	9	+	"slices"
	10	+	"strings"
	11	+	"time"
	12	+	)
	13	+
	14	+	const (
	15	+	nameNormal = iota
	16	+	nameDotDir
	17	+	)
	18	+
	19	+	// buildFileList turns the sender's source arguments into file-list entries
	20	+	// (send_file_list). Errors about individual sources are reported to the
	21	+	// client and recorded in s.ioError.
	22	+	func (s session) buildFileList(args []string) ([]fileEntry, error) {
	23	+	var files []*fileEntry
	24	+	flags := 0
	25	+	if s.opts.recurse {
	26	+	flags = flagContentDir
	27	+	}
	28	+	for _, arg := range args {
	29	+	fbuf := strings.TrimLeft(arg, "/")
	30	+	nameType := nameNormal
	31	+	switch {
	32	+	case fbuf == "" \|\| strings.HasSuffix(fbuf, "/"):
	33	+	if fbuf == "./" {
	34	+	fbuf = "."
	35	+	} else {
	36	+	fbuf += "."
	37	+	}
	38	+	nameType = nameDotDir
	39	+	case fbuf == "." \|\| strings.HasSuffix(fbuf, "/."):
	40	+	nameType = nameDotDir
	41	+	}
	42	+	dir, fn := "", fbuf
	43	+	if i := strings.LastIndexByte(fbuf, '/'); i >= 0 {
	44	+	dir, fn = fbuf[:i], fbuf[i+1:]
	45	+	}
	46	+	if fn == "" {
	47	+	fn = "."
	48	+	nameType = nameDotDir
	49	+	}
	50	+	dir, err := cleanSourcePath(dir)
	51	+	if err != nil {
	52	+	return nil, err
	53	+	}
	54	+	if fn == ".." \|\| strings.Contains(fn, "\x00") {
	55	+	return nil, fmt.Errorf("invalid source path: %s", arg)
	56	+	}
	57	+	src := dir
	58	+	if fn != "." {
	59	+	src = path.Join(dir, fn)
	60	+	}
	61	+
	62	+	info, err := s.fs.Stat(src)
	63	+	if err != nil {
	64	+	if errors.Is(err, fs.ErrNotExist) {
	65	+	s.xferError("link_stat %q failed: No such file or directory (2)", "/"+src)
	66	+	} else {
	67	+	s.ioError \|= ioErrGeneral
	68	+	s.xferError("link_stat %q failed: %v", "/"+src, err)
	69	+	}
	70	+	continue
	71	+	}
	72	+	if nameType != nameDotDir && s.filters.excluded(fn, info.IsDir, nil) {
	73	+	continue
	74	+	}
	75	+	if info.IsDir && !s.opts.xferDirs {
	76	+	s.info("skipping directory %s", fn)
	77	+	continue
	78	+	}
	79	+
	80	+	if s.opts.recurse \|\| (s.opts.xferDirs && nameType != nameNormal) {
	81	+	top := s.newEntry(fn, src, info, flagTopDir\|flagContentDir\|flags)
	82	+	files = append(files, top)
	83	+	if info.IsDir {
	84	+	children, err := s.listDir(fn, src, dir, flags)
	85	+	var mergeErr *mergeFileError
	86	+	if errors.As(err, &mergeErr) {
	87	+	return nil, err
	88	+	}
	89	+	if err != nil {
	90	+	s.ioError \|= ioErrGeneral
	91	+	s.xferError("opendir %q failed: %v", "/"+src, err)
	92	+	continue
	93	+	}
	94	+	files = append(files, children...)
	95	+	}
	96	+	} else {
	97	+	files = append(files, s.newEntry(fn, src, info, flags))
	98	+	}
	99	+	}
	100	+	return files, nil
	101	+	}
	102	+
	103	+	// listDir returns the entries below a source directory, with names under
	104	+	// prefix. Directories missing from the listing are implied by their
	105	+	// contents, as object stores don't keep them. root is the storage path of
	106	+	// the transfer root.
	107	+	func (s session) listDir(prefix, src, root string, flags int) ([]fileEntry, error) {
	108	+	entries, err := s.fs.ReadDir(src, s.opts.recurse)
	109	+	if err != nil {
	110	+	return nil, err
	111	+	}
	112	+	join := func(name string) string {
	113	+	if prefix == "." {
	114	+	return name
	115	+	}
	116	+	return prefix + "/" + name
	117	+	}
	118	+
	119	+	byName := make(map[string]FileInfo, len(entries))
	120	+	for _, e := range entries {
	121	+	name, err := cleanSourcePath(e.Name)
	122	+	if err != nil \|\| name == "" {
	123	+	continue
	124	+	}
	125	+	if !s.opts.recurse && strings.Contains(name, "/") {
	126	+	continue
	127	+	}
	128	+	e.Name = name
	129	+	byName[name] = e
	130	+	if s.opts.recurse {
	131	+	for d := path.Dir(name); d != "."; d = path.Dir(d) {
	132	+	if _, ok := byName[d]; ok {
	133	+	break
	134	+	}
	135	+	byName[d] = FileInfo{Name: d, IsDir: true, ModTime: e.ModTime}
	136	+	}
	137	+	}
	138	+	}
	139	+	names := make([]string, 0, len(byName))
	140	+	regular := make(map[string]bool, len(byName))
	141	+	for name, e := range byName {
	142	+	names = append(names, name)
	143	+	regular[name] = !e.IsDir
	144	+	}
	145	+	slices.Sort(names)
	146	+	scopes := s.filters.newScopeTree(root, src, s.mergeReader(src, regular))
	147	+
	148	+	var files []*fileEntry
	149	+	var excludedDirs []string
	150	+	for _, name := range names {
	151	+	e := byName[name]
	152	+	if slices.ContainsFunc(excludedDirs, func(d string) bool { return strings.HasPrefix(name, d+"/") }) {
	153	+	continue
	154	+	}
	155	+	full := join(name)
	156	+	sc, err := scopes.scope(path.Dir(name))
	157	+	if err != nil {
	158	+	return nil, err
	159	+	}
	160	+	if s.filters.excluded(full, e.IsDir, sc) {
	161	+	if e.IsDir {
	162	+	excludedDirs = append(excludedDirs, name)
	163	+	}
	164	+	continue
	165	+	}
	166	+	files = append(files, s.newEntry(full, path.Join(src, name), e, flags))
	167	+	}
	168	+	return files, nil
	169	+	}
	170	+
	171	+	func (s session) newEntry(name, src string, info FileInfo, flags int) fileEntry {
	172	+	f := newFileEntry(name)
	173	+	if name == "." {
	174	+	f.dirname, f.basename = "", "."
	175	+	}
	176	+	f.src = src
	177	+	f.mtime = info.ModTime.Unix()
	178	+	f.nsec = uint32(info.ModTime.Nanosecond())
	179	+	if info.ModTime.IsZero() {
	180	+	f.mtime, f.nsec = time.Now().Unix(), 0
	181	+	}
	182	+	if info.IsDir {
	183	+	f.mode = sIFDIR \| 0o755
	184	+	f.size = 4096
	185	+	f.flags = flags
	186	+	} else {
	187	+	f.mode = sIFREG \| 0o644
	188	+	f.size = info.Size
	189	+	}
	190	+	return f
	191	+	}
	192	+
	193	+	// cleanSourcePath normalizes a path inside the storage, refusing anything
	194	+	// that would leave it.
	195	+	func cleanSourcePath(p string) (string, error) {
	196	+	var parts []string
	197	+	for _, c := range strings.Split(p, "/") {
	198	+	switch c {
	199	+	case "", ".":
	200	+	case "..":
	201	+	return "", fmt.Errorf("path escapes the transfer root: %s", p)
	202	+	default:
	203	+	parts = append(parts, c)
	204	+	}
	205	+	}
	206	+	return strings.Join(parts, "/"), nil
	207	+	}
	208	+
	209	+	// runSender is the server side of a download (do_server_sender).
	210	+	func (s *session) runSender(args []string) error {
	211	+	start := time.Now()
	212	+	if err := s.recvFilterList(); err != nil {
	213	+	return err
	214	+	}
	215	+
	216	+	srcs := args
	217	+	if len(srcs) > 0 {
	218	+	// The first argument is the "." the client uses as a placeholder
	219	+	// for the working directory.
	220	+	srcs = srcs[1:]
	221	+	}
	222	+	if len(srcs) == 0 && (s.opts.recurse \|\| s.opts.xferDirs) {
	223	+	srcs = []string{"."}
	224	+	}
	225	+
	226	+	files, err := s.buildFileList(srcs)
	227	+	if err != nil {
	228	+	return s.fatal(exitFileSelect, err)
	229	+	}
	230	+	if s.opts.alwaysChecksum {
	231	+	for _, f := range files {
	232	+	if f.isReg() {
	233	+	if f.sum, err = s.fileChecksum(f.src); err != nil {
	234	+	s.ioError \|= ioErrGeneral
	235	+	s.xferError("failed to checksum %q: %v", "/"+f.src, err)
	236	+	}
	237	+	}
	238	+	}
	239	+	}
	240	+	buildTime := time.Since(start)
	241	+
	242	+	start = time.Now()
	243	+	codec := s.newFlistCodec()
	244	+	var totalSize int64
	245	+	for _, f := range files {
	246	+	if err := codec.sendEntry(s.c, f); err != nil {
	247	+	return err
	248	+	}
	249	+	if f.isReg() \|\| f.isLink() {
	250	+	totalSize += f.size
	251	+	}
	252	+	}
	253	+	ioError := s.ioError
	254	+	if s.opts.ignoreErrors {
	255	+	ioError = 0
	256	+	}
	257	+	if ioError != 0 && !s.safeFlist {
	258	+	if err := codec.sendEnd(s.c, 0); err != nil {
	259	+	return err
	260	+	}
	261	+	} else if err := codec.sendEnd(s.c, ioError); err != nil {
	262	+	return err
	263	+	}
	264	+	sortFileList(files, s.c.protocol, false)
	265	+	if err := s.sendIDLists(); err != nil {
	266	+	return err
	267	+	}
	268	+	if s.c.protocol < 30 {
	269	+	if err := s.c.writeInt32(ioError); err != nil {
	270	+	return err
	271	+	}
	272	+	} else if !s.safeFlist && ioError != 0 {
	273	+	if err := s.c.writeMsgInt(msgIOError, ioError); err != nil {
	274	+	return err
	275	+	}
	276	+	}
	277	+	if err := s.c.flush(); err != nil {
	278	+	return err
	279	+	}
	280	+	xferTime := time.Since(start)
	281	+
	282	+	if len(files) == 0 {
	283	+	return s.finish()
	284	+	}
	285	+
	286	+	if err := s.sendFiles(files); err != nil {
	287	+	return err
	288	+	}
	289	+
	290	+	if err := s.c.writeVarlong30(s.c.bytesRead(), 3); err != nil {
	291	+	return err
	292	+	}
	293	+	if err := s.c.writeVarlong30(s.c.bytesWritten(), 3); err != nil {
	294	+	return err
	295	+	}
	296	+	if err := s.c.writeVarlong30(totalSize, 3); err != nil {
	297	+	return err
	298	+	}
	299	+	if s.c.protocol >= 29 {
	300	+	if err := s.c.writeVarlong30(max(buildTime.Milliseconds(), 1), 3); err != nil {
	301	+	return err
	302	+	}
	303	+	if err := s.c.writeVarlong30(xferTime.Milliseconds(), 3); err != nil {
	304	+	return err
	305	+	}
	306	+	}
	307	+	if err := s.readFinalGoodbye(true); err != nil {
	308	+	return err
	309	+	}
	310	+	return s.finish()
	311	+	}
	312	+
	313	+	func (s *session) sendIDLists() error {
	314	+	if s.opts.numericIDs {
	315	+	return nil
	316	+	}
	317	+	for _, want := range []bool{s.opts.preserveUID, s.opts.preserveGID} {
	318	+	if !want {
	319	+	continue
	320	+	}
	321	+	// Every entry is owned by id 0, which is never listed by number.
	322	+	if err := s.c.writeVarint30(0); err != nil {
	323	+	return err
	324	+	}
	325	+	if s.xmitID0Names {
	326	+	if err := s.c.writeByte(4); err != nil {
	327	+	return err
	328	+	}
	329	+	if err := s.c.write([]byte("root")); err != nil {
	330	+	return err
	331	+	}
	332	+	}
	333	+	}
	334	+	return nil
	335	+	}
	336	+
	337	+	// sendFiles answers the receiver's requests until it is done
	338	+	// (send_files).
	339	+	func (s session) sendFiles(files []fileEntry) error {
	340	+	phase := 0
	341	+	maxPhase := 1
	342	+	if s.c.protocol >= 29 {
	343	+	maxPhase = 2
	344	+	}
	345	+	savedIOError := s.ioError
	346	+	tok := newTokenSender(s.c, s.compression, s.compressLevel)
	347	+	for {
	348	+	req, err := s.readNdxAndAttrs(len(files))
	349	+	if err != nil {
	350	+	return err
	351	+	}
	352	+	if req.ndx == ndxDone {
	353	+	phase++
	354	+	if phase > maxPhase {
	355	+	break
	356	+	}
	357	+	if err := s.c.writeNdx(ndxDone); err != nil {
	358	+	return err
	359	+	}
	360	+	continue
	361	+	}
	362	+	f := files[req.ndx]
	363	+	if req.iflags&itemTransfer != 0 && !f.isReg() {
	364	+	return s.fatal(exitProtocol, fmt.Errorf("received request to transfer non-regular file: %d", req.ndx))
	365	+	}
	366	+	if req.iflags&itemTransfer == 0 {
	367	+	if err := s.writeNdxAndAttrs(req); err != nil {
	368	+	return err
	369	+	}
	370	+	continue
	371	+	}
	372	+	if phase == 2 {
	373	+	return s.fatal(exitProtocol, errors.New("got transfer request in phase 2"))
	374	+	}
	375	+	if s.opts.dryRun {
	376	+	if err := s.writeNdxAndAttrs(req); err != nil {
	377	+	return err
	378	+	}
	379	+	continue
	380	+	}
	381	+
	382	+	head, sums, err := s.receiveSums()
	383	+	if err != nil {
	384	+	return s.fatal(exitProtocol, err)
	385	+	}
	386	+	if err := s.sendFile(f, req, head, sums, tok); err != nil {
	387	+	return err
	388	+	}
	389	+	}
	390	+	if s.ioError != savedIOError && s.c.protocol >= 30 {
	391	+	if err := s.c.writeMsgInt(msgIOError, s.ioError); err != nil {
	392	+	return err
	393	+	}
	394	+	}
	395	+	return s.c.writeNdx(ndxDone)
	396	+	}
	397	+
	398	+	func (s session) sendFile(f fileEntry, req ndxRequest, head sumHead, sums []sumBuf, tok tokenSender) error {
	399	+	file, info, err := s.fs.Open(f.src)
	400	+	if err != nil {
	401	+	if errors.Is(err, fs.ErrNotExist) {
	402	+	s.ioError \|= ioErrVanished
	403	+	s.warning("file has vanished: %q", "/"+f.src)
	404	+	} else {
	405	+	s.ioError \|= ioErrGeneral
	406	+	s.xferError("send_files failed to open %q: %v", "/"+f.src, err)
	407	+	}
	408	+	if s.c.protocol >= 30 {
	409	+	return s.c.writeMsgInt(msgNoSend, req.ndx)
	410	+	}
	411	+	return nil
	412	+	}
	413	+	defer func() { _ = file.Close() }()
	414	+
	415	+	if err := s.writeNdxAndAttrs(req); err != nil {
	416	+	return err
	417	+	}
	418	+	if err := s.c.writeSumHead(head); err != nil {
	419	+	return err
	420	+	}
	421	+	inplace := (s.inplacePartial && req.fnamecmpType == fnamecmpPartialDir) \|\|
	422	+	(s.opts.inplace && req.fnamecmpType == fnamecmpFname)
	423	+	readSize := max(3*int64(head.blength), maxMapSize)
	424	+	m := &matcher{
	425	+	head: head,
	426	+	sums: sums,
	427	+	cs: &s.cs,
	428	+	tok: tok,
	429	+	buf: newMapFile(file, info.Size, readSize, head.blength),
	430	+	sum: s.cs.newXferSum(),
	431	+	inplace: inplace,
	432	+	}
	433	+	if err := m.run(info.Size); err != nil {
	434	+	return err
	435	+	}
	436	+	sum := m.sum.Sum(nil)
	437	+	if m.buf.err != nil {
	438	+	s.ioError \|= ioErrGeneral
	439	+	s.xferError("read errors mapping %q: %v", "/"+f.src, m.buf.err)
	440	+	// A checksum the receiver can't match makes it discard the file.
	441	+	allZero := !slices.ContainsFunc(sum, func(b byte) bool { return b != 0 })
	442	+	clear(sum)
	443	+	if allZero {
	444	+	sum[len(sum)-1]++
	445	+	}
	446	+	}
	447	+	s.stats.literal += m.literal
	448	+	s.stats.matched += m.matched
	449	+	return s.c.write(sum)
	450	+	}
	451	+
	452	+	// receiveSums reads the block signatures the generator computed for its
	453	+	// copy of a file.
	454	+	func (s *session) receiveSums() (sumHead, []sumBuf, error) {
	455	+	head, err := s.c.readSumHead(s.cs.xfer.size())
	456	+	if err != nil {
	457	+	return head, nil, err
	458	+	}
	459	+	sums := make([]sumBuf, head.count)
	460	+	sum2 := make([]byte, int(head.count)*int(head.s2length))
	461	+	var offset int64
	462	+	for i := range sums {
	463	+	v, err := s.c.readInt32()
	464	+	if err != nil {
	465	+	return head, nil, err
	466	+	}
	467	+	b := sum2[iint(head.s2length) : (i+1)int(head.s2length)]
	468	+	if err := s.c.readFull(b); err != nil {
	469	+	return head, nil, err
	470	+	}
	471	+	l := head.blength
	472	+	if int32(i) == head.count-1 && head.remainder != 0 {
	473	+	l = head.remainder
	474	+	}
	475	+	sums[i] = sumBuf{offset: offset, len: l, sum1: uint32(v), sum2: b}
	476	+	offset += int64(l)
	477	+	}
	478	+	return head, sums, nil
	479	+	}
	480	+
	481	+	func (s *session) fileChecksum(name string) ([]byte, error) {
	482	+	f, info, err := s.fs.Open(name)
	483	+	if err != nil {
	484	+	return nil, err
	485	+	}
	486	+	defer func() { _ = f.Close() }()
	487	+	h := s.cs.newFileSum()
	488	+	if _, err := io.Copy(h, io.NewSectionReader(f, 0, info.Size)); err != nil {
	489	+	return nil, err
	490	+	}
	491	+	return h.Sum(nil), nil
	492	+	}

...	...		@@ -0,0 +1,810 @@
	1	+	package rsync
	2	+
	3	+	import (
	4	+	"crypto/rand"
	5	+	"encoding/binary"
	6	+	"errors"
	7	+	"fmt"
	8	+	"io"
	9	+	"log/slog"
	10	+	"slices"
	11	+	"strings"
	12	+	"sync"
	13	+
	14	+	"github.com/picosh/pico/pkg/rsync/rsyncopts"
	15	+	)
	16	+
	17	+	// Config configures a server session.
	18	+	type Config struct {
	19	+	FS FS
	20	+	Logger *slog.Logger
	21	+	// Stderr receives errors raised before the protocol is running.
	22	+	Stderr io.Writer
	23	+	}
	24	+
	25	+	// ExitError carries the exit code the client should see. Its message has
	26	+	// already been delivered to the client.
	27	+	type ExitError struct {
	28	+	Code int
	29	+	Err error
	30	+	}
	31	+
	32	+	func (e *ExitError) Error() string {
	33	+	if e.Err == nil {
	34	+	return fmt.Sprintf("rsync exited with code %d", e.Code)
	35	+	}
	36	+	return e.Err.Error()
	37	+	}
	38	+
	39	+	func (e *ExitError) Unwrap() error { return e.Err }
	40	+
	41	+	// options holds the parsed command line in the form the session uses.
	42	+	type options struct {
	43	+	sender bool
	44	+	recurse bool
	45	+	xferDirs bool
	46	+	dryRun bool
	47	+	preserveUID bool
	48	+	preserveGID bool
	49	+	preserveLinks bool
	50	+	preserveDevs bool
	51	+	preserveSpecs bool
	52	+	preserveTimes bool
	53	+	preserveAtimes bool
	54	+	numericIDs bool
	55	+	alwaysChecksum bool
	56	+	ignoreTimes bool
	57	+	sizeOnly bool
	58	+	updateOnly bool
	59	+	ignoreExisting bool
	60	+	ignoreNonExist bool
	61	+	wholeFile bool
	62	+	inplace bool
	63	+	delayUpdates bool
	64	+	removeSource bool
	65	+	deleteMode bool
	66	+	deleteBefore bool
	67	+	deleteDuring int
	68	+	deleteAfter bool
	69	+	deleteExcluded bool
	70	+	pruneEmptyDirs bool
	71	+	ignoreErrors bool
	72	+	cvsExclude bool
	73	+	maxDelete int
	74	+	maxSize int64
	75	+	minSize int64
	76	+	blockSize int32
	77	+	infoName int
	78	+	infoDel int
	79	+	infoNonreg int
	80	+	}
	81	+
	82	+	type stats struct {
	83	+	literal int64
	84	+	matched int64
	85	+	}
	86	+
	87	+	// session is one rsync server run.
	88	+	type session struct {
	89	+	fs FS
	90	+	logger *slog.Logger
	91	+	c *conn
	92	+	opts options
	93	+
	94	+	cs checksums
	95	+	compression compression
	96	+	compressLevel int
	97	+	filters *filterList
	98	+	varintFlags bool
	99	+	safeFlist bool
	100	+	xmitID0Names bool
	101	+	inplacePartial bool
	102	+
	103	+	mu sync.Mutex
	104	+	ioError int32
	105	+	gotXferError bool
	106	+	stats stats
	107	+	}
	108	+
	109	+	// Serve runs the server side of an rsync transfer over rw, which carries the
	110	+	// SSH session's stdin and stdout. args is the command line after "rsync".
	111	+	func Serve(cfg Config, rw io.ReadWriter, args []string) error {
	112	+	logger := cfg.Logger
	113	+	if logger == nil {
	114	+	logger = slog.New(slog.DiscardHandler)
	115	+	}
	116	+	stderr := cfg.Stderr
	117	+	if stderr == nil {
	118	+	stderr = io.Discard
	119	+	}
	120	+	early := func(code int, err error) error {
	121	+	_, _ = fmt.Fprintf(stderr, "rsync: %v\n", err)
	122	+	return &ExitError{Code: code, Err: err}
	123	+	}
	124	+
	125	+	c := newConn(rw)
	126	+	defer func() { _ = c.close() }()
	127	+	pc, err := rsyncopts.ParseArguments(args)
	128	+	if err != nil {
	129	+	return early(exitSyntax, err)
	130	+	}
	131	+	if pc.Options.ProtectArgs() {
	132	+	// The real arguments follow on stdin, NUL-separated.
	133	+	extra, err := readProtectedArgs(c)
	134	+	if err != nil {
	135	+	return early(exitProtocol, err)
	136	+	}
	137	+	if pc, err = rsyncopts.ParseArguments(append(slices.Clone(args), extra...)); err != nil {
	138	+	return early(exitSyntax, err)
	139	+	}
	140	+	}
	141	+	o := pc.Options
	142	+	if !o.Server() {
	143	+	return early(exitSyntax, errors.New("only server mode is supported"))
	144	+	}
	145	+	if err := checkSupported(o); err != nil {
	146	+	return early(exitUnsupported, err)
	147	+	}
	148	+
	149	+	s := &session{
	150	+	fs: cfg.FS,
	151	+	logger: logger,
	152	+	c: c,
	153	+	filters: &filterList{},
	154	+	}
	155	+	s.opts = options{
	156	+	sender: o.Sender(),
	157	+	recurse: o.Recurse(),
	158	+	xferDirs: o.XferDirs(),
	159	+	dryRun: o.DryRun(),
	160	+	preserveUID: o.PreserveUid(),
	161	+	preserveGID: o.PreserveGid(),
	162	+	preserveLinks: o.PreserveLinks(),
	163	+	preserveDevs: o.PreserveDevices(),
	164	+	preserveSpecs: o.PreserveSpecials(),
	165	+	preserveTimes: o.PreserveMTimes(),
	166	+	preserveAtimes: o.PreserveAtimes(),
	167	+	numericIDs: o.NumericIDs(),
	168	+	alwaysChecksum: o.AlwaysChecksum(),
	169	+	ignoreTimes: o.IgnoreTimes(),
	170	+	sizeOnly: o.SizeOnly(),
	171	+	updateOnly: o.UpdateOnly(),
	172	+	ignoreExisting: o.IgnoreExisting(),
	173	+	ignoreNonExist: o.IgnoreNonExisting(),
	174	+	wholeFile: o.WholeFile(),
	175	+	inplace: o.Inplace(),
	176	+	delayUpdates: o.DelayUpdates(),
	177	+	removeSource: o.RemoveSourceFiles() != 0,
	178	+	deleteMode: o.DeleteMode(),
	179	+	deleteBefore: o.DeleteBefore(),
	180	+	deleteDuring: o.DeleteDuring(),
	181	+	deleteAfter: o.DeleteAfter(),
	182	+	deleteExcluded: o.DeleteExcluded(),
	183	+	pruneEmptyDirs: o.PruneEmptyDirs(),
	184	+	ignoreErrors: o.IgnoreErrors(),
	185	+	cvsExclude: o.CvsExclude(),
	186	+	maxDelete: o.MaxDelete(),
	187	+	maxSize: o.MaxSize(),
	188	+	minSize: o.MinSize(),
	189	+	blockSize: int32(o.BlockSize()),
	190	+	infoName: o.Info(rsyncopts.INFO_NAME),
	191	+	infoDel: o.Info(rsyncopts.INFO_DEL),
	192	+	infoNonreg: o.Info(rsyncopts.INFO_NONREG),
	193	+	}
	194	+	s.filters.deleteExcluded = s.opts.deleteExcluded
	195	+	s.filters.otherSide = ruleSenderSide
	196	+	if s.opts.sender {
	197	+	s.filters.otherSide = ruleReceiverSide
	198	+	}
	199	+
	200	+	if err := s.setupProtocol(o); err != nil {
	201	+	var exit *ExitError
	202	+	if errors.As(err, &exit) {
	203	+	return early(exit.Code, exit.Err)
	204	+	}
	205	+	return early(exitProtocol, err)
	206	+	}
	207	+	s.filters.cvsPerishable = s.c.protocol >= 30
	208	+
	209	+	if err := c.w.setMultiplex(true); err != nil {
	210	+	return &ExitError{Code: exitStreamIO, Err: err}
	211	+	}
	212	+	if s.c.protocol >= 30 {
	213	+	c.r.multiplex = true
	214	+	}
	215	+	c.r.onMsg = s.handleMsg
	216	+
	217	+	logger.Debug("rsync session starting",
	218	+	"protocol", c.protocol,
	219	+	"sender", s.opts.sender,
	220	+	"checksum", s.cs.xfer,
	221	+	"compression", s.compression,
	222	+	)
	223	+
	224	+	if s.opts.sender {
	225	+	err = s.runSender(pc.RemainingArgs)
	226	+	} else {
	227	+	err = s.runReceiver(pc.RemainingArgs)
	228	+	}
	229	+	if err == nil {
	230	+	return nil
	231	+	}
	232	+	var exit *ExitError
	233	+	if errors.As(err, &exit) {
	234	+	return exit
	235	+	}
	236	+	// An unexpected failure, most likely the connection going away.
	237	+	return s.fatal(exitStreamIO, err)
	238	+	}
	239	+
	240	+	// checkSupported refuses options that would change the wire format in ways
	241	+	// we don't implement, or that make no sense for a storage backend.
	242	+	func checkSupported(o *rsyncopts.Options) error {
	243	+	switch {
	244	+	case o.PreserveHardLinks():
	245	+	return errors.New("--hard-links is not supported")
	246	+	case o.PreserveACLs():
	247	+	return errors.New("--acls is not supported")
	248	+	case o.PreserveXattrs():
	249	+	return errors.New("--xattrs is not supported")
	250	+	case o.PreserveCrtimes():
	251	+	return errors.New("--crtimes is not supported")
	252	+	case o.RelativePaths():
	253	+	return errors.New("--relative is not supported")
	254	+	case o.AppendMode() != 0:
	255	+	return errors.New("--append is not supported")
	256	+	case len(o.BasisDirs()) > 0:
	257	+	return errors.New("--link-dest, --copy-dest and --compare-dest are not supported")
	258	+	case o.BatchMode():
	259	+	return errors.New("batch mode is not supported")
	260	+	case o.FilesFrom() != "":
	261	+	return errors.New("--files-from is not supported")
	262	+	case o.Iconv() != "":
	263	+	return errors.New("--iconv is not supported")
	264	+	case o.MissingArgs() == 2:
	265	+	return errors.New("--delete-missing-args is not supported")
	266	+	case o.Sender() && o.RemoveSourceFiles() != 0:
	267	+	return errors.New("--remove-source-files is not supported when downloading")
	268	+	}
	269	+	return nil
	270	+	}
	271	+
	272	+	// readProtectedArgs reads the arguments a --secluded-args client sends
	273	+	// before the protocol starts. The first one is the program name.
	274	+	func readProtectedArgs(c *conn) ([]string, error) {
	275	+	var args []string
	276	+	var cur []byte
	277	+	for {
	278	+	b, err := c.readByte()
	279	+	if err != nil {
	280	+	return nil, err
	281	+	}
	282	+	if b != 0 {
	283	+	cur = append(cur, b)
	284	+	continue
	285	+	}
	286	+	if len(cur) == 0 {
	287	+	break
	288	+	}
	289	+	args = append(args, string(cur))
	290	+	cur = cur[:0]
	291	+	}
	292	+	if len(args) > 0 {
	293	+	args = args[1:]
	294	+	}
	295	+	return args, nil
	296	+	}
	297	+
	298	+	// setupProtocol runs the handshake that precedes the transfer
	299	+	// (compat.c:setup_protocol).
	300	+	func (s session) setupProtocol(o rsyncopts.Options) error {
	301	+	c := s.c
	302	+	version := maxProtocol
	303	+	if v := o.ProtocolVersion(); v > 0 && v < version {
	304	+	version = v
	305	+	}
	306	+	if err := c.writeInt32(int32(version)); err != nil {
	307	+	return err
	308	+	}
	309	+	if err := c.flush(); err != nil {
	310	+	return err
	311	+	}
	312	+	remote, err := c.readInt32()
	313	+	if err != nil {
	314	+	return err
	315	+	}
	316	+	if int(remote) < version {
	317	+	version = int(remote)
	318	+	}
	319	+	if version < minProtocol {
	320	+	return &ExitError{Code: exitProtocol, Err: fmt.Errorf("protocol version %d is too old, %d or newer is required", remote, minProtocol)}
	321	+	}
	322	+	c.protocol = version
	323	+
	324	+	if s.opts.deleteMode && !s.opts.deleteBefore && s.opts.deleteDuring == 0 && !s.opts.deleteAfter {
	325	+	if version < 30 {
	326	+	s.opts.deleteBefore = true
	327	+	} else {
	328	+	s.opts.deleteDuring = 1
	329	+	}
	330	+	}
	331	+
	332	+	negotiate := false
	333	+	if version >= 30 {
	334	+	info := clientInfo(o.ShellCmd())
	335	+	flags := 0
	336	+	if strings.Contains(info, "f") {
	337	+	flags \|= cfSafeFlist
	338	+	}
	339	+	if strings.Contains(info, "x") {
	340	+	flags \|= cfAvoidXattrOptim
	341	+	}
	342	+	if strings.Contains(info, "C") {
	343	+	flags \|= cfChksumSeedFix
	344	+	}
	345	+	if strings.Contains(info, "I") {
	346	+	flags \|= cfInplacePartialDir
	347	+	}
	348	+	if strings.Contains(info, "u") {
	349	+	flags \|= cfID0Names
	350	+	}
	351	+	if strings.Contains(info, "v") {
	352	+	negotiate = true
	353	+	flags \|= cfVarintFlistFlags
	354	+	}
	355	+	if strings.Contains(info, "V") {
	356	+	flags \|= cfVarintFlistFlags
	357	+	err = c.writeByte(byte(flags))
	358	+	} else {
	359	+	err = c.writeVarint(int32(flags))
	360	+	}
	361	+	if err != nil {
	362	+	return err
	363	+	}
	364	+	s.cs.properSeedOrder = flags&cfChksumSeedFix != 0
	365	+	s.varintFlags = flags&cfVarintFlistFlags != 0
	366	+	s.xmitID0Names = flags&cfID0Names != 0
	367	+	s.safeFlist = flags&cfSafeFlist != 0 \|\| version >= 31
	368	+	s.inplacePartial = flags&cfInplacePartialDir != 0
	369	+	}
	370	+
	371	+	checksumChoice := o.ChecksumChoice()
	372	+	compressChoice := o.CompressChoice()
	373	+	wantCompress := o.Compress() \|\| compressChoice != ""
	374	+	if negotiate {
	375	+	if checksumChoice == "" {
	376	+	if err := c.writeVstring(nameList(checksumNames)); err != nil {
	377	+	return err
	378	+	}
	379	+	}
	380	+	if wantCompress && compressChoice == "" {
	381	+	if err := c.writeVstring(nameList(compressNames)); err != nil {
	382	+	return err
	383	+	}
	384	+	}
	385	+	if err := c.flush(); err != nil {
	386	+	return err
	387	+	}
	388	+	}
	389	+
	390	+	// Pick the transfer checksum.
	391	+	switch {
	392	+	case checksumChoice != "":
	393	+	xfer, file, _ := strings.Cut(checksumChoice, ",")
	394	+	if file == "" {
	395	+	file = xfer
	396	+	}
	397	+	if s.cs.xfer, err = checksumByChoice(xfer, version); err != nil {
	398	+	return &ExitError{Code: exitUnsupported, Err: err}
	399	+	}
	400	+	if s.cs.file, err = checksumByChoice(file, version); err != nil {
	401	+	return &ExitError{Code: exitUnsupported, Err: err}
	402	+	}
	403	+	case negotiate:
	404	+	list, err := c.readVstring(256)
	405	+	if err != nil {
	406	+	return err
	407	+	}
	408	+	name, ok := negotiateName(checksumNames, list)
	409	+	if !ok {
	410	+	return &ExitError{Code: exitUnsupported, Err: fmt.Errorf("failed to negotiate a checksum choice, client list: %s", list)}
	411	+	}
	412	+	s.cs.xfer, _ = parseChecksumName(name)
	413	+	s.cs.file = s.cs.xfer
	414	+	default:
	415	+	s.cs.xfer, _ = checksumByChoice("auto", version)
	416	+	s.cs.file = s.cs.xfer
	417	+	}
	418	+
	419	+	// Pick the compression.
	420	+	s.compression = compressNone
	421	+	switch {
	422	+	case compressChoice != "":
	423	+	if s.compression, err = parseCompressName(compressChoice); err != nil {
	424	+	return &ExitError{Code: exitUnsupported, Err: err}
	425	+	}
	426	+	case wantCompress && negotiate:
	427	+	list, err := c.readVstring(256)
	428	+	if err != nil {
	429	+	return err
	430	+	}
	431	+	name, ok := negotiateName(compressNames, list)
	432	+	if !ok {
	433	+	return &ExitError{Code: exitUnsupported, Err: fmt.Errorf("failed to negotiate a compress choice, client list: %s", list)}
	434	+	}
	435	+	s.compression, _ = parseCompressName(name)
	436	+	case wantCompress:
	437	+	s.compression = compressZlib
	438	+	}
	439	+	s.compressLevel = 6
	440	+	if lvl := o.CompressLevel(); s.compression != compressNone && lvl >= 0 {
	441	+	if lvl == 0 {
	442	+	s.compression = compressNone
	443	+	} else {
	444	+	s.compressLevel = min(lvl, 9)
	445	+	}
	446	+	}
	447	+
	448	+	seed := int32(o.ChecksumSeed())
	449	+	if seed == 0 {
	450	+	var b [4]byte
	451	+	_, _ = rand.Read(b[:])
	452	+	seed = int32(binary.LittleEndian.Uint32(b[:]) & 0x7FFFFFFF)
	453	+	if seed == 0 {
	454	+	seed = 1
	455	+	}
	456	+	}
	457	+	s.cs.seed = seed
	458	+	if err := c.writeInt32(seed); err != nil {
	459	+	return err
	460	+	}
	461	+	if s.cs.xfer == csumNone {
	462	+	s.opts.wholeFile = true
	463	+	}
	464	+	return c.flush()
	465	+	}
	466	+
	467	+	// clientInfo extracts the capability letters the client appends to its -e
	468	+	// option, as in "-e.LsfxCIvu".
	469	+	func clientInfo(shellCmd string) string {
	470	+	if i := strings.IndexByte(shellCmd, '.'); i >= 0 {
	471	+	return shellCmd[i+1:]
	472	+	}
	473	+	return shellCmd
	474	+	}
	475	+
	476	+	func checksumByChoice(name string, protocol int) (csumType, error) {
	477	+	if name == "auto" {
	478	+	if protocol >= 30 {
	479	+	return csumMD5, nil
	480	+	}
	481	+	return csumMD4Old, nil
	482	+	}
	483	+	return parseChecksumName(name)
	484	+	}
	485	+
	486	+	func nameList[T any](items []struct {
	487	+	name string
	488	+	typ T
	489	+	}) string {
	490	+	var names []string
	491	+	seen := map[any]bool{}
	492	+	for _, it := range items {
	493	+	if seen[it.typ] {
	494	+	continue
	495	+	}
	496	+	seen[it.typ] = true
	497	+	names = append(names, it.name)
	498	+	}
	499	+	return strings.Join(names, " ")
	500	+	}
	501	+
	502	+	// negotiateName picks our most preferred name that the client also listed.
	503	+	func negotiateName[T any](items []struct {
	504	+	name string
	505	+	typ T
	506	+	}, list string) (string, bool) {
	507	+	theirs := strings.Fields(list)
	508	+	for _, it := range items {
	509	+	if slices.Contains(theirs, it.name) {
	510	+	return it.name, true
	511	+	}
	512	+	}
	513	+	return "", false
	514	+	}
	515	+
	516	+	// handleMsg processes an out-of-band message from the client.
	517	+	func (s *session) handleMsg(tag byte, data []byte) error {
	518	+	switch tag {
	519	+	case msgIOError:
	520	+	if len(data) != 4 {
	521	+	return fmt.Errorf("invalid multi-message %d:%d", tag, len(data))
	522	+	}
	523	+	s.addIOError(int32(binary.LittleEndian.Uint32(data)) & ioErrMask)
	524	+	case msgNoop:
	525	+	case msgErrorExit:
	526	+	code := 0
	527	+	if len(data) == 4 {
	528	+	code = int(int32(binary.LittleEndian.Uint32(data)))
	529	+	}
	530	+	return &remoteExitError{code: code}
	531	+	case msgInfo, msgError, msgErrorXfer, msgWarning:
	532	+	s.logger.Info("rsync client message", "msg", strings.TrimSpace(string(data)))
	533	+	case msgNoSend:
	534	+	// The sender couldn't open a file we asked for and has reported
	535	+	// why; nothing will arrive for it.
	536	+	if len(data) != 4 {
	537	+	return fmt.Errorf("invalid multi-message %d:%d", tag, len(data))
	538	+	}
	539	+	case msgSuccess, msgDeleted:
	540	+	default:
	541	+	return fmt.Errorf("unexpected tag %d", tag)
	542	+	}
	543	+	return nil
	544	+	}
	545	+
	546	+	func (s *session) addIOError(v int32) {
	547	+	s.mu.Lock()
	548	+	s.ioError \|= v
	549	+	s.mu.Unlock()
	550	+	}
	551	+
	552	+	func (s *session) who() string {
	553	+	if s.opts.sender {
	554	+	return "sender"
	555	+	}
	556	+	return "receiver"
	557	+	}
	558	+
	559	+	func (s *session) send(tag byte, msg string) {
	560	+	if s.c.protocol < 30 {
	561	+	switch tag {
	562	+	case msgError:
	563	+	tag = msgErrorXfer
	564	+	case msgWarning:
	565	+	tag = msgInfo
	566	+	}
	567	+	}
	568	+	if err := s.c.writeMsg(tag, []byte(msg)); err != nil {
	569	+	s.logger.Debug("rsync message not delivered", "err", err)
	570	+	}
	571	+	}
	572	+
	573	+	func (s *session) info(format string, args ...any) {
	574	+	s.send(msgInfo, fmt.Sprintf(format, args...)+"\n")
	575	+	}
	576	+
	577	+	func (s *session) warning(format string, args ...any) {
	578	+	s.send(msgWarning, fmt.Sprintf(format, args...)+"\n")
	579	+	}
	580	+
	581	+	// xferError reports a failure affecting one file; the run continues and
	582	+	// ends with a partial-transfer exit code.
	583	+	func (s *session) xferError(format string, args ...any) {
	584	+	s.mu.Lock()
	585	+	s.gotXferError = true
	586	+	s.mu.Unlock()
	587	+	s.send(msgErrorXfer, fmt.Sprintf("rsync: [%s] %s\n", s.who(), fmt.Sprintf(format, args...)))
	588	+	}
	589	+
	590	+	// fatal reports err to the client and ends the run.
	591	+	func (s *session) fatal(code int, err error) error {
	592	+	var remote *remoteExitError
	593	+	if errors.As(err, &remote) {
	594	+	return &ExitError{Code: remote.code, Err: err}
	595	+	}
	596	+	s.logger.Error("rsync session failed", "err", err)
	597	+	if !errors.Is(err, io.EOF) && !errors.Is(err, io.ErrUnexpectedEOF) {
	598	+	s.send(msgError, fmt.Sprintf("rsync: [%s] %v\n", s.who(), err))
	599	+	if s.c.protocol >= 31 {
	600	+	_ = s.c.writeMsgInt(msgErrorExit, int32(code))
	601	+	}
	602	+	_ = s.c.flush()
	603	+	}
	604	+	return &ExitError{Code: code, Err: err}
	605	+	}
	606	+
	607	+	// finish ends a run that completed, reporting a partial transfer if any
	608	+	// file failed.
	609	+	func (s *session) finish() error {
	610	+	if err := s.c.flush(); err != nil {
	611	+	return err
	612	+	}
	613	+	s.mu.Lock()
	614	+	ioError, xferErr := s.ioError, s.gotXferError
	615	+	s.mu.Unlock()
	616	+	code := 0
	617	+	switch {
	618	+	case ioError&ioErrGeneral != 0 \|\| xferErr:
	619	+	code = exitPartial
	620	+	case ioError&ioErrVanished != 0:
	621	+	code = exitVanished
	622	+	case ioError&ioErrDelLimit != 0:
	623	+	code = exitDelLimit
	624	+	}
	625	+	if code == 0 {
	626	+	return nil
	627	+	}
	628	+	return &ExitError{Code: code, Err: fmt.Errorf("some files were not transferred (code %d)", code)}
	629	+	}
	630	+
	631	+	func (s session) newFlistCodec() flistCodec {
	632	+	return &flistCodec{
	633	+	protocol: s.c.protocol,
	634	+	varintFlags: s.varintFlags,
	635	+	safeFlist: s.safeFlist,
	636	+	preserveUID: s.opts.preserveUID,
	637	+	preserveGID: s.opts.preserveGID,
	638	+	preserveLinks: s.opts.preserveLinks,
	639	+	preserveDevs: s.opts.preserveDevs,
	640	+	preserveSpecs: s.opts.preserveSpecs,
	641	+	preserveAtime: s.opts.preserveAtimes,
	642	+	alwaysSum: s.opts.alwaysChecksum,
	643	+	sumLen: s.cs.file.size(),
	644	+	}
	645	+	}
	646	+
	647	+	// recvFilterList reads the client's filter rules (exclude.c:recv_filter_list).
	648	+	func (s *session) recvFilterList() error {
	649	+	receiverWants := s.opts.pruneEmptyDirs \|\| (s.opts.deleteMode && (!s.opts.deleteExcluded \|\| s.c.protocol >= 29))
	650	+	if s.opts.sender \|\| receiverWants {
	651	+	for {
	652	+	n, err := s.c.readInt32()
	653	+	if err != nil {
	654	+	return err
	655	+	}
	656	+	if n == 0 {
	657	+	break
	658	+	}
	659	+	if n < 0 \|\| n >= maxPathLen*2 {
	660	+	return s.fatal(exitProtocol, fmt.Errorf("filter rule too long: %d", n))
	661	+	}
	662	+	line := make([]byte, n)
	663	+	if err := s.c.readFull(line); err != nil {
	664	+	return err
	665	+	}
	666	+	if err := s.filters.parse(string(line), s.c.protocol < 29); err != nil {
	667	+	return s.fatal(exitSyntax, err)
	668	+	}
	669	+	}
	670	+	}
	671	+	if s.opts.cvsExclude && (s.opts.sender \|\| s.c.protocol < 29) {
	672	+	if err := s.filters.parse(":C", false); err != nil {
	673	+	return s.fatal(exitSyntax, err)
	674	+	}
	675	+	s.filters.rules = append(s.filters.rules, s.filters.cvsIgnoreRules()...)
	676	+	}
	677	+	return nil
	678	+	}
	679	+
	680	+	type ndxRequest struct {
	681	+	ndx int32
	682	+	iflags int
	683	+	fnamecmpType byte
	684	+	xname string
	685	+	hasXname bool
	686	+	}
	687	+
	688	+	// readNdxAndAttrs reads the next file index and its item flags
	689	+	// (rsync.c:read_ndx_and_attrs).
	690	+	func (s *session) readNdxAndAttrs(count int) (ndxRequest, error) {
	691	+	c := s.c
	692	+	for {
	693	+	ndx, err := c.readNdx()
	694	+	if err != nil {
	695	+	return ndxRequest{}, err
	696	+	}
	697	+	if ndx == ndxDone {
	698	+	return ndxRequest{ndx: ndx}, nil
	699	+	}
	700	+	if ndx == ndxDelStats {
	701	+	var counts [5]int32
	702	+	for i := range counts {
	703	+	if counts[i], err = c.readVarint(); err != nil {
	704	+	return ndxRequest{}, err
	705	+	}
	706	+	}
	707	+	if s.opts.sender {
	708	+	// Echo the receiver's deletion stats back to the client.
	709	+	if err := s.writeDelStats(counts); err != nil {
	710	+	return ndxRequest{}, err
	711	+	}
	712	+	}
	713	+	continue
	714	+	}
	715	+	if ndx < 0 \|\| int(ndx) > count {
	716	+	return ndxRequest{}, s.fatal(exitProtocol, fmt.Errorf("invalid file index: %d (%d - %d)", ndx, ndxDone, count-1))
	717	+	}
	718	+	req := ndxRequest{ndx: ndx, iflags: itemTransfer \| itemMissingData, fnamecmpType: fnamecmpFname}
	719	+	if c.protocol >= 29 {
	720	+	v, err := c.readShortint()
	721	+	if err != nil {
	722	+	return ndxRequest{}, err
	723	+	}
	724	+	req.iflags = int(v)
	725	+	}
	726	+	// The protocol 29 keep-alive.
	727	+	if c.protocol < 30 && int(ndx) == count && req.iflags == itemIsNew {
	728	+	continue
	729	+	}
	730	+	if int(ndx) == count {
	731	+	return ndxRequest{}, s.fatal(exitProtocol, fmt.Errorf("invalid file index: %d", ndx))
	732	+	}
	733	+	if req.iflags&itemBasisTypeFollows != 0 {
	734	+	b, err := c.readByte()
	735	+	if err != nil {
	736	+	return ndxRequest{}, err
	737	+	}
	738	+	req.fnamecmpType = b
	739	+	}
	740	+	if req.iflags&itemXnameFollows != 0 {
	741	+	if req.xname, err = c.readVstring(maxPathLen); err != nil {
	742	+	return ndxRequest{}, err
	743	+	}
	744	+	req.hasXname = true
	745	+	}
	746	+	return req, nil
	747	+	}
	748	+	}
	749	+
	750	+	func (s *session) writeNdxAndAttrs(req ndxRequest) error {
	751	+	if err := s.c.writeNdx(req.ndx); err != nil {
	752	+	return err
	753	+	}
	754	+	if s.c.protocol < 29 {
	755	+	return nil
	756	+	}
	757	+	if err := s.c.writeShortint(uint16(req.iflags)); err != nil {
	758	+	return err
	759	+	}
	760	+	if req.iflags&itemBasisTypeFollows != 0 {
	761	+	if err := s.c.writeByte(req.fnamecmpType); err != nil {
	762	+	return err
	763	+	}
	764	+	}
	765	+	if req.iflags&itemXnameFollows != 0 {
	766	+	return s.c.writeVstring(req.xname)
	767	+	}
	768	+	return nil
	769	+	}
	770	+
	771	+	func (s *session) writeDelStats(counts [5]int32) error {
	772	+	if err := s.c.writeNdx(ndxDelStats); err != nil {
	773	+	return err
	774	+	}
	775	+	for _, v := range counts {
	776	+	if err := s.c.writeVarint(v); err != nil {
	777	+	return err
	778	+	}
	779	+	}
	780	+	return nil
	781	+	}
	782	+
	783	+	// readFinalGoodbye waits for the client's end-of-run index
	784	+	// (main.c:read_final_goodbye).
	785	+	func (s *session) readFinalGoodbye(sender bool) error {
	786	+	c := s.c
	787	+	var ndx int32
	788	+	var err error
	789	+	if c.protocol < 29 {
	790	+	ndx, err = c.readInt32()
	791	+	} else {
	792	+	var req ndxRequest
	793	+	req, err = s.readNdxAndAttrs(0)
	794	+	ndx = req.ndx
	795	+	if err == nil && c.protocol >= 31 && ndx == ndxDone && sender {
	796	+	if err = c.writeNdx(ndxDone); err != nil {
	797	+	return err
	798	+	}
	799	+	req, err = s.readNdxAndAttrs(0)
	800	+	ndx = req.ndx
	801	+	}
	802	+	}
	803	+	if err != nil {
	804	+	return err
	805	+	}
	806	+	if ndx != ndxDone {
	807	+	return s.fatal(exitProtocol, fmt.Errorf("invalid packet at end of run (%d)", ndx))
	808	+	}
	809	+	return nil
	810	+	}

...	...		@@ -0,0 +1,189 @@
	1	+	package rsync
	2	+
	3	+	import (
	4	+	"encoding/binary"
	5	+	"fmt"
	6	+	"io"
	7	+	)
	8	+
	9	+	// sumHead describes the block signatures of a basis file.
	10	+	type sumHead struct {
	11	+	count int32
	12	+	blength int32
	13	+	s2length int32
	14	+	remainder int32
	15	+	}
	16	+
	17	+	type sumBuf struct {
	18	+	offset int64
	19	+	len int32
	20	+	sum1 uint32
	21	+	sum2 []byte
	22	+	chain int32
	23	+	same bool // matched at the same offset during an in-place update
	24	+	}
	25	+
	26	+	func (c *conn) writeSumHead(h sumHead) error {
	27	+	if err := c.writeInt32(h.count); err != nil {
	28	+	return err
	29	+	}
	30	+	if err := c.writeInt32(h.blength); err != nil {
	31	+	return err
	32	+	}
	33	+	if err := c.writeInt32(h.s2length); err != nil {
	34	+	return err
	35	+	}
	36	+	return c.writeInt32(h.remainder)
	37	+	}
	38	+
	39	+	func (c *conn) readSumHead(xferLen int) (sumHead, error) {
	40	+	var h sumHead
	41	+	var err error
	42	+	maxBlength := int32(maxBlockSize)
	43	+	if c.protocol < 30 {
	44	+	maxBlength = oldMaxBlockSize
	45	+	}
	46	+	if h.count, err = c.readInt32(); err != nil {
	47	+	return h, err
	48	+	}
	49	+	if h.count < 0 {
	50	+	return h, fmt.Errorf("invalid checksum count %d", h.count)
	51	+	}
	52	+	if h.blength, err = c.readInt32(); err != nil {
	53	+	return h, err
	54	+	}
	55	+	if h.blength < 0 \|\| h.blength > maxBlength {
	56	+	return h, fmt.Errorf("invalid block length %d", h.blength)
	57	+	}
	58	+	if h.count > 0 && h.blength == 0 {
	59	+	return h, fmt.Errorf("invalid zero block length")
	60	+	}
	61	+	if h.s2length, err = c.readInt32(); err != nil {
	62	+	return h, err
	63	+	}
	64	+	if h.s2length < 0 \|\| int(h.s2length) > xferLen {
	65	+	return h, fmt.Errorf("invalid checksum length %d", h.s2length)
	66	+	}
	67	+	if h.remainder, err = c.readInt32(); err != nil {
	68	+	return h, err
	69	+	}
	70	+	if h.remainder < 0 \|\| h.remainder > h.blength {
	71	+	return h, fmt.Errorf("invalid remainder length %d", h.remainder)
	72	+	}
	73	+	return h, nil
	74	+	}
	75	+
	76	+	// sumSizesSqroot picks the block size (about the square root of the file
	77	+	// length, rounded to a multiple of 8) and the strong checksum length.
	78	+	func sumSizesSqroot(length int64, protocol int, fixedBlock int32, csumLength int, xferLen int) (sumHead, error) {
	79	+	maxS2 := min(sumLength, xferLen)
	80	+	var blength int32
	81	+	switch {
	82	+	case fixedBlock > 0:
	83	+	blength = fixedBlock
	84	+	case length <= blockSize*blockSize:
	85	+	blength = blockSize
	86	+	default:
	87	+	maxBlength := int32(maxBlockSize)
	88	+	if protocol < 30 {
	89	+	maxBlength = oldMaxBlockSize
	90	+	}
	91	+	c := int64(1)
	92	+	for l := length; ; {
	93	+	l >>= 2
	94	+	if l == 0 {
	95	+	break
	96	+	}
	97	+	c <<= 1
	98	+	}
	99	+	if c >= int64(maxBlength) {
	100	+	blength = maxBlength
	101	+	} else {
	102	+	var b int64
	103	+	for {
	104	+	b \|= c
	105	+	if length < b*b {
	106	+	b &^= c
	107	+	}
	108	+	c >>= 1
	109	+	if c < 8 {
	110	+	break
	111	+	}
	112	+	}
	113	+	blength = int32(max(b, blockSize))
	114	+	}
	115	+	}
	116	+
	117	+	var s2length int
	118	+	if csumLength == sumLength {
	119	+	s2length = maxS2
	120	+	} else {
	121	+	b := blocksumBias
	122	+	for l := length; ; {
	123	+	l >>= 1
	124	+	if l == 0 {
	125	+	break
	126	+	}
	127	+	b += 2
	128	+	}
	129	+	for c := blength; ; {
	130	+	c >>= 1
	131	+	if c == 0 \|\| b == 0 {
	132	+	break
	133	+	}
	134	+	b--
	135	+	}
	136	+	s2length = (b + 1 - 32 + 7) / 8
	137	+	s2length = max(s2length, csumLength)
	138	+	s2length = min(s2length, maxS2)
	139	+	}
	140	+
	141	+	remainder := int32(length % int64(blength))
	142	+	count := length / int64(blength)
	143	+	if remainder != 0 {
	144	+	count++
	145	+	}
	146	+	if count > 0x7FFFFFFF {
	147	+	return sumHead{}, fmt.Errorf("file is too large for checksum sending")
	148	+	}
	149	+	return sumHead{
	150	+	count: int32(count),
	151	+	blength: blength,
	152	+	s2length: int32(s2length),
	153	+	remainder: remainder,
	154	+	}, nil
	155	+	}
	156	+
	157	+	// encodeSums computes the block signatures of a basis file in their wire
	158	+	// form. They are built up front so a read error can't leave a half-written
	159	+	// signature on the stream.
	160	+	func encodeSums(r io.ReaderAt, length int64, head sumHead, cs *checksums) ([]byte, error) {
	161	+	out := make([]byte, 0, int(head.count)*(4+int(head.s2length)))
	162	+	buf := make([]byte, head.blength)
	163	+	var sum2 []byte
	164	+	var offset int64
	165	+	for i := int32(0); i < head.count; i++ {
	166	+	n := min(int64(head.blength), length-offset)
	167	+	block := buf[:n]
	168	+	if err := readFullAt(r, block, offset); err != nil {
	169	+	return nil, err
	170	+	}
	171	+	offset += n
	172	+	out = binary.LittleEndian.AppendUint32(out, checksum1(block))
	173	+	sum2 = cs.blockSum(block, sum2[:0])
	174	+	out = append(out, sum2[:head.s2length]...)
	175	+	}
	176	+	return out, nil
	177	+	}
	178	+
	179	+	// readFullAt fills p from r at off, treating a short read as an error.
	180	+	func readFullAt(r io.ReaderAt, p []byte, off int64) error {
	181	+	n, err := r.ReadAt(p, off)
	182	+	if n == len(p) {
	183	+	return nil
	184	+	}
	185	+	if err == nil \|\| err == io.EOF {
	186	+	err = io.ErrUnexpectedEOF
	187	+	}
	188	+	return err
	189	+	}

...	...		@@ -0,0 +1,532 @@
	1	+	package rsync
	2	+
	3	+	import (
	4	+	"bytes"
	5	+	stdflate "compress/flate"
	6	+	"errors"
	7	+	"fmt"
	8	+	"io"
	9	+
	10	+	"github.com/klauspost/compress/flate"
	11	+	)
	12	+
	13	+	// compression identifies the negotiated token compression.
	14	+	type compression int
	15	+
	16	+	const (
	17	+	compressNone compression = iota
	18	+	compressZlib
	19	+	compressZlibx
	20	+	)
	21	+
	22	+	// compressNames is our preference order for negotiation. zlib adds matched
	23	+	// blocks to the deflate history; zlibx does not.
	24	+	var compressNames = []struct {
	25	+	name string
	26	+	typ compression
	27	+	}{
	28	+	{"zlibx", compressZlibx},
	29	+	{"zlib", compressZlib},
	30	+	{"none", compressNone},
	31	+	}
	32	+
	33	+	func parseCompressName(name string) (compression, error) {
	34	+	for _, c := range compressNames {
	35	+	if c.name == name {
	36	+	return c.typ, nil
	37	+	}
	38	+	}
	39	+	return 0, fmt.Errorf("unknown compress name: %s", name)
	40	+	}
	41	+
	42	+	// tokenSender emits one file's delta. token is a block index for a match, -1
	43	+	// at the end of the file, or -2 to send literal data without a token.
	44	+	// matched holds the matched block's data.
	45	+	type tokenSender interface {
	46	+	send(token int32, literal, matched []byte) error
	47	+	}
	48	+
	49	+	// tokenReceiver decodes one file's delta. recv returns a positive length
	50	+	// with literal data, -(block+1) for a match, or 0 at the end of the file.
	51	+	// see is called with each matched block's data.
	52	+	type tokenReceiver interface {
	53	+	recv() (int32, []byte, error)
	54	+	see(data []byte)
	55	+	}
	56	+
	57	+	func newTokenSender(c *conn, comp compression, level int) tokenSender {
	58	+	if comp == compressNone {
	59	+	return &simpleTokenSender{c: c}
	60	+	}
	61	+	return &deflateTokenSender{c: c, zlib: comp == compressZlib, level: level, lastToken: -1}
	62	+	}
	63	+
	64	+	func newTokenReceiver(c *conn, comp compression) tokenReceiver {
	65	+	if comp == compressNone {
	66	+	return &simpleTokenReceiver{c: c, buf: make([]byte, chunkSize)}
	67	+	}
	68	+	return &deflateTokenReceiver{c: c, zlib: comp == compressZlib, out: make([]byte, chunkSize)}
	69	+	}
	70	+
	71	+	type simpleTokenSender struct{ c *conn }
	72	+
	73	+	func (s *simpleTokenSender) send(token int32, literal, _ []byte) error {
	74	+	for len(literal) > 0 {
	75	+	n := min(len(literal), chunkSize)
	76	+	if err := s.c.writeInt32(int32(n)); err != nil {
	77	+	return err
	78	+	}
	79	+	if err := s.c.write(literal[:n]); err != nil {
	80	+	return err
	81	+	}
	82	+	literal = literal[n:]
	83	+	}
	84	+	if token == -2 {
	85	+	return nil
	86	+	}
	87	+	return s.c.writeInt32(-(token + 1))
	88	+	}
	89	+
	90	+	type simpleTokenReceiver struct {
	91	+	c *conn
	92	+	buf []byte
	93	+	residue int32
	94	+	}
	95	+
	96	+	func (s *simpleTokenReceiver) recv() (int32, []byte, error) {
	97	+	if s.residue == 0 {
	98	+	i, err := s.c.readInt32()
	99	+	if err != nil \|\| i <= 0 {
	100	+	return i, nil, err
	101	+	}
	102	+	s.residue = i
	103	+	}
	104	+	n := min(int32(len(s.buf)), s.residue)
	105	+	s.residue -= n
	106	+	if err := s.c.readFull(s.buf[:n]); err != nil {
	107	+	return 0, nil, err
	108	+	}
	109	+	return n, s.buf[:n], nil
	110	+	}
	111	+
	112	+	func (s *simpleTokenReceiver) see([]byte) {}
	113	+
	114	+	// Flag bytes of the compressed token stream.
	115	+	const (
	116	+	tokEndFlag = 0
	117	+	tokLong = 0x20
	118	+	tokRunLong = 0x21
	119	+	tokDeflatedData = 0x40
	120	+	tokRel = 0x80
	121	+	tokRunRel = 0xc0
	122	+	maxTokenIndex = 0x7ffffffe
	123	+	deflateWindow = 32 * 1024
	124	+	)
	125	+
	126	+	var syncMarker = []byte{0, 0, 0xff, 0xff}
	127	+
	128	+	// deflateTokenSender produces rsync's compressed token stream: literal data
	129	+	// is raw deflate, sync-flushed before every token with the trailing
	130	+	// 00 00 ff ff marker stripped.
	131	+	type deflateTokenSender struct {
	132	+	c *conn
	133	+	zlib bool
	134	+	level int
	135	+
	136	+	w *flate.Writer
	137	+	out bytes.Buffer
	138	+
	139	+	lastToken, runStart, lastRunEnd int32
	140	+	flushPending bool
	141	+
	142	+	history []byte
	143	+	needDict bool
	144	+	}
	145	+
	146	+	func (d *deflateTokenSender) send(token int32, literal, matched []byte) error {
	147	+	switch {
	148	+	case d.lastToken == -1:
	149	+	if err := d.resetStream(); err != nil {
	150	+	return err
	151	+	}
	152	+	d.lastRunEnd = 0
	153	+	d.runStart = token
	154	+	d.flushPending = false
	155	+	case d.lastToken == -2:
	156	+	d.runStart = token
	157	+	case len(literal) != 0 \|\| token != d.lastToken+1 \|\| token >= d.runStart+65536:
	158	+	if err := d.writeRun(); err != nil {
	159	+	return err
	160	+	}
	161	+	d.lastRunEnd = d.lastToken
	162	+	d.runStart = token
	163	+	}
	164	+	d.lastToken = token
	165	+
	166	+	if len(literal) != 0 \|\| d.flushPending {
	167	+	if err := d.deflate(literal, token != -2); err != nil {
	168	+	return err
	169	+	}
	170	+	d.flushPending = token == -2
	171	+	}
	172	+
	173	+	if token == -1 {
	174	+	return d.c.writeByte(tokEndFlag)
	175	+	}
	176	+	if token != -2 && d.zlib {
	177	+	d.addHistory(matched, d.c.protocol)
	178	+	d.needDict = true
	179	+	}
	180	+	return nil
	181	+	}
	182	+
	183	+	func (d *deflateTokenSender) resetStream() error {
	184	+	d.history = d.history[:0]
	185	+	d.needDict = false
	186	+	d.out.Reset()
	187	+	if d.w == nil {
	188	+	w, err := flate.NewWriter(&d.out, d.level)
	189	+	if err != nil {
	190	+	return err
	191	+	}
	192	+	d.w = w
	193	+	return nil
	194	+	}
	195	+	// Reset would reuse the last dictionary, so clear it explicitly.
	196	+	d.w.ResetDict(&d.out, nil)
	197	+	return nil
	198	+	}
	199	+
	200	+	func (d *deflateTokenSender) writeRun() error {
	201	+	r := d.runStart - d.lastRunEnd
	202	+	n := d.lastToken - d.runStart
	203	+	if r >= 0 && r <= 63 {
	204	+	flag := byte(tokRel)
	205	+	if n != 0 {
	206	+	flag = tokRunRel
	207	+	}
	208	+	if err := d.c.writeByte(flag + byte(r)); err != nil {
	209	+	return err
	210	+	}
	211	+	} else {
	212	+	flag := byte(tokLong)
	213	+	if n != 0 {
	214	+	flag = tokRunLong
	215	+	}
	216	+	if err := d.c.writeByte(flag); err != nil {
	217	+	return err
	218	+	}
	219	+	if err := d.c.writeInt32(d.runStart); err != nil {
	220	+	return err
	221	+	}
	222	+	}
	223	+	if n != 0 {
	224	+	return d.c.write([]byte{byte(n), byte(n >> 8)})
	225	+	}
	226	+	return nil
	227	+	}
	228	+
	229	+	func (d *deflateTokenSender) deflate(literal []byte, flush bool) error {
	230	+	if d.needDict {
	231	+	// The receiver's window holds the matched blocks too, so restart
	232	+	// the compressor with the same history.
	233	+	d.w.ResetDict(&d.out, d.window())
	234	+	d.needDict = false
	235	+	}
	236	+	if len(literal) > 0 {
	237	+	if _, err := d.w.Write(literal); err != nil {
	238	+	return err
	239	+	}
	240	+	d.addHistory(literal, 31)
	241	+	}
	242	+	if flush {
	243	+	if err := d.w.Flush(); err != nil {
	244	+	return err
	245	+	}
	246	+	if !bytes.HasSuffix(d.out.Bytes(), syncMarker) {
	247	+	return errors.New("rsync: deflate flush did not end in a sync marker")
	248	+	}
	249	+	d.out.Truncate(d.out.Len() - len(syncMarker))
	250	+	}
	251	+	data := d.out.Bytes()
	252	+	for len(data) > 0 {
	253	+	n := min(len(data), maxDataCount)
	254	+	if err := d.c.write([]byte{byte(tokDeflatedData + n>>8), byte(n)}); err != nil {
	255	+	return err
	256	+	}
	257	+	if err := d.c.write(data[:n]); err != nil {
	258	+	return err
	259	+	}
	260	+	data = data[n:]
	261	+	}
	262	+	d.out.Reset()
	263	+	return nil
	264	+	}
	265	+
	266	+	// addHistory appends data to the deflate history. Before protocol 31 rsync
	267	+	// fed every 64K piece of a long matched block from its start, and both
	268	+	// sides must reproduce that.
	269	+	func (d *deflateTokenSender) addHistory(data []byte, protocol int) {
	270	+	d.history = appendHistory(d.history, data, protocol)
	271	+	}
	272	+
	273	+	func (d *deflateTokenSender) window() []byte {
	274	+	if len(d.history) > deflateWindow {
	275	+	return d.history[len(d.history)-deflateWindow:]
	276	+	}
	277	+	return d.history
	278	+	}
	279	+
	280	+	func appendHistory(history, data []byte, protocol int) []byte {
	281	+	toklen := len(data)
	282	+	src := data
	283	+	for toklen > 0 {
	284	+	n := min(toklen, 0xffff)
	285	+	history = append(history, src[:n]...)
	286	+	if protocol >= 31 {
	287	+	src = src[n:]
	288	+	}
	289	+	toklen -= n
	290	+	}
	291	+	if len(history) > 2*deflateWindow {
	292	+	history = append(history[:0], history[len(history)-deflateWindow:]...)
	293	+	}
	294	+	return history
	295	+	}
	296	+
	297	+	type deflateRecvState int
	298	+
	299	+	const (
	300	+	rInit deflateRecvState = iota
	301	+	rIdle
	302	+	rRunning
	303	+	rInflating
	304	+	)
	305	+
	306	+	// deflateTokenReceiver decodes the compressed token stream. Each run of
	307	+	// DEFLATED_DATA chunks is inflated as one segment: the source appends the
	308	+	// stripped sync marker when the next flag arrives and then reports EOF, and
	309	+	// the inflater is restarted with the current history for the next segment.
	310	+	type deflateTokenReceiver struct {
	311	+	c *conn
	312	+	zlib bool
	313	+
	314	+	state deflateRecvState
	315	+	rxToken int32
	316	+	rxRun int32
	317	+	savedFlag int
	318	+	hasSaved bool
	319	+
	320	+	src segmentSource
	321	+	inflate io.ReadCloser
	322	+	history []byte
	323	+	out []byte
	324	+	}
	325	+
	326	+	func (d *deflateTokenReceiver) recv() (int32, []byte, error) {
	327	+	for {
	328	+	switch d.state {
	329	+	case rInit:
	330	+	d.history = d.history[:0]
	331	+	d.rxToken = 0
	332	+	d.state = rIdle
	333	+
	334	+	case rIdle:
	335	+	var flag int
	336	+	if d.hasSaved {
	337	+	flag, d.hasSaved = d.savedFlag, false
	338	+	} else {
	339	+	b, err := d.c.readByte()
	340	+	if err != nil {
	341	+	return 0, nil, err
	342	+	}
	343	+	flag = int(b)
	344	+	}
	345	+	if flag&0xC0 == tokDeflatedData {
	346	+	if err := d.src.start(d.c, flag); err != nil {
	347	+	return 0, nil, err
	348	+	}
	349	+	if d.inflate == nil {
	350	+	d.inflate = stdflate.NewReaderDict(&d.src, d.window())
	351	+	} else if err := d.inflate.(stdflate.Resetter).Reset(&d.src, d.window()); err != nil {
	352	+	return 0, nil, err
	353	+	}
	354	+	d.state = rInflating
	355	+	continue
	356	+	}
	357	+	if flag == tokEndFlag {
	358	+	d.state = rInit
	359	+	return 0, nil, nil
	360	+	}
	361	+	return d.tokenNum(flag)
	362	+
	363	+	case rInflating:
	364	+	n, err := d.inflate.Read(d.out)
	365	+	if n > 0 {
	366	+	d.history = appendHistory(d.history, d.out[:n], 31)
	367	+	return int32(n), d.out[:n], nil
	368	+	}
	369	+	if err == nil {
	370	+	continue
	371	+	}
	372	+	if d.src.err != nil {
	373	+	return 0, nil, d.src.err
	374	+	}
	375	+	if !errors.Is(err, io.ErrUnexpectedEOF) \|\| !d.src.done() {
	376	+	return 0, nil, fmt.Errorf("rsync: inflate failed: %w", err)
	377	+	}
	378	+	d.savedFlag, d.hasSaved = d.src.next, true
	379	+	d.state = rIdle
	380	+
	381	+	case rRunning:
	382	+	if d.rxRun <= 0 \|\| d.rxToken >= maxTokenIndex {
	383	+	return 0, nil, errors.New("rsync: invalid token number in compressed stream")
	384	+	}
	385	+	d.rxToken++
	386	+	d.rxRun--
	387	+	if d.rxRun == 0 {
	388	+	d.state = rIdle
	389	+	}
	390	+	return -1 - d.rxToken, nil, nil
	391	+	}
	392	+	}
	393	+	}
	394	+
	395	+	func (d *deflateTokenReceiver) tokenNum(flag int) (int32, []byte, error) {
	396	+	invalid := errors.New("rsync: invalid token number in compressed stream")
	397	+	if flag&tokRel != 0 {
	398	+	incr := int32(flag & 0x3f)
	399	+	if d.rxToken > maxTokenIndex-incr {
	400	+	return 0, nil, invalid
	401	+	}
	402	+	d.rxToken += incr
	403	+	flag >>= 6
	404	+	} else {
	405	+	v, err := d.c.readInt32()
	406	+	if err != nil {
	407	+	return 0, nil, err
	408	+	}
	409	+	if v < 0 \|\| v > maxTokenIndex {
	410	+	return 0, nil, invalid
	411	+	}
	412	+	d.rxToken = v
	413	+	}
	414	+	if flag&1 != 0 {
	415	+	lo, err := d.c.readByte()
	416	+	if err != nil {
	417	+	return 0, nil, err
	418	+	}
	419	+	hi, err := d.c.readByte()
	420	+	if err != nil {
	421	+	return 0, nil, err
	422	+	}
	423	+	d.rxRun = int32(lo) + int32(hi)<<8
	424	+	if d.rxRun <= 0 \|\| d.rxToken > maxTokenIndex-d.rxRun {
	425	+	return 0, nil, invalid
	426	+	}
	427	+	d.state = rRunning
	428	+	}
	429	+	return -1 - d.rxToken, nil, nil
	430	+	}
	431	+
	432	+	func (d *deflateTokenReceiver) see(data []byte) {
	433	+	if d.zlib {
	434	+	d.history = appendHistory(d.history, data, d.c.protocol)
	435	+	}
	436	+	}
	437	+
	438	+	func (d *deflateTokenReceiver) window() []byte {
	439	+	if len(d.history) > deflateWindow {
	440	+	return d.history[len(d.history)-deflateWindow:]
	441	+	}
	442	+	return d.history
	443	+	}
	444	+
	445	+	// segmentSource feeds one segment of DEFLATED_DATA chunks to the inflater,
	446	+	// reading further chunks from the connection on demand.
	447	+	type segmentSource struct {
	448	+	c *conn
	449	+	buf []byte
	450	+	pos int
	451	+	marker int // bytes of the sync marker handed out, -1 before the segment ends
	452	+	next int // flag byte that ended the segment
	453	+	err error
	454	+	scratch [maxDataCount]byte
	455	+	}
	456	+
	457	+	func (s segmentSource) start(c conn, flag int) error {
	458	+	s.c = c
	459	+	s.marker = -1
	460	+	s.err = nil
	461	+	return s.readChunk(flag)
	462	+	}
	463	+
	464	+	func (s *segmentSource) readChunk(flag int) error {
	465	+	lo, err := s.c.readByte()
	466	+	if err != nil {
	467	+	return err
	468	+	}
	469	+	n := (flag&0x3f)<<8 + int(lo)
	470	+	if err := s.c.readFull(s.scratch[:n]); err != nil {
	471	+	return err
	472	+	}
	473	+	s.buf = s.scratch[:n]
	474	+	s.pos = 0
	475	+	return nil
	476	+	}
	477	+
	478	+	func (s *segmentSource) done() bool { return s.marker == len(syncMarker) }
	479	+
	480	+	func (s *segmentSource) ReadByte() (byte, error) {
	481	+	for {
	482	+	if s.marker >= 0 {
	483	+	if s.marker == len(syncMarker) {
	484	+	return 0, io.EOF
	485	+	}
	486	+	b := syncMarker[s.marker]
	487	+	s.marker++
	488	+	return b, nil
	489	+	}
	490	+	if s.pos < len(s.buf) {
	491	+	b := s.buf[s.pos]
	492	+	s.pos++
	493	+	return b, nil
	494	+	}
	495	+	b, err := s.c.readByte()
	496	+	if err != nil {
	497	+	s.err = err
	498	+	return 0, err
	499	+	}
	500	+	if int(b)&0xC0 == tokDeflatedData {
	501	+	if err := s.readChunk(int(b)); err != nil {
	502	+	s.err = err
	503	+	return 0, err
	504	+	}
	505	+	continue
	506	+	}
	507	+	s.next = int(b)
	508	+	s.marker = 0
	509	+	}
	510	+	}
	511	+
	512	+	func (s *segmentSource) Read(p []byte) (int, error) {
	513	+	n := 0
	514	+	for n < len(p) {
	515	+	if s.marker < 0 && s.pos < len(s.buf) {
	516	+	m := copy(p[n:], s.buf[s.pos:])
	517	+	s.pos += m
	518	+	n += m
	519	+	continue
	520	+	}
	521	+	if n > 0 {
	522	+	return n, nil
	523	+	}
	524	+	b, err := s.ReadByte()
	525	+	if err != nil {
	526	+	return n, err
	527	+	}
	528	+	p[n] = b
	529	+	n++
	530	+	}
	531	+	return n, nil
	532	+	}

...	...		@@ -0,0 +1,497 @@
	1	+	package rsync
	2	+
	3	+	import (
	4	+	"bytes"
	5	+	"errors"
	6	+	"io"
	7	+	"path"
	8	+	"slices"
	9	+	"strings"
	10	+	"testing"
	11	+	"time"
	12	+	)
	13	+
	14	+	type loopback struct {
	15	+	bytes.Buffer
	16	+	}
	17	+
	18	+	func newTestConn(t testing.T, protocol int) (conn, *loopback) {
	19	+	var buf loopback
	20	+	c := newConn(&buf)
	21	+	c.protocol = protocol
	22	+	t.Cleanup(func() { _ = c.close() })
	23	+	return c, &buf
	24	+	}
	25	+
	26	+	func TestVarintRoundTrip(t *testing.T) {
	27	+	values := []int32{0, 1, 0x7f, 0x80, 0xff, 0x3fff, 0x4000, 0x1fffff, 0x200000, 0x0fffffff, 0x10000000, 0x7fffffff, -1, -2, -0x80000000}
	28	+	for _, v := range values {
	29	+	c, _ := newTestConn(t, 31)
	30	+	if err := c.writeVarint(v); err != nil {
	31	+	t.Fatal(err)
	32	+	}
	33	+	if err := c.flush(); err != nil {
	34	+	t.Fatal(err)
	35	+	}
	36	+	got, err := c.readVarint()
	37	+	if err != nil \|\| got != v {
	38	+	t.Errorf("varint %d: got %d, %v", v, got, err)
	39	+	}
	40	+	}
	41	+	if got := appendVarint(nil, 0x80); !bytes.Equal(got, []byte{0x80, 0x80}) {
	42	+	t.Errorf("varint 0x80 encodes as % x", got)
	43	+	}
	44	+	}
	45	+
	46	+	func TestVarlongRoundTrip(t *testing.T) {
	47	+	values := []int64{0, 1, 0xffffff, 0x1000000, 0x7fffffff, 0x80000000, 1 << 40, 1<<62 + 12345, -1}
	48	+	for _, min := range []int{3, 4} {
	49	+	for _, v := range values {
	50	+	c, _ := newTestConn(t, 31)
	51	+	if err := c.writeVarlong(v, min); err != nil {
	52	+	t.Fatal(err)
	53	+	}
	54	+	if err := c.flush(); err != nil {
	55	+	t.Fatal(err)
	56	+	}
	57	+	got, err := c.readVarlong(min)
	58	+	if err != nil \|\| got != v {
	59	+	t.Errorf("varlong(%d) %d: got %d, %v", min, v, got, err)
	60	+	}
	61	+	}
	62	+	}
	63	+	}
	64	+
	65	+	func TestLongintRoundTrip(t *testing.T) {
	66	+	for _, v := range []int64{0, 0x7fffffff, 0x80000000, 1 << 40, -5} {
	67	+	c, _ := newTestConn(t, 29)
	68	+	if err := c.writeLongint(v); err != nil {
	69	+	t.Fatal(err)
	70	+	}
	71	+	if err := c.flush(); err != nil {
	72	+	t.Fatal(err)
	73	+	}
	74	+	got, err := c.readLongint()
	75	+	if err != nil \|\| got != v {
	76	+	t.Errorf("longint %d: got %d, %v", v, got, err)
	77	+	}
	78	+	}
	79	+	}
	80	+
	81	+	func TestNdxRoundTrip(t *testing.T) {
	82	+	seq := []int32{0, 1, 2, 5, 300, 299, 40000, 70000, 3, ndxDone, ndxDelStats, ndxFlistEOF, 7, ndxDone}
	83	+	for _, proto := range []int{29, 30} {
	84	+	c, _ := newTestConn(t, proto)
	85	+	for _, v := range seq {
	86	+	if err := c.writeNdx(v); err != nil {
	87	+	t.Fatal(err)
	88	+	}
	89	+	}
	90	+	if err := c.flush(); err != nil {
	91	+	t.Fatal(err)
	92	+	}
	93	+	for _, v := range seq {
	94	+	got, err := c.readNdx()
	95	+	if err != nil \|\| got != v {
	96	+	t.Fatalf("protocol %d: ndx %d: got %d, %v", proto, v, got, err)
	97	+	}
	98	+	}
	99	+	}
	100	+	}
	101	+
	102	+	func TestMultiplexedMessages(t *testing.T) {
	103	+	c, buf := newTestConn(t, 31)
	104	+	if err := c.w.setMultiplex(true); err != nil {
	105	+	t.Fatal(err)
	106	+	}
	107	+	_ = c.writeInt32(42)
	108	+	_ = c.writeMsg(msgInfo, []byte("hello\n"))
	109	+	_ = c.writeInt32(43)
	110	+	if err := c.flush(); err != nil {
	111	+	t.Fatal(err)
	112	+	}
	113	+
	114	+	r, _ := newTestConn(t, 31)
	115	+	r.r.r.Reset(bytes.NewReader(buf.Bytes()))
	116	+	r.r.multiplex = true
	117	+	var msgs []string
	118	+	r.r.onMsg = func(tag byte, data []byte) error {
	119	+	msgs = append(msgs, string(data))
	120	+	return nil
	121	+	}
	122	+	a, err := r.readInt32()
	123	+	if err != nil {
	124	+	t.Fatal(err)
	125	+	}
	126	+	b, err := r.readInt32()
	127	+	if err != nil {
	128	+	t.Fatal(err)
	129	+	}
	130	+	if a != 42 \|\| b != 43 \|\| !slices.Equal(msgs, []string{"hello\n"}) {
	131	+	t.Fatalf("got %d %d %q", a, b, msgs)
	132	+	}
	133	+	if _, err := r.readByte(); err != io.ErrUnexpectedEOF {
	134	+	t.Fatalf("want EOF, got %v", err)
	135	+	}
	136	+	}
	137	+
	138	+	// A generator can still be writing when the session gives up and closes the
	139	+	// writer; it has to get an error instead of waiting on a queue nobody drains.
	140	+	func TestMuxWriterAfterClose(t *testing.T) {
	141	+	m := newMuxWriter(io.Discard)
	142	+	if err := m.Close(); err != nil {
	143	+	t.Fatal(err)
	144	+	}
	145	+	done := make(chan error, 1)
	146	+	go func() {
	147	+	chunk := make([]byte, muxBatchSize)
	148	+	for range 2 * muxMaxQueued / muxBatchSize {
	149	+	if _, err := m.Write(chunk); err != nil {
	150	+	done <- err
	151	+	return
	152	+	}
	153	+	}
	154	+	done <- m.Flush()
	155	+	}()
	156	+	select {
	157	+	case err := <-done:
	158	+	if err == nil {
	159	+	t.Fatal("writes after Close succeeded")
	160	+	}
	161	+	case <-time.After(5 * time.Second):
	162	+	t.Fatal("write after Close blocked")
	163	+	}
	164	+	}
	165	+
	166	+	func TestChecksum1(t *testing.T) {
	167	+	// Bytes above 0x7f count as negative, as in rsync's C code.
	168	+	if got := checksum1([]byte{0xff, 1, 2, 3, 4, 5}); got != 0x001d000e {
	169	+	t.Errorf("checksum1 = %#x", got)
	170	+	}
	171	+	if got := checksum1(nil); got != 0 {
	172	+	t.Errorf("checksum1(nil) = %#x", got)
	173	+	}
	174	+	}
	175	+
	176	+	func TestSumSizes(t *testing.T) {
	177	+	tests := []struct {
	178	+	length int64
	179	+	proto int
	180	+	want sumHead
	181	+	}{
	182	+	{0, 31, sumHead{0, 700, 2, 0}},
	183	+	{1, 31, sumHead{1, 700, 2, 1}},
	184	+	{1_000_000, 31, sumHead{1000, 1000, 2, 0}},
	185	+	{100 << 20, 31, sumHead{10240, 10240, 3, 0}},
	186	+	{100<<20 + 1, 29, sumHead{10241, 10240, 3, 1}},
	187	+	}
	188	+	for _, tt := range tests {
	189	+	got, err := sumSizesSqroot(tt.length, tt.proto, 0, shortSumLength, 16)
	190	+	if err != nil \|\| got != tt.want {
	191	+	t.Errorf("sumSizes(%d, %d) = %+v, %v; want %+v", tt.length, tt.proto, got, err, tt.want)
	192	+	}
	193	+	}
	194	+	got, _ := sumSizesSqroot(1_000_000, 31, 0, sumLength, 8)
	195	+	if got.s2length != 8 {
	196	+	t.Errorf("full checksum length capped at digest size: got %d", got.s2length)
	197	+	}
	198	+	}
	199	+
	200	+	func TestFileNameOrder(t *testing.T) {
	201	+	mk := func(name string, dir bool) *fileEntry {
	202	+	f := newFileEntry(name)
	203	+	if name == "." {
	204	+	f.dirname, f.basename = "", "."
	205	+	}
	206	+	f.mode = sIFREG
	207	+	if dir {
	208	+	f.mode = sIFDIR
	209	+	}
	210	+	return f
	211	+	}
	212	+	entries := func() []*fileEntry {
	213	+	return []*fileEntry{
	214	+	mk("b.txt", false), mk("a/x", false), mk("a", true), mk(".", true),
	215	+	mk("a.b", false), mk("a-b", false), mk("a/y/z", false), mk("a/y", true), mk("a/w", false),
	216	+	}
	217	+	}
	218	+	names := func(fs []*fileEntry) []string {
	219	+	var out []string
	220	+	for _, f := range fs {
	221	+	out = append(out, f.name())
	222	+	}
	223	+	return out
	224	+	}
	225	+
	226	+	modern := entries()
	227	+	sortFileList(modern, 31, false)
	228	+	want := []string{".", "a-b", "a.b", "b.txt", "a", "a/w", "a/x", "a/y", "a/y/z"}
	229	+	if got := names(modern); !slices.Equal(got, want) {
	230	+	t.Errorf("protocol 31 order:\n got %q\nwant %q", got, want)
	231	+	}
	232	+
	233	+	old := entries()
	234	+	sortFileList(old, 28, false)
	235	+	want = []string{".", "a", "a-b", "a.b", "a/w", "a/x", "a/y", "a/y/z", "b.txt"}
	236	+	if got := names(old); !slices.Equal(got, want) {
	237	+	t.Errorf("protocol 28 order:\n got %q\nwant %q", got, want)
	238	+	}
	239	+	}
	240	+
	241	+	func TestDuplicateNames(t *testing.T) {
	242	+	a := newFileEntry("x")
	243	+	a.mode = sIFREG
	244	+	b := newFileEntry("x")
	245	+	b.mode = sIFDIR
	246	+	c := newFileEntry("x")
	247	+	c.mode = sIFREG
	248	+	files := []*fileEntry{a, b, c}
	249	+	sortFileList(files, 31, true)
	250	+	var active []*fileEntry
	251	+	for _, f := range files {
	252	+	if !f.inactive {
	253	+	active = append(active, f)
	254	+	}
	255	+	}
	256	+	if len(active) != 1 \|\| !active[0].isDir() {
	257	+	t.Fatalf("want only the directory kept, got %d entries", len(active))
	258	+	}
	259	+	}
	260	+
	261	+	func TestWildmatch(t *testing.T) {
	262	+	tests := []struct {
	263	+	pattern, text string
	264	+	want bool
	265	+	}{
	266	+	{"foo", "foo", true},
	267	+	{"foo", "bar", false},
	268	+	{"???", "foo", true},
	269	+	{"??", "foo", false},
	270	+	{"*", "foo", true},
	271	+	{"f*", "foo", true},
	272	+	{"*f", "foo", false},
	273	+	{"obar", "foobar", true},
	274	+	{"*ab", "aaaaaaabababab", true},
	275	+	{`foo\`, "foo", true},
	276	+	{`foo\*bar`, "foobar", false},
	277	+	{"*[al]?", "ball", true},
	278	+	{"[ten]", "ten", false},
	279	+	{"**[!te]", "ten", true},
	280	+	{"**[!ten]", "ten", false},
	281	+	{"t[a-g]n", "ten", true},
	282	+	{"t[!a-g]n", "ten", false},
	283	+	{"t[^a-g]n", "ton", true},
	284	+	{"a[]]b", "a]b", true},
	285	+	{"[[:digit:]]x", "7x", true},
	286	+	{"[[:alpha:]]x", "7x", false},
	287	+	{"[[:bogus:]]", "a", false},
	288	+	{"*", "a/b", false},
	289	+	{"**", "a/b", true},
	290	+	{"a/*", "a/b/c", false},
	291	+	{"a/**", "a/b/c", true},
	292	+	{"**/c", "a/b/c", true},
	293	+	{"a?b", "a/b", false},
	294	+	{"[a/]b", "/b", false},
	295	+	}
	296	+	for _, tt := range tests {
	297	+	if got := wildmatch(tt.pattern, tt.text); got != tt.want {
	298	+	t.Errorf("wildmatch(%q, %q) = %v, want %v", tt.pattern, tt.text, got, tt.want)
	299	+	}
	300	+	}
	301	+	}
	302	+
	303	+	func mapReader(files map[string]string) func(string) ([]byte, error) {
	304	+	return func(name string) ([]byte, error) {
	305	+	data, ok := files[name]
	306	+	if !ok {
	307	+	return nil, nil
	308	+	}
	309	+	return []byte(data), nil
	310	+	}
	311	+	}
	312	+
	313	+	func TestPerDirMergeRules(t *testing.T) {
	314	+	files := map[string]string{
	315	+	".rsync-filter": "- /site/scanned.txt\n- /outside.txt\n",
	316	+	"site/.rsync-filter": "# comment\n; comment\n- *.log\nP protected.txt\nH hidden.txt\nexclude long.txt\r\n",
	317	+	"site/sub/.rsync-filter": "+ important.log\n- /anchored.txt\n",
	318	+	"site/clear/.rsync-filter": "!\n- z\n",
	319	+	"site/.cvsignore": "a.o b.o\n",
	320	+	"site/sub/.cvsignore": "! c.o",
	321	+	}
	322	+	tests := []struct {
	323	+	otherSide int
	324	+	name string
	325	+	want int
	326	+	}{
	327	+	{ruleSenderSide, "a.log", -1},
	328	+	{ruleSenderSide, "sub/x.log", -1},
	329	+	{ruleSenderSide, "sub/important.log", 1},
	330	+	{ruleSenderSide, "long.txt", -1},
	331	+	{ruleSenderSide, "scanned.txt", -1},
	332	+	{ruleSenderSide, "outside.txt", 0},
	333	+	{ruleSenderSide, "sub/anchored.txt", -1},
	334	+	{ruleSenderSide, "anchored.txt", 0},
	335	+	{ruleSenderSide, "sub/deeper/anchored.txt", 0},
	336	+	{ruleSenderSide, "clear/a.log", 0},
	337	+	{ruleSenderSide, "clear/z", -1},
	338	+	{ruleSenderSide, ".rsync-filter", -1},
	339	+	{ruleSenderSide, "a.o", -1},
	340	+	{ruleSenderSide, "sub/a.o", 0},
	341	+	{ruleSenderSide, "sub/c.o", -1},
	342	+	{ruleSenderSide, "protected.txt", -1},
	343	+	{ruleSenderSide, "hidden.txt", 0},
	344	+	{ruleReceiverSide, "protected.txt", 0},
	345	+	{ruleReceiverSide, "hidden.txt", -1},
	346	+	}
	347	+	for _, tt := range tests {
	348	+	l := filterList{otherSide: tt.otherSide}
	349	+	for _, rule := range []string{":e /.rsync-filter", ": .rsync-filter", ":C"} {
	350	+	if err := l.parse(rule, false); err != nil {
	351	+	t.Fatalf("parse %q: %v", rule, err)
	352	+	}
	353	+	}
	354	+	if len(l.merges) != 2 {
	355	+	t.Fatalf("duplicate merge rule kept: %d merges", len(l.merges))
	356	+	}
	357	+	sc, err := l.newScopeTree("site", "site", mapReader(files)).scope(path.Dir(tt.name))
	358	+	if err != nil {
	359	+	t.Fatal(err)
	360	+	}
	361	+	if got := l.check(tt.name, false, sc); got != tt.want {
	362	+	t.Errorf("otherSide=%d check(%q) = %d, want %d", tt.otherSide, tt.name, got, tt.want)
	363	+	}
	364	+	}
	365	+	}
	366	+
	367	+	func TestPerDirMergeDeleteExcluded(t *testing.T) {
	368	+	l := filterList{otherSide: ruleSenderSide, deleteExcluded: true}
	369	+	if err := l.parse(": .rsync-filter", false); err != nil {
	370	+	t.Fatal(err)
	371	+	}
	372	+	files := map[string]string{"site/.rsync-filter": "- excluded.txt\nP protected.txt\n"}
	373	+	sc, err := l.newScopeTree("site", "site", mapReader(files)).scope(".")
	374	+	if err != nil {
	375	+	t.Fatal(err)
	376	+	}
	377	+	if l.excluded("excluded.txt", false, sc) {
	378	+	t.Error("--delete-excluded should let the receiver delete excluded files")
	379	+	}
	380	+	if !l.excluded("protected.txt", false, sc) {
	381	+	t.Error("protect rules still apply with --delete-excluded")
	382	+	}
	383	+	}
	384	+
	385	+	func TestPerDirMergeNoInherit(t *testing.T) {
	386	+	l := filterList{otherSide: ruleSenderSide}
	387	+	if err := l.parse(":n /.rsync-filter", false); err != nil {
	388	+	t.Fatal(err)
	389	+	}
	390	+	files := map[string]string{
	391	+	".rsync-filter": "- *.root\n",
	392	+	"site/.rsync-filter": "- *.tmp\n",
	393	+	}
	394	+	scopes := l.newScopeTree("site", "site", mapReader(files))
	395	+	for _, tt := range []struct {
	396	+	name string
	397	+	want int
	398	+	}{{"x.tmp", -1}, {"sub/x.tmp", 0}, {"x.root", 0}} {
	399	+	sc, err := scopes.scope(path.Dir(tt.name))
	400	+	if err != nil {
	401	+	t.Fatal(err)
	402	+	}
	403	+	if got := l.check(tt.name, false, sc); got != tt.want {
	404	+	t.Errorf("check(%q) = %d, want %d", tt.name, got, tt.want)
	405	+	}
	406	+	}
	407	+	}
	408	+
	409	+	func TestPerDirMergeErrors(t *testing.T) {
	410	+	for _, rule := range []string{". rules", "merge rules", ":! .rsync-filter", ": x/"} {
	411	+	var l filterList
	412	+	if err := l.parse(rule, false); err == nil {
	413	+	t.Errorf("parse %q: want an error", rule)
	414	+	}
	415	+	}
	416	+	for _, contents := range []string{
	417	+	"merge other\n",
	418	+	": .nested\n",
	419	+	"dir-merge .nested\n",
	420	+	"bogus\n",
	421	+	"-foo\n",
	422	+	"! trailing\n",
	423	+	"- " + strings.Repeat("x", maxPathLen) + "\n",
	424	+	} {
	425	+	l := filterList{otherSide: ruleSenderSide}
	426	+	if err := l.parse(": .rsync-filter", false); err != nil {
	427	+	t.Fatal(err)
	428	+	}
	429	+	files := map[string]string{"site/.rsync-filter": contents}
	430	+	_, err := l.newScopeTree("site", "site", mapReader(files)).scope(".")
	431	+	var mergeErr *mergeFileError
	432	+	if !errors.As(err, &mergeErr) {
	433	+	t.Errorf("merge file %q: want a mergeFileError, got %v", contents, err)
	434	+	}
	435	+	}
	436	+
	437	+	l := filterList{otherSide: ruleSenderSide}
	438	+	if err := l.parse(": .rsync-filter", false); err != nil {
	439	+	t.Fatal(err)
	440	+	}
	441	+	readErr := errors.New("backend down")
	442	+	_, err := l.newScopeTree("site", "site", func(string) ([]byte, error) { return nil, readErr }).scope(".")
	443	+	if !errors.Is(err, readErr) {
	444	+	t.Errorf("read error not reported: %v", err)
	445	+	}
	446	+
	447	+	for _, pattern := range []string{". rules", ": rules"} {
	448	+	var l filterList
	449	+	if err := l.parse(pattern, true); err != nil {
	450	+	t.Fatal(err)
	451	+	}
	452	+	if l.check(pattern, false, nil) != -1 {
	453	+	t.Errorf("protocol 28 pattern %q should remain an exclusion", pattern)
	454	+	}
	455	+	}
	456	+	}
	457	+
	458	+	func TestFilterRules(t *testing.T) {
	459	+	var l filterList
	460	+	for _, rule := range []string{"+ keep.log", "- .log", "- /build/", "- node_modules/", "- docs/.tmp", "-! *.html"} {
	461	+	if err := l.parse(rule, false); err != nil {
	462	+	t.Fatalf("parse %q: %v", rule, err)
	463	+	}
	464	+	}
	465	+	tests := []struct {
	466	+	name string
	467	+	isDir bool
	468	+	want int
	469	+	}{
	470	+	{"keep.log", false, 1},
	471	+	{"sub/keep.log", false, 1},
	472	+	{"x.log", false, -1},
	473	+	{"sub/x.log", false, -1},
	474	+	{"build", true, -1},
	475	+	{"build", false, -1}, // "-! *.html" excludes anything not ending in .html
	476	+	{"sub/build", true, -1},
	477	+	{"src/node_modules", true, -1},
	478	+	{"docs/a.tmp", false, -1},
	479	+	{"index.html", false, 0},
	480	+	}
	481	+	for _, tt := range tests {
	482	+	if got := l.check(tt.name, tt.isDir, nil); got != tt.want {
	483	+	t.Errorf("check(%q, dir=%v) = %d, want %d", tt.name, tt.isDir, got, tt.want)
	484	+	}
	485	+	}
	486	+
	487	+	var old filterList
	488	+	_ = old.parse("*.o", true)
	489	+	_ = old.parse("+ keep.o", true)
	490	+	if old.check("x.o", false, nil) != -1 {
	491	+	t.Error("protocol 28 bare pattern is an exclude")
	492	+	}
	493	+	_ = old.parse("!", true)
	494	+	if len(old.rules) != 0 {
	495	+	t.Error("! clears the list")
	496	+	}
	497	+	}

...	...		@@ -3,175 +3,121 @@ package rsync
3	3		import (
4	4		"errors"
5	5		"fmt"
	6	+	"io"
6	7		"io/fs"
7	8		"os"
8	9		"path"
9		-	"slices"
10	10		"strings"
11	11
12	12		"github.com/picosh/pico/pkg/pssh"
13		-	"github.com/picosh/pico/pkg/rsync-receiver/rsyncopts"
14		-	"github.com/picosh/pico/pkg/rsync-receiver/rsyncreceiver"
15		-	"github.com/picosh/pico/pkg/rsync-receiver/rsyncsender"
16		-	rsyncutils "github.com/picosh/pico/pkg/rsync-receiver/utils"
	13	+	"github.com/picosh/pico/pkg/rsync"
17	14		"github.com/picosh/pico/pkg/send/utils"
18	15		)
19	16
20		-	type handler struct {
21		-	session *pssh.SSHServerConnSession
22		-	writeHandler utils.CopyFromClientHandler
23		-	root string
24		-	recursive bool
25		-	ignoreTimes bool
	17	+	// storageFS exposes a CopyFromClientHandler as an rsync.FS. Handlers take
	18	+	// absolute paths and differ in how they name listing results, so this
	19	+	// normalizes both directions.
	20	+	type storageFS struct {
	21	+	session *pssh.SSHServerConnSession
	22	+	handler utils.CopyFromClientHandler
26	23		}
27	24
28		-	func (h *handler) List(rPath string) ([]fs.FileInfo, error) {
29		-	isDir := false
30		-	if rPath == "." {
31		-	rPath = "/"
32		-	isDir = true
33		-	}
	25	+	func abs(name string) string { return "/" + strings.TrimPrefix(name, "/") }
34	26
35		-	list, err := h.writeHandler.List(h.session, rPath, isDir, h.recursive)
36		-	if err != nil {
37		-	return nil, err
	27	+	// isSelf reports whether a listing entry stands for the listed path itself
	28	+	// rather than something below it.
	29	+	func isSelf(info os.FileInfo) bool {
	30	+	switch info.Name() {
	31	+	case "", ".", "/":
	32	+	return info.IsDir()
38	33		}
	34	+	return false
	35	+	}
39	36
40		-	var dirs []string
41		-
42		-	var newList []fs.FileInfo
43		-
44		-	for _, f := range list {
45		-	if !f.IsDir() && f.Size() == 0 {
46		-	continue
47		-	}
48		-
49		-	fname := f.Name()
50		-	if strings.HasPrefix(f.Name(), "/") {
51		-	fname = path.Join(rPath, f.Name())
52		-	}
53		-
54		-	if fname == "" && !f.IsDir() {
55		-	fname = path.Base(rPath)
56		-	}
57		-
58		-	newFile := &utils.VirtualFile{
59		-	FName: fname,
60		-	FIsDir: f.IsDir(),
61		-	FSize: f.Size(),
62		-	FModTime: f.ModTime(),
63		-	FSys: f.Sys(),
64		-	}
65		-
66		-	newList = append(newList, newFile)
67		-
68		-	parts := strings.Split(newFile.Name(), string(os.PathSeparator))
69		-	lastDir := newFile.Name()
70		-	for i := 0; i < len(parts); i++ {
71		-	lastDir, _ = path.Split(lastDir)
72		-	if lastDir == "" {
73		-	continue
74		-	}
75		-
76		-	lastDir = lastDir[:len(lastDir)-1]
77		-	dirs = append(dirs, lastDir)
78		-	}
	37	+	func (s *storageFS) Stat(name string) (rsync.FileInfo, error) {
	38	+	if name == "" {
	39	+	return rsync.FileInfo{IsDir: true}, nil
79	40		}
80		-
81		-	for _, dir := range dirs {
82		-	newList = append(newList, &utils.VirtualFile{
83		-	FName: dir,
84		-	FIsDir: true,
85		-	})
	41	+	entries, err := s.handler.List(s.session, abs(name), false, false)
	42	+	if err != nil {
	43	+	return rsync.FileInfo{}, err
86	44		}
87		-
88		-	slices.Reverse(newList)
89		-
90		-	onlyEmpty := true
91		-	for _, f := range newList {
92		-	if f.Name() != "" {
93		-	onlyEmpty = false
	45	+	for _, e := range entries {
	46	+	if isSelf(e) {
	47	+	return rsync.FileInfo{Name: name, ModTime: e.ModTime(), IsDir: true}, nil
94	48		}
95	49		}
96		-
97		-	if len(newList) == 0 \|\| onlyEmpty {
98		-	return nil, errors.New("no files to send, the directory may not exist or could be empty")
99		-	}
100		-
101		-	return newList, nil
102		-	}
103		-
104		-	func (h handler) Read(file rsyncutils.SenderFile) (os.FileInfo, rsyncutils.ReaderAtCloser, error) {
105		-	filePath := file.WPath
106		-
107		-	if strings.HasSuffix(h.root, file.WPath) {
108		-	filePath = h.root
109		-	} else if !strings.HasPrefix(filePath, h.root) {
110		-	filePath = path.Join(h.root, file.Path, file.WPath)
	50	+	switch {
	51	+	case len(entries) == 0:
	52	+	return rsync.FileInfo{}, fmt.Errorf("%s: %w", name, fs.ErrNotExist)
	53	+	case len(entries) == 1 && !entries[0].IsDir() && path.Base(entries[0].Name()) == path.Base(name):
	54	+	e := entries[0]
	55	+	return rsync.FileInfo{Name: name, Size: e.Size(), ModTime: e.ModTime()}, nil
	56	+	default:
	57	+	// The handler listed the contents of a directory.
	58	+	return rsync.FileInfo{Name: name, IsDir: true}, nil
111	59		}
112		-
113		-	return h.writeHandler.Read(h.session, &utils.FileEntry{Filepath: filePath})
114	60		}
115	61
116		-	func (h handler) Put(file rsyncutils.ReceiverFile) (int64, error) {
117		-	fileEntry := &utils.FileEntry{
118		-	Filepath: path.Join("/", h.root, file.Name),
119		-	Mode: fs.FileMode(0600),
120		-	Size: file.Length,
121		-	Mtime: file.ModTime.Unix(),
122		-	Atime: file.ModTime.Unix(),
123		-	}
124		-	fileEntry.Reader = file.Reader
125		-
126		-	msg, err := h.writeHandler.Write(h.session, fileEntry)
	62	+	func (s *storageFS) ReadDir(dir string, recursive bool) ([]rsync.FileInfo, error) {
	63	+	entries, err := s.handler.List(s.session, abs(dir), true, recursive)
127	64		if err != nil {
128		-	errMsg := fmt.Sprintf("%s\r\n", err.Error())
129		-	_, err = h.session.Stderr().Write([]byte(errMsg))
	65	+	return nil, err
130	66		}
131		-	if msg != "" {
132		-	nMsg := fmt.Sprintf("%s\r\n", msg)
133		-	_, err = h.session.Stderr().Write([]byte(nMsg))
	67	+	out := make([]rsync.FileInfo, 0, len(entries))
	68	+	for _, e := range entries {
	69	+	if isSelf(e) {
	70	+	continue
	71	+	}
	72	+	name := strings.Trim(e.Name(), "/")
	73	+	if name == "" {
	74	+	continue
	75	+	}
	76	+	out = append(out, rsync.FileInfo{Name: name, Size: e.Size(), ModTime: e.ModTime(), IsDir: e.IsDir()})
134	77		}
135		-	return 0, err
	78	+	return out, nil
136	79		}
137	80
138		-	func (h handler) Remove(willReceive []rsyncutils.ReceiverFile) error {
139		-	entries, err := h.writeHandler.List(h.session, path.Join("/", h.root), true, true)
	81	+	func (s *storageFS) Open(name string) (rsync.File, rsync.FileInfo, error) {
	82	+	info, r, err := s.handler.Read(s.session, &utils.FileEntry{Filepath: abs(name)})
140	83		if err != nil {
141		-	return err
	84	+	return nil, rsync.FileInfo{}, err
142	85		}
143		-
144		-	var toDelete []string
145		-
146		-	for _, entry := range entries {
147		-	exists := slices.ContainsFunc(willReceive, func(rf *rsyncutils.ReceiverFile) bool {
148		-	return rf.Name == entry.Name()
149		-	})
150		-
151		-	if !exists && path.Base(entry.Name()) != "._pico_keep_dir" {
152		-	toDelete = append(toDelete, entry.Name())
153		-	}
	86	+	if r == nil {
	87	+	return nil, rsync.FileInfo{}, fmt.Errorf("%s: %w", name, fs.ErrNotExist)
154	88		}
	89	+	return r, rsync.FileInfo{Name: name, Size: info.Size(), ModTime: info.ModTime()}, nil
	90	+	}
155	91
156		-	// Sort by path depth descending so children are deleted before parents.
157		-	// This ensures directories are empty before we try to remove them.
158		-	slices.SortFunc(toDelete, func(a, b string) int {
159		-	depthA := strings.Count(a, "/")
160		-	depthB := strings.Count(b, "/")
161		-	return depthB - depthA
	92	+	func (s *storageFS) Put(name string, info rsync.FileInfo, r io.Reader) (string, error) {
	93	+	msg, err := s.handler.Write(s.session, &utils.FileEntry{
	94	+	Filepath: abs(name),
	95	+	Mode: fs.FileMode(0600),
	96	+	Size: info.Size,
	97	+	Mtime: info.ModTime.Unix(),
	98	+	Atime: info.ModTime.Unix(),
	99	+	Reader: r,
162	100		})
	101	+	if errors.Is(err, utils.ErrFileIgnored) {
	102	+	return msg, ignoredError{err}
	103	+	}
	104	+	return msg, err
	105	+	}
163	106
164		-	var errs []error
	107	+	// ignoredError marks a handler's ignored file for rsync, keeping the
	108	+	// handler's message.
	109	+	type ignoredError struct{ error }
165	110
166		-	for _, file := range toDelete {
167		-	errs = append(errs, h.writeHandler.Delete(h.session, &utils.FileEntry{Filepath: path.Join("/", h.root, file)}))
168		-	_, err = fmt.Fprintf(h.session.Stderr(), "deleting %s\r\n", file)
169		-	errs = append(errs, err)
170		-	}
	111	+	func (e ignoredError) Is(target error) bool { return target == rsync.ErrFileIgnored }
	112	+
	113	+	func (e ignoredError) Unwrap() error { return e.error }
171	114
172		-	return errors.Join(errs...)
	115	+	func (s *storageFS) Remove(name string, isDir bool) error {
	116	+	return s.handler.Delete(s.session, &utils.FileEntry{Filepath: abs(name)})
173	117		}
174	118
	119	+	// Middleware serves rsync clients ("rsync --server ...") from the handler's
	120	+	// storage.
175	121		func Middleware(writeHandler utils.CopyFromClientHandler) pssh.SSHServerMiddleware {
176	122		return func(sshHandler pssh.SSHServerHandler) pssh.SSHServerHandler {
177	123		return func(session *pssh.SSHServerConnSession) error {

...	...		@@ -192,66 +138,26 @@ func Middleware(writeHandler utils.CopyFromClientHandler) pssh.SSHServerMiddlewa
192	138		}
193	139		}()
194	140
195		-	cmdFlags := session.Command()
196		-	flgs := cmdFlags[1:]
197		-	for idx, f := range flgs {
198		-	// openrsync sends "delete-before" when the client provided "delete"
199		-	flgs[idx] = strings.ReplaceAll(f, "delete-before", "delete")
	141	+	err := rsync.Serve(rsync.Config{
	142	+	FS: &storageFS{session: session, handler: writeHandler},
	143	+	Logger: logger,
	144	+	Stderr: session.Stderr(),
	145	+	}, session, cmd[1:])
	146	+	if err == nil {
	147	+	return nil
200	148		}
201	149
202		-	optsCtx, err := rsyncopts.ParseArguments(cmdFlags[1:], true)
203		-	if err != nil {
204		-	_, _ = fmt.Fprintf(session.Stderr(), "ERROR: error parsing rsync arguments: %s\r\n", err.Error())
205		-	return err
	150	+	// The client has already been told what went wrong; what is
	151	+	// left is to hand it rsync's exit code.
	152	+	code := 1
	153	+	var exit *rsync.ExitError
	154	+	if errors.As(err, &exit) {
	155	+	code = exit.Code
206	156		}
207		-
208		-	if optsCtx.Options.Compress() {
209		-	err := fmt.Errorf("compression is currently unsupported")
210		-	_, _ = fmt.Fprintf(session.Stderr(), "error: %s\r\n", err.Error())
211		-	return err
212		-	}
213		-
214		-	if optsCtx.Options.AlwaysChecksum() {
215		-	err := fmt.Errorf("checksum is currently unsupported")
216		-	_, _ = fmt.Fprintf(session.Stderr(), "error: %s\r\n", err.Error())
217		-	return err
218		-	}
219		-
220		-	if len(optsCtx.RemainingArgs) != 2 {
221		-	err := fmt.Errorf("missing source and destination arguments")
222		-	_, _ = fmt.Fprintf(session.Stderr(), "error: %s\r\n", err.Error())
223		-	return err
224		-	}
225		-
226		-	root := strings.TrimPrefix(optsCtx.RemainingArgs[len(optsCtx.RemainingArgs)-1], "/")
227		-	if root == "" {
228		-	root = "/"
229		-	}
230		-
231		-	fileHandler := &handler{
232		-	session: session,
233		-	writeHandler: writeHandler,
234		-	root: root,
235		-	recursive: optsCtx.Options.Recurse(),
236		-	ignoreTimes: !optsCtx.Options.PreserveMTimes(),
237		-	}
238		-
239		-	for _, arg := range cmd {
240		-	if arg == "--sender" {
241		-	err := rsyncsender.ClientRun(logger, optsCtx.Options, session, fileHandler, []string{fileHandler.root}, true)
242		-	if err != nil {
243		-	logger.Error("error running rsync sender", "err", err)
244		-	}
245		-	return err
246		-	}
247		-	}
248		-
249		-	err = rsyncreceiver.ClientRun(logger, optsCtx.Options, session, fileHandler, []string{fileHandler.root}, true)
250		-	if err != nil {
251		-	logger.Error("error running rsync receiver", "err", err)
252		-	}
253		-
254		-	return err
	157	+	logger.Error("rsync transfer failed", "err", err, "code", code)
	158	+	_ = session.Exit(code)
	159	+	_ = session.Close()
	160	+	return nil
255	161		}
256	162		}
257	163		}

...	...		@@ -3,15 +3,18 @@ package rsync
3	3		import (
4	4		"bytes"
5	5		"context"
	6	+	"errors"
6	7		"io"
7	8		"io/fs"
8	9		"log/slog"
9	10		"os"
	11	+	"slices"
	12	+	"strings"
10	13		"testing"
11	14		"time"
12	15
13	16		"github.com/picosh/pico/pkg/pssh"
14		-	rsyncutils "github.com/picosh/pico/pkg/rsync-receiver/utils"
	17	+	"github.com/picosh/pico/pkg/rsync"
15	18		"github.com/picosh/pico/pkg/send/utils"
16	19		"golang.org/x/crypto/ssh"
17	20		)

...	...		@@ -20,46 +23,62 @@ import (
20	23		type mockFileInfo struct {
21	24		name string
22	25		size int64
23		-	mode fs.FileMode
24	26		modTime time.Time
25	27		isDir bool
26	28		}
27	29
28		-	func (m *mockFileInfo) Name() string { return m.name }
29		-	func (m *mockFileInfo) Size() int64 { return m.size }
30		-	func (m *mockFileInfo) Mode() fs.FileMode { return m.mode }
	30	+	func (m *mockFileInfo) Name() string { return m.name }
	31	+	func (m *mockFileInfo) Size() int64 { return m.size }
	32	+	func (m *mockFileInfo) Mode() fs.FileMode {
	33	+	if m.isDir {
	34	+	return fs.ModeDir \| 0o755
	35	+	}
	36	+	return 0o644
	37	+	}
31	38		func (m *mockFileInfo) ModTime() time.Time { return m.modTime }
32	39		func (m *mockFileInfo) IsDir() bool { return m.isDir }
33	40		func (m *mockFileInfo) Sys() any { return nil }
34	41
35		-	// mockWriteHandler implements utils.CopyFromClientHandler for testing.
36		-	// It records the order of Delete calls to verify deletion order.
	42	+	// mockWriteHandler implements utils.CopyFromClientHandler and records the
	43	+	// paths it was called with.
37	44		type mockWriteHandler struct {
38		-	entries []os.FileInfo
39		-	deleteCalls []string
40		-	deleteErrors map[string]error
	45	+	lists map[string][]os.FileInfo
	46	+	files map[string]string
	47	+	calls []string
	48	+	written map[string]string
41	49		}
42	50
43	51		func (m mockWriteHandler) Delete(_ pssh.SSHServerConnSession, entry *utils.FileEntry) error {
44		-	m.deleteCalls = append(m.deleteCalls, entry.Filepath)
45		-	if m.deleteErrors != nil {
46		-	if err, ok := m.deleteErrors[entry.Filepath]; ok {
47		-	return err
48		-	}
49		-	}
	52	+	m.calls = append(m.calls, "delete "+entry.Filepath)
50	53		return nil
51	54		}
52	55
53		-	func (m mockWriteHandler) Write(_ pssh.SSHServerConnSession, _ *utils.FileEntry) (string, error) {
	56	+	func (m mockWriteHandler) Write(_ pssh.SSHServerConnSession, entry *utils.FileEntry) (string, error) {
	57	+	m.calls = append(m.calls, "write "+entry.Filepath)
	58	+	b, err := io.ReadAll(entry.Reader)
	59	+	if err != nil {
	60	+	return "", err
	61	+	}
	62	+	if m.written == nil {
	63	+	m.written = map[string]string{}
	64	+	}
	65	+	m.written[entry.Filepath] = string(b)
54	66		return "", nil
55	67		}
56	68
57		-	func (m mockWriteHandler) Read(_ pssh.SSHServerConnSession, _ *utils.FileEntry) (os.FileInfo, utils.ReadAndReaderAtCloser, error) {
58		-	return nil, nil, nil
	69	+	func (m mockWriteHandler) Read(_ pssh.SSHServerConnSession, entry *utils.FileEntry) (os.FileInfo, utils.ReadAndReaderAtCloser, error) {
	70	+	m.calls = append(m.calls, "read "+entry.Filepath)
	71	+	data, ok := m.files[entry.Filepath]
	72	+	if !ok {
	73	+	return nil, nil, os.ErrNotExist
	74	+	}
	75	+	info := &mockFileInfo{name: entry.Filepath, size: int64(len(data))}
	76	+	return info, utils.NopReadAndReaderAtCloser(strings.NewReader(data)), nil
59	77		}
60	78
61		-	func (m mockWriteHandler) List(_ pssh.SSHServerConnSession, _ string, _ bool, _ bool) ([]os.FileInfo, error) {
62		-	return m.entries, nil
	79	+	func (m mockWriteHandler) List(_ pssh.SSHServerConnSession, fpath string, isDir bool, recursive bool) ([]os.FileInfo, error) {
	80	+	m.calls = append(m.calls, "list "+fpath)
	81	+	return m.lists[fpath], nil
63	82		}
64	83
65	84		func (m mockWriteHandler) GetLogger(_ pssh.SSHServerConnSession) *slog.Logger {

...	...		@@ -87,9 +106,8 @@ func (m *mockChannel) Stderr() io.ReadWriter { return m.stderr }
87	106		var _ ssh.Channel = (*mockChannel)(nil)
88	107
89	108		// newMockSession creates a mock SSHServerConnSession for testing.
90		-	func newMockSession() (pssh.SSHServerConnSession, bytes.Buffer) {
91		-	stderr := &bytes.Buffer{}
92		-	channel := &mockChannel{stderr: stderr}
	109	+	func newMockSession() *pssh.SSHServerConnSession {
	110	+	channel := &mockChannel{stderr: &bytes.Buffer{}}
93	111
94	112		ctx, cancel := context.WithCancel(context.Background())
95	113		logger := slog.Default()

...	...		@@ -100,138 +118,93 @@ func newMockSession() (pssh.SSHServerConnSession, bytes.Buffer) {
100	118		},
101	119		}, server)
102	120
103		-	session := &pssh.SSHServerConnSession{
	121	+	return &pssh.SSHServerConnSession{
104	122		Channel: channel,
105	123		SSHServerConn: serverConn,
106	124		Ctx: ctx,
107	125		CancelFunc: cancel,
108	126		}
109		-
110		-	return session, stderr
111	127		}
112	128
113		-	func TestRemove_DeletesChildrenBeforeParents(t *testing.T) {
114		-	session, _ := newMockSession()
115		-	mockHandler := &mockWriteHandler{
116		-	entries: []os.FileInfo{
117		-	&mockFileInfo{name: "a", isDir: true},
118		-	&mockFileInfo{name: "a/file.txt", size: 100},
119		-	&mockFileInfo{name: "b/c", isDir: true},
120		-	&mockFileInfo{name: "b/c/deep.txt", size: 50},
121		-	&mockFileInfo{name: "b", isDir: true},
	129	+	func TestReadDirNormalizesListing(t *testing.T) {
	130	+	handler := &mockWriteHandler{lists: map[string][]os.FileInfo{
	131	+	"/site": {
	132	+	&mockFileInfo{name: "", isDir: true},
	133	+	&mockFileInfo{name: "index.html", size: 10},
	134	+	&mockFileInfo{name: "/css/site.css", size: 5},
	135	+	&mockFileInfo{name: "css", isDir: true},
	136	+	&mockFileInfo{name: "empty.txt"},
122	137		},
123		-	}
	138	+	}}
	139	+	fsys := &storageFS{session: newMockSession(), handler: handler}
124	140
125		-	h := &handler{
126		-	session: session,
127		-	writeHandler: mockHandler,
128		-	root: "test",
129		-	}
130		-
131		-	err := h.Remove([]*rsyncutils.ReceiverFile{})
	141	+	entries, err := fsys.ReadDir("site", true)
132	142		if err != nil {
133		-	t.Fatalf("Remove() returned error: %v", err)
	143	+	t.Fatal(err)
134	144		}
135		-
136		-	if len(mockHandler.deleteCalls) != 5 {
137		-	t.Fatalf("expected 5 delete calls, got %d: %v", len(mockHandler.deleteCalls), mockHandler.deleteCalls)
	145	+	var names []string
	146	+	for _, e := range entries {
	147	+	names = append(names, e.Name)
	148	+	}
	149	+	want := []string{"index.html", "css/site.css", "css", "empty.txt"}
	150	+	if !slices.Equal(names, want) {
	151	+	t.Errorf("got %q, want %q", names, want)
138	152		}
	153	+	}
139	154
140		-	indexOfA := -1
141		-	indexOfAFile := -1
142		-	indexOfB := -1
143		-	indexOfBC := -1
144		-	indexOfBCDeep := -1
	155	+	func TestStat(t *testing.T) {
	156	+	handler := &mockWriteHandler{lists: map[string][]os.FileInfo{
	157	+	"/site": {&mockFileInfo{name: "", isDir: true}},
	158	+	"/site/index.html": {&mockFileInfo{name: "index.html", size: 42}},
	159	+	"/": {&mockFileInfo{name: "/", isDir: true}, &mockFileInfo{name: "hello.md"}},
	160	+	"/posts": {&mockFileInfo{name: "a.md"}, &mockFileInfo{name: "b.md"}},
	161	+	}}
	162	+	fsys := &storageFS{session: newMockSession(), handler: handler}
145	163
146		-	for i, path := range mockHandler.deleteCalls {
147		-	switch path {
148		-	case "/test/a":
149		-	indexOfA = i
150		-	case "/test/a/file.txt":
151		-	indexOfAFile = i
152		-	case "/test/b":
153		-	indexOfB = i
154		-	case "/test/b/c":
155		-	indexOfBC = i
156		-	case "/test/b/c/deep.txt":
157		-	indexOfBCDeep = i
158		-	}
	164	+	if info, err := fsys.Stat("site"); err != nil \|\| !info.IsDir {
	165	+	t.Errorf("site: %+v, %v", info, err)
159	166		}
160		-
161		-	if indexOfAFile > indexOfA {
162		-	t.Errorf("a/file.txt (index %d) should be deleted before a (index %d)", indexOfAFile, indexOfA)
	167	+	if info, err := fsys.Stat("site/index.html"); err != nil \|\| info.IsDir \|\| info.Size != 42 {
	168	+	t.Errorf("site/index.html: %+v, %v", info, err)
163	169		}
164		-	if indexOfBCDeep > indexOfBC {
165		-	t.Errorf("b/c/deep.txt (index %d) should be deleted before b/c (index %d)", indexOfBCDeep, indexOfBC)
	170	+	if info, err := fsys.Stat(""); err != nil \|\| !info.IsDir {
	171	+	t.Errorf("root: %+v, %v", info, err)
166	172		}
167		-	if indexOfBC > indexOfB {
168		-	t.Errorf("b/c (index %d) should be deleted before b (index %d)", indexOfBC, indexOfB)
	173	+	if info, err := fsys.Stat("posts"); err != nil \|\| !info.IsDir {
	174	+	t.Errorf("posts: %+v, %v", info, err)
169	175		}
170		-	}
171		-
172		-	func TestRemove_IgnoresPicoKeepDir(t *testing.T) {
173		-	session, _ := newMockSession()
174		-	mockHandler := &mockWriteHandler{
175		-	entries: []os.FileInfo{
176		-	&mockFileInfo{name: "dir", isDir: true},
177		-	&mockFileInfo{name: "dir/._pico_keep_dir", size: 0},
178		-	&mockFileInfo{name: "dir/file.txt", size: 100},
179		-	},
	176	+	if _, err := fsys.Stat("nope"); !errors.Is(err, fs.ErrNotExist) {
	177	+	t.Errorf("nope: %v", err)
180	178		}
	179	+	}
181	180
182		-	h := &handler{
183		-	session: session,
184		-	writeHandler: mockHandler,
185		-	root: "test",
186		-	}
	181	+	func TestHandlerPathsAreAbsolute(t *testing.T) {
	182	+	handler := &mockWriteHandler{files: map[string]string{"/site/a.txt": "hello"}}
	183	+	fsys := &storageFS{session: newMockSession(), handler: handler}
187	184
188		-	err := h.Remove([]*rsyncutils.ReceiverFile{})
	185	+	f, info, err := fsys.Open("site/a.txt")
189	186		if err != nil {
190		-	t.Fatalf("Remove() returned error: %v", err)
	187	+	t.Fatal(err)
191	188		}
192		-
193		-	for _, path := range mockHandler.deleteCalls {
194		-	if path == "/test/dir/._pico_keep_dir" {
195		-	t.Error("._pico_keep_dir should not be in delete list")
196		-	}
	189	+	buf := make([]byte, info.Size)
	190	+	if _, err := f.ReadAt(buf, 0); err != nil \|\| string(buf) != "hello" {
	191	+	t.Fatalf("read %q, %v", buf, err)
197	192		}
198		-
199		-	if len(mockHandler.deleteCalls) != 2 {
200		-	t.Errorf("expected 2 delete calls (dir, dir/file.txt), got %d: %v", len(mockHandler.deleteCalls), mockHandler.deleteCalls)
	193	+	if _, err := fsys.Put("site/b.txt", fsInfo(5), strings.NewReader("world")); err != nil {
	194	+	t.Fatal(err)
201	195		}
202		-	}
203		-
204		-	func TestRemove_OnlyDeletesFilesNotInWillReceive(t *testing.T) {
205		-	session, _ := newMockSession()
206		-	mockHandler := &mockWriteHandler{
207		-	entries: []os.FileInfo{
208		-	&mockFileInfo{name: "a.txt", size: 100},
209		-	&mockFileInfo{name: "b.txt", size: 100},
210		-	&mockFileInfo{name: "c.txt", size: 100},
211		-	},
	196	+	if err := fsys.Remove("site/c.txt", false); err != nil {
	197	+	t.Fatal(err)
212	198		}
213		-
214		-	h := &handler{
215		-	session: session,
216		-	writeHandler: mockHandler,
217		-	root: "test",
	199	+	want := []string{"read /site/a.txt", "write /site/b.txt", "delete /site/c.txt"}
	200	+	if !slices.Equal(handler.calls, want) {
	201	+	t.Errorf("calls %q, want %q", handler.calls, want)
218	202		}
219		-
220		-	willReceive := []*rsyncutils.ReceiverFile{
221		-	{Name: "a.txt"},
222		-	{Name: "c.txt"},
223		-	}
224		-
225		-	err := h.Remove(willReceive)
226		-	if err != nil {
227		-	t.Fatalf("Remove() returned error: %v", err)
228		-	}
229		-
230		-	if len(mockHandler.deleteCalls) != 1 {
231		-	t.Fatalf("expected 1 delete call, got %d: %v", len(mockHandler.deleteCalls), mockHandler.deleteCalls)
	203	+	if handler.written["/site/b.txt"] != "world" {
	204	+	t.Errorf("written %q", handler.written)
232	205		}
	206	+	}
233	207
234		-	if mockHandler.deleteCalls[0] != "/test/b.txt" {
235		-	t.Errorf("expected to delete /test/b.txt, got %s", mockHandler.deleteCalls[0])
236		-	}
	208	+	func fsInfo(size int64) rsync.FileInfo {
	209	+	return rsync.FileInfo{Size: size, ModTime: time.Unix(1700000000, 0)}
237	210		}

...	...		@@ -9,6 +9,7 @@ import (
9	9		"path/filepath"
10	10		"regexp"
11	11		"strconv"
	12	+	"strings"
12	13
13	14		"github.com/picosh/pico/pkg/pssh"
14	15		"github.com/picosh/pico/pkg/send/utils"

...	...		@@ -85,19 +86,27 @@ func copyFromClient(session *pssh.SSHServerConnSession, info Info, handler utils
85	86		// accepts the header
86	87		_, _ = session.Write(utils.NULL)
87	88
	89	+	reader := utils.NewLimitReader(r, int(size))
88	90		result, err := handler.Write(session, &utils.FileEntry{
89	91		Filepath: filepath.Join(path, name),
90	92		Mode: fs.FileMode(mode),
91	93		Size: size,
92	94		Mtime: mtime,
93	95		Atime: atime,
94		-	Reader: utils.NewLimitReader(r, int(size)),
	96	+	Reader: reader,
95	97		})
	98	+	// A handler that rejects a file may not read all of it. Skip
	99	+	// the rest so the next header is read from the right place.
	100	+	if _, derr := io.Copy(io.Discard, reader); derr != nil {
	101	+	return fmt.Errorf("failed to read file: %q: %w", name, derr)
	102	+	}
96	103
97		-	if err == nil {
	104	+	switch {
	105	+	case err == nil:
98	106		writeSuccess = append(writeSuccess, result)
99		-	} else {
	107	+	case errors.Is(err, utils.ErrFileIgnored):
100	108		writeErrors = append(writeErrors, err)
	109	+	default:
101	110		fmt.Printf("failed to write file: %q: %v\n", name, err)
102	111		}
103	112

...	...		@@ -106,6 +115,13 @@ func copyFromClient(session *pssh.SSHServerConnSession, info Info, handler utils
106	115
107	116		mtime = 0
108	117		atime = 0
	118	+	if err != nil && !errors.Is(err, utils.ErrFileIgnored) {
	119	+	// A warning reply makes the client report the file and
	120	+	// exit non-zero once the rest are sent.
	121	+	msg := strings.ReplaceAll(err.Error(), "\n", " ")
	122	+	_, _ = fmt.Fprintf(session, "\x01%s\n", msg)
	123	+	continue
	124	+	}
109	125		// says 'hey im done'
110	126		_, _ = session.Write(utils.NULL)
111	127		continue

...	...		@@ -2,14 +2,17 @@ package sftp
2	2
3	3		import (
4	4		"bytes"
	5	+	"crypto/sha256"
5	6		"errors"
6	7		"fmt"
7	8		"io"
8	9		"io/fs"
9	10		"os"
10	11		"path/filepath"
11		-
12	12		"slices"
	13	+	"strings"
	14	+	"sync"
	15	+	"time"
13	16
14	17		"github.com/picosh/pico/pkg/pssh"
15	18		"github.com/picosh/pico/pkg/send/utils"

...	...		@@ -33,6 +36,41 @@ func (f listerat) ListAt(ls []os.FileInfo, offset int64) (int, error) {
33	36		type handler struct {
34	37		session *pssh.SSHServerConnSession
35	38		writeHandler utils.CopyFromClientHandler
	39	+
	40	+	mu sync.Mutex
	41	+	pending map[string]*pendingFile
	42	+	// ignored counts open handles to files the handler ignores. A stat of
	43	+	// one is refused so a mounted filesystem reports that it wasn't saved.
	44	+	ignored map[string]int
	45	+	// hidden holds stored files renamed away while open, by the open file
	46	+	// that will replace them. They look deleted to this session.
	47	+	hidden map[string]*pendingFile
	48	+	}
	49	+
	50	+	func (f handler) pendingFile(name string) pendingFile {
	51	+	f.mu.Lock()
	52	+	defer f.mu.Unlock()
	53	+	return f.pending[name]
	54	+	}
	55	+
	56	+	func (f *handler) isHidden(name string) bool {
	57	+	f.mu.Lock()
	58	+	defer f.mu.Unlock()
	59	+	return f.hidden[name] != nil
	60	+	}
	61	+
	62	+	// release returns the files p was renamed from that are still waiting on
	63	+	// it, and shows them again. The caller holds f.mu.
	64	+	func (f handler) release(p pendingFile) []renamedFile {
	65	+	var out []renamedFile
	66	+	for _, old := range p.replaces {
	67	+	if f.hidden[old.path] == p {
	68	+	delete(f.hidden, old.path)
	69	+	out = append(out, old)
	70	+	}
	71	+	}
	72	+	p.replaces = nil
	73	+	return out
36	74		}
37	75
38	76		func (f handler) Filecmd(r sftp.Request) error {

...	...		@@ -42,8 +80,34 @@ func (f handler) Filecmd(r sftp.Request) error {
42	80
43	81		if r.Method == "Rmdir" {
44	82		entry.Mode = os.ModeDir
	83	+	return f.writeHandler.Delete(f.session, entry)
45	84		}
46	85
	86	+	f.mu.Lock()
	87	+	hidden := f.hidden[r.Filepath] != nil
	88	+	p, ok := f.pending[r.Filepath]
	89	+	var replaced []renamedFile
	90	+	if ok {
	91	+	delete(f.pending, r.Filepath)
	92	+	replaced = f.release(p)
	93	+	}
	94	+	f.mu.Unlock()
	95	+	if hidden && !ok {
	96	+	return os.ErrNotExist
	97	+	}
	98	+	// Removing a renamed file removes what it was renamed from.
	99	+	for _, old := range replaced {
	100	+	if err := f.deleteIfUnchanged(old.path, old.sum); err != nil {
	101	+	return err
	102	+	}
	103	+	}
	104	+	if ok {
	105	+	// An open file may have been renamed over a stored one.
	106	+	stored, _, err := f.stored(entry, false)
	107	+	if err != nil \|\| !stored {
	108	+	return err
	109	+	}
	110	+	}
47	111		return f.writeHandler.Delete(f.session, entry)
48	112		case "Mkdir":
49	113		entry := toFileEntry(r)

...	...		@@ -53,7 +117,17 @@ func (f handler) Filecmd(r sftp.Request) error {
53	117		_, err := f.writeHandler.Write(f.session, entry)
54	118
55	119		return err
	120	+	case "Rename":
	121	+	return f.rename(r.Filepath, r.Target, false)
56	122		case "Setstat":
	123	+	// Only an open file's size can change, as when an editor
	124	+	// truncates what it is writing.
	125	+	if p := f.pendingFile(r.Filepath); p != nil && r.AttrFlags().Size {
	126	+	f.mu.Lock()
	127	+	p.dirty = true
	128	+	f.mu.Unlock()
	129	+	p.buf.truncate(int64(r.Attributes().Size))
	130	+	}
57	131		return nil
58	132		}
59	133		return errors.New("unsupported")

...	...		@@ -64,10 +138,29 @@ func (f handler) Filelist(r sftp.Request) (sftp.ListerAt, error) {
64	138		case "List", "Stat":
65	139		list := r.Method == "List"
66	140
	141	+	if !list {
	142	+	f.mu.Lock()
	143	+	p, ignored := f.pending[r.Filepath], f.ignored[r.Filepath] > 0
	144	+	hidden := f.hidden[r.Filepath] != nil
	145	+	f.mu.Unlock()
	146	+	if ignored {
	147	+	return nil, os.ErrPermission
	148	+	}
	149	+	if hidden && p == nil {
	150	+	return nil, os.ErrNotExist
	151	+	}
	152	+	if p != nil {
	153	+	return listerat{p.info(filepath.Base(r.Filepath))}, nil
	154	+	}
	155	+	}
	156	+
67	157		listData, err := f.writeHandler.List(f.session, r.Filepath, list, false)
68	158		if err != nil {
69	159		return nil, err
70	160		}
	161	+	if list {
	162	+	listData = f.withPending(r.Filepath, listData)
	163	+	}
71	164
72	165		// an empty string or exact match from filepath base name is what we want
73	166		if !list {

...	...		@@ -92,6 +185,291 @@ func (f handler) Filelist(r sftp.Request) (sftp.ListerAt, error) {
92	185		return nil, errors.New("unsupported")
93	186		}
94	187
	188	+	// PosixRename is a rename that replaces an existing file at the target.
	189	+	func (f handler) PosixRename(r sftp.Request) error {
	190	+	return f.rename(r.Filepath, r.Target, true)
	191	+	}
	192	+
	193	+	// rename moves a file or a directory. A handler that implements
	194	+	// utils.FileRenamer moves a file in place. Otherwise each file is written
	195	+	// to its new path through the handler, which applies its usual checks, and
	196	+	// the sources are deleted once every write succeeded.
	197	+	func (f *handler) rename(from, to string, replace bool) error {
	198	+	if from == to {
	199	+	return nil
	200	+	}
	201	+	if strings.HasPrefix(to, from+"/") {
	202	+	return fmt.Errorf("cannot move %s into itself", from)
	203	+	}
	204	+	if moved, err := f.renamePending(from, to, replace); moved \|\| err != nil {
	205	+	return err
	206	+	}
	207	+	if f.isHidden(from) {
	208	+	return os.ErrNotExist
	209	+	}
	210	+	toHidden := f.isHidden(to)
	211	+
	212	+	isFile, _, err := f.stored(&utils.FileEntry{Filepath: from}, false)
	213	+	if err != nil {
	214	+	return err
	215	+	}
	216	+	files, dirs := []string{"."}, []string(nil)
	217	+	if !isFile {
	218	+	if files, dirs, err = f.tree(from); err != nil {
	219	+	return err
	220	+	}
	221	+	if len(files)+len(dirs) == 0 {
	222	+	if ok, err := f.isDir(from); err != nil \|\| !ok {
	223	+	return errors.Join(os.ErrNotExist, err)
	224	+	}
	225	+	}
	226	+	}
	227	+	if !replace \|\| !isFile {
	228	+	exists, _, err := f.stored(&utils.FileEntry{Filepath: to}, false)
	229	+	if err != nil {
	230	+	return err
	231	+	}
	232	+	exists = exists && !toHidden
	233	+	belowFiles, belowDirs, err := f.tree(to)
	234	+	if err != nil {
	235	+	return err
	236	+	}
	237	+	if exists \|\| len(belowFiles)+len(belowDirs) > 0 {
	238	+	return os.ErrExist
	239	+	}
	240	+	}
	241	+
	242	+	if isFile {
	243	+	if r, ok := f.writeHandler.(utils.FileRenamer); ok {
	244	+	err := r.Rename(f.session, &utils.FileEntry{Filepath: from}, &utils.FileEntry{Filepath: to})
	245	+	if !errors.Is(err, errors.ErrUnsupported) {
	246	+	if err == nil {
	247	+	f.mu.Lock()
	248	+	delete(f.hidden, to)
	249	+	f.mu.Unlock()
	250	+	}
	251	+	return err
	252	+	}
	253	+	}
	254	+	}
	255	+
	256	+	if !isFile {
	257	+	for _, d := range append([]string{"."}, dirs...) {
	258	+	if _, err := f.writeHandler.Write(f.session, &utils.FileEntry{Filepath: filepath.Join(to, d), Mode: os.ModeDir}); err != nil {
	259	+	return err
	260	+	}
	261	+	}
	262	+	}
	263	+	var written []string
	264	+	sums := make([][]byte, len(files))
	265	+	for i, name := range files {
	266	+	dst := filepath.Join(to, name)
	267	+	sum, err := f.copyFile(filepath.Join(from, name), dst)
	268	+	if err != nil {
	269	+	for _, w := range written {
	270	+	_ = f.writeHandler.Delete(f.session, &utils.FileEntry{Filepath: w})
	271	+	}
	272	+	return err
	273	+	}
	274	+	sums[i] = sum
	275	+	written = append(written, dst)
	276	+	}
	277	+	for i, name := range files {
	278	+	if err := f.deleteIfUnchanged(filepath.Join(from, name), sums[i]); err != nil {
	279	+	return err
	280	+	}
	281	+	}
	282	+	if !isFile {
	283	+	// Children before parents.
	284	+	slices.SortFunc(dirs, func(a, b string) int { return strings.Count(b, "/") - strings.Count(a, "/") })
	285	+	for _, d := range append(dirs, ".") {
	286	+	_ = f.writeHandler.Delete(f.session, &utils.FileEntry{Filepath: filepath.Join(from, d), Mode: os.ModeDir})
	287	+	}
	288	+	}
	289	+	f.mu.Lock()
	290	+	delete(f.hidden, to)
	291	+	f.mu.Unlock()
	292	+	return nil
	293	+	}
	294	+
	295	+	// renamePending moves a file that is open for writing, which happens when
	296	+	// sshfs delivers the close after a rename. Its contents are stored at the
	297	+	// new path when it closes, and a stored source is kept until then.
	298	+	func (f *handler) renamePending(from, to string, replace bool) (bool, error) {
	299	+	f.mu.Lock()
	300	+	p, ok := f.pending[from]
	301	+	if !ok {
	302	+	defer f.mu.Unlock()
	303	+	for name := range f.pending {
	304	+	if strings.HasPrefix(name, from+"/") {
	305	+	return false, fmt.Errorf("cannot move %s while %s is open for writing", from, name)
	306	+	}
	307	+	}
	308	+	return false, nil
	309	+	}
	310	+	_, toOpen := f.pending[to]
	311	+	toHidden := f.hidden[to] != nil
	312	+	stored := p.stored
	313	+	f.mu.Unlock()
	314	+	if v, ok := f.writeHandler.(utils.WriteValidator); ok {
	315	+	if err := v.ValidateWrite(f.session, &utils.FileEntry{Filepath: to, Mode: p.entry.Mode}); err != nil {
	316	+	return true, err
	317	+	}
	318	+	}
	319	+	if !replace {
	320	+	exists, _, err := f.stored(&utils.FileEntry{Filepath: to}, false)
	321	+	if err != nil {
	322	+	return true, err
	323	+	}
	324	+	if (exists && !toHidden) \|\| toOpen {
	325	+	return true, os.ErrExist
	326	+	}
	327	+	}
	328	+
	329	+	var sum []byte
	330	+	if stored {
	331	+	var err error
	332	+	if sum, stored, err = f.checksum(from); err != nil {
	333	+	return true, err
	334	+	}
	335	+	}
	336	+
	337	+	f.mu.Lock()
	338	+	defer f.mu.Unlock()
	339	+	delete(f.pending, from)
	340	+	delete(f.hidden, to)
	341	+	f.pending[to] = p
	342	+	p.path = to
	343	+	p.dirty = true
	344	+	if stored {
	345	+	p.replaces = append(p.replaces, renamedFile{path: from, sum: sum})
	346	+	f.hidden[from] = p
	347	+	}
	348	+	p.stored = false
	349	+	return true, nil
	350	+	}
	351	+
	352	+	// copyFile copies a stored file and returns the checksum of what it copied.
	353	+	func (f *handler) copyFile(from, to string) ([]byte, error) {
	354	+	info, r, err := f.writeHandler.Read(f.session, &utils.FileEntry{Filepath: from})
	355	+	if err != nil {
	356	+	return nil, err
	357	+	}
	358	+	defer func() { _ = r.Close() }()
	359	+	h := sha256.New()
	360	+	msg, err := f.writeHandler.Write(f.session, &utils.FileEntry{
	361	+	Filepath: to,
	362	+	Mode: info.Mode().Perm(),
	363	+	Size: info.Size(),
	364	+	Mtime: info.ModTime().Unix(),
	365	+	Atime: info.ModTime().Unix(),
	366	+	Reader: io.TeeReader(r, h),
	367	+	})
	368	+	if err != nil {
	369	+	return nil, fmt.Errorf("%s: %w", to, err)
	370	+	}
	371	+	if msg != "" {
	372	+	_, _ = fmt.Fprintf(f.session.Stderr(), "%s\r\n", msg)
	373	+	}
	374	+	return h.Sum(nil), nil
	375	+	}
	376	+
	377	+	// checksum hashes the stored file at name.
	378	+	func (f *handler) checksum(name string) ([]byte, bool, error) {
	379	+	stored, data, err := f.stored(&utils.FileEntry{Filepath: name}, true)
	380	+	if err != nil \|\| !stored {
	381	+	return nil, stored, err
	382	+	}
	383	+	sum := sha256.Sum256(data)
	384	+	return sum[:], true, nil
	385	+	}
	386	+
	387	+	// deleteIfUnchanged deletes a stored file unless another session replaced
	388	+	// it since it had sum. Storage has no conditional delete, so a write
	389	+	// between the check and the delete can still be lost.
	390	+	func (f *handler) deleteIfUnchanged(name string, sum []byte) error {
	391	+	cur, stored, err := f.checksum(name)
	392	+	if err != nil \|\| !stored \|\| !bytes.Equal(cur, sum) {
	393	+	return err
	394	+	}
	395	+	return f.writeHandler.Delete(f.session, &utils.FileEntry{Filepath: name})
	396	+	}
	397	+
	398	+	// tree returns the files and directories below dir, relative to it.
	399	+	func (f *handler) tree(dir string) (files, dirs []string, err error) {
	400	+	entries, err := f.writeHandler.List(f.session, dir, true, true)
	401	+	if err != nil {
	402	+	return nil, nil, err
	403	+	}
	404	+	for _, e := range entries {
	405	+	name := strings.Trim(e.Name(), "/")
	406	+	switch {
	407	+	case name == "" \|\| name == ".":
	408	+	case e.IsDir():
	409	+	dirs = append(dirs, name)
	410	+	default:
	411	+	files = append(files, name)
	412	+	}
	413	+	}
	414	+	return files, dirs, nil
	415	+	}
	416	+
	417	+	func (f *handler) isDir(name string) (bool, error) {
	418	+	entries, err := f.writeHandler.List(f.session, filepath.Dir(name), true, false)
	419	+	if err != nil {
	420	+	return false, err
	421	+	}
	422	+	base := filepath.Base(name)
	423	+	return slices.ContainsFunc(entries, func(e os.FileInfo) bool {
	424	+	return e.IsDir() && strings.Trim(e.Name(), "/") == base
	425	+	}), nil
	426	+	}
	427	+
	428	+	// withPending adds the files open for writing in dir to its listing and
	429	+	// drops the ones renamed away.
	430	+	func (f *handler) withPending(dir string, listData []os.FileInfo) []os.FileInfo {
	431	+	f.mu.Lock()
	432	+	defer f.mu.Unlock()
	433	+	for name := range f.hidden {
	434	+	if filepath.Dir(name) == filepath.Clean(dir) {
	435	+	base := filepath.Base(name)
	436	+	listData = slices.DeleteFunc(listData, func(info os.FileInfo) bool { return info.Name() == base })
	437	+	}
	438	+	}
	439	+	for name, p := range f.pending {
	440	+	if filepath.Dir(name) != filepath.Clean(dir) {
	441	+	continue
	442	+	}
	443	+	base := filepath.Base(name)
	444	+	listData = slices.DeleteFunc(listData, func(info os.FileInfo) bool { return info.Name() == base })
	445	+	listData = append(listData, p.info(base))
	446	+	}
	447	+	return listData
	448	+	}
	449	+
	450	+	// stored returns the contents of the file at name if storage has it.
	451	+	// Contents are only read when withContents is set.
	452	+	func (f handler) stored(entry utils.FileEntry, withContents bool) (bool, []byte, error) {
	453	+	list, err := f.writeHandler.List(f.session, entry.Filepath, false, false)
	454	+	if err != nil {
	455	+	return false, nil, err
	456	+	}
	457	+	base := filepath.Base(entry.Filepath)
	458	+	if !slices.ContainsFunc(list, func(info os.FileInfo) bool { return !info.IsDir() && info.Name() == base }) {
	459	+	return false, nil, nil
	460	+	}
	461	+	if !withContents {
	462	+	return true, nil, nil
	463	+	}
	464	+	_, r, err := f.writeHandler.Read(f.session, entry)
	465	+	if err != nil {
	466	+	return false, nil, err
	467	+	}
	468	+	defer func() { _ = r.Close() }()
	469	+	data, err := io.ReadAll(r)
	470	+	return true, data, err
	471	+	}
	472	+
95	473		func toFileEntry(r sftp.Request) utils.FileEntry {
96	474		attrs := r.Attributes()
97	475		var size int64 = 0

...	...		@@ -115,19 +493,54 @@ func toFileEntry(r sftp.Request) utils.FileEntry {
115	493		return entry
116	494		}
117	495
	496	+	// Filewrite opens a file for writing. Nothing is stored until the last
	497	+	// handle to it closes, so a rejected upload leaves storage untouched.
118	498		func (f handler) Filewrite(r sftp.Request) (io.WriterAt, error) {
119	499		entry := toFileEntry(r)
120		-	entry.Reader = bytes.NewReader([]byte{})
121		-
122		-	_, err := f.writeHandler.Write(f.session, entry)
123		-	if err != nil {
124		-	return nil, err
	500	+	if v, ok := f.writeHandler.(utils.WriteValidator); ok {
	501	+	if err := v.ValidateWrite(f.session, entry); err != nil {
	502	+	if errors.Is(err, utils.ErrFileIgnored) {
	503	+	_, _ = fmt.Fprintf(f.session.Stderr(), "%s\r\n", err)
	504	+	f.mu.Lock()
	505	+	f.ignored[r.Filepath]++
	506	+	f.mu.Unlock()
	507	+	return ignoredWriter{handler: f, path: r.Filepath}, nil
	508	+	}
	509	+	return nil, err
	510	+	}
125	511		}
126	512
127		-	buf := &buffer{}
128		-	entry.Reader = buf
	513	+	trunc := r.Pflags().Trunc
	514	+	f.mu.Lock()
	515	+	defer f.mu.Unlock()
	516	+	if p, ok := f.pending[r.Filepath]; ok {
	517	+	p.handles++
	518	+	if trunc {
	519	+	p.buf.truncate(0)
	520	+	p.dirty = true
	521	+	}
	522	+	return &pendingWriter{handler: f, file: p}, nil
	523	+	}
129	524
130		-	return fakeWrite{fileEntry: entry, buf: buf, handler: f}, nil
	525	+	// A file renamed away while open is gone, and this one takes its path.
	526	+	var stored bool
	527	+	var data []byte
	528	+	if f.hidden[r.Filepath] != nil {
	529	+	delete(f.hidden, r.Filepath)
	530	+	trunc = true
	531	+	} else {
	532	+	// Opening without truncating keeps what is stored, as touch does.
	533	+	var err error
	534	+	if stored, data, err = f.stored(entry, !trunc); err != nil {
	535	+	return nil, err
	536	+	}
	537	+	}
	538	+	p := &pendingFile{path: r.Filepath, entry: entry, buf: &buffer{}, modTime: time.Now(), handles: 1, stored: stored, dirty: trunc}
	539	+	if _, err := p.buf.WriteAt(data, 0); err != nil {
	540	+	return nil, err
	541	+	}
	542	+	f.pending[r.Filepath] = p
	543	+	return &pendingWriter{handler: f, file: p}, nil
131	544		}
132	545
133	546		func (f handler) Fileread(r sftp.Request) (io.ReaderAt, error) {

...	...		@@ -135,6 +548,13 @@ func (f handler) Fileread(r sftp.Request) (io.ReaderAt, error) {
135	548		return nil, os.ErrInvalid
136	549		}
137	550
	551	+	if p := f.pendingFile(r.Filepath); p != nil {
	552	+	return bytes.NewReader(p.buf.bytes()), nil
	553	+	}
	554	+	if f.isHidden(r.Filepath) {
	555	+	return nil, os.ErrNotExist
	556	+	}
	557	+
138	558		fileEntry := toFileEntry(r)
139	559		_, reader, err := f.writeHandler.Read(f.session, fileEntry)
140	560

...	...		@@ -152,6 +572,13 @@ func (f handlererr) Filecmd(r sftp.Request) error {
152	572		}
153	573		return err
154	574		}
	575	+	func (f handlererr) PosixRename(r sftp.Request) error {
	576	+	err := f.Handler.PosixRename(r)
	577	+	if err != nil {
	578	+	_, _ = fmt.Fprintln(f.Handler.session.Stderr(), err)
	579	+	}
	580	+	return err
	581	+	}
155	582		func (f handlererr) Filelist(r sftp.Request) (sftp.ListerAt, error) {
156	583		result, err := f.Handler.Filelist(r)
157	584		if err != nil {

...	...		@@ -45,6 +45,9 @@ func Middleware(writeHandler utils.CopyFromClientHandler) pssh.SSHServerMiddlewa
45	45		Handler: &handler{
46	46		session: session,
47	47		writeHandler: writeHandler,
	48	+	pending: map[string]*pendingFile{},
	49	+	ignored: map[string]int{},
	50	+	hidden: map[string]*pendingFile{},
48	51		},
49	52		}
50	53

...	...		@@ -1,9 +1,10 @@
1	1		package sftp
2	2
3	3		import (
	4	+	"bytes"
4	5		"fmt"
5		-	"io"
6	6		"sync"
	7	+	"time"
7	8
8	9		"github.com/picosh/pico/pkg/send/utils"
9	10		)

...	...		@@ -11,7 +12,6 @@ import (
11	12		type buffer struct {
12	13		buf []byte
13	14		m sync.Mutex
14		-	off int
15	15		}
16	16
17	17		func (b *buffer) WriteAt(p []byte, pos int64) (n int, err error) {

...	...		@@ -20,58 +20,136 @@ func (b *buffer) WriteAt(p []byte, pos int64) (n int, err error) {
20	20		b.m.Lock()
21	21		defer b.m.Unlock()
22	22		if int64(len(b.buf)) < expLen {
	23	+	oldLen := len(b.buf)
23	24		if int64(cap(b.buf)) < expLen {
24	25		newBuf := make([]byte, expLen)
25	26		copy(newBuf, b.buf)
26	27		b.buf = newBuf
27	28		}
	29	+	// A truncated buffer keeps its old bytes past its length.
28	30		b.buf = b.buf[:expLen]
	31	+	clear(b.buf[oldLen:])
29	32		}
30	33		copy(b.buf[pos:], p)
31	34		return pLen, nil
32	35		}
33	36
34		-	func (b *buffer) Read(p []byte) (n int, err error) {
	37	+	func (b *buffer) truncate(size int64) {
35	38		b.m.Lock()
36	39		defer b.m.Unlock()
37		-	if len(b.buf) <= b.off {
38		-	if len(p) == 0 {
39		-	return 0, nil
40		-	}
41		-	return 0, io.EOF
	40	+	if size < int64(len(b.buf)) {
	41	+	b.buf = b.buf[:size]
	42	+	return
42	43		}
43		-	n = copy(p, b.buf[b.off:])
44		-	b.off += n
45		-	return n, nil
	44	+	b.buf = append(b.buf, make([]byte, size-int64(len(b.buf)))...)
46	45		}
47	46
48		-	func (b *buffer) Close() error {
49		-	b.buf = []byte{}
50		-	return nil
	47	+	func (b *buffer) bytes() []byte {
	48	+	b.m.Lock()
	49	+	defer b.m.Unlock()
	50	+	return bytes.Clone(b.buf)
51	51		}
52	52
53		-	type fakeWrite struct {
54		-	fileEntry *utils.FileEntry
55		-	handler *handler
56		-	buf *buffer
	53	+	// pendingFile is a file open for writing. Editors over sshfs stat a file
	54	+	// right after creating it, so it is listed from here until the last handle
	55	+	// closes and its contents are written.
	56	+	type pendingFile struct {
	57	+	path string
	58	+	entry *utils.FileEntry
	59	+	buf *buffer
	60	+	modTime time.Time
	61	+	handles int
	62	+	// stored is set when storage had the file at open. Unless a handle
	63	+	// truncated or wrote to it, it's left as it is.
	64	+	stored bool
	65	+	dirty bool
	66	+	// replaces holds stored files this one was renamed from while open.
	67	+	// They are deleted once it is stored at its new path.
	68	+	replaces []renamedFile
57	69		}
58	70
59		-	func (f fakeWrite) WriteAt(p []byte, off int64) (int, error) {
60		-	return f.buf.WriteAt(p, off)
	71	+	// renamedFile is a stored file renamed away, with the checksum it had then.
	72	+	// Another session may replace it before it's deleted.
	73	+	type renamedFile struct {
	74	+	path string
	75	+	sum []byte
61	76		}
62	77
63		-	func (f fakeWrite) Close() error {
64		-	msg, err := f.handler.writeHandler.Write(f.handler.session, f.fileEntry)
65		-	if err != nil {
66		-	errMsg := fmt.Sprintf("%s\r\n", err.Error())
67		-	_, err = f.handler.session.Stderr().Write([]byte(errMsg))
	78	+	func (p pendingFile) info(name string) utils.VirtualFile {
	79	+	return &utils.VirtualFile{
	80	+	FName: name,
	81	+	FSize: int64(len(p.buf.bytes())),
	82	+	FModTime: p.modTime,
68	83		}
69		-	if msg != "" {
70		-	nMsg := fmt.Sprintf("%s\r\n", msg)
71		-	_, err = f.handler.session.Stderr().Write([]byte(nMsg))
	84	+	}
	85	+
	86	+	type pendingWriter struct {
	87	+	handler *handler
	88	+	file *pendingFile
	89	+	}
	90	+
	91	+	func (w *pendingWriter) WriteAt(p []byte, off int64) (int, error) {
	92	+	w.handler.mu.Lock()
	93	+	w.file.dirty = true
	94	+	w.handler.mu.Unlock()
	95	+	return w.file.buf.WriteAt(p, off)
	96	+	}
	97	+
	98	+	func (w *pendingWriter) Close() error {
	99	+	h, f := w.handler, w.file
	100	+	h.mu.Lock()
	101	+	f.handles--
	102	+	// A file removed while open is dropped.
	103	+	commit := f.handles == 0 && h.pending[f.path] == f
	104	+	if commit {
	105	+	delete(h.pending, f.path)
	106	+	}
	107	+	name := f.path
	108	+	h.mu.Unlock()
	109	+	if !commit \|\| (f.stored && !f.dirty) {
	110	+	return nil
72	111		}
	112	+
	113	+	data := f.buf.bytes()
	114	+	entry := *f.entry
	115	+	entry.Filepath = name
	116	+	entry.Reader = bytes.NewReader(data)
	117	+	entry.Size = int64(len(data))
	118	+	msg, err := h.writeHandler.Write(h.session, &entry)
	119	+	h.mu.Lock()
	120	+	replaced := h.release(f)
	121	+	h.mu.Unlock()
73	122		if err != nil {
	123	+	_, _ = fmt.Fprintf(h.session.Stderr(), "%s\r\n", err)
74	124		return err
75	125		}
76		-	return f.buf.Close()
	126	+	if msg != "" {
	127	+	_, _ = fmt.Fprintf(h.session.Stderr(), "%s\r\n", msg)
	128	+	}
	129	+	for _, old := range replaced {
	130	+	if err := h.deleteIfUnchanged(old.path, old.sum); err != nil {
	131	+	_, _ = fmt.Fprintf(h.session.Stderr(), "%s\r\n", err)
	132	+	return err
	133	+	}
	134	+	}
	135	+	return nil
	136	+	}
	137	+
	138	+	// ignoredWriter accepts a file the handler ignores by policy, so a
	139	+	// transfer moves on to the next one.
	140	+	type ignoredWriter struct {
	141	+	handler *handler
	142	+	path string
	143	+	}
	144	+
	145	+	func (ignoredWriter) WriteAt(p []byte, _ int64) (int, error) { return len(p), nil }
	146	+
	147	+	func (w ignoredWriter) Close() error {
	148	+	w.handler.mu.Lock()
	149	+	defer w.handler.mu.Unlock()
	150	+	w.handler.ignored[w.path]--
	151	+	if w.handler.ignored[w.path] <= 0 {
	152	+	delete(w.handler.ignored, w.path)
	153	+	}
	154	+	return nil
77	155		}

...	...		@@ -2,6 +2,7 @@ package utils
2	2
3	3		import (
4	4		"encoding/base64"
	5	+	"errors"
5	6		"fmt"
6	7		"io"
7	8		"io/fs"

...	...		@@ -16,6 +17,9 @@ import (
16	17		// NULL is an array with a single NULL byte.
17	18		var NULL = []byte{'\x00'}
18	19
	20	+	// ErrFileIgnored marks a file deliberately excluded by upload policy.
	21	+	var ErrFileIgnored = errors.New("file ignored")
	22	+
19	23		// FileEntry is an Entry that reads from a Reader, defining a file and
20	24		// its contents.
21	25		type FileEntry struct {

...	...		@@ -66,6 +70,20 @@ type CopyFromClientHandler interface {
66	70		Validate(*pssh.SSHServerConnSession) error
67	71		}
68	72
	73	+	// WriteValidator is implemented by handlers that can check a write before
	74	+	// its contents arrive, so SFTP can refuse a file when it is opened.
	75	+	type WriteValidator interface {
	76	+	ValidateWrite(pssh.SSHServerConnSession, FileEntry) error
	77	+	}
	78	+
	79	+	// FileRenamer is implemented by handlers that can move a stored file in
	80	+	// place, replacing any file at the destination. It returns
	81	+	// errors.ErrUnsupported for a move it can't make, and SFTP copies the file
	82	+	// instead.
	83	+	type FileRenamer interface {
	84	+	Rename(s pssh.SSHServerConnSession, from, to FileEntry) error
	85	+	}
	86	+
69	87		func KeyText(session *pssh.SSHServerConnSession) (string, error) {
70	88		if session.PublicKey() == nil {
71	89		return "", fmt.Errorf("session doesn't have public key")

...	...		@@ -2,6 +2,7 @@ package router
2	2
3	3		import (
4	4		"context"
	5	+	"errors"
5	6		"fmt"
6	7		"log/slog"
7	8		"net"

...	...		@@ -9,6 +10,7 @@ import (
9	10		"net/http/pprof"
10	11		"regexp"
11	12		"strings"
	13	+	"time"
12	14
13	15		"github.com/hashicorp/golang-lru/v2/expirable"
14	16		"github.com/picosh/pico/pkg/db"

...	...		@@ -75,7 +77,7 @@ func (hc *ApiConfig) HasPrivilegedAccess(apiToken string) bool {
75	77		}
76	78
77	79		func (hc ApiConfig) HasPlusOrSpace(user db.User, space string) bool {
78		-	return hc.Dbpool.HasFeatureByUser(user.ID, "plus") \|\| hc.Dbpool.HasFeatureByUser(user.ID, space)
	80	+	return db.HasPlusOrSpace(hc.Dbpool, user.ID, space)
79	81		}
80	82
81	83		func (hc *ApiConfig) CreateCtx(prevCtx context.Context, subdomain string) context.Context {

...	...		@@ -196,25 +198,66 @@ func GetSubdomain(r *http.Request) string {
196	198
197	199		var txtCache = expirable.NewLRU[string, string](2048, nil, shared.CacheTimeout)
198	200
199		-	func GetCustomDomain(host string, space string) string {
200		-	txt := fmt.Sprintf("_%s.%s", space, host)
201		-	record, found := txtCache.Get(txt)
202		-	if found {
203		-	return record
	201	+	// txtMissCache remembers hosts without a TXT record for a short time, so
	202	+	// unknown hosts don't cost a lookup per request and a record that was just
	203	+	// added starts working quickly.
	204	+	var txtMissCache = expirable.NewLRU[string, struct{}](2048, nil, 30*time.Second)
	205	+
	206	+	var lookupTXT = net.LookupTXT
	207	+
	208	+	// ErrNoTXTRecord is returned when a host has no TXT record for a service.
	209	+	var ErrNoTXTRecord = errors.New("no TXT record")
	210	+
	211	+	// TXTRecordName is the record that points host at a site on space.
	212	+	func TXTRecordName(host, space string) string {
	213	+	return fmt.Sprintf("_%s.%s", space, host)
	214	+	}
	215	+
	216	+	// LookupTXTRecords asks DNS for host's records on space, bypassing the
	217	+	// cache. No records returns ErrNoTXTRecord.
	218	+	func LookupTXTRecords(host, space string) ([]string, error) {
	219	+	records, err := lookupTXT(TXTRecordName(host, space))
	220	+	var dnsErr *net.DNSError
	221	+	if errors.As(err, &dnsErr) && dnsErr.IsNotFound {
	222	+	return nil, ErrNoTXTRecord
	223	+	}
	224	+	if err != nil {
	225	+	return nil, err
204	226		}
	227	+	if len(records) == 0 {
	228	+	return nil, ErrNoTXTRecord
	229	+	}
	230	+	return records, nil
	231	+	}
205	232
206		-	records, err := net.LookupTXT(txt)
	233	+	// LookupCustomDomain returns the site host's first record on space names,
	234	+	// bypassing the cache.
	235	+	func LookupCustomDomain(host, space string) (string, error) {
	236	+	records, err := LookupTXTRecords(host, space)
207	237		if err != nil {
208		-	return ""
	238	+	return "", err
209	239		}
	240	+	return strings.TrimSpace(records[0]), nil
	241	+	}
210	242
211		-	for _, v := range records {
212		-	rec := strings.TrimSpace(v)
213		-	txtCache.Add(txt, rec)
214		-	return rec
	243	+	func GetCustomDomain(host string, space string) string {
	244	+	txt := TXTRecordName(host, space)
	245	+	if record, found := txtCache.Get(txt); found {
	246	+	return record
	247	+	}
	248	+	if _, missed := txtMissCache.Get(txt); missed {
	249	+	return ""
215	250		}
216	251
217		-	return ""
	252	+	record, err := LookupCustomDomain(host, space)
	253	+	if errors.Is(err, ErrNoTXTRecord) {
	254	+	txtMissCache.Add(txt, struct{}{})
	255	+	}
	256	+	if err != nil {
	257	+	return ""
	258	+	}
	259	+	txtCache.Add(txt, record)
	260	+	return record
218	261		}
219	262
220	263		func GetApiToken(r *http.Request) string {

...	...		@@ -0,0 +1,52 @@
	1	+	package router
	2	+
	3	+	import (
	4	+	"errors"
	5	+	"net"
	6	+	"testing"
	7	+	)
	8	+
	9	+	func TestGetCustomDomainCachesMisses(t *testing.T) {
	10	+	records := map[string][]string{}
	11	+	var lookups int
	12	+	var fail error
	13	+	lookupTXT = func(name string) ([]string, error) {
	14	+	lookups++
	15	+	if fail != nil {
	16	+	return nil, fail
	17	+	}
	18	+	if r, ok := records[name]; ok {
	19	+	return r, nil
	20	+	}
	21	+	return nil, &net.DNSError{Err: "no such host", Name: name, IsNotFound: true}
	22	+	}
	23	+	t.Cleanup(func() {
	24	+	lookupTXT = net.LookupTXT
	25	+	txtCache.Purge()
	26	+	txtMissCache.Purge()
	27	+	})
	28	+
	29	+	if got := GetCustomDomain("example.com", "pgs"); got != "" \|\| lookups != 1 {
	30	+	t.Fatalf("got %q after %d lookups", got, lookups)
	31	+	}
	32	+	if got := GetCustomDomain("example.com", "pgs"); got != "" \|\| lookups != 1 {
	33	+	t.Fatalf("missing record was looked up again: %q after %d lookups", got, lookups)
	34	+	}
	35	+
	36	+	// The uncached lookup sees a record as soon as it exists.
	37	+	records["_pgs.example.com"] = []string{" antonio-blog "}
	38	+	if got, err := LookupCustomDomain("example.com", "pgs"); got != "antonio-blog" \|\| err != nil {
	39	+	t.Fatalf("LookupCustomDomain = %q, %v", got, err)
	40	+	}
	41	+
	42	+	// Failures other than a missing record are not cached.
	43	+	fail = errors.New("i/o timeout")
	44	+	if got := GetCustomDomain("other.com", "pgs"); got != "" {
	45	+	t.Fatalf("got %q", got)
	46	+	}
	47	+	fail = nil
	48	+	records["_pgs.other.com"] = []string{"antonio-other"}
	49	+	if got := GetCustomDomain("other.com", "pgs"); got != "antonio-other" {
	50	+	t.Fatalf("lookup after a transient failure got %q", got)
	51	+	}
	52	+	}

...	...		@@ -0,0 +1,301 @@
	1	+	package storage_test
	2	+
	3	+	import (
	4	+	"bytes"
	5	+	"crypto/rand"
	6	+	"errors"
	7	+	"fmt"
	8	+	"io"
	9	+	"io/fs"
	10	+	"log/slog"
	11	+	"slices"
	12	+	"strings"
	13	+	"testing"
	14	+	"time"
	15	+
	16	+	"github.com/picosh/pico/pkg/storage"
	17	+	"github.com/picosh/pico/pkg/storage/storagetest"
	18	+	)
	19	+
	20	+	func backends(t testing.T) map[string]func(t testing.T) storage.StorageServe {
	21	+	return map[string]func(t *testing.T) storage.StorageServe{
	22	+	"fs": func(t *testing.T) storage.StorageServe {
	23	+	st, err := storage.NewStorageFS(slog.New(slog.DiscardHandler), t.TempDir())
	24	+	if err != nil {
	25	+	t.Fatal(err)
	26	+	}
	27	+	return st
	28	+	},
	29	+	"memory": func(t *testing.T) storage.StorageServe {
	30	+	st, err := storage.NewStorageMemory(map[string]map[string]string{})
	31	+	if err != nil {
	32	+	t.Fatal(err)
	33	+	}
	34	+	return st
	35	+	},
	36	+	}
	37	+	}
	38	+
	39	+	func put(t *testing.T, st storage.StorageServe, b storage.Bucket, name string, data []byte, mtime time.Time) {
	40	+	t.Helper()
	41	+	_, size, err := st.PutObject(b, name, bytes.NewReader(data), &storage.ObjectInfo{LastModified: mtime})
	42	+	if err != nil {
	43	+	t.Fatalf("put %s: %v", name, err)
	44	+	}
	45	+	if size != int64(len(data)) {
	46	+	t.Fatalf("put %s: size %d, want %d", name, size, len(data))
	47	+	}
	48	+	}
	49	+
	50	+	// entry renders a listing entry so listings compare as sorted strings.
	51	+	func entry(name string, isDir bool, size int64, mtime time.Time) string {
	52	+	if isDir {
	53	+	return name + "/"
	54	+	}
	55	+	return fmt.Sprintf("%s %d %d", name, size, mtime.Unix())
	56	+	}
	57	+
	58	+	func list(t *testing.T, st storage.StorageServe, b storage.Bucket, dir string, recursive bool) []string {
	59	+	t.Helper()
	60	+	infos, err := st.ListObjects(b, dir, recursive)
	61	+	if err != nil {
	62	+	t.Fatalf("list %s: %v", dir, err)
	63	+	}
	64	+	out := []string{}
	65	+	for _, info := range infos {
	66	+	out = append(out, entry(info.Name(), info.IsDir(), info.Size(), info.ModTime()))
	67	+	}
	68	+	slices.Sort(out)
	69	+	return out
	70	+	}
	71	+
	72	+	func TestStorageConformance(t *testing.T) {
	73	+	for name, newStorage := range backends(t) {
	74	+	t.Run(name, func(t *testing.T) {
	75	+	st := newStorage(t)
	76	+	b := storagetest.Bucket(t, st, "conformance")
	77	+
	78	+	t1 := time.Unix(1_700_000_000, 0)
	79	+	t2 := time.Unix(1_700_100_000, 0)
	80	+	put(t, st, b, "/proj/index.html", []byte("<html></html>"), t1)
	81	+	put(t, st, b, "/proj/css/site.css", []byte("body{}"), t2)
	82	+	put(t, st, b, "/proj/css/deep/x.css", []byte("a{}"), t1)
	83	+	put(t, st, b, "/other.txt", []byte("other"), t2)
	84	+	before := time.Now().Add(-time.Second)
	85	+	put(t, st, b, "/proj/now.txt", []byte("now"), time.Time{})
	86	+
	87	+	t.Run("get", func(t *testing.T) {
	88	+	r, info, err := st.GetObject(b, "/proj/index.html")
	89	+	if err != nil {
	90	+	t.Fatal(err)
	91	+	}
	92	+	defer func() { _ = r.Close() }()
	93	+	data, _ := io.ReadAll(r)
	94	+	if string(data) != "<html></html>" \|\| info.Size != 13 \|\| !info.LastModified.Equal(t1) {
	95	+	t.Fatalf("got %q size=%d mtime=%v", data, info.Size, info.LastModified)
	96	+	}
	97	+
	98	+	_, info, err = st.GetObject(b, "/proj/now.txt")
	99	+	if err != nil {
	100	+	t.Fatal(err)
	101	+	}
	102	+	if info.LastModified.Before(before) \|\| info.LastModified.After(time.Now().Add(time.Second)) {
	103	+	t.Fatalf("object stored without an mtime has mtime %v", info.LastModified)
	104	+	}
	105	+
	106	+	if _, _, err := st.GetObject(b, "/proj/missing.txt"); err == nil {
	107	+	t.Fatal("missing object returned no error")
	108	+	}
	109	+	})
	110	+
	111	+	t.Run("stat", func(t *testing.T) {
	112	+	info, err := st.StatObject(b, "/proj/css/site.css")
	113	+	if err != nil {
	114	+	t.Fatal(err)
	115	+	}
	116	+	if info.Size != 6 \|\| !info.LastModified.Equal(t2) {
	117	+	t.Fatalf("size=%d mtime=%v", info.Size, info.LastModified)
	118	+	}
	119	+	for _, missing := range []string{"/proj/missing.txt", "/proj/css", "/proj/css/"} {
	120	+	if _, err := st.StatObject(b, missing); !errors.Is(err, fs.ErrNotExist) {
	121	+	t.Errorf("StatObject(%q) returned %v, want fs.ErrNotExist", missing, err)
	122	+	}
	123	+	}
	124	+	})
	125	+
	126	+	t.Run("list", func(t *testing.T) {
	127	+	now := list(t, st, b, "/proj/now.txt", false)[0]
	128	+	tests := []struct {
	129	+	dir string
	130	+	recursive bool
	131	+	want []string
	132	+	}{
	133	+	{"/proj/index.html", false, []string{entry("index.html", false, 13, t1)}},
	134	+	{"/proj", false, []string{"/"}},
	135	+	{"/proj/", false, []string{"css/", entry("index.html", false, 13, t1), now}},
	136	+	{"/proj/", true, []string{
	137	+	"css/",
	138	+	entry("css/deep/x.css", false, 3, t1),
	139	+	"css/deep/",
	140	+	entry("css/site.css", false, 6, t2),
	141	+	entry("index.html", false, 13, t1),
	142	+	now,
	143	+	}},
	144	+	{"/proj/css/", true, []string{entry("deep/x.css", false, 3, t1), "deep/", entry("site.css", false, 6, t2)}},
	145	+	{"/missing", false, []string{}},
	146	+	{"/missing/", true, []string{}},
	147	+	}
	148	+	for _, tt := range tests {
	149	+	want := slices.Clone(tt.want)
	150	+	slices.Sort(want)
	151	+	if got := list(t, st, b, tt.dir, tt.recursive); !slices.Equal(got, want) {
	152	+	t.Errorf("list(%q, recursive=%v)\n got %q\nwant %q", tt.dir, tt.recursive, got, want)
	153	+	}
	154	+	}
	155	+	})
	156	+
	157	+	t.Run("overwrite", func(t *testing.T) {
	158	+	put(t, st, b, "/proj/css/site.css", []byte("body{color:red}"), t1)
	159	+	r, info, err := st.GetObject(b, "/proj/css/site.css")
	160	+	if err != nil {
	161	+	t.Fatal(err)
	162	+	}
	163	+	data, _ := io.ReadAll(r)
	164	+	_ = r.Close()
	165	+	if string(data) != "body{color:red}" \|\| !info.LastModified.Equal(t1) {
	166	+	t.Fatalf("got %q mtime=%v", data, info.LastModified)
	167	+	}
	168	+	})
	169	+
	170	+	t.Run("random access", func(t *testing.T) {
	171	+	data := make([]byte, 4<<20+12345)
	172	+	_, _ = rand.Read(data)
	173	+	put(t, st, b, "/proj/big.bin", data, t2)
	174	+
	175	+	r, info, err := st.GetObject(b, "/proj/big.bin")
	176	+	if err != nil {
	177	+	t.Fatal(err)
	178	+	}
	179	+	defer func() { _ = r.Close() }()
	180	+	if info.Size != int64(len(data)) \|\| !info.LastModified.Equal(t2) {
	181	+	t.Fatalf("size=%d mtime=%v", info.Size, info.LastModified)
	182	+	}
	183	+	for _, off := range []int64{0, 1<<20 - 10, 5, int64(len(data)) - 100, 3 << 20} {
	184	+	buf := make([]byte, 64<<10)
	185	+	n, err := r.ReadAt(buf, off)
	186	+	want := data[off:min(off+int64(len(buf)), int64(len(data)))]
	187	+	if !bytes.Equal(buf[:n], want) {
	188	+	t.Fatalf("ReadAt(%d) returned wrong data", off)
	189	+	}
	190	+	if n < len(buf) && err != io.EOF {
	191	+	t.Fatalf("short ReadAt(%d) returned %v", off, err)
	192	+	}
	193	+	}
	194	+	if _, err := r.Seek(int64(len(data))-1000, io.SeekStart); err != nil {
	195	+	t.Fatal(err)
	196	+	}
	197	+	tail, err := io.ReadAll(r)
	198	+	if err != nil \|\| !bytes.Equal(tail, data[len(data)-1000:]) {
	199	+	t.Fatalf("read after seek: %d bytes, %v", len(tail), err)
	200	+	}
	201	+	if _, err := r.Seek(0, io.SeekStart); err != nil {
	202	+	t.Fatal(err)
	203	+	}
	204	+	all, err := io.ReadAll(r)
	205	+	if err != nil \|\| !bytes.Equal(all, data) {
	206	+	t.Fatalf("full read: %d bytes, %v", len(all), err)
	207	+	}
	208	+	})
	209	+
	210	+	t.Run("failed upload", func(t *testing.T) {
	211	+	body := io.MultiReader(strings.NewReader(strings.Repeat("x", 1000)), errReader{})
	212	+	_, _, err := st.PutObject(b, "/proj/broken.txt", body, &storage.ObjectInfo{})
	213	+	if err == nil {
	214	+	t.Fatal("upload from a failing reader succeeded")
	215	+	}
	216	+	r, _, err := st.GetObject(b, "/proj/broken.txt")
	217	+	if err == nil {
	218	+	data, _ := io.ReadAll(r)
	219	+	_ = r.Close()
	220	+	t.Fatalf("failed upload left an object of %d bytes", len(data))
	221	+	}
	222	+	})
	223	+
	224	+	t.Run("delete", func(t *testing.T) {
	225	+	if err := st.DeleteObject(b, "/proj/css/deep/x.css"); err != nil {
	226	+	t.Fatal(err)
	227	+	}
	228	+	if err := st.DeleteObject(b, "/proj/css/deep/x.css"); err != nil {
	229	+	t.Fatalf("deleting a missing object: %v", err)
	230	+	}
	231	+	if _, _, err := st.GetObject(b, "/proj/css/deep/x.css"); err == nil {
	232	+	t.Fatal("deleted object still readable")
	233	+	}
	234	+	if _, err := st.StatObject(b, "/proj/css/deep/x.css"); !errors.Is(err, fs.ErrNotExist) {
	235	+	t.Fatalf("StatObject after delete returned %v", err)
	236	+	}
	237	+	got := list(t, st, b, "/proj/css/", true)
	238	+	want := []string{"deep/", entry("site.css", false, 15, t1)}
	239	+	if !slices.Equal(got, want) {
	240	+	t.Fatalf("after delete got %q, want %q", got, want)
	241	+	}
	242	+	})
	243	+
	244	+	t.Run("directories", func(t *testing.T) {
	245	+	if err := st.PutDir(b, "/proj/empty/nested"); err != nil {
	246	+	t.Fatal(err)
	247	+	}
	248	+	if got, want := list(t, st, b, "/proj/empty/", true), []string{"nested/"}; !slices.Equal(got, want) {
	249	+	t.Fatalf("list after PutDir got %q, want %q", got, want)
	250	+	}
	251	+	if got := list(t, st, b, "/proj/empty/nested", false); !slices.Equal(got, []string{"/"}) {
	252	+	t.Fatalf("empty directory not found: %q", got)
	253	+	}
	254	+	if _, err := st.StatObject(b, "/proj/empty/nested"); !errors.Is(err, fs.ErrNotExist) {
	255	+	t.Fatalf("StatObject on a directory returned %v", err)
	256	+	}
	257	+
	258	+	if err := st.DeleteObject(b, "/proj/empty"); err == nil {
	259	+	t.Fatal("deleting a non-empty directory succeeded")
	260	+	}
	261	+	for _, dir := range []string{"/proj/empty/nested", "/proj/empty", "/proj/css/deep"} {
	262	+	if err := st.DeleteObject(b, dir); err != nil {
	263	+	t.Fatalf("delete %s: %v", dir, err)
	264	+	}
	265	+	if got := list(t, st, b, dir, false); len(got) != 0 {
	266	+	t.Fatalf("%s still listed as %q", dir, got)
	267	+	}
	268	+	}
	269	+
	270	+	put(t, st, b, "/solo/only.txt", []byte("x"), t1)
	271	+	if err := st.DeleteObject(b, "/solo/only.txt"); err != nil {
	272	+	t.Fatal(err)
	273	+	}
	274	+	if got := list(t, st, b, "/", false); !slices.Contains(got, "solo/") {
	275	+	t.Fatalf("directory went with its last file: %q", got)
	276	+	}
	277	+	if err := st.DeleteObject(b, "/solo"); err != nil {
	278	+	t.Fatal(err)
	279	+	}
	280	+	if got := list(t, st, b, "/", false); slices.Contains(got, "solo/") {
	281	+	t.Fatalf("deleted directory still listed: %q", got)
	282	+	}
	283	+	})
	284	+
	285	+	t.Run("quota", func(t *testing.T) {
	286	+	size, err := st.GetBucketQuota(b)
	287	+	if err != nil {
	288	+	t.Fatal(err)
	289	+	}
	290	+	want := uint64(13 + 15 + 5 + 3 + 4<<20 + 12345)
	291	+	if size != want {
	292	+	t.Fatalf("quota %d, want %d", size, want)
	293	+	}
	294	+	})
	295	+	})
	296	+	}
	297	+	}
	298	+
	299	+	type errReader struct{}
	300	+
	301	+	func (errReader) Read([]byte) (int, error) { return 0, fmt.Errorf("connection reset") }

...	...		@@ -1,8 +1,6 @@
1	1		package storage
2	2
3	3		import (
4		-	"crypto/md5"
5		-	"encoding/hex"
6	4		"fmt"
7	5		"io"
8	6		"io/fs"

...	...		@@ -37,6 +35,10 @@ func dirSize(path string) (int64, error) {
37	35		return size, err
38	36		}
39	37
	38	+	// legacyKeepDir is the marker pgs used to write to keep empty directories.
	39	+	// Listings hide it until cmd/scripts/clean-keep-dirs has removed them all.
	40	+	const legacyKeepDir = "._pico_keep_dir"
	41	+
40	42		type StorageFS struct {
41	43		Dir string
42	44		Logger *slog.Logger

...	...		@@ -120,31 +122,30 @@ func (s *StorageFS) GetObject(bucket Bucket, fpath string) (utils.ReadAndReaderA
120	122		_ = dat.Close()
121	123		return nil, objInfo, err
122	124		}
	125	+	fillObjectInfo(objInfo, info)
	126	+	return dat, objInfo, nil
	127	+	}
	128	+
	129	+	// fillObjectInfo sets an object's size, mtime and ETag from its file. The
	130	+	// ETag is built from size and mtime, as web servers do, so serving a file
	131	+	// doesn't mean reading it twice.
	132	+	func fillObjectInfo(objInfo *ObjectInfo, info os.FileInfo) {
123	133		objInfo.Size = info.Size()
124	134		objInfo.LastModified = info.ModTime()
	135	+	objInfo.ETag = fmt.Sprintf("%x-%x", info.ModTime().UnixNano(), info.Size())
	136	+	}
125	137
126		-	etag := ""
127		-	// only generate etag if file is less than 10MB
128		-	if info.Size() <= int64(10*MB) {
129		-	// calculate etag
130		-	h := md5.New()
131		-	if _, err := io.Copy(h, dat); err != nil {
132		-	_ = dat.Close()
133		-	return nil, objInfo, err
134		-	}
135		-	md5Sum := h.Sum(nil)
136		-	etag = hex.EncodeToString(md5Sum)
137		-
138		-	// reset os.File reader
139		-	_, err = dat.Seek(0, io.SeekStart)
140		-	if err != nil {
141		-	_ = dat.Close()
142		-	return nil, objInfo, err
143		-	}
	138	+	func (s StorageFS) StatObject(bucket Bucket, fpath string) (ObjectInfo, error) {
	139	+	info, err := os.Stat(filepath.Join(bucket.Path, fpath))
	140	+	if err != nil {
	141	+	return nil, err
144	142		}
145		-
146		-	objInfo.ETag = etag
147		-	return dat, objInfo, nil
	143	+	if info.IsDir() {
	144	+	return nil, fmt.Errorf("%s is a directory: %w", fpath, fs.ErrNotExist)
	145	+	}
	146	+	objInfo := &ObjectInfo{ContentType: mime.GetMimeType(fpath)}
	147	+	fillObjectInfo(objInfo, info)
	148	+	return objInfo, nil
148	149		}
149	150
150	151		func (s StorageFS) PutObject(bucket Bucket, fpath string, contents io.Reader, info ObjectInfo) (string, int64, error) {

...	...		@@ -175,51 +176,18 @@ func (s *StorageFS) PutObject(bucket Bucket, fpath string, contents io.Reader, i
175	176		return loc, size, nil
176	177		}
177	178
	179	+	func (s *StorageFS) PutDir(bucket Bucket, dir string) error {
	180	+	return os.MkdirAll(filepath.Join(bucket.Path, dir), os.ModePerm)
	181	+	}
	182	+
178	183		func (s *StorageFS) DeleteObject(bucket Bucket, fpath string) error {
179	184		loc := filepath.Join(bucket.Path, fpath)
	185	+	// A leftover marker would keep an otherwise empty directory in place.
	186	+	_ = os.Remove(filepath.Join(loc, legacyKeepDir))
180	187		err := os.Remove(loc)
181		-	if err != nil {
182		-	if os.IsNotExist(err) {
183		-	return nil
184		-	}
	188	+	if err != nil && !os.IsNotExist(err) {
185	189		return err
186	190		}
187		-
188		-	// traverse up the folder tree and remove all empty folders
189		-	dir := filepath.Dir(loc)
190		-	for dir != "" {
191		-	f, err := os.Open(dir)
192		-	if err != nil {
193		-	s.Logger.Info("open dir", "dir", dir, "err", err)
194		-	break
195		-	}
196		-	defer func() {
197		-	_ = f.Close()
198		-	}()
199		-
200		-	// https://stackoverflow.com/a/30708914
201		-	contents, err := f.Readdirnames(-1)
202		-	if err != nil {
203		-	s.Logger.Info("read dir", "dir", dir, "err", err)
204		-	break
205		-	}
206		-	if len(contents) > 0 {
207		-	break
208		-	}
209		-
210		-	err = os.Remove(dir)
211		-	if err != nil {
212		-	s.Logger.Info("remove dir", "dir", dir, "err", err)
213		-	break
214		-	}
215		-	fp := strings.Split(dir, "/")
216		-	prefix := ""
217		-	if strings.HasPrefix(loc, "/") {
218		-	prefix = "/"
219		-	}
220		-	dir = prefix + filepath.Join(fp[:len(fp)-1]...)
221		-	}
222		-
223	191		return nil
224	192		}
225	193

...	...		@@ -270,6 +238,9 @@ func (s *StorageFS) ListObjects(bucket Bucket, dir string, recursive bool) ([]os
270	238		if err != nil {
271	239		return err
272	240		}
	241	+	if d.Name() == legacyKeepDir {
	242	+	return nil
	243	+	}
273	244		info, err := d.Info()
274	245		if err != nil {
275	246		return nil

...	...		@@ -299,6 +270,9 @@ func (s *StorageFS) ListObjects(bucket Bucket, dir string, recursive bool) ([]os
299	270		return fileList, nil
300	271		}
301	272		for _, d := range fls {
	273	+	if d.Name() == legacyKeepDir {
	274	+	continue
	275	+	}
302	276		info, err := d.Info()
303	277		if err != nil {
304	278		continue

...	...		@@ -179,10 +179,10 @@ func TestFsAdapter(t *testing.T) {
179	179		t.Fatal("file should have been deleted")
180	180		}
181	181
182		-	// ensure containing folder was also deleted
	182	+	// the containing folder stays, as it would on any filesystem
183	183		_, err = os.Stat(filepath.Join(bucket.Path, "nice"))
184		-	if !os.IsNotExist(err) {
185		-	t.Fatal("containing folder should have been deleted")
	184	+	if err != nil {
	185	+	t.Fatal("containing folder should have been kept")
186	186		}
187	187
188	188		// it should not error if file doesn't exist

...	...		@@ -200,18 +200,14 @@ func TestFsAdapter(t *testing.T) {
200	200		t.Fatal(err)
201	201		}
202	202
203		-	// delete deeply nested file and all parent folders that are now empty
	203	+	// delete deeply nested file; its now empty parent folders stay
204	204		err = st.DeleteObject(bucket, "here/yes/we/can.txt")
205	205		if err != nil {
206	206		t.Fatal(err)
207	207		}
208		-	_, err = os.Stat(filepath.Join(bucket.Path, "here"))
209		-	if os.IsNotExist(err) {
210		-	t.Fatal("this folder had multiple files and should not have been deleted")
211		-	}
212		-	_, err = os.Stat(filepath.Join(bucket.Path, "here/yes"))
213		-	if !os.IsNotExist(err) {
214		-	t.Fatal("containing folder should have been deleted")
	208	+	_, err = os.Stat(filepath.Join(bucket.Path, "here/yes/we"))
	209	+	if err != nil {
	210	+	t.Fatal("containing folder should have been kept")
215	211		}
216	212
217	213		// delete bucket even with file contents

...	...		@@ -220,3 +216,38 @@ func TestFsAdapter(t *testing.T) {
220	216		t.Fatal(err)
221	217		}
222	218		}
	219	+
	220	+	func TestFsLegacyKeepDir(t *testing.T) {
	221	+	st, err := NewStorageFS(slog.New(slog.DiscardHandler), t.TempDir())
	222	+	if err != nil {
	223	+	t.Fatal(err)
	224	+	}
	225	+	bucket, err := st.UpsertBucket("legacy")
	226	+	if err != nil {
	227	+	t.Fatal(err)
	228	+	}
	229	+	for _, name := range []string{"proj/empty/" + legacyKeepDir, "proj/index.html"} {
	230	+	if _, _, err := st.PutObject(bucket, name, strings.NewReader(""), &ObjectInfo{}); err != nil {
	231	+	t.Fatal(err)
	232	+	}
	233	+	}
	234	+
	235	+	for _, recursive := range []bool{false, true} {
	236	+	infos, err := st.ListObjects(bucket, "proj/", recursive)
	237	+	if err != nil {
	238	+	t.Fatal(err)
	239	+	}
	240	+	for _, info := range infos {
	241	+	if strings.Contains(info.Name(), legacyKeepDir) {
	242	+	t.Fatalf("listing (recursive=%v) shows %s", recursive, info.Name())
	243	+	}
	244	+	}
	245	+	}
	246	+
	247	+	if err := st.DeleteObject(bucket, "proj/empty"); err != nil {
	248	+	t.Fatalf("deleting a directory holding only a marker: %v", err)
	249	+	}
	250	+	if _, err := os.Stat(filepath.Join(bucket.Path, "proj/empty")); !os.IsNotExist(err) {
	251	+	t.Fatalf("directory still exists: %v", err)
	252	+	}
	253	+	}

...	...		@@ -4,7 +4,9 @@ import (
4	4		"bytes"
5	5		"fmt"
6	6		"io"
	7	+	"io/fs"
7	8		"os"
	9	+	"path"
8	10		"path/filepath"
9	11		"strings"
10	12		"sync"

...	...		@@ -21,6 +23,7 @@ func (s *seekableReader) Close() error { return nil }
21	23
22	24		type StorageMemory struct {
23	25		storage map[string]map[string]string
	26	+	mtimes map[string]map[string]time.Time
24	27		mu sync.RWMutex
25	28		}
26	29

...	...		@@ -28,11 +31,43 @@ var _ StorageServe = &StorageMemory{}
28	31		var _ StorageServe = (*StorageMemory)(nil)
29	32
30	33		func NewStorageMemory(st map[string]map[string]string) (*StorageMemory, error) {
	34	+	now := time.Now()
	35	+	mtimes := map[string]map[string]time.Time{}
	36	+	for bucket, objects := range st {
	37	+	mtimes[bucket] = map[string]time.Time{}
	38	+	for key := range objects {
	39	+	mtimes[bucket][key] = now
	40	+	}
	41	+	}
31	42		return &StorageMemory{
32	43		storage: st,
	44	+	mtimes: mtimes,
33	45		}, nil
34	46		}
35	47
	48	+	func memoryKey(fpath string) string {
	49	+	if !strings.HasPrefix(fpath, "/") {
	50	+	return "/" + fpath
	51	+	}
	52	+	return fpath
	53	+	}
	54	+
	55	+	// dirKey is the key that keeps a directory listed while it's empty. Keys are
	56	+	// flat like S3's, so a directory is a "dir/" key with no contents.
	57	+	func dirKey(dir string) string {
	58	+	return strings.TrimSuffix(memoryKey(dir), "/") + "/"
	59	+	}
	60	+
	61	+	// hasPrefix reports whether any key other than skip starts with prefix.
	62	+	func hasPrefix(objects map[string]string, prefix, skip string) bool {
	63	+	for key := range objects {
	64	+	if key != skip && strings.HasPrefix(key, prefix) {
	65	+	return true
	66	+	}
	67	+	}
	68	+	return false
	69	+	}
	70	+
36	71		func (s *StorageMemory) GetBucket(name string) (Bucket, error) {
37	72		s.mu.RLock()
38	73		defer s.mu.RUnlock()

...	...		@@ -60,6 +95,7 @@ func (s *StorageMemory) UpsertBucket(name string) (Bucket, error) {
60	95		defer s.mu.Unlock()
61	96
62	97		s.storage[name] = map[string]string{}
	98	+	s.mtimes[name] = map[string]time.Time{}
63	99		return bucket, nil
64	100		}
65	101

...	...		@@ -80,6 +116,7 @@ func (s *StorageMemory) DeleteBucket(bucket Bucket) error {
80	116		defer s.mu.Unlock()
81	117
82	118		delete(s.storage, bucket.Path)
	119	+	delete(s.mtimes, bucket.Path)
83	120		return nil
84	121		}
85	122

...	...		@@ -87,23 +124,34 @@ func (s *StorageMemory) GetObject(bucket Bucket, fpath string) (utils.ReadAndRea
87	124		s.mu.RLock()
88	125		defer s.mu.RUnlock()
89	126
90		-	if !strings.HasPrefix(fpath, "/") {
91		-	fpath = "/" + fpath
92		-	}
93		-
94		-	objInfo := &ObjectInfo{
95		-	LastModified: time.Time{},
96		-	}
	127	+	fpath = memoryKey(fpath)
	128	+	objInfo := &ObjectInfo{}
97	129
98	130		dat, ok := s.storage[bucket.Path][fpath]
99		-	if !ok {
	131	+	if !ok \|\| strings.HasSuffix(fpath, "/") {
100	132		return nil, objInfo, fmt.Errorf("object does not exist: %s", fpath)
101	133		}
102	134
103	135		objInfo.Size = int64(len([]byte(dat)))
	136	+	objInfo.LastModified = s.mtimes[bucket.Path][fpath]
104	137		return &seekableReader{bytes.NewReader([]byte(dat))}, objInfo, nil
105	138		}
106	139
	140	+	func (s StorageMemory) StatObject(bucket Bucket, fpath string) (ObjectInfo, error) {
	141	+	s.mu.RLock()
	142	+	defer s.mu.RUnlock()
	143	+
	144	+	fpath = memoryKey(fpath)
	145	+	dat, ok := s.storage[bucket.Path][fpath]
	146	+	if !ok \|\| strings.HasSuffix(fpath, "/") {
	147	+	return nil, fmt.Errorf("object does not exist: %s: %w", fpath, fs.ErrNotExist)
	148	+	}
	149	+	return &ObjectInfo{
	150	+	Size: int64(len(dat)),
	151	+	LastModified: s.mtimes[bucket.Path][fpath],
	152	+	}, nil
	153	+	}
	154	+
107	155		func (s StorageMemory) PutObject(bucket Bucket, fpath string, contents io.Reader, info ObjectInfo) (string, int64, error) {
108	156		s.mu.Lock()
109	157		defer s.mu.Unlock()

...	...		@@ -113,15 +161,55 @@ func (s *StorageMemory) PutObject(bucket Bucket, fpath string, contents io.Reade
113	161		return "", 0, err
114	162		}
115	163
	164	+	fpath = memoryKey(fpath)
	165	+	mtime := info.LastModified
	166	+	if mtime.IsZero() {
	167	+	mtime = time.Now()
	168	+	}
116	169		s.storage[bucket.Path][fpath] = string(d)
	170	+	if s.mtimes[bucket.Path] == nil {
	171	+	s.mtimes[bucket.Path] = map[string]time.Time{}
	172	+	}
	173	+	s.mtimes[bucket.Path][fpath] = mtime
117	174		return fmt.Sprintf("%s%s", bucket.Path, fpath), int64(len(d)), nil
118	175		}
119	176
	177	+	func (s *StorageMemory) PutDir(bucket Bucket, dir string) error {
	178	+	s.mu.Lock()
	179	+	defer s.mu.Unlock()
	180	+
	181	+	key := dirKey(dir)
	182	+	if key == "/" {
	183	+	return nil
	184	+	}
	185	+	s.storage[bucket.Path][key] = ""
	186	+	s.mtimes[bucket.Path][key] = time.Now()
	187	+	return nil
	188	+	}
	189	+
120	190		func (s *StorageMemory) DeleteObject(bucket Bucket, fpath string) error {
121	191		s.mu.Lock()
122	192		defer s.mu.Unlock()
123	193
124		-	delete(s.storage[bucket.Path], fpath)
	194	+	objects := s.storage[bucket.Path]
	195	+	mtimes := s.mtimes[bucket.Path]
	196	+	fpath = memoryKey(fpath)
	197	+	if _, ok := objects[fpath]; ok && !strings.HasSuffix(fpath, "/") {
	198	+	delete(objects, fpath)
	199	+	delete(mtimes, fpath)
	200	+	if parent := dirKey(path.Dir(fpath)); parent != "/" && !hasPrefix(objects, parent, "") {
	201	+	objects[parent] = ""
	202	+	mtimes[parent] = time.Now()
	203	+	}
	204	+	return nil
	205	+	}
	206	+
	207	+	key := dirKey(fpath)
	208	+	if hasPrefix(objects, key, key) {
	209	+	return fmt.Errorf("%s: directory not empty", fpath)
	210	+	}
	211	+	delete(objects, key)
	212	+	delete(mtimes, key)
125	213		return nil
126	214		}
127	215

...	...		@@ -136,76 +224,71 @@ func (s *StorageMemory) ListBuckets() ([]string, error) {
136	224		return buckets, nil
137	225		}
138	226
	227	+	// ListObjects follows StorageFS: a path naming an object lists that object,
	228	+	// a directory path without a trailing slash lists only the directory itself,
	229	+	// and a path ending in a slash lists its contents relative to it.
139	230		func (s *StorageMemory) ListObjects(bucket Bucket, dir string, recursive bool) ([]os.FileInfo, error) {
140	231		s.mu.RLock()
141	232		defer s.mu.RUnlock()
142	233
143		-	var fileList []os.FileInfo
144		-
145		-	resolved := dir
146		-
147		-	if !strings.HasPrefix(resolved, "/") {
148		-	resolved = "/" + resolved
149		-	}
150		-
	234	+	fileList := []os.FileInfo{}
	235	+	resolved := memoryKey(dir)
151	236		objects := s.storage[bucket.Path]
152		-	// dir is actually an object
153		-	oval, ok := objects[resolved]
154		-	if ok {
	237	+	mtimes := s.mtimes[bucket.Path]
	238	+
	239	+	if val, ok := objects[resolved]; ok && !strings.HasSuffix(resolved, "/") {
155	240		fileList = append(fileList, &utils.VirtualFile{
156	241		FName: filepath.Base(resolved),
157		-	FIsDir: false,
158		-	FSize: int64(len([]byte(oval))),
159		-	FModTime: time.Time{},
	242	+	FSize: int64(len(val)),
	243	+	FModTime: mtimes[resolved],
160	244		})
161	245		return fileList, nil
162	246		}
163	247
164		-	for key, val := range objects {
165		-	if !strings.HasPrefix(key, resolved) {
166		-	continue
	248	+	if !strings.HasSuffix(resolved, "/") {
	249	+	for key := range objects {
	250	+	if strings.HasPrefix(key, resolved+"/") {
	251	+	fileList = append(fileList, &utils.VirtualFile{FName: "", FIsDir: true})
	252	+	break
	253	+	}
167	254		}
	255	+	return fileList, nil
	256	+	}
168	257
169		-	rep := strings.Replace(key, resolved, "", 1)
170		-	fdir := filepath.Dir(rep)
171		-	fname := filepath.Base(rep)
172		-	paths := strings.Split(fdir, "/")
173		-
174		-	if fdir == "/" {
175		-	ffname := filepath.Base(resolved)
176		-	fileList = append(fileList, &utils.VirtualFile{
177		-	FName: ffname,
178		-	FIsDir: true,
179		-	})
	258	+	seen := map[string]bool{}
	259	+	addDir := func(name string) {
	260	+	if !seen[name] {
	261	+	seen[name] = true
	262	+	fileList = append(fileList, &utils.VirtualFile{FName: name, FIsDir: true})
180	263		}
181		-
182		-	for _, p := range paths {
183		-	if p == "" \|\| p == "/" \|\| p == "." {
	264	+	}
	265	+	for key, val := range objects {
	266	+	rel, ok := strings.CutPrefix(key, resolved)
	267	+	if !ok \|\| rel == "" {
	268	+	continue
	269	+	}
	270	+	if !recursive {
	271	+	if d, _, nested := strings.Cut(rel, "/"); nested {
	272	+	addDir(d)
184	273		continue
185	274		}
186		-	fileList = append(fileList, &utils.VirtualFile{
187		-	FName: p,
188		-	FIsDir: true,
189		-	})
190	275		}
191		-
192		-	trimRes := strings.TrimSuffix(resolved, "/")
193		-	dirKey := filepath.Dir(key)
194		-	if recursive {
195		-	fileList = append(fileList, &utils.VirtualFile{
196		-	FName: fname,
197		-	FIsDir: false,
198		-	FSize: int64(len([]byte(val))),
199		-	FModTime: time.Time{},
200		-	})
201		-	} else if resolved == dirKey \|\| trimRes == dirKey {
202		-	fileList = append(fileList, &utils.VirtualFile{
203		-	FName: fname,
204		-	FIsDir: false,
205		-	FSize: int64(len([]byte(val))),
206		-	FModTime: time.Time{},
207		-	})
	276	+	name, isDir := strings.CutSuffix(rel, "/")
	277	+	start := filepath.Dir(name)
	278	+	if isDir {
	279	+	start = name
208	280		}
	281	+	for d := start; d != "." && !seen[d]; d = filepath.Dir(d) {
	282	+	addDir(d)
	283	+	}
	284	+	if isDir {
	285	+	continue
	286	+	}
	287	+	fileList = append(fileList, &utils.VirtualFile{
	288	+	FName: rel,
	289	+	FSize: int64(len(val)),
	290	+	FModTime: mtimes[key],
	291	+	})
209	292		}
210	293
211	294		return fileList, nil

...	...		@@ -31,7 +31,14 @@ type BucketStorage interface {
31	31
32	32		type ObjectStorage interface {
33	33		GetObject(bucket Bucket, fpath string) (utils.ReadAndReaderAtCloser, *ObjectInfo, error)
	34	+	// StatObject returns an object's info without opening it.
	35	+	StatObject(bucket Bucket, fpath string) (*ObjectInfo, error)
34	36		PutObject(bucket Bucket, fpath string, contents io.Reader, info *ObjectInfo) (string, int64, error)
	37	+	// PutDir creates a directory, and its parents, that is listed even
	38	+	// while it's empty.
	39	+	PutDir(bucket Bucket, dir string) error
	40	+	// DeleteObject removes a file or an empty directory. Deleting a missing
	41	+	// path succeeds. A directory stays after its last file is deleted.
35	42		DeleteObject(bucket Bucket, fpath string) error
36	43		ListObjects(bucket Bucket, dir string, recursive bool) ([]os.FileInfo, error)
37	44		}

...	...		@@ -0,0 +1,24 @@
	1	+	// Package storagetest provides storage helpers for tests.
	2	+	package storagetest
	3	+
	4	+	import (
	5	+	"testing"
	6	+
	7	+	"github.com/picosh/pico/pkg/storage"
	8	+	)
	9	+
	10	+	// Bucket creates an empty bucket and removes it when the test ends.
	11	+	func Bucket(t testing.TB, st storage.StorageServe, name string) storage.Bucket {
	12	+	t.Helper()
	13	+	if b, err := st.GetBucket(name); err == nil {
	14	+	if err := st.DeleteBucket(b); err != nil {
	15	+	t.Fatal(err)
	16	+	}
	17	+	}
	18	+	b, err := st.UpsertBucket(name)
	19	+	if err != nil {
	20	+	t.Fatal(err)
	21	+	}
	22	+	t.Cleanup(func() { _ = st.DeleteBucket(b) })
	23	+	return b
	24	+	}

...	...		@@ -39,12 +39,19 @@ func (m *UsageInfo) getKv() []Kv {
39	39		}
40	40		kv := []Kv{
41	41		{Key: label, Value: fmt.Sprintf("%d", m.stats.Num)},
42		-	{Key: "oldest", Value: m.stats.FirstCreatedAt.Format(time.DateOnly)},
43		-	{Key: "newest", Value: m.stats.LastestCreatedAt.Format(time.DateOnly)},
	42	+	{Key: "oldest", Value: formatUsageDate(m.stats.FirstCreatedAt)},
	43	+	{Key: "newest", Value: formatUsageDate(m.stats.LastestCreatedAt)},
44	44		}
45	45		return kv
46	46		}
47	47
	48	+	func formatUsageDate(t time.Time) string {
	49	+	if t.IsZero() {
	50	+	return "-"
	51	+	}
	52	+	return t.Format(time.DateOnly)
	53	+	}
	54	+
48	55		type UserInfo struct {
49	56		shared *SharedModel
50	57		}

...	...		@@ -153,13 +160,42 @@ func (m *FeaturesList) getFeaturesKv() []Kv {
153	160		}
154	161
155	162		type ServicesList struct {
156		-	plusFf *db.FeatureFlag
157		-	pgsFf *db.FeatureFlag
158		-	proseFf *db.FeatureFlag
	163	+	shared *SharedModel
	164	+	data [][]string
	165	+	}
	166	+
	167	+	func NewServicesList(shrd SharedModel) ServicesList {
	168	+	return &ServicesList{shared: shrd}
	169	+	}
	170	+
	171	+	func (m *ServicesList) HandleEvent(ev vaxis.Event, phase vxfw.EventPhase) (vxfw.Command, error) {
	172	+	switch ev.(type) {
	173	+	case vxfw.Init:
	174	+	m.fetchServices()
	175	+	return vxfw.RedrawCmd{}, nil
	176	+	}
	177	+	return nil, nil
159	178		}
160	179
161		-	func NewServicesList(plusFf db.FeatureFlag, pgsFf db.FeatureFlag, proseFf db.FeatureFlag) ServicesList {
162		-	return &ServicesList{plusFf: plusFf, pgsFf: pgsFf, proseFf: proseFf}
	180	+	func (m *ServicesList) fetchServices() {
	181	+	dbpool := m.shared.Dbpool
	182	+	userID := m.shared.User.ID
	183	+	status := func(space, missing string) string {
	184	+	if db.HasPlusOrSpace(dbpool, userID, space) {
	185	+	return "active"
	186	+	}
	187	+	return missing
	188	+	}
	189	+	m.data = [][]string{
	190	+	{"name", "status"},
	191	+	{"pipe", "active"},
	192	+	{"pastes", status("pastes", "invite/pico+")},
	193	+	{"prose", status("prose", "invite/pico+")},
	194	+	{"pages", status("pgs", "invite/pico+")},
	195	+	{"tuns", status("tuns", "pico+")},
	196	+	{"irc bouncer", status("bouncer", "pico+")},
	197	+	{"feeds", status("feeds", "pico+")},
	198	+	}
163	199		}
164	200
165	201		func (m *ServicesList) Draw(ctx vxfw.DrawContext) (vxfw.Surface, error) {

...	...		@@ -167,7 +203,7 @@ func (m *ServicesList) Draw(ctx vxfw.DrawContext) (vxfw.Surface, error) {
167	203		brd := NewBorder(services)
168	204		brd.Label = "services"
169	205		brd.Style = vaxis.Style{Foreground: purp}
170		-	servicesHeight := 7
	206	+	servicesHeight := 8
171	207		return brd.Draw(vxfw.DrawContext{
172	208		Characters: ctx.Characters,
173	209		Max: vxfw.Size{

...	...		@@ -178,42 +214,8 @@ func (m *ServicesList) Draw(ctx vxfw.DrawContext) (vxfw.Surface, error) {
178	214		}
179	215
180	216		func (m *ServicesList) getServiceKv() []Kv {
181		-	hasPlus := m.plusFf != nil && m.plusFf.IsValid()
182		-	hasPgs := m.pgsFf != nil && m.pgsFf.IsValid()
183		-	hasProse := m.proseFf != nil && m.proseFf.IsValid()
184		-	pagesStatus := "invite/pico+"
185		-	if hasPlus \|\| hasPgs {
186		-	pagesStatus = "active"
187		-	}
188		-	proseStatus := "invite/pico+"
189		-	if hasPlus \|\| hasProse {
190		-	proseStatus = "active"
191		-	}
192		-	data := [][]string{
193		-	{"name", "status"},
194		-	{"pipe", "active"},
195		-	{"prose", proseStatus},
196		-	{"pages", pagesStatus},
197		-	}
198		-
199		-	if hasPlus {
200		-	data = append(
201		-	data,
202		-	[]string{"tuns", "active"},
203		-	[]string{"irc bouncer", "active"},
204		-	[]string{"rss-to-email", "active"},
205		-	)
206		-	} else {
207		-	data = append(
208		-	data,
209		-	[]string{"tuns", "pico+"},
210		-	[]string{"irc bouncer", "pico+"},
211		-	[]string{"rss-to-email", "pico+"},
212		-	)
213		-	}
214		-
215	217		kv := []Kv{}
216		-	for idx, d := range data {
	218	+	for idx, d := range m.data {
217	219		value := d[1]
218	220		var style vaxis.Style
219	221		if idx == 0 {

...	...		@@ -143,7 +143,7 @@ func (m *AddInvitePage) Draw(ctx vxfw.DrawContext) (vxfw.Surface, error) {
143	143		root := vxfw.NewSurface(w, h, m)
144	144		ah := 0
145	145
146		-	header := text.New("Invite a user to pico! Invitee must already have a pico account. This grants them `pgs` and `prose` access on their respective free tiers. Only pico+ members or users that received an invite can invite other users.")
	146	+	header := text.New("Invite a user to pico! Invitee must already have a pico account. This grants them `pgs`, `prose` and `pastes` access on their respective free tiers. Only pico+ members or users with an active invite can invite other users.")
147	147		headerSurf, _ := header.Draw(ctx)
148	148		root.AddChild(0, ah, headerSurf)
149	149		ah += int(headerSurf.Size.Height) + 1

...	...		@@ -13,7 +13,7 @@ var menuChoices = []string{
13	13		"invite",
14	14		"tokens",
15	15		"logs",
16		-	"access_logs",
	16	+	"access-logs",
17	17		"analytics",
18	18		"pages",
19	19		"tuns",

...	...		@@ -28,6 +28,7 @@ type MenuPage struct {
28	28
29	29		list list.Dynamic
30	30		features *FeaturesList
	31	+	services *ServicesList
31	32		stats *db.UserStats
32	33		}
33	34

...	...		@@ -50,6 +51,7 @@ func NewMenuPage(shrd SharedModel) MenuPage {
50	51		m := &MenuPage{shared: shrd}
51	52		m.list = list.Dynamic{Builder: getMenuWidget, DrawCursor: true}
52	53		m.features = NewFeaturesList(shrd)
	54	+	m.services = NewServicesList(shrd)
53	55		return m
54	56		}
55	57

...	...		@@ -74,8 +76,10 @@ func (m *MenuPage) HandleEvent(ev vaxis.Event, phase vxfw.EventPhase) (vxfw.Comm
74	76		case PageIn:
75	77		_ = m.fetchUsageStats()
76	78		cmd, _ := m.features.HandleEvent(vxfw.Init{}, phase)
	79	+	servicesCmd, _ := m.services.HandleEvent(vxfw.Init{}, phase)
77	80		return vxfw.BatchCmd([]vxfw.Command{
78	81		cmd,
	82	+	servicesCmd,
79	83		vxfw.FocusWidgetCmd(&m.list),
80	84		}), nil
81	85		case vaxis.Key:

...	...		@@ -105,11 +109,7 @@ func (m *MenuPage) Draw(ctx vxfw.DrawContext) (vxfw.Surface, error) {
105	109		},
106	110		})
107	111
108		-	services, _ := NewServicesList(
109		-	m.shared.PlusFeatureFlag,
110		-	m.shared.PgsFeatureFlag,
111		-	m.shared.ProseFeatureFlag,
112		-	).Draw(ctx)
	112	+	services, _ := m.services.Draw(ctx)
113	113		features, _ := m.features.Draw(ctx)
114	114
115	115		leftPane := NewGroupStack([]vxfw.Surface{

...	...		@@ -127,10 +127,12 @@ func (m *MenuPage) Draw(ctx vxfw.DrawContext) (vxfw.Surface, error) {
127	127		if m.stats != nil {
128	128		pages, _ := NewUsageInfo("pages", &m.stats.Pages).Draw(ctx)
129	129		prose, _ := NewUsageInfo("prose", &m.stats.Prose).Draw(ctx)
130		-	feeds, _ := NewUsageInfo("rss-to-email", &m.stats.Feeds).Draw(ctx)
	130	+	pastes, _ := NewUsageInfo("pastes", &m.stats.Pastes).Draw(ctx)
	131	+	feeds, _ := NewUsageInfo("feeds", &m.stats.Feeds).Draw(ctx)
131	132		right = append(right,
132	133		pages,
133	134		prose,
	135	+	pastes,
134	136		feeds,
135	137		)
136	138		}

...	...		@@ -31,8 +31,6 @@ type SharedModel struct {
31	31		InvitedBy string
32	32		PlusFeatureFlag *db.FeatureFlag
33	33		BouncerFeatureFlag *db.FeatureFlag
34		-	PgsFeatureFlag *db.FeatureFlag
35		-	ProseFeatureFlag *db.FeatureFlag
36	34		Impersonator string
37	35		App *vxfw.App
38	36		}

...	...		@@ -276,12 +274,6 @@ func initData(shrd *SharedModel) error {
276	274		bff, _ := FindFeatureFlag(shrd, "bouncer")
277	275		shrd.BouncerFeatureFlag = bff
278	276
279		-	pff, _ := FindFeatureFlag(shrd, "pgs")
280		-	shrd.PgsFeatureFlag = pff
281		-
282		-	prosef, _ := FindFeatureFlag(shrd, "prose")
283		-	shrd.ProseFeatureFlag = prosef
284		-
285	277		if shrd.User != nil {
286	278		invitedBy, _ := shrd.Dbpool.FindWhoInvitedUser(user.ID)
287	279		shrd.InvitedBy = invitedBy

...	...		@@ -366,7 +358,7 @@ func NewTui(opts vaxis.Options, shrd *SharedModel) error {
366	358		"analytics": NewAnalyticsPage(shrd),
367	359		"chat": NewChatPage(shrd),
368	360		"tuns": NewTunsPage(shrd),
369		-	"access_logs": NewAccessLogsPage(shrd),
	361	+	"access-logs": NewAccessLogsPage(shrd),
370	362		"pages": NewPagesPage(shrd),
371	363		}
372	364		root := &App{

...	...		@@ -0,0 +1,10 @@
	1	+	INSERT INTO feature_flags (user_id, name, expires_at)
	2	+	SELECT DISTINCT ON (ff.user_id) ff.user_id, 'pastes', ff.expires_at
	3	+	FROM feature_flags ff
	4	+	JOIN invites ON invites.to_user_id = ff.user_id
	5	+	WHERE ff.name = 'prose'
	6	+	AND NOT EXISTS (
	7	+	SELECT 1 FROM feature_flags existing
	8	+	WHERE existing.user_id = ff.user_id AND existing.name = 'pastes'
	9	+	)
	10	+	ORDER BY ff.user_id, ff.expires_at DESC;

Commit 4e51b51

Commit `4e51b51`