feat(pico): add dns-check and speed up pgs serving

...	...		@@ -88,6 +88,7 @@ PGS_STORAGE_DIR=.storage
88	88		PGS_DEBUG=1
89	89		PGS_CACHE_TTL=600s
90	90		PGS_CACHE_MAX_ITEMS=0
	91	+	PGS_CACHE_MAX_BODY_SIZE=0
91	92
92	93		PICO_CADDYFILE=./caddy/Caddyfile.pico
93	94		PICO_V4=

...	...		@@ -28,13 +28,13 @@ func main() {
28	28		}()
29	29		cfg := pgs.NewPgsConfig(logger, nil, nil, drain)
30	30		proxy := newProxyServe(cfg.Logger)
31		-	httpCache := pgs.NewPgsHttpCache(cfg, proxy)
	31	+	httpCache, cache := pgs.NewPgsHttpCache(cfg, proxy)
32	32		cacher := &cachedHttp{
33	33		Logger: cfg.Logger,
34	34		Cache: httpCache,
35	35		}
36	36
37		-	go pgs.CacheMgmt(ctx, cfg.CacheClearingQueue, cfg, httpCache.Cache)
	37	+	go pgs.CacheMgmt(ctx, cfg, cache, nil)
38	38
39	39		portStr := fmt.Sprintf(":%s", cfg.WebPort)
40	40		cfg.Logger.Info(

...	...		@@ -233,8 +233,11 @@ func calcRoutes(projectName, fp string, userRedirects []RedirectRule) []HttpRe
233	233		case "wildcard":
234	234		fallthrough
235	235		case "variable":
236		-	rr := regexp.MustCompile(fromMatcher)
237		-	match = rr.FindStringSubmatch(fp)
	236	+	// The pattern comes from the user's _redirects, so an invalid one
	237	+	// is a rule that never matches.
	238	+	if rr, err := regexp.Compile(fromMatcher); err == nil {
	239	+	match = rr.FindStringSubmatch(fp)
	240	+	}
238	241		case "none":
239	242		fallthrough
240	243		default:

...	...		@@ -80,6 +80,12 @@ func (c *Cmd) notice() {
80	80		}
81	81		}
82	82
	83	+	// purge clears what the web servers cache about a project after a command
	84	+	// changed it.
	85	+	func (c *Cmd) purge(projectName string) {
	86	+	c.Cfg.CacheClearingQueue <- getSurrogateKey(c.User.Name, projectName)
	87	+	}
	88	+
83	89		func (c *Cmd) RmProjectAssets(projectName string) error {
84	90		bucketName := shared.GetAssetBucketName(c.User.ID)
85	91		bucket, err := c.Store.GetBucket(bucketName)

...	...		@@ -98,6 +104,9 @@ func (c *Cmd) RmProjectAssets(projectName string) error {
98	104		return nil
99	105		}
100	106		c.output(fmt.Sprintf("found (%d) assets for project (%s), removing", len(fileList), projectName))
	107	+	if c.Write {
	108	+	defer c.purge(projectName)
	109	+	}
101	110
102	111		for _, file := range fileList {
103	112		if file.IsDir() {

...	...		@@ -291,6 +300,9 @@ func (c *Cmd) unlink(projectName string) error {
291	300		if err != nil {
292	301		return err
293	302		}
	303	+	if c.Write {
	304	+	c.purge(project.Name)
	305	+	}
294	306		c.output(fmt.Sprintf("(%s) unlinked", project.Name))
295	307
296	308		return nil

...	...		@@ -365,6 +377,9 @@ func (c *Cmd) link(projectName, linkTo string) error {
365	377		if err != nil {
366	378		return err
367	379		}
	380	+	if c.Write {
	381	+	c.purge(projectName)
	382	+	}
368	383
369	384		out := fmt.Sprintf("(%s) might have orphaned assets, removing", projectName)
370	385		c.output(out)

...	...		@@ -463,6 +478,7 @@ func (c *Cmd) prune(prefix string, keepNumLatest int) error {
463	478		if err != nil {
464	479		return err
465	480		}
	481	+	c.purge(project.Name)
466	482		}
467	483		}
468	484

...	...		@@ -500,6 +516,7 @@ func (c *Cmd) rm(projectName string) error {
500	516		if err != nil {
501	517		return err
502	518		}
	519	+	c.purge(project.Name)
503	520		}
504	521		} else {
505	522		msg := fmt.Sprintf("(%s) project record not found for user (%s)", projectName, c.User.Name)

...	...		@@ -524,7 +541,10 @@ func (c *Cmd) acl(projectName, aclType string, acls []string) error {
524	541		Data: acls,
525	542		}
526	543		if c.Write {
527		-	return c.Dbpool.UpdateProjectAcl(c.User.ID, projectName, acl)
	544	+	if err := c.Dbpool.UpdateProjectAcl(c.User.ID, projectName, acl); err != nil {
	545	+	return err
	546	+	}
	547	+	c.purge(projectName)
528	548		}
529	549		return nil
530	550		}

...	...		@@ -0,0 +1,63 @@
	1	+	package pgs
	2	+
	3	+	import (
	4	+	"bytes"
	5	+	"io"
	6	+	"log/slog"
	7	+	"testing"
	8	+
	9	+	"github.com/picosh/pico/pkg/shared"
	10	+	"github.com/picosh/pico/pkg/storage"
	11	+	)
	12	+
	13	+	type fakeCmdSession struct{ bytes.Buffer }
	14	+
	15	+	func (s *fakeCmdSession) Exit(int) error { return nil }
	16	+	func (s *fakeCmdSession) Close() error { return nil }
	17	+	func (s *fakeCmdSession) Stderr() io.ReadWriter { return &s.Buffer }
	18	+
	19	+	// Commands that change how a project is served purge its cached pages and
	20	+	// records.
	21	+	func TestCliPurges(t *testing.T) {
	22	+	logger := slog.New(slog.DiscardHandler)
	23	+	dbpool := NewPgsDb(logger)
	24	+	user := dbpool.Users[0]
	25	+	if _, err := dbpool.InsertProject(user.ID, "other", "other"); err != nil {
	26	+	t.Fatal(err)
	27	+	}
	28	+	st, err := storage.NewStorageMemory(map[string]map[string]string{
	29	+	shared.GetAssetBucketName(user.ID): {"/test/index.html": "hello"},
	30	+	})
	31	+	if err != nil {
	32	+	t.Fatal(err)
	33	+	}
	34	+	cfg := NewPgsConfig(logger, dbpool, st, discardPubsub{})
	35	+	cmd := &Cmd{User: user, Session: &fakeCmdSession{}, Log: logger, Store: st, Dbpool: dbpool, Write: true, Cfg: cfg}
	36	+
	37	+	tests := []struct {
	38	+	name string
	39	+	run func() error
	40	+	}{
	41	+	{"acl", func() error { return cmd.acl("test", "private", nil) }},
	42	+	{"link", func() error { return cmd.link("test", "other") }},
	43	+	{"unlink", func() error { return cmd.unlink("test") }},
	44	+	{"rm", func() error { return cmd.rm("test") }},
	45	+	}
	46	+	want := getSurrogateKey(user.Name, "test")
	47	+	for _, tt := range tests {
	48	+	if err := tt.run(); err != nil {
	49	+	t.Fatalf("%s: %v", tt.name, err)
	50	+	}
	51	+	select {
	52	+	case got := <-cfg.CacheClearingQueue:
	53	+	if got != want {
	54	+	t.Fatalf("%s purged %q, want %q", tt.name, got, want)
	55	+	}
	56	+	default:
	57	+	t.Fatalf("%s purged nothing", tt.name)
	58	+	}
	59	+	for len(cfg.CacheClearingQueue) > 0 {
	60	+	<-cfg.CacheClearingQueue
	61	+	}
	62	+	}
	63	+	}

...	...		@@ -15,6 +15,7 @@ import (
15	15		type PgsConfig struct {
16	16		CacheTTL time.Duration
17	17		CacheMaxItems int
	18	+	CacheMaxBodySize int64
18	19		Domain string
19	20		MaxAssetSize int64
20	21		MaxSize uint64

...	...		@@ -65,6 +66,10 @@ func (c *PgsConfig) StaticPath(fname string) string {
65	66		var maxSize = uint64(50 * shared.MB)
66	67		var maxAssetSize = int64(10 * shared.MB)
67	68
	69	+	// Larger responses stream from storage instead of being held in the cache.
	70	+	// Zero means no limit.
	71	+	var cacheMaxBodySize = int64(0)
	72	+
68	73		// Needs to be small for caching files like _headers and _redirects.
69	74		var maxSpecialFileSize = int64(5 * shared.KB)
70	75

...	...		@@ -81,6 +86,10 @@ func NewPgsConfig(logger *slog.Logger, dbpool pgsdb.PgsDB, st storage.StorageSer
81	86		if cacheMaxItemsStr != "" {
82	87		cacheMaxItems, _ = strconv.Atoi(cacheMaxItemsStr)
83	88		}
	89	+	cacheMaxBody, err := strconv.ParseInt(shared.GetEnv("PGS_CACHE_MAX_BODY_SIZE", ""), 10, 64)
	90	+	if err != nil {
	91	+	cacheMaxBody = cacheMaxBodySize
	92	+	}
84	93
85	94		sshHost := shared.GetEnv("PGS_SSH_HOST", "0.0.0.0")
86	95		sshPort := shared.GetEnv("PGS_SSH_PORT", "2222")

...	...		@@ -88,6 +97,7 @@ func NewPgsConfig(logger *slog.Logger, dbpool pgsdb.PgsDB, st storage.StorageSer
88	97		cfg := PgsConfig{
89	98		CacheTTL: cacheTTL,
90	99		CacheMaxItems: cacheMaxItems,
	100	+	CacheMaxBodySize: cacheMaxBody,
91	101		Domain: domain,
92	102		MaxAssetSize: maxAssetSize,
93	103		MaxSize: maxSize,

...	...		@@ -3,6 +3,7 @@ package pgsdb
3	3		import (
4	4		"fmt"
5	5		"log/slog"
	6	+	"slices"
6	7		"time"
7	8
8	9		"github.com/google/uuid"

...	...		@@ -149,11 +150,22 @@ func (me MemoryDB) UpsertProject(userID, projectName, projectDir string) (db.P
149	150		}
150	151
151	152		func (me *MemoryDB) LinkToProject(userID, projectID, projectDir string, commit bool) error {
152		-	return errNotImpl
	153	+	for _, project := range me.Projects {
	154	+	if project.ID == projectID && project.UserID == userID {
	155	+	if commit {
	156	+	project.ProjectDir = projectDir
	157	+	}
	158	+	return nil
	159	+	}
	160	+	}
	161	+	return fmt.Errorf("project not found by id %s", projectID)
153	162		}
154	163
155	164		func (me *MemoryDB) RemoveProject(projectID string) error {
156		-	return errNotImpl
	165	+	me.Projects = slices.DeleteFunc(me.Projects, func(project *db.Project) bool {
	166	+	return project.ID == projectID
	167	+	})
	168	+	return nil
157	169		}
158	170
159	171		func (me MemoryDB) FindProjectByName(userID, name string) (db.Project, error) {

...	...		@@ -172,7 +184,13 @@ func (me MemoryDB) FindProjectByName(userID, name string) (db.Project, error)
172	184		}
173	185
174	186		func (me MemoryDB) FindProjectLinks(userID, name string) ([]db.Project, error) {
175		-	return []*db.Project{}, errNotImpl
	187	+	links := []*db.Project{}
	188	+	for _, project := range me.Projects {
	189	+	if project.UserID == userID && project.ProjectDir == name && project.Name != name {
	190	+	links = append(links, project)
	191	+	}
	192	+	}
	193	+	return links, nil
176	194		}
177	195
178	196		func (me MemoryDB) FindProjectsByPrefix(userID, prefix string) ([]db.Project, error) {

...	...		@@ -195,7 +213,12 @@ func (me MemoryDB) FindProjects(userID string) ([]db.Project, error) {
195	213		}
196	214
197	215		func (me *MemoryDB) UpdateProjectAcl(userID, name string, acl db.ProjectAcl) error {
198		-	return errNotImpl
	216	+	project, err := me.FindProjectByName(userID, name)
	217	+	if err != nil {
	218	+	return err
	219	+	}
	220	+	project.Acl = acl
	221	+	return nil
199	222		}
200	223
201	224		func (me *MemoryDB) RegisterAdmin(username, pubkey, pubkeyName string) error {

...	...		@@ -92,24 +92,20 @@ func toDisplayEntries(entries []os.FileInfo) []dirEntryDisplay {
92	92		return displayEntries
93	93		}
94	94
95		-	func shouldGenerateListing(st storage.StorageServe, bucket storage.Bucket, projectDir string, path string) bool {
	95	+	// dirListing returns the entries to list for a path, and false when the path
	96	+	// is not a directory or has an index.html to serve instead.
	97	+	func dirListing(st storage.StorageServe, bucket storage.Bucket, projectDir string, path string) ([]os.FileInfo, bool) {
96	98		dirPath := projectDir + path
97		-	if path == "/" {
98		-	dirPath = projectDir + "/"
99		-	}
100	99
101	100		entries, err := st.ListObjects(bucket, dirPath, false)
102	101		if err != nil \|\| len(entries) == 0 {
103		-	return false
	102	+	return nil, false
104	103		}
105	104
106		-	indexPath := dirPath + "index.html"
107		-	obj, _, err := st.GetObject(bucket, indexPath)
108		-	if err != nil {
109		-	return true
	105	+	if _, err := st.StatObject(bucket, dirPath+"index.html"); err == nil {
	106	+	return nil, false
110	107		}
111		-	_ = obj.Close()
112		-	return false
	108	+	return entries, true
113	109		}
114	110
115	111		func generateDirectoryHTML(path string, entries []os.FileInfo) string {

...	...		@@ -211,10 +211,10 @@ func TestShouldGenerateListing(t *testing.T) {
211	211		st, _ := storage.NewStorageMemory(fixture.Storage)
212	212		bucket := storage.Bucket{Name: "testbucket", Path: "testbucket"}
213	213
214		-	result := shouldGenerateListing(st, bucket, "project", fixture.Path)
	214	+	_, result := dirListing(st, bucket, "project", fixture.Path)
215	215
216	216		if result != fixture.Expected {
217		-	t.Errorf("shouldGenerateListing(%q) = %v, want %v", fixture.Path, result, fixture.Expected)
	217	+	t.Errorf("dirListing(%q) = %v, want %v", fixture.Path, result, fixture.Expected)
218	218		}
219	219		})
220	220		}

...	...		@@ -2,6 +2,8 @@ package pgs
2	2
3	3		import (
4	4		"fmt"
	5	+	"log/slog"
	6	+	"regexp"
5	7		"slices"
6	8		"strings"
7	9		)

...	...		@@ -82,6 +84,27 @@ func parseHeaderText(text string) ([]*HeaderRule, error) {
82	84		return rules, nil
83	85		}
84	86
	87	+	// headerMatcher is a _headers rule with its path compiled.
	88	+	type headerMatcher struct {
	89	+	path *regexp.Regexp
	90	+	headers []*HeaderLine
	91	+	}
	92	+
	93	+	// compileHeaderRules compiles each rule's path once, when the rules are
	94	+	// cached, and drops rules whose path is not a valid pattern.
	95	+	func compileHeaderRules(rules []HeaderRule, logger slog.Logger) []headerMatcher {
	96	+	matchers := make([]headerMatcher, 0, len(rules))
	97	+	for _, rule := range rules {
	98	+	re, err := regexp.Compile(rule.Path)
	99	+	if err != nil {
	100	+	logger.Error("skipping _headers rule with invalid path", "path", rule.Path, "err", err)
	101	+	continue
	102	+	}
	103	+	matchers = append(matchers, headerMatcher{path: re, headers: rule.Headers})
	104	+	}
	105	+	return matchers
	106	+	}
	107	+
85	108		func parseLine(line string) (*HeaderLine, error) {
86	109		rule := &HeaderLine{}
87	110

...	...		@@ -109,10 +109,10 @@ func (p *PromCacheMetrics) AddUpstreamRequest() {
109	109		p.UpstreamReq.Add(1)
110	110		}
111	111
112		-	func NewPgsHttpCache(cfg PgsConfig, upstream http.Handler) httpcache.HttpCache {
	112	+	func NewPgsHttpCache(cfg PgsConfig, upstream http.Handler) (httpcache.HttpCache, *SiteCache) {
113	113		ttl := cfg.CacheTTL
114	114		metrics := NewPromCacheMetrics(cfg.Logger, prometheus.DefaultRegisterer)
115		-	cache := expirable.NewLRU(cfg.CacheMaxItems, metrics.EvictCacheItem, ttl)
	115	+	cache := NewSiteCache(cfg.CacheMaxItems, ttl, metrics.EvictCacheItem)
116	116		httpCache := &httpcache.HttpCache{
117	117		Ttl: ttl,
118	118		Logger: cfg.Logger,

...	...		@@ -123,22 +123,24 @@ func NewPgsHttpCache(cfg PgsConfig, upstream http.Handler) httpcache.HttpCache
123	123		TxtPrefix: cfg.TxtPrefix,
124	124		},
125	125		CacheMetrics: metrics,
	126	+	MaxBodySize: cfg.CacheMaxBodySize,
126	127		}
127	128		httpCache.Logger.Info(
128	129		"httpcache initiated",
129	130		"storageType", "expirable.LRU",
130	131		"ttl", ttl,
131	132		"maxItems", cfg.CacheMaxItems,
	133	+	"maxBodySize", cfg.CacheMaxBodySize,
132	134		)
133		-	return httpCache
	135	+	return httpCache, cache
134	136		}
135	137
136	138		func StartApiServer(cfg *PgsConfig) {
137	139		ctx := context.Background()
138	140
139	141		router := NewWebRouter(cfg)
140		-	httpCache := NewPgsHttpCache(router.Cfg, router)
141		-	go CacheMgmt(ctx, cfg.CacheClearingQueue, cfg, httpCache.Cache)
	142	+	httpCache, cache := NewPgsHttpCache(router.Cfg, router)
	143	+	go CacheMgmt(ctx, cfg, cache, router.clearSiteCache)
142	144
143	145		portStr := fmt.Sprintf(":%s", cfg.WebPort)
144	146		cfg.Logger.Info(

...	...		@@ -160,20 +162,32 @@ type WebRouter struct {
160	162		RootRouter *http.ServeMux
161	163		UserRouter *http.ServeMux
162	164		RedirectsCache expirable.LRU[string, []RedirectRule]
163		-	HeadersCache expirable.LRU[string, []HeaderRule]
	165	+	HeadersCache *expirable.LRU[string, []headerMatcher]
	166	+	SiteCache expirable.LRU[string, siteInfo]
	167	+	}
	168	+
	169	+	// siteInfo is what serving a site needs from the database and storage.
	170	+	type siteInfo struct {
	171	+	user *db.User
	172	+	project *db.Project
	173	+	bucket storage.Bucket
	174	+	plus *db.FeatureFlag
	175	+	}
	176	+
	177	+	func (s *siteInfo) hasPicoPlus() bool {
	178	+	return s.plus != nil && s.plus.ExpiresAt.After(time.Now())
164	179		}
165	180
166	181		func NewWebRouter(cfg PgsConfig) WebRouter {
167		-	router := newWebRouter(cfg)
168		-	go router.WatchCacheClear()
169		-	return router
	182	+	return newWebRouter(cfg)
170	183		}
171	184
172	185		func newWebRouter(cfg PgsConfig) WebRouter {
173	186		router := &WebRouter{
174	187		Cfg: cfg,
175	188		RedirectsCache: expirable.NewLRU[string, []*RedirectRule](2048, nil, shared.CacheTimeout),
176		-	HeadersCache: expirable.NewLRU[string, []*HeaderRule](2048, nil, shared.CacheTimeout),
	189	+	HeadersCache: expirable.NewLRU[string, []headerMatcher](2048, nil, shared.CacheTimeout),
	190	+	SiteCache: expirable.NewLRU[string, *siteInfo](2048, nil, shared.CacheTimeout),
177	191		}
178	192		router.initRouters()
179	193		return router

...	...		@@ -199,14 +213,51 @@ func (web WebRouter) ServeHTTP(w http.ResponseWriter, r http.Request) {
199	213		mux.ServeHTTP(w, r.WithContext(ctx))
200	214		}
201	215
202		-	func (web *WebRouter) WatchCacheClear() {
203		-	for key := range web.Cfg.CacheClearingQueue {
204		-	web.Cfg.Logger.Info("lru cache clear request", "key", key)
205		-	rKey := filepath.Join(key, "_redirects")
206		-	web.RedirectsCache.Remove(rKey)
207		-	hKey := filepath.Join(key, "_headers")
208		-	web.HeadersCache.Remove(hKey)
	216	+	// clearSiteCache drops what is cached about a site: its database records and
	217	+	// its parsed _redirects and _headers. The key is the surrogate key that
	218	+	// uploads and the CLI purge, or "*" for every site.
	219	+	func (web *WebRouter) clearSiteCache(key string) {
	220	+	web.Cfg.Logger.Info("lru cache clear request", "key", key)
	221	+	if key == "*" {
	222	+	web.SiteCache.Purge()
	223	+	web.RedirectsCache.Purge()
	224	+	web.HeadersCache.Purge()
	225	+	return
209	226		}
	227	+	web.SiteCache.Remove(key)
	228	+	web.RedirectsCache.Remove(filepath.Join(key, "_redirects"))
	229	+	web.HeadersCache.Remove(filepath.Join(key, "_headers"))
	230	+	}
	231	+
	232	+	// findSite looks up the user, project, bucket and pico+ flag behind a
	233	+	// subdomain. Only complete results are cached, so a site that is created or
	234	+	// fixed later is found on its next request.
	235	+	func (web WebRouter) findSite(props router.SubdomainProps, logger slog.Logger) (siteInfo, int, string) {
	236	+	key := getSurrogateKey(props.Username, props.ProjectName)
	237	+	if site, ok := web.SiteCache.Get(key); ok {
	238	+	return site, http.StatusOK, ""
	239	+	}
	240	+
	241	+	user, err := web.Cfg.DB.FindUserByName(props.Username)
	242	+	if err != nil {
	243	+	logger.Info("user not found")
	244	+	return nil, http.StatusNotFound, "user not found"
	245	+	}
	246	+	project, err := web.Cfg.DB.FindProjectByName(user.ID, props.ProjectName)
	247	+	if err != nil {
	248	+	logger.Info("project not found")
	249	+	return nil, http.StatusNotFound, "project not found"
	250	+	}
	251	+	bucket, err := web.Cfg.Storage.GetBucket(shared.GetAssetBucketName(user.ID))
	252	+	if err != nil {
	253	+	logger.Error("bucket not found", "err", err)
	254	+	return nil, http.StatusNotFound, "bucket not found"
	255	+	}
	256	+	plus, _ := web.Cfg.DB.FindFeature(user.ID, "plus")
	257	+
	258	+	site := &siteInfo{user: user, project: project, bucket: bucket, plus: plus}
	259	+	web.SiteCache.Add(key, site)
	260	+	return site, http.StatusOK, ""
210	261		}
211	262
212	263		func (web *WebRouter) initRouters() {

...	...		@@ -376,7 +427,10 @@ func (web WebRouter) checkHandler(w http.ResponseWriter, r http.Request) {
376	427		w.WriteHeader(http.StatusNotFound)
377	428		}
378	429
379		-	func CacheMgmt(ctx context.Context, notify chan string, cfg *PgsConfig, cacher httpcache.Cacher) {
	430	+	// CacheMgmt applies the purges published on the cache-drain pipe. onPurge,
	431	+	// when set, clears whatever else is cached about the site; it runs first so
	432	+	// the http cache isn't refilled from stale records.
	433	+	func CacheMgmt(ctx context.Context, cfg PgsConfig, cache SiteCache, onPurge func(site string)) {
380	434		cfg.Logger.Info("cache mgmt initiated")
381	435		for {
382	436		scanner := bufio.NewScanner(cfg.Pubsub)

...	...		@@ -384,20 +438,16 @@ func CacheMgmt(ctx context.Context, notify chan string, cfg *PgsConfig, cacher h
384	438		for scanner.Scan() {
385	439		subdomain := strings.TrimSpace(scanner.Text())
386	440		cfg.Logger.Info("received cache-drain item", "subdomain", subdomain)
387		-	notify <- subdomain
	441	+	if onPurge != nil {
	442	+	onPurge(subdomain)
	443	+	}
388	444
389	445		if subdomain == "*" {
390		-	cacher.Purge()
	446	+	cache.Purge()
391	447		cfg.Logger.Info("successfully cleared cache from remote cli request")
392	448		continue
393	449		}
394		-
395		-	for _, key := range cacher.Keys() {
396		-	if strings.HasPrefix(key, subdomain) {
397		-	cfg.Logger.Info("deleting cache item", "subdomain", subdomain, "key", key)
398		-	_ = cacher.Remove(key)
399		-	}
400		-	}
	450	+	cache.PurgeSite(subdomain)
401	451		}
402	452		}
403	453		}

...	...		@@ -533,27 +583,15 @@ func (web WebRouter) ServeAsset(fname string, opts storage.ImgProcessOpts, has
533	583		"user", props.Username,
534	584		)
535	585
536		-	user, err := web.Cfg.DB.FindUserByName(props.Username)
537		-	if err != nil {
538		-	logger.Info("user not found")
539		-	http.Error(w, "user not found", http.StatusNotFound)
	586	+	site, status, msg := web.findSite(props, logger)
	587	+	if site == nil {
	588	+	http.Error(w, msg, status)
540	589		return
541	590		}
	591	+	user, project, bucket := site.user, site.project, site.bucket
542	592
543	593		logger = logger.With(
544	594		"userId", user.ID,
545		-	)
546		-
547		-	var bucket storage.Bucket
548		-	bucket, err = web.Cfg.Storage.GetBucket(shared.GetAssetBucketName(user.ID))
549		-	project, perr := web.Cfg.DB.FindProjectByName(user.ID, props.ProjectName)
550		-	if perr != nil {
551		-	logger.Info("project not found")
552		-	http.Error(w, "project not found", http.StatusNotFound)
553		-	return
554		-	}
555		-
556		-	logger = logger.With(
557	595		"projectId", project.ID,
558	596		"project", project.Name,
559	597		)

...	...		@@ -589,20 +627,6 @@ func (web WebRouter) ServeAsset(fname string, opts storage.ImgProcessOpts, has
589	627		return
590	628		}
591	629
592		-	if err != nil {
593		-	logger.Error("bucket not found", "err", err)
594		-	http.Error(w, "bucket not found", http.StatusNotFound)
595		-	return
596		-	}
597		-
598		-	hasPicoPlus := false
599		-	ff, _ := web.Cfg.DB.FindFeature(user.ID, "plus")
600		-	if ff != nil {
601		-	if ff.ExpiresAt.After(time.Now()) {
602		-	hasPicoPlus = true
603		-	}
604		-	}
605		-
606	630		asset := &ApiAssetHandler{
607	631		WebRouter: web,
608	632		Logger: logger,

...	...		@@ -615,7 +639,7 @@ func (web WebRouter) ServeAsset(fname string, opts storage.ImgProcessOpts, has
615	639		Filepath: fname,
616	640		Bucket: bucket,
617	641		ImgProcessOpts: opts,
618		-	HasPicoPlus: hasPicoPlus,
	642	+	HasPicoPlus: site.hasPicoPlus(),
619	643		HttpPass: project.Acl.Type == "http-pass",
620	644		}
621	645

...	...		@@ -7,7 +7,6 @@ import (
7	7		"net/http"
8	8		"net/url"
9	9		"path/filepath"
10		-	"regexp"
11	10		"strconv"
12	11		"strings"
13	12

...	...		@@ -42,7 +41,7 @@ func (h ApiAssetHandler) ServeHTTP(w http.ResponseWriter, r http.Request) {
42	41		logger := h.Logger
43	42		var redirects []*RedirectRule
44	43
45		-	redirectsCacheKey := filepath.Join(getSurrogateKey(h.UserID, h.ProjectDir), "_redirects")
	44	+	redirectsCacheKey := filepath.Join(getSurrogateKey(h.Username, h.ProjectDir), "_redirects")
46	45		logger.Info("looking for _redirects in lru cache", "key", redirectsCacheKey)
47	46		if cachedRedirects, found := h.RedirectsCache.Get(redirectsCacheKey); found {
48	47		logger.Info("_redirects found in lru cache", "key", redirectsCacheKey)

...	...		@@ -104,11 +103,9 @@ func (h ApiAssetHandler) ServeHTTP(w http.ResponseWriter, r http.Request) {
104	103		// before redirecting, this saves a hop that will just end up a 404
105	104		if !hasProtocol(fp.Filepath) && strings.HasSuffix(fp.Filepath, "/") {
106	105		next := filepath.Join(h.ProjectDir, fp.Filepath, "index.html")
107		-	obj, _, err := h.Cfg.Storage.GetObject(h.Bucket, next)
108		-	if err != nil {
	106	+	if _, err := h.Cfg.Storage.StatObject(h.Bucket, next); err != nil {
109	107		continue
110	108		}
111		-	_ = obj.Close()
112	109		}
113	110		logger.Info(
114	111		"redirecting request",

...	...		@@ -177,25 +174,21 @@ func (h ApiAssetHandler) ServeHTTP(w http.ResponseWriter, r http.Request) {
177	174		}
178	175
179	176		if assetFilepath == "" {
180		-	if shouldGenerateListing(h.Cfg.Storage, h.Bucket, h.ProjectDir, "/"+fpath) {
	177	+	if entries, ok := dirListing(h.Cfg.Storage, h.Bucket, h.ProjectDir, "/"+fpath); ok {
181	178		logger.Info(
182	179		"generating directory listing",
183	180		"path", fpath,
184	181		)
185		-	dirPath := h.ProjectDir + "/" + fpath
186		-	entries, err := h.Cfg.Storage.ListObjects(h.Bucket, dirPath, false)
187		-	if err == nil {
188		-	requestPath := "/" + fpath
189		-	if !strings.HasSuffix(requestPath, "/") {
190		-	requestPath += "/"
191		-	}
192		-
193		-	html := generateDirectoryHTML(requestPath, entries)
194		-	w.Header().Set("content-type", "text/html")
195		-	w.WriteHeader(http.StatusOK)
196		-	_, _ = w.Write([]byte(html))
197		-	return
	182	+	requestPath := "/" + fpath
	183	+	if !strings.HasSuffix(requestPath, "/") {
	184	+	requestPath += "/"
198	185		}
	186	+
	187	+	html := generateDirectoryHTML(requestPath, entries)
	188	+	w.Header().Set("content-type", "text/html")
	189	+	w.WriteHeader(http.StatusOK)
	190	+	_, _ = w.Write([]byte(html))
	191	+	return
199	192		}
200	193
201	194		logger.Info(

...	...		@@ -210,9 +203,9 @@ func (h ApiAssetHandler) ServeHTTP(w http.ResponseWriter, r http.Request) {
210	203		_ = contents.Close()
211	204		}()
212	205
213		-	var headers []*HeaderRule
	206	+	var headers []headerMatcher
214	207
215		-	headersCacheKey := filepath.Join(getSurrogateKey(h.UserID, h.ProjectDir), "_headers")
	208	+	headersCacheKey := filepath.Join(getSurrogateKey(h.Username, h.ProjectDir), "_headers")
216	209		logger.Info("looking for _headers in lru cache", "key", headersCacheKey)
217	210		if cachedHeaders, found := h.HeadersCache.Get(headersCacheKey); found {
218	211		logger.Info("_headers found in lru", "key", headersCacheKey)

...	...		@@ -238,21 +231,20 @@ func (h ApiAssetHandler) ServeHTTP(w http.ResponseWriter, r http.Request) {
238	231		return
239	232		}
240	233
241		-	headers, err = parseHeaderText(buf.String())
	234	+	rules, err := parseHeaderText(buf.String())
242	235		if err != nil {
243	236		logger.Error("could not parse header text", "err", err.Error())
244	237		}
	238	+	headers = compileHeaderRules(rules, logger)
245	239		}
246	240
247	241		h.HeadersCache.Add(headersCacheKey, headers)
248	242		}
249	243
250	244		userHeaders := []*HeaderLine{}
251		-	for _, headerRule := range headers {
252		-	rr := regexp.MustCompile(headerRule.Path)
253		-	match := rr.FindStringSubmatch(assetFilepath)
254		-	if len(match) > 0 {
255		-	userHeaders = headerRule.Headers
	245	+	for _, rule := range headers {
	246	+	if rule.path.MatchString(assetFilepath) {
	247	+	userHeaders = rule.headers
256	248		}
257	249		}
258	250

...	...		@@ -5,8 +5,11 @@ import (
5	5		"fmt"
6	6		"io"
7	7		"log/slog"
	8	+	"strings"
	9	+	"sync"
8	10		"time"
9	11
	12	+	"github.com/hashicorp/golang-lru/v2/expirable"
10	13		"github.com/picosh/pico/pkg/shared"
11	14		"github.com/picosh/utils/pipe"
12	15		)

...	...		@@ -104,7 +107,6 @@ func CreateSubCacheDrain(ctx context.Context, logger slog.Logger) pipe.Reconne
104	107		// from the cache).
105	108		func purgeCache(cfg *PgsConfig, writer io.Writer, surrogate string) error {
106	109		cfg.Logger.Info("purging cache", "surrogate", surrogate)
107		-	time.Sleep(1 * time.Second)
108	110		_, err := writer.Write([]byte(surrogate + "\n"))
109	111		return err
110	112		}

...	...		@@ -112,3 +114,64 @@ func purgeCache(cfg *PgsConfig, writer io.Writer, surrogate string) error {
112	114		func purgeAllCache(cfg *PgsConfig, writer io.Writer) error {
113	115		return purgeCache(cfg, writer, "*")
114	116		}
	117	+
	118	+	// SiteCache holds the http cache's responses and indexes their keys by site,
	119	+	// so purging one site doesn't scan the whole cache.
	120	+	type SiteCache struct {
	121	+	*expirable.LRU[string, []byte]
	122	+	mu sync.Mutex
	123	+	sites map[string]map[string]struct{}
	124	+	}
	125	+
	126	+	func NewSiteCache(size int, ttl time.Duration, onEvict func(key string, value []byte)) *SiteCache {
	127	+	c := &SiteCache{sites: map[string]map[string]struct{}{}}
	128	+	c.LRU = expirable.NewLRU(size, func(key string, value []byte) {
	129	+	c.unindex(key)
	130	+	if onEvict != nil {
	131	+	onEvict(key, value)
	132	+	}
	133	+	}, ttl)
	134	+	return c
	135	+	}
	136	+
	137	+	// cacheKeySite returns the site a PgsCacheKey belongs to.
	138	+	func cacheKeySite(key string) string {
	139	+	site, _, _ := strings.Cut(key, "__")
	140	+	return site
	141	+	}
	142	+
	143	+	func (c *SiteCache) Add(key string, value []byte) bool {
	144	+	evicted := c.LRU.Add(key, value)
	145	+	site := cacheKeySite(key)
	146	+	c.mu.Lock()
	147	+	defer c.mu.Unlock()
	148	+	if c.sites[site] == nil {
	149	+	c.sites[site] = map[string]struct{}{}
	150	+	}
	151	+	c.sites[site][key] = struct{}{}
	152	+	return evicted
	153	+	}
	154	+
	155	+	// PurgeSite removes every cached response for a site. The index lock is
	156	+	// released before touching the LRU, whose eviction callback takes it.
	157	+	func (c *SiteCache) PurgeSite(site string) {
	158	+	c.mu.Lock()
	159	+	keys := c.sites[site]
	160	+	delete(c.sites, site)
	161	+	c.mu.Unlock()
	162	+	for key := range keys {
	163	+	c.Remove(key)
	164	+	}
	165	+	}
	166	+
	167	+	func (c *SiteCache) unindex(key string) {
	168	+	site := cacheKeySite(key)
	169	+	c.mu.Lock()
	170	+	defer c.mu.Unlock()
	171	+	if keys, ok := c.sites[site]; ok {
	172	+	delete(keys, key)
	173	+	if len(keys) == 0 {
	174	+	delete(c.sites, site)
	175	+	}
	176	+	}
	177	+	}

...	...		@@ -0,0 +1,42 @@
	1	+	package pgs
	2	+
	3	+	import (
	4	+	"slices"
	5	+	"testing"
	6	+	"time"
	7	+	)
	8	+
	9	+	func TestSiteCachePurgeSite(t *testing.T) {
	10	+	evicted := []string{}
	11	+	c := NewSiteCache(0, time.Minute, func(key string, _ []byte) { evicted = append(evicted, key) })
	12	+	keys := []string{
	13	+	"antonio-blog__GET__/",
	14	+	"antonio-blog__GET__/about",
	15	+	"antonio-blog2__GET__/",
	16	+	"antonio-blog-v2__GET__/",
	17	+	}
	18	+	for _, key := range keys {
	19	+	c.Add(key, []byte("x"))
	20	+	}
	21	+
	22	+	c.PurgeSite("antonio-blog")
	23	+	got := c.Keys()
	24	+	slices.Sort(got)
	25	+	want := []string{"antonio-blog-v2__GET__/", "antonio-blog2__GET__/"}
	26	+	if !slices.Equal(got, want) {
	27	+	t.Fatalf("after purge got %q, want %q", got, want)
	28	+	}
	29	+	if len(evicted) != 2 {
	30	+	t.Fatalf("eviction callback ran %d times, want 2", len(evicted))
	31	+	}
	32	+
	33	+	// Entries that leave the cache on their own leave the index too.
	34	+	c.Remove("antonio-blog2__GET__/")
	35	+	if _, ok := c.sites["antonio-blog2"]; ok {
	36	+	t.Fatal("removed entry still indexed")
	37	+	}
	38	+	c.Purge()
	39	+	if len(c.sites) != 0 {
	40	+	t.Fatalf("index holds %d sites after a full purge", len(c.sites))
	41	+	}
	42	+	}

...	...		@@ -6,6 +6,7 @@ import (
6	6		"net/http"
7	7		"net/http/httptest"
8	8		"strings"
	9	+	"sync/atomic"
9	10		"testing"
10	11		"time"
11	12

...	...		@@ -413,6 +414,34 @@ func TestApiBasic(t *testing.T) {
413	414		},
414	415		},
415	416		},
	417	+	{
	418	+	name: "headers-invalid-path-skipped",
	419	+	path: "/test.html",
	420	+	want: "hello world!",
	421	+	status: http.StatusOK,
	422	+	contentType: "text/html",
	423	+	wantCacheCtrl: "public, max-age=31536000, immutable",
	424	+
	425	+	storage: map[string]map[string]string{
	426	+	bucketName: {
	427	+	"/test/test.html": "hello world!",
	428	+	"/test/_headers": "/[\n\tx-broken: yes\n/*\n\tcache-control: public, max-age=31536000, immutable",
	429	+	},
	430	+	},
	431	+	},
	432	+	{
	433	+	name: "redirects-invalid-pattern-skipped",
	434	+	path: "/foo(",
	435	+	want: "404 not found",
	436	+	status: http.StatusNotFound,
	437	+	contentType: "text/plain; charset=utf-8",
	438	+
	439	+	storage: map[string]map[string]string{
	440	+	bucketName: {
	441	+	"/test/_redirects": "/foo( /index.html 301",
	442	+	},
	443	+	},
	444	+	},
416	445		}
417	446
418	447		for _, tc := range tt {

...	...		@@ -748,3 +777,106 @@ func TestDirectoryListing(t *testing.T) {
748	777		// })
749	778		// }
750	779		// }
	780	+
	781	+	// The purge an upload sends must drop the site's parsed _redirects, or the
	782	+	// old rules keep being served and cached.
	783	+	func TestRulesCacheClearedByUploadPurge(t *testing.T) {
	784	+	logger := slog.New(slog.DiscardHandler)
	785	+	dbpool := NewPgsDb(logger)
	786	+	user := dbpool.Users[0]
	787	+	bucketName := shared.GetAssetBucketName(user.ID)
	788	+	st, err := storage.NewStorageMemory(map[string]map[string]string{
	789	+	bucketName: {
	790	+	"/test/old.html": "old",
	791	+	"/test/new.html": "new",
	792	+	"/test/_redirects": "/page /old.html 200",
	793	+	},
	794	+	})
	795	+	if err != nil {
	796	+	t.Fatal(err)
	797	+	}
	798	+	cfg := NewPgsConfig(logger, dbpool, st, discardPubsub{})
	799	+	cfg.Domain = "pgs.test"
	800	+	router := newWebRouter(cfg)
	801	+
	802	+	get := func() string {
	803	+	rec := httptest.NewRecorder()
	804	+	router.ServeHTTP(rec, httptest.NewRequest("GET", dbpool.mkpath("/page"), nil))
	805	+	return strings.TrimSpace(rec.Body.String())
	806	+	}
	807	+	if got := get(); got != "old" {
	808	+	t.Fatalf("before upload got %q", got)
	809	+	}
	810	+
	811	+	bucket, err := st.GetBucket(bucketName)
	812	+	if err != nil {
	813	+	t.Fatal(err)
	814	+	}
	815	+	if _, _, err := st.PutObject(bucket, "/test/_redirects", strings.NewReader("/page /new.html 200"), &storage.ObjectInfo{}); err != nil {
	816	+	t.Fatal(err)
	817	+	}
	818	+	router.clearSiteCache(getSurrogateKey(user.Name, "test"))
	819	+	if got := get(); got != "new" {
	820	+	t.Fatalf("after upload got %q", got)
	821	+	}
	822	+	}
	823	+
	824	+	// countingDB counts the lookups serving a site makes.
	825	+	type countingDB struct {
	826	+	*PgsDb
	827	+	lookups atomic.Int64
	828	+	}
	829	+
	830	+	func (c countingDB) FindUserByName(name string) (db.User, error) {
	831	+	c.lookups.Add(1)
	832	+	return c.PgsDb.FindUserByName(name)
	833	+	}
	834	+
	835	+	func (c countingDB) FindProjectByName(userID, name string) (db.Project, error) {
	836	+	c.lookups.Add(1)
	837	+	return c.PgsDb.FindProjectByName(userID, name)
	838	+	}
	839	+
	840	+	func (c countingDB) FindFeature(userID, name string) (db.FeatureFlag, error) {
	841	+	c.lookups.Add(1)
	842	+	return c.PgsDb.FindFeature(userID, name)
	843	+	}
	844	+
	845	+	// A site's database records are looked up once and again only after a purge.
	846	+	func TestSiteCache(t *testing.T) {
	847	+	logger := slog.New(slog.DiscardHandler)
	848	+	dbpool := &countingDB{PgsDb: NewPgsDb(logger)}
	849	+	user := dbpool.Users[0]
	850	+	st, err := storage.NewStorageMemory(map[string]map[string]string{
	851	+	shared.GetAssetBucketName(user.ID): {"/test/index.html": "hello"},
	852	+	})
	853	+	if err != nil {
	854	+	t.Fatal(err)
	855	+	}
	856	+	cfg := NewPgsConfig(logger, dbpool, st, discardPubsub{})
	857	+	cfg.Domain = "pgs.test"
	858	+	router := newWebRouter(cfg)
	859	+
	860	+	get := func() {
	861	+	t.Helper()
	862	+	rec := httptest.NewRecorder()
	863	+	router.ServeHTTP(rec, httptest.NewRequest("GET", dbpool.mkpath("/"), nil))
	864	+	if rec.Code != http.StatusOK {
	865	+	t.Fatalf("got status %d", rec.Code)
	866	+	}
	867	+	}
	868	+	get()
	869	+	first := dbpool.lookups.Load()
	870	+	if first == 0 {
	871	+	t.Fatal("serving made no lookups")
	872	+	}
	873	+	get()
	874	+	if n := dbpool.lookups.Load(); n != first {
	875	+	t.Fatalf("second request made %d more lookups", n-first)
	876	+	}
	877	+	router.clearSiteCache(getSurrogateKey(user.Name, "test"))
	878	+	get()
	879	+	if n := dbpool.lookups.Load(); n != 2*first {
	880	+	t.Fatalf("request after purge made %d lookups, want %d", n-first, first)
	881	+	}
	882	+	}

...	...		@@ -42,6 +42,8 @@ type Cmd struct {
42	42		Log *slog.Logger
43	43		Dbpool db.DB
44	44		Write bool
	45	+
	46	+	lookupTXT func(host, space string) ([]string, error)
45	47		}
46	48
47	49		func (c *Cmd) output(out string) {

...	...		@@ -49,13 +51,14 @@ func (c *Cmd) output(out string) {
49	51		}
50	52
51	53		func (c *Cmd) help() {
52		-	helpStr := "Commands: [help, user, logs, access_logs, chat, not-found]\n"
	54	+	helpStr := "Commands: [help, user, logs, access-logs, chat, not-found, dns-check]\n"
53	55		helpStr += "help - this message\n"
54	56		helpStr += "user - display user information (returns name, id, account created, pico+ expiration)\n"
55	57		helpStr += "logs - stream user logs\n"
56		-	helpStr += "access_logs - fetch access logs from the last 30 days\n"
	58	+	helpStr += "access-logs - fetch access logs from the last 30 days\n"
57	59		helpStr += "chat - IRC chat (must enable pty with `-t` to the SSH command)\n"
58	60		helpStr += "not-found - return all status 404 requests for a host (hostname.com [year\|month])\n"
	61	+	helpStr += "dns-check - check a domain's TXT records for pgs, prose and tuns custom domains (hostname.com or *.hostname.com)\n"
59	62		c.output(helpStr)
60	63		}
61	64

...	...		@@ -96,7 +99,7 @@ func (c *Cmd) notFound(host, interval string) error {
96	99		return nil
97	100		}
98	101
99		-	func (c *Cmd) access_logs(_ context.Context) error {
	102	+	func (c *Cmd) accessLogs(_ context.Context) error {
100	103		fromDate := time.Now().AddDate(0, 0, -30)
101	104		logs, err := c.Dbpool.FindAccessLogs(c.User.ID, &fromDate)
102	105		if err != nil {

...	...		@@ -264,8 +267,12 @@ func Middleware(handler *CliHandler) pssh.SSHServerMiddleware {
264	267		sesh.Fatal(err)
265	268		}
266	269		return nil
267		-	case "access_logs":
268		-	err = opts.access_logs(sesh.Context())
	270	+	case "dns-check":
	271	+	sesh.Fatal(fmt.Errorf("must provide a domain name (`dns-check example.com`)"))
	272	+	return nil
	273	+	// access_logs is the command's old name.
	274	+	case "access-logs", "access_logs":
	275	+	err = opts.accessLogs(sesh.Context())
269	276		if err != nil {
270	277		sesh.Fatal(err)
271	278		}

...	...		@@ -275,6 +282,14 @@ func Middleware(handler *CliHandler) pssh.SSHServerMiddleware {
275	282		}
276	283		}
277	284
	285	+	if cmd == "dns-check" {
	286	+	err = opts.dnsCheck(args[1])
	287	+	if err != nil {
	288	+	sesh.Fatal(err)
	289	+	}
	290	+	return nil
	291	+	}
	292	+
278	293		if cmd == "not-found" {
279	294		if len(args) < 3 {
280	295		sesh.Fatal(fmt.Errorf("must provide host name and interval (`month` or `year`)"))

...	...		@@ -0,0 +1,149 @@
	1	+	package pico
	2	+
	3	+	import (
	4	+	"crypto/rand"
	5	+	"encoding/hex"
	6	+	"errors"
	7	+	"fmt"
	8	+	"strings"
	9	+
	10	+	"github.com/picosh/pico/pkg/db"
	11	+	"github.com/picosh/pico/pkg/shared/router"
	12	+	"golang.org/x/crypto/ssh"
	13	+	)
	14	+
	15	+	// customDomainService is a service that serves a domain whose TXT record
	16	+	// names it. check judges the records the way the service does and describes
	17	+	// the result.
	18	+	type customDomainService struct {
	19	+	space string
	20	+	check func(c *Cmd, records []string) string
	21	+	wildcard bool
	22	+	}
	23	+
	24	+	var customDomainServices = []customDomainService{
	25	+	{space: "pgs", check: siteCheck(resolvePgsSite)},
	26	+	{space: "prose", check: siteCheck(resolveProseSite)},
	27	+	// tuns accepts a match under either prefix; _sish is what sish uses.
	28	+	{space: "tuns", check: checkTunsRecords, wildcard: true},
	29	+	{space: "sish", check: checkTunsRecords, wildcard: true},
	30	+	}
	31	+
	32	+	// siteCheck checks a record whose value names a site. Like the services,
	33	+	// it reads only the first record.
	34	+	func siteCheck(resolve func(dbpool db.DB, value string) (db.User, string, error)) func(c Cmd, records []string) string {
	35	+	return func(c *Cmd, records []string) string {
	36	+	value := strings.TrimSpace(records[0])
	37	+	user, site, err := resolve(c.Dbpool, value)
	38	+	if err != nil {
	39	+	return fmt.Sprintf("%q is invalid: %v", value, err)
	40	+	}
	41	+	line := fmt.Sprintf("%q is valid, serving %s", value, site)
	42	+	if user.ID != c.User.ID {
	43	+	line += " (not your account)"
	44	+	}
	45	+	return line
	46	+	}
	47	+	}
	48	+
	49	+	func resolvePgsSite(dbpool db.DB, value string) (*db.User, string, error) {
	50	+	props, err := router.GetProjectFromSubdomain(value)
	51	+	if err != nil {
	52	+	return nil, "", err
	53	+	}
	54	+	user, err := dbpool.FindUserByName(props.Username)
	55	+	if err != nil {
	56	+	return nil, "", fmt.Errorf("user %q not found", props.Username)
	57	+	}
	58	+	if _, err := dbpool.FindProjectByName(user.ID, props.ProjectName); err != nil {
	59	+	return user, "", fmt.Errorf("user %q has no project %q", user.Name, props.ProjectName)
	60	+	}
	61	+	return user, fmt.Sprintf("pgs project %q of %s", props.ProjectName, user.Name), nil
	62	+	}
	63	+
	64	+	func resolveProseSite(dbpool db.DB, value string) (*db.User, string, error) {
	65	+	user, err := dbpool.FindUserByName(value)
	66	+	if err != nil {
	67	+	return nil, "", fmt.Errorf("user %q not found", value)
	68	+	}
	69	+	return user, fmt.Sprintf("prose blog of %s", user.Name), nil
	70	+	}
	71	+
	72	+	// checkTunsRecords looks for a record holding the user's name or the SHA256
	73	+	// fingerprint of one of their keys. tuns compares the whole record, so it
	74	+	// must match exactly.
	75	+	func checkTunsRecords(c *Cmd, records []string) string {
	76	+	keys, err := c.Dbpool.FindKeysByUser(c.User)
	77	+	if err != nil {
	78	+	return fmt.Sprintf("could not load your keys: %v", err)
	79	+	}
	80	+	names := map[string]string{}
	81	+	for _, k := range keys {
	82	+	pk, _, _, _, err := ssh.ParseAuthorizedKey([]byte(k.Key))
	83	+	if err != nil {
	84	+	continue
	85	+	}
	86	+	names[ssh.FingerprintSHA256(pk)] = k.Name
	87	+	}
	88	+
	89	+	for _, r := range records {
	90	+	if r == c.User.Name {
	91	+	return fmt.Sprintf("%q is valid, tunnels from any of your keys can use this domain", r)
	92	+	}
	93	+	if name, ok := names[r]; ok {
	94	+	return fmt.Sprintf("%q is valid, tunnels from your key %q can use this domain", r, name)
	95	+	}
	96	+	}
	97	+	for _, r := range records {
	98	+	trimmed := strings.TrimSpace(r)
	99	+	if _, ok := names[trimmed]; ok \|\| trimmed == c.User.Name {
	100	+	return fmt.Sprintf("%q is invalid: it has surrounding whitespace, and tuns needs an exact match", r)
	101	+	}
	102	+	}
	103	+	quoted := make([]string, len(records))
	104	+	for i, r := range records {
	105	+	quoted[i] = fmt.Sprintf("%q", r)
	106	+	}
	107	+	return fmt.Sprintf("%s is invalid: no record is your username or the SHA256 fingerprint of one of your keys", strings.Join(quoted, ", "))
	108	+	}
	109	+
	110	+	// dnsCheck reports, for each service, whether domain has a TXT record and
	111	+	// whether the service would accept it.
	112	+	func (c *Cmd) dnsCheck(domain string) error {
	113	+	domain = strings.TrimSuffix(strings.ToLower(strings.TrimSpace(domain)), ".")
	114	+	base, wildcard := strings.CutPrefix(domain, "*.")
	115	+	if base == "" \|\| strings.ContainsAny(base, "/:@ *") {
	116	+	return fmt.Errorf("expected a domain name like example.com or *.example.com")
	117	+	}
	118	+	lookup := c.lookupTXT
	119	+	if lookup == nil {
	120	+	lookup = router.LookupTXTRecords
	121	+	}
	122	+
	123	+	host := domain
	124	+	if wildcard {
	125	+	// tuns verifies a wildcard by looking up a random name under it.
	126	+	label := make([]byte, 6)
	127	+	_, _ = rand.Read(label)
	128	+	host = hex.EncodeToString(label) + "." + base
	129	+	}
	130	+
	131	+	for _, svc := range customDomainServices {
	132	+	name := router.TXTRecordName(host, svc.space)
	133	+	if wildcard && !svc.wildcard {
	134	+	c.output(fmt.Sprintf("%s: wildcard domains are not supported", router.TXTRecordName(domain, svc.space)))
	135	+	continue
	136	+	}
	137	+	records, err := lookup(host, svc.space)
	138	+	switch {
	139	+	case errors.Is(err, router.ErrNoTXTRecord):
	140	+	c.output(fmt.Sprintf("%s: no TXT record", name))
	141	+	case err != nil:
	142	+	c.output(fmt.Sprintf("%s: lookup failed: %v", name, err))
	143	+	default:
	144	+	c.output(fmt.Sprintf("%s: %s", name, svc.check(c, records)))
	145	+	}
	146	+	}
	147	+	c.output("pgs and prose remember a missing record for 30 seconds and a found record for 2 minutes. tuns checks when a tunnel connects.")
	148	+	return nil
	149	+	}

...	...		@@ -0,0 +1,162 @@
	1	+	package pico
	2	+
	3	+	import (
	4	+	"bytes"
	5	+	"crypto/ed25519"
	6	+	"fmt"
	7	+	"io"
	8	+	"log/slog"
	9	+	"strings"
	10	+	"testing"
	11	+
	12	+	"github.com/picosh/pico/pkg/db"
	13	+	"github.com/picosh/pico/pkg/shared"
	14	+	"github.com/picosh/pico/pkg/shared/router"
	15	+	"golang.org/x/crypto/ssh"
	16	+	)
	17	+
	18	+	type fakeSession struct{ bytes.Buffer }
	19	+
	20	+	func (s *fakeSession) Exit(int) error { return nil }
	21	+	func (s *fakeSession) Close() error { return nil }
	22	+	func (s *fakeSession) Stderr() io.ReadWriter { return &s.Buffer }
	23	+
	24	+	type dnsCheckDB struct {
	25	+	db.DB
	26	+	users []*db.User
	27	+	projects map[string]bool
	28	+	keys []*db.PublicKey
	29	+	}
	30	+
	31	+	func (d dnsCheckDB) FindKeysByUser(user db.User) ([]*db.PublicKey, error) {
	32	+	keys := []*db.PublicKey{}
	33	+	for _, k := range d.keys {
	34	+	if k.UserID == user.ID {
	35	+	keys = append(keys, k)
	36	+	}
	37	+	}
	38	+	return keys, nil
	39	+	}
	40	+
	41	+	func (d dnsCheckDB) FindUserByName(name string) (db.User, error) {
	42	+	for _, u := range d.users {
	43	+	if u.Name == name {
	44	+	return u, nil
	45	+	}
	46	+	}
	47	+	return nil, fmt.Errorf("user not found")
	48	+	}
	49	+
	50	+	func (d dnsCheckDB) FindProjectByName(userID, name string) (db.Project, error) {
	51	+	if d.projects[userID+"/"+name] {
	52	+	return &db.Project{UserID: userID, Name: name}, nil
	53	+	}
	54	+	return nil, fmt.Errorf("project not found")
	55	+	}
	56	+
	57	+	func TestDNSCheck(t *testing.T) {
	58	+	me := &db.User{ID: "1", Name: "antonio"}
	59	+	other := &db.User{ID: "2", Name: "erock"}
	60	+	edKey, _, err := ed25519.GenerateKey(nil)
	61	+	if err != nil {
	62	+	t.Fatal(err)
	63	+	}
	64	+	pk, err := ssh.NewPublicKey(edKey)
	65	+	if err != nil {
	66	+	t.Fatal(err)
	67	+	}
	68	+	fingerprint := ssh.FingerprintSHA256(pk)
	69	+	dbpool := &dnsCheckDB{
	70	+	users: []*db.User{me, other},
	71	+	projects: map[string]bool{"1/blog": true},
	72	+	keys: []*db.PublicKey{{UserID: me.ID, Name: "laptop", Key: shared.KeyForKeyText(pk)}},
	73	+	}
	74	+
	75	+	tests := []struct {
	76	+	name string
	77	+	domain string
	78	+	records map[string][]string
	79	+	want []string
	80	+	}{
	81	+	{"no records", "", map[string][]string{}, []string{
	82	+	"_pgs.example.com: no TXT record",
	83	+	"_prose.example.com: no TXT record",
	84	+	"_tuns.example.com: no TXT record",
	85	+	"_sish.example.com: no TXT record",
	86	+	}},
	87	+	{"valid", "", map[string][]string{
	88	+	"pgs": {"antonio-blog"},
	89	+	"prose": {"erock"},
	90	+	"sish": {"SHA256:someone-else", fingerprint},
	91	+	}, []string{
	92	+	`_pgs.example.com: "antonio-blog" is valid, serving pgs project "blog" of antonio`,
	93	+	`_prose.example.com: "erock" is valid, serving prose blog of erock (not your account)`,
	94	+	fmt.Sprintf(`_sish.example.com: %q is valid, tunnels from your key "laptop" can use this domain`, fingerprint),
	95	+	}},
	96	+	{"invalid", "", map[string][]string{
	97	+	"pgs": {"antonio-blgo"},
	98	+	"prose": {"nobody"},
	99	+	"sish": {"SHA256:someone-else"},
	100	+	}, []string{
	101	+	`_pgs.example.com: "antonio-blgo" is invalid: user "antonio" has no project "blgo"`,
	102	+	`_prose.example.com: "nobody" is invalid: user "nobody" not found`,
	103	+	`_sish.example.com: "SHA256:someone-else" is invalid: no record is your username or the SHA256 fingerprint of one of your keys`,
	104	+	}},
	105	+	{"tuns record with whitespace", "", map[string][]string{"sish": {fingerprint + " "}}, []string{
	106	+	fmt.Sprintf(`_sish.example.com: %q is invalid: it has surrounding whitespace, and tuns needs an exact match`, fingerprint+" "),
	107	+	}},
	108	+	{"tuns username", "", map[string][]string{"tuns": {"antonio"}}, []string{
	109	+	`_tuns.example.com: "antonio" is valid, tunnels from any of your keys can use this domain`,
	110	+	"_sish.example.com: no TXT record",
	111	+	}},
	112	+	{"wildcard", "*.example.com", map[string][]string{"pgs": {"antonio-blog"}, "sish": {"antonio"}}, []string{
	113	+	"_pgs.*.example.com: wildcard domains are not supported\r\n",
	114	+	"_prose.*.example.com: wildcard domains are not supported\r\n",
	115	+	`.example.com: "antonio" is valid, tunnels from any of your keys can use this domain`,
	116	+	"_tuns.",
	117	+	}},
	118	+	}
	119	+	for _, tt := range tests {
	120	+	t.Run(tt.name, func(t *testing.T) {
	121	+	session := &fakeSession{}
	122	+	cmd := &Cmd{
	123	+	User: me,
	124	+	Session: session,
	125	+	Log: slog.New(slog.DiscardHandler),
	126	+	Dbpool: dbpool,
	127	+	lookupTXT: func(host, space string) ([]string, error) {
	128	+	if tt.domain == "" && host != "example.com" {
	129	+	t.Fatalf("looked up %q", host)
	130	+	}
	131	+	if tt.domain != "" && (!strings.HasSuffix(host, ".example.com") \|\| strings.Contains(host, "*")) {
	132	+	t.Fatalf("looked up %q for %q", host, tt.domain)
	133	+	}
	134	+	if v, ok := tt.records[space]; ok {
	135	+	return v, nil
	136	+	}
	137	+	return nil, router.ErrNoTXTRecord
	138	+	},
	139	+	}
	140	+	domain := tt.domain
	141	+	if domain == "" {
	142	+	domain = " Example.COM. "
	143	+	}
	144	+	if err := cmd.dnsCheck(domain); err != nil {
	145	+	t.Fatal(err)
	146	+	}
	147	+	out := session.String()
	148	+	for _, line := range tt.want {
	149	+	if !strings.Contains(out, line) {
	150	+	t.Errorf("missing %q in output:\n%s", line, out)
	151	+	}
	152	+	}
	153	+	})
	154	+	}
	155	+
	156	+	cmd := &Cmd{User: me, Session: &fakeSession{}, Dbpool: dbpool}
	157	+	for _, bad := range []string{"https://example.com/", ".", "a..example.com"} {
	158	+	if err := cmd.dnsCheck(bad); err == nil {
	159	+	t.Errorf("%q was accepted as a domain", bad)
	160	+	}
	161	+	}
	162	+	}

...	...		@@ -1,7 +1,6 @@
1	1		package httpcache
2	2
3	3		import (
4		-	"encoding/json"
5	4		"log/slog"
6	5		"net/http"
7	6		"net/http/httptest"

...	...		@@ -241,7 +240,7 @@ func TestCacheVary(t *testing.T) {
241	240		// VaryRequestHeaders snapshots the request header values that were present
242	241		// when this entry was cached, keyed by the lowercase header name.
243	242		cv.VaryRequestHeaders = map[string]string{"accept-encoding": "gzip"}
244		-	cacheValue, _ := json.Marshal(cv)
	243	+	cacheValue := cv.encode()
245	244		handler.Cache.Add(cacheKey, cacheValue)
246	245
247	246		respMatch, _ := tc.DoWithHeaders(req, map[string][]string{

...	...		@@ -385,7 +384,7 @@ func TestCacheValidation(t *testing.T) {
385	384		cv := testCacheValue(250 * time.Second)
386	385		cv.Header["ETag"] = []string{"ccc"}
387	386		cv.Header["Last-Modified"] = []string{actualStr}
388		-	cacheValue, _ := json.Marshal(cv)
	387	+	cacheValue := cv.encode()
389	388		handler.Cache.Add(cacheKey, cacheValue)
390	389
391	390		t.Run(tt.name, func(t *testing.T) {

...	...		@@ -423,7 +422,7 @@ func TestCacheAge(t *testing.T) {
423	422
424	423		req, _ := http.NewRequest("GET", tc.cachedServer.URL+"/test", nil)
425	424		cacheKey := handler.GetCacheKey(req)
426		-	cacheValue, _ := json.Marshal(testCacheValue(250 * time.Second))
	425	+	cacheValue := testCacheValue(250 * time.Second).encode()
427	426		handler.Cache.Add(cacheKey, cacheValue)
428	427
429	428		resp, _ := tc.Do(req)

...	...		@@ -509,7 +508,7 @@ func TestCacheRequestDirectives(t *testing.T) {
509	508
510	509		req, _ := http.NewRequest("GET", tc.cachedServer.URL+"/test", nil)
511	510		cacheKey := handler.GetCacheKey(req)
512		-	cacheValue, _ := json.Marshal(testCacheValue(250 * time.Second))
	511	+	cacheValue := testCacheValue(250 * time.Second).encode()
513	512		handler.Cache.Add(cacheKey, cacheValue)
514	513
515	514		t.Run(tt.name, func(t *testing.T) {

...	...		@@ -762,7 +761,7 @@ func TestCache304NotModifiedMerge(t *testing.T) {
762	761		staleCv.Header["ETag"] = []string{"\"abc\""}
763	762		staleCv.Header["Cache-Control"] = []string{"max-age=60, must-revalidate"}
764	763		staleCv.Body = []byte("original body")
765		-	cacheData, _ := json.Marshal(staleCv)
	764	+	cacheData := staleCv.encode()
766	765		handler.Cache.Add(cacheKey, cacheData)
767	766
768	767		// First request with If-None-Match triggers validation; origin returns 304

...	...		@@ -890,7 +889,7 @@ func TestCache304NoBody(t *testing.T) {
890	889		cv.Header["ETag"] = []string{"\"abc\""}
891	890		cv.Header["Cache-Control"] = []string{"max-age=60, must-revalidate"}
892	891		cv.Body = []byte("original body")
893		-	cacheData, _ := json.Marshal(cv)
	892	+	cacheData := cv.encode()
894	893		handler.Cache.Add(cacheKey, cacheData)
895	894
896	895		// Trigger revalidation — upstream returns 304 with a spurious body.

...	...		@@ -944,7 +943,7 @@ func TestCache304IncludesCachedHeaders(t *testing.T) {
944	943		cv.Header["Content-Type"] = []string{"text/html; charset=utf-8"}
945	944		cv.Header["Cache-Control"] = []string{"max-age=300"}
946	945		cv.Body = []byte("<h1>hello</h1>")
947		-	cacheData, _ := json.Marshal(cv)
	946	+	cacheData := cv.encode()
948	947		handler.Cache.Add(cacheKey, cacheData)
949	948
950	949		// Send conditional request that triggers a 304 from the cache layer

...	...		@@ -1075,7 +1074,7 @@ func TestCacheMustRevalidateRevalidationHeaders(t *testing.T) {
1075	1074		}
1076	1075		cv.Header["Cache-Control"] = []string{"max-age=60, must-revalidate"}
1077	1076		cv.Body = []byte("cached body")
1078		-	cacheData, _ := json.Marshal(cv)
	1077	+	cacheData := cv.encode()
1079	1078		handler.Cache.Add(cacheKey, cacheData)
1080	1079
1081	1080		resp, _ := tc.Do(req)

...	...		@@ -1103,3 +1102,66 @@ func TestCacheMustRevalidateRevalidationHeaders(t *testing.T) {
1103	1102		})
1104	1103		}
1105	1104		}
	1105	+
	1106	+	// A range request that misses must not leave a partial response cached for
	1107	+	// requests of the whole resource.
	1108	+	func TestCacheRangeNotStored(t *testing.T) {
	1109	+	body := strings.Repeat("0123456789", 100)
	1110	+	upstream := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
	1111	+	http.ServeContent(w, r, "file.txt", time.Unix(1_700_000_000, 0), strings.NewReader(body))
	1112	+	})
	1113	+	handler := NewHttpCache(slog.New(slog.DiscardHandler), upstream)
	1114	+
	1115	+	req := httptest.NewRequest("GET", "http://example.com/file.txt", nil)
	1116	+	req.Header.Set("Range", "bytes=0-9")
	1117	+	rec := httptest.NewRecorder()
	1118	+	handler.ServeHTTP(rec, req)
	1119	+	if rec.Code != http.StatusPartialContent \|\| rec.Body.String() != body[:10] {
	1120	+	t.Fatalf("range request got %d %q", rec.Code, rec.Body.String())
	1121	+	}
	1122	+
	1123	+	for _, want := range []string{"miss", "hit"} {
	1124	+	rec = httptest.NewRecorder()
	1125	+	handler.ServeHTTP(rec, httptest.NewRequest("GET", "http://example.com/file.txt", nil))
	1126	+	if rec.Code != http.StatusOK \|\| rec.Body.String() != body {
	1127	+	t.Fatalf("full request got %d with %d bytes", rec.Code, rec.Body.Len())
	1128	+	}
	1129	+	if status := rec.Header().Get("cache-status"); !strings.Contains(status, want) {
	1130	+	t.Fatalf("full request cache-status %q, want %s", status, want)
	1131	+	}
	1132	+	}
	1133	+	}
	1134	+
	1135	+	// Bodies over MaxBodySize stream through and are not stored, whether or not
	1136	+	// upstream announces their length.
	1137	+	func TestCacheMaxBodySize(t *testing.T) {
	1138	+	body := strings.Repeat("x", 1000)
	1139	+	for name, setLength := range map[string]bool{"content-length": true, "chunked": false} {
	1140	+	t.Run(name, func(t *testing.T) {
	1141	+	upstream := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
	1142	+	if setLength {
	1143	+	w.Header().Set("content-length", strconv.Itoa(len(body)))
	1144	+	}
	1145	+	for i := 0; i < len(body); i += 100 {
	1146	+	_, _ = w.Write([]byte(body[i : i+100]))
	1147	+	}
	1148	+	})
	1149	+	handler := NewHttpCache(slog.New(slog.DiscardHandler), upstream)
	1150	+	handler.MaxBodySize = 250
	1151	+
	1152	+	for range 2 {
	1153	+	rec := httptest.NewRecorder()
	1154	+	handler.ServeHTTP(rec, httptest.NewRequest("GET", "http://example.com/big", nil))
	1155	+	if rec.Code != http.StatusOK \|\| rec.Body.String() != body {
	1156	+	t.Fatalf("got %d with %d bytes", rec.Code, rec.Body.Len())
	1157	+	}
	1158	+	if status := rec.Header().Get("cache-status"); !strings.Contains(status, "miss") {
	1159	+	t.Fatalf("cache-status %q, want a miss", status)
	1160	+	}
	1161	+	}
	1162	+	if handler.Cache.Len() != 0 {
	1163	+	t.Fatalf("stored %d entries", handler.Cache.Len())
	1164	+	}
	1165	+	})
	1166	+	}
	1167	+	}

...	...		@@ -1,26 +1,77 @@
1	1		package httpcache
2	2
3	3		import (
	4	+	"encoding/binary"
	5	+	"encoding/json"
	6	+	"errors"
4	7		"net/http"
	8	+	"strconv"
5	9		"strings"
6	10		"time"
7	11		)
8	12
	13	+	// responseWriter buffers an upstream response so it can be stored. A body
	14	+	// larger than maxBody is not stored: what was buffered is sent and the rest
	15	+	// streams straight through.
9	16		type responseWriter struct {
10	17		http.ResponseWriter
11	18		statusCode int
12	19		body []byte
	20	+	maxBody int64
	21	+	streaming bool
	22	+	// streamStatus is the cache-status sent with a streamed response.
	23	+	streamStatus string
13	24		}
14	25
15	26		func (rw *responseWriter) WriteHeader(code int) {
16	27		rw.statusCode = code
	28	+	size, err := strconv.ParseInt(rw.Header().Get("content-length"), 10, 64)
	29	+	if err == nil && rw.tooLarge(size) {
	30	+	_ = rw.stream()
	31	+	}
17	32		}
18	33
19	34		func (rw *responseWriter) Write(data []byte) (int, error) {
	35	+	if !rw.streaming && rw.tooLarge(int64(len(rw.body)+len(data))) {
	36	+	if err := rw.stream(); err != nil {
	37	+	return 0, err
	38	+	}
	39	+	}
	40	+	if rw.streaming {
	41	+	return rw.ResponseWriter.Write(data)
	42	+	}
20	43		rw.body = append(rw.body, data...)
21	44		return len(data), nil
22	45		}
23	46
	47	+	// Flush passes through once the response is streaming. Flushing a buffered
	48	+	// response would send its headers before it is known whether it is stored.
	49	+	func (rw *responseWriter) Flush() {
	50	+	if rw.streaming {
	51	+	_ = http.NewResponseController(rw.ResponseWriter).Flush()
	52	+	}
	53	+	}
	54	+
	55	+	func (rw *responseWriter) tooLarge(size int64) bool {
	56	+	return rw.maxBody > 0 && size > rw.maxBody
	57	+	}
	58	+
	59	+	func (rw *responseWriter) stream() error {
	60	+	if rw.streaming {
	61	+	return nil
	62	+	}
	63	+	rw.streaming = true
	64	+	rw.Header().Set("cache-status", rw.streamStatus)
	65	+	rw.ResponseWriter.WriteHeader(rw.StatusCode())
	66	+	body := rw.body
	67	+	rw.body = nil
	68	+	if len(body) == 0 {
	69	+	return nil
	70	+	}
	71	+	_, err := rw.ResponseWriter.Write(body)
	72	+	return err
	73	+	}
	74	+
24	75		// Body returns the captured response body.
25	76		func (rw *responseWriter) Body() []byte {
26	77		return rw.body

...	...		@@ -77,8 +128,33 @@ func (rw responseWriter) ToCacheValue(r http.Request) *CacheValue {
77	128
78	129		type CacheValue struct {
79	130		Header map[string][]string `json:"headers"`
80		-	Body []byte `json:"body"`
	131	+	Body []byte `json:"-"`
81	132		CreatedAt time.Time `json:"created_at"`
82	133		StatusCode int `json:"status_code"`
83	134		VaryRequestHeaders map[string]string `json:"vary_request_headers,omitempty"`
84	135		}
	136	+
	137	+	// encode stores the body after the JSON metadata so a cache hit decodes only
	138	+	// the metadata.
	139	+	func (cv *CacheValue) encode() []byte {
	140	+	meta, _ := json.Marshal(cv)
	141	+	out := make([]byte, 0, binary.MaxVarintLen64+len(meta)+len(cv.Body))
	142	+	out = binary.AppendUvarint(out, uint64(len(meta)))
	143	+	out = append(out, meta...)
	144	+	return append(out, cv.Body...)
	145	+	}
	146	+
	147	+	// decodeCacheValue reads an entry written by encode. The body shares memory
	148	+	// with data, which the cache never modifies.
	149	+	func decodeCacheValue(data []byte) (*CacheValue, error) {
	150	+	n, k := binary.Uvarint(data)
	151	+	if k <= 0 \|\| n > uint64(len(data)-k) {
	152	+	return nil, errors.New("corrupt cache entry")
	153	+	}
	154	+	cv := &CacheValue{}
	155	+	if err := json.Unmarshal(data[k:k+int(n)], cv); err != nil {
	156	+	return nil, err
	157	+	}
	158	+	cv.Body = data[k+int(n):]
	159	+	return cv, nil
	160	+	}

...	...		@@ -1,7 +1,6 @@
1	1		package httpcache
2	2
3	3		import (
4		-	"encoding/json"
5	4		"errors"
6	5		"fmt"
7	6		"log/slog"

...	...		@@ -52,6 +51,9 @@ type HttpCache struct {
52	51		Upstream http.Handler
53	52		Cache Cacher
54	53		Logger *slog.Logger
	54	+	// MaxBodySize is the largest body that is stored. Larger responses
	55	+	// stream through uncached. Zero means no limit.
	56	+	MaxBodySize int64
55	57		}
56	58
57	59		func NewHttpCache(log slog.Logger, upstream http.Handler) HttpCache {

...	...		@@ -78,6 +80,10 @@ func (c HttpCache) ServeHTTP(w http.ResponseWriter, r http.Request) {
78	80		cacheKey := c.GetCacheKey(r)
79	81		log := c.Logger.With("cache_key", cacheKey)
80	82
	83	+	// A partial response must not be stored under the key of the whole
	84	+	// resource, so a range request that misses goes straight upstream.
	85	+	hasRange := r.Header.Get("range") != ""
	86	+
81	87		err := c.maybeUseCache(cacheKey, w, r)
82	88		if err == nil {
83	89		log.Info("cache hit")

...	...		@@ -85,6 +91,15 @@ func (c HttpCache) ServeHTTP(w http.ResponseWriter, r http.Request) {
85	91		return
86	92		}
87	93
	94	+	if hasRange {
	95	+	log.Info("cache miss for range request, passing upstream", "err", err)
	96	+	c.AddCacheMiss()
	97	+	w.Header().Set("cache-status", cacheStatusMiss(cacheKey, false))
	98	+	c.Upstream.ServeHTTP(w, r)
	99	+	c.AddUpstreamRequest()
	100	+	return
	101	+	}
	102	+
88	103		// RFC 9111 5.2.1.7 only-if-cached - don't store new responses
89	104		cacheContState := parseCacheControl(r.Header.Get("cache-control"))
90	105		onlyIfCached := cacheContState.onlyIfCache

...	...		@@ -105,8 +120,7 @@ func (c HttpCache) ServeHTTP(w http.ResponseWriter, r http.Request) {
105	120
106	121		if errors.Is(err, ErrMustRevalidate) {
107	122		if cachedData, exists := c.Cache.Get(cacheKey); exists {
108		-	var cachedValue CacheValue
109		-	if json.Unmarshal(cachedData, &cachedValue) == nil {
	123	+	if cachedValue, err := decodeCacheValue(cachedData); err == nil {
110	124		if etag := getHeader(cachedValue.Header, "etag"); etag != "" {
111	125		r.Header.Set("if-none-match", etag)
112	126		}

...	...		@@ -119,9 +133,17 @@ func (c HttpCache) ServeHTTP(w http.ResponseWriter, r http.Request) {
119	133
120	134		log.Info("cache miss, requesting upstream", "err", err)
121	135		c.AddCacheMiss()
122		-	wrapped := &responseWriter{ResponseWriter: w}
	136	+	wrapped := &responseWriter{
	137	+	ResponseWriter: w,
	138	+	maxBody: c.MaxBodySize,
	139	+	streamStatus: cacheStatusMiss(cacheKey, false),
	140	+	}
123	141		c.Upstream.ServeHTTP(wrapped, r)
124	142		c.AddUpstreamRequest()
	143	+	if wrapped.streaming {
	144	+	log.Info("not cachable", "err", "response body too large")
	145	+	return
	146	+	}
125	147
126	148		// RFC 9111 4.3.4 304 Not Modified
127	149		// https://www.rfc-editor.org/rfc/rfc9111.html#section-4.3.4

...	...		@@ -135,10 +157,9 @@ func (c HttpCache) ServeHTTP(w http.ResponseWriter, r http.Request) {
135	157		return
136	158		}
137	159
138		-	var cacheValue CacheValue
139		-	err = json.Unmarshal(existingData, &cacheValue)
	160	+	cacheValue, err := decodeCacheValue(existingData)
140	161		if err != nil {
141		-	log.Error("json unmarshal", "err", err)
	162	+	log.Error("decode cache entry", "err", err)
142	163		wrapped.Send()
143	164		return
144	165		}

...	...		@@ -161,7 +182,7 @@ func (c HttpCache) ServeHTTP(w http.ResponseWriter, r http.Request) {
161	182		}
162	183		// Revalidation refreshes the entry -- reset CreatedAt so it's fresh again.
163	184		cacheValue.CreatedAt = time.Now()
164		-	enc, _ := json.Marshal(cacheValue)
	185	+	enc := cacheValue.encode()
165	186		log.Info("updating cached headers from 304 response")
166	187		c.Cache.Remove(cacheKey)
167	188		c.Cache.Add(cacheKey, enc)

...	...		@@ -172,9 +193,9 @@ func (c HttpCache) ServeHTTP(w http.ResponseWriter, r http.Request) {
172	193		// updated cached entry and return 304 if it still matches.
173	194		r.Header.Set("if-none-match", clientIfNoneMatch)
174	195		r.Header.Set("if-modified-since", clientIfModifiedSince)
175		-	valid := c.handleValidation(r, &cacheValue)
	196	+	valid := c.handleValidation(r, cacheValue)
176	197		if valid {
177		-	hdr := stripForbiddenHeaders(w, &cacheValue)
	198	+	hdr := stripForbiddenHeaders(w, cacheValue)
178	199		ageDur := calcAge(cacheValue.CreatedAt)
179	200		hdr.Set("age", strconv.Itoa(int(ageDur.Seconds())+1))
180	201		hdr.Set("cache-status", cacheStatusStale(cacheKey, wrapped.StatusCode()))

...	...		@@ -187,15 +208,14 @@ func (c HttpCache) ServeHTTP(w http.ResponseWriter, r http.Request) {
187	208		// Client request was unconditional (or conditional but no longer matches)
188	209		// serve the full cached response.
189	210		log.Info("serving full cached response to client")
190		-	serveCache(w, c.Ttl, cacheKey, &cacheValue)
	211	+	serveCache(w, c.Ttl, cacheKey, cacheValue)
191	212		return
192	213		}
193	214
194	215		err = isResponseCachable(r, wrapped)
195	216		if err == nil {
196	217		log.Info("storing cache")
197		-	nextValue := wrapped.ToCacheValue(r)
198		-	enc, _ := json.Marshal(nextValue)
	218	+	enc := wrapped.ToCacheValue(r).encode()
199	219		c.Cache.Remove(cacheKey)
200	220		c.Cache.Add(cacheKey, enc)
201	221		c.AddCacheItem(float64(len(enc)))

...	...		@@ -409,10 +429,11 @@ func isResponseCachable(r http.Request, resp responseWriter) error {
409	429		// 206, 300, 301,
410	430		// 308, 404, 405,
411	431		// 410, 414, 501.
	432	+	// 206 is left out: the cache key does not include the range.
412	433		func isCacheableStatusCode(code int) bool {
413	434		switch code {
414	435		case http.StatusOK, http.StatusNonAuthoritativeInfo, http.StatusNoContent,
415		-	http.StatusPartialContent, http.StatusMultipleChoices, http.StatusMovedPermanently,
	436	+	http.StatusMultipleChoices, http.StatusMovedPermanently,
416	437		http.StatusPermanentRedirect, http.StatusNotFound, http.StatusMethodNotAllowed,
417	438		http.StatusGone, http.StatusRequestURITooLong, http.StatusNotImplemented:
418	439		return true

...	...		@@ -559,14 +580,13 @@ func (c HttpCache) maybeUseCache(cacheKey string, w http.ResponseWriter, r htt
559	580		return fmt.Errorf("no cache stored")
560	581		}
561	582
562		-	var cacheValue CacheValue
563		-	err := json.Unmarshal(data, &cacheValue)
	583	+	cacheValue, err := decodeCacheValue(data)
564	584		if err != nil {
565		-	return fmt.Errorf("json unmarshal: %w", err)
	585	+	return fmt.Errorf("decode cache entry: %w", err)
566	586		}
567	587
568	588		// RFC 9111 4.1 Vary - check if request matches cached Vary values
569		-	if !matchVary(r, &cacheValue) {
	589	+	if !matchVary(r, cacheValue) {
570	590		return fmt.Errorf("vary mismatch")
571	591		}
572	592

...	...		@@ -624,9 +644,9 @@ func (c HttpCache) maybeUseCache(cacheKey string, w http.ResponseWriter, r htt
624	644		// RFC 9111 4.3 Validation - check validation headers first
625	645		// RFC 9110 13 Conditional Requests
626	646		// https://www.rfc-editor.org/rfc/rfc9110.html#section-13
627		-	valid := c.handleValidation(r, &cacheValue)
	647	+	valid := c.handleValidation(r, cacheValue)
628	648		if valid {
629		-	hdr := stripForbiddenHeaders(w, &cacheValue)
	649	+	hdr := stripForbiddenHeaders(w, cacheValue)
630	650		ageDur := calcAge(cacheValue.CreatedAt)
631	651		hdr.Set("age", strconv.Itoa(int(ageDur.Seconds())+1))
632	652		hdr.Set("cache-status", cacheStatusHit(cacheKey, freshness.Seconds()))

...	...		@@ -658,7 +678,7 @@ func (c HttpCache) maybeUseCache(cacheKey string, w http.ResponseWriter, r htt
658	678		return fmt.Errorf("response older than request max-age")
659	679		}
660	680
661		-	serveCache(w, freshness, cacheKey, &cacheValue)
	681	+	serveCache(w, freshness, cacheKey, cacheValue)
662	682		return nil
663	683		}
664	684

...	...		@@ -349,10 +349,12 @@ func (s *SSHServer) ListenAndServe() error {
349	349		},
350	350		}, []string{"command"})
351	351
352		-	go func() {
353		-	mux := http.NewServeMux()
354		-	mux.Handle("/metrics", promhttp.Handler())
	352	+	// The handler reads the default registerer, so build it before the
	353	+	// listener is published rather than in the goroutine.
	354	+	mux := http.NewServeMux()
	355	+	mux.Handle("/metrics", promhttp.Handler())
355	356
	357	+	go func() {
356	358		srv := &http.Server{Addr: s.Config.PromListenAddr, Handler: mux}
357	359
358	360		go func() {

...	...		@@ -2,6 +2,7 @@ package router
2	2
3	3		import (
4	4		"context"
	5	+	"errors"
5	6		"fmt"
6	7		"log/slog"
7	8		"net"

...	...		@@ -9,6 +10,7 @@ import (
9	10		"net/http/pprof"
10	11		"regexp"
11	12		"strings"
	13	+	"time"
12	14
13	15		"github.com/hashicorp/golang-lru/v2/expirable"
14	16		"github.com/picosh/pico/pkg/db"

...	...		@@ -196,25 +198,66 @@ func GetSubdomain(r *http.Request) string {
196	198
197	199		var txtCache = expirable.NewLRU[string, string](2048, nil, shared.CacheTimeout)
198	200
199		-	func GetCustomDomain(host string, space string) string {
200		-	txt := fmt.Sprintf("_%s.%s", space, host)
201		-	record, found := txtCache.Get(txt)
202		-	if found {
203		-	return record
	201	+	// txtMissCache remembers hosts without a TXT record for a short time, so
	202	+	// unknown hosts don't cost a lookup per request and a record that was just
	203	+	// added starts working quickly.
	204	+	var txtMissCache = expirable.NewLRU[string, struct{}](2048, nil, 30*time.Second)
	205	+
	206	+	var lookupTXT = net.LookupTXT
	207	+
	208	+	// ErrNoTXTRecord is returned when a host has no TXT record for a service.
	209	+	var ErrNoTXTRecord = errors.New("no TXT record")
	210	+
	211	+	// TXTRecordName is the record that points host at a site on space.
	212	+	func TXTRecordName(host, space string) string {
	213	+	return fmt.Sprintf("_%s.%s", space, host)
	214	+	}
	215	+
	216	+	// LookupTXTRecords asks DNS for host's records on space, bypassing the
	217	+	// cache. No records returns ErrNoTXTRecord.
	218	+	func LookupTXTRecords(host, space string) ([]string, error) {
	219	+	records, err := lookupTXT(TXTRecordName(host, space))
	220	+	var dnsErr *net.DNSError
	221	+	if errors.As(err, &dnsErr) && dnsErr.IsNotFound {
	222	+	return nil, ErrNoTXTRecord
	223	+	}
	224	+	if err != nil {
	225	+	return nil, err
204	226		}
	227	+	if len(records) == 0 {
	228	+	return nil, ErrNoTXTRecord
	229	+	}
	230	+	return records, nil
	231	+	}
205	232
206		-	records, err := net.LookupTXT(txt)
	233	+	// LookupCustomDomain returns the site host's first record on space names,
	234	+	// bypassing the cache.
	235	+	func LookupCustomDomain(host, space string) (string, error) {
	236	+	records, err := LookupTXTRecords(host, space)
207	237		if err != nil {
208		-	return ""
	238	+	return "", err
209	239		}
	240	+	return strings.TrimSpace(records[0]), nil
	241	+	}
210	242
211		-	for _, v := range records {
212		-	rec := strings.TrimSpace(v)
213		-	txtCache.Add(txt, rec)
214		-	return rec
	243	+	func GetCustomDomain(host string, space string) string {
	244	+	txt := TXTRecordName(host, space)
	245	+	if record, found := txtCache.Get(txt); found {
	246	+	return record
	247	+	}
	248	+	if _, missed := txtMissCache.Get(txt); missed {
	249	+	return ""
215	250		}
216	251
217		-	return ""
	252	+	record, err := LookupCustomDomain(host, space)
	253	+	if errors.Is(err, ErrNoTXTRecord) {
	254	+	txtMissCache.Add(txt, struct{}{})
	255	+	}
	256	+	if err != nil {
	257	+	return ""
	258	+	}
	259	+	txtCache.Add(txt, record)
	260	+	return record
218	261		}
219	262
220	263		func GetApiToken(r *http.Request) string {

...	...		@@ -0,0 +1,52 @@
	1	+	package router
	2	+
	3	+	import (
	4	+	"errors"
	5	+	"net"
	6	+	"testing"
	7	+	)
	8	+
	9	+	func TestGetCustomDomainCachesMisses(t *testing.T) {
	10	+	records := map[string][]string{}
	11	+	var lookups int
	12	+	var fail error
	13	+	lookupTXT = func(name string) ([]string, error) {
	14	+	lookups++
	15	+	if fail != nil {
	16	+	return nil, fail
	17	+	}
	18	+	if r, ok := records[name]; ok {
	19	+	return r, nil
	20	+	}
	21	+	return nil, &net.DNSError{Err: "no such host", Name: name, IsNotFound: true}
	22	+	}
	23	+	t.Cleanup(func() {
	24	+	lookupTXT = net.LookupTXT
	25	+	txtCache.Purge()
	26	+	txtMissCache.Purge()
	27	+	})
	28	+
	29	+	if got := GetCustomDomain("example.com", "pgs"); got != "" \|\| lookups != 1 {
	30	+	t.Fatalf("got %q after %d lookups", got, lookups)
	31	+	}
	32	+	if got := GetCustomDomain("example.com", "pgs"); got != "" \|\| lookups != 1 {
	33	+	t.Fatalf("missing record was looked up again: %q after %d lookups", got, lookups)
	34	+	}
	35	+
	36	+	// The uncached lookup sees a record as soon as it exists.
	37	+	records["_pgs.example.com"] = []string{" antonio-blog "}
	38	+	if got, err := LookupCustomDomain("example.com", "pgs"); got != "antonio-blog" \|\| err != nil {
	39	+	t.Fatalf("LookupCustomDomain = %q, %v", got, err)
	40	+	}
	41	+
	42	+	// Failures other than a missing record are not cached.
	43	+	fail = errors.New("i/o timeout")
	44	+	if got := GetCustomDomain("other.com", "pgs"); got != "" {
	45	+	t.Fatalf("got %q", got)
	46	+	}
	47	+	fail = nil
	48	+	records["_pgs.other.com"] = []string{"antonio-other"}
	49	+	if got := GetCustomDomain("other.com", "pgs"); got != "antonio-other" {
	50	+	t.Fatalf("lookup after a transient failure got %q", got)
	51	+	}
	52	+	}

...	...		@@ -13,7 +13,7 @@ var menuChoices = []string{
13	13		"invite",
14	14		"tokens",
15	15		"logs",
16		-	"access_logs",
	16	+	"access-logs",
17	17		"analytics",
18	18		"pages",
19	19		"tuns",

...	...		@@ -366,7 +366,7 @@ func NewTui(opts vaxis.Options, shrd *SharedModel) error {
366	366		"analytics": NewAnalyticsPage(shrd),
367	367		"chat": NewChatPage(shrd),
368	368		"tuns": NewTunsPage(shrd),
369		-	"access_logs": NewAccessLogsPage(shrd),
	369	+	"access-logs": NewAccessLogsPage(shrd),
370	370		"pages": NewPagesPage(shrd),
371	371		}
372	372		root := &App{

Commit a1973f3

Commit `a1973f3`