Eric Bower
·
2025-06-09
web.go
1package pgs
2
3import (
4 "bufio"
5 "context"
6 "fmt"
7 "html/template"
8 "log/slog"
9 "net/http"
10 "net/url"
11 "os"
12 "path/filepath"
13 "regexp"
14 "strings"
15 "time"
16
17 _ "net/http/pprof"
18
19 "github.com/darkweak/souin/configurationtypes"
20 "github.com/darkweak/souin/pkg/middleware"
21 "github.com/darkweak/souin/plugins/souin/storages"
22 "github.com/darkweak/storages/core"
23 "github.com/gorilla/feeds"
24 "github.com/hashicorp/golang-lru/v2/expirable"
25 "github.com/picosh/pico/pkg/cache"
26 "github.com/picosh/pico/pkg/db"
27 sst "github.com/picosh/pico/pkg/pobj/storage"
28 "github.com/picosh/pico/pkg/shared"
29 "github.com/picosh/pico/pkg/shared/storage"
30 "github.com/prometheus/client_golang/prometheus/promhttp"
31 "google.golang.org/protobuf/proto"
32)
33
34type CachedHttp struct {
35 handler *middleware.SouinBaseHandler
36 routes *WebRouter
37}
38
39func (c *CachedHttp) ServeHTTP(writer http.ResponseWriter, req *http.Request) {
40 _ = c.handler.ServeHTTP(writer, req, func(w http.ResponseWriter, r *http.Request) error {
41 c.routes.ServeHTTP(w, r)
42 return nil
43 })
44}
45
46func SetupCache(cfg *PgsConfig) *middleware.SouinBaseHandler {
47 ttl := configurationtypes.Duration{Duration: cfg.CacheTTL}
48 stale := configurationtypes.Duration{Duration: cfg.CacheTTL * 2}
49 c := &middleware.BaseConfiguration{
50 API: configurationtypes.API{
51 Prometheus: configurationtypes.APIEndpoint{
52 Enable: true,
53 },
54 },
55 DefaultCache: &configurationtypes.DefaultCache{
56 TTL: ttl,
57 Stale: stale,
58 Otter: configurationtypes.CacheProvider{
59 Uuid: fmt.Sprintf("OTTER-%s", stale),
60 Configuration: map[string]interface{}{},
61 },
62 Regex: configurationtypes.Regex{
63 Exclude: "/check|/_metrics",
64 },
65 MaxBodyBytes: uint64(cfg.MaxAssetSize),
66 DefaultCacheControl: cfg.CacheControl,
67 },
68 }
69 c.SetLogger(&CompatLogger{Logger: cfg.Logger})
70 storages.InitFromConfiguration(c)
71 return middleware.NewHTTPCacheHandler(c)
72}
73
74func StartApiServer(cfg *PgsConfig) {
75 ctx := context.Background()
76
77 httpCache := SetupCache(cfg)
78 routes := NewWebRouter(cfg)
79 cacher := &CachedHttp{
80 handler: httpCache,
81 routes: routes,
82 }
83
84 go routes.CacheMgmt(ctx, httpCache, cfg.CacheClearingQueue)
85
86 portStr := fmt.Sprintf(":%s", cfg.WebPort)
87 cfg.Logger.Info(
88 "starting server on port",
89 "port", cfg.WebPort,
90 "domain", cfg.Domain,
91 )
92 err := http.ListenAndServe(portStr, cacher)
93 cfg.Logger.Error(
94 "listen and serve",
95 "err", err.Error(),
96 )
97}
98
99type HasPerm = func(proj *db.Project) bool
100
101type WebRouter struct {
102 Cfg *PgsConfig
103 RootRouter *http.ServeMux
104 UserRouter *http.ServeMux
105 RedirectsCache *expirable.LRU[string, []*RedirectRule]
106 HeadersCache *expirable.LRU[string, []*HeaderRule]
107}
108
109func NewWebRouter(cfg *PgsConfig) *WebRouter {
110 router := &WebRouter{
111 Cfg: cfg,
112 RedirectsCache: expirable.NewLRU[string, []*RedirectRule](2048, nil, cache.CacheTimeout),
113 HeadersCache: expirable.NewLRU[string, []*HeaderRule](2048, nil, cache.CacheTimeout),
114 }
115 router.initRouters()
116 go router.WatchCacheClear()
117 return router
118}
119
120func (web *WebRouter) WatchCacheClear() {
121 for key := range web.Cfg.CacheClearingQueue {
122 web.Cfg.Logger.Info("lru cache clear request", "key", key)
123 rKey := filepath.Join(key, "_redirects")
124 web.RedirectsCache.Remove(rKey)
125 hKey := filepath.Join(key, "_headers")
126 web.HeadersCache.Remove(hKey)
127 }
128}
129
130func (web *WebRouter) initRouters() {
131 // ensure legacy router is disabled
132 // GODEBUG=httpmuxgo121=0
133
134 // root domain
135 rootRouter := http.NewServeMux()
136 rootRouter.HandleFunc("GET /check", web.checkHandler)
137 rootRouter.HandleFunc("GET /_metrics", promhttp.Handler().ServeHTTP)
138 rootRouter.Handle("GET /main.css", web.serveFile("main.css", "text/css"))
139 rootRouter.Handle("GET /favicon-16x16.png", web.serveFile("favicon-16x16.png", "image/png"))
140 rootRouter.Handle("GET /favicon.ico", web.serveFile("favicon.ico", "image/x-icon"))
141 rootRouter.Handle("GET /robots.txt", web.serveFile("robots.txt", "text/plain"))
142
143 rootRouter.Handle("GET /rss/updated", web.createRssHandler("updated_at"))
144 rootRouter.Handle("GET /rss", web.createRssHandler("created_at"))
145 rootRouter.Handle("GET /{$}", web.createPageHandler("html/marketing.page.tmpl"))
146 web.RootRouter = rootRouter
147
148 // subdomain or custom domains
149 userRouter := http.NewServeMux()
150 userRouter.HandleFunc("GET /{fname...}", web.AssetRequest(WebPerm))
151 userRouter.HandleFunc("GET /{$}", web.AssetRequest(WebPerm))
152 web.UserRouter = userRouter
153}
154
155func (web *WebRouter) serveFile(file string, contentType string) http.HandlerFunc {
156 return func(w http.ResponseWriter, r *http.Request) {
157 logger := web.Cfg.Logger
158 cfg := web.Cfg
159
160 contents, err := os.ReadFile(cfg.StaticPath(fmt.Sprintf("public/%s", file)))
161 if err != nil {
162 logger.Error(
163 "could not read file",
164 "fname", file,
165 "err", err.Error(),
166 )
167 http.Error(w, "file not found", 404)
168 }
169
170 w.Header().Add("Content-Type", contentType)
171
172 _, err = w.Write(contents)
173 if err != nil {
174 logger.Error(
175 "could not write http response",
176 "file", file,
177 "err", err.Error(),
178 )
179 }
180 }
181}
182
183func renderTemplate(cfg *PgsConfig, templates []string) (*template.Template, error) {
184 files := make([]string, len(templates))
185 copy(files, templates)
186 files = append(
187 files,
188 cfg.StaticPath("html/footer.partial.tmpl"),
189 cfg.StaticPath("html/marketing-footer.partial.tmpl"),
190 cfg.StaticPath("html/base.layout.tmpl"),
191 )
192
193 ts, err := template.New("base").ParseFiles(files...)
194 if err != nil {
195 return nil, err
196 }
197 return ts, nil
198}
199
200func (web *WebRouter) createPageHandler(fname string) http.HandlerFunc {
201 return func(w http.ResponseWriter, r *http.Request) {
202 logger := web.Cfg.Logger
203 cfg := web.Cfg
204 ts, err := renderTemplate(cfg, []string{cfg.StaticPath(fname)})
205
206 if err != nil {
207 logger.Error(
208 "could not render template",
209 "fname", fname,
210 "err", err.Error(),
211 )
212 http.Error(w, err.Error(), http.StatusInternalServerError)
213 return
214 }
215
216 data := shared.PageData{
217 Site: shared.SitePageData{Domain: template.URL(cfg.Domain), HomeURL: "/"},
218 }
219 err = ts.Execute(w, data)
220 if err != nil {
221 logger.Error(
222 "could not execute template",
223 "fname", fname,
224 "err", err.Error(),
225 )
226 http.Error(w, err.Error(), http.StatusInternalServerError)
227 }
228 }
229}
230
231func (web *WebRouter) checkHandler(w http.ResponseWriter, r *http.Request) {
232 dbpool := web.Cfg.DB
233 cfg := web.Cfg
234 logger := web.Cfg.Logger
235
236 hostDomain := r.URL.Query().Get("domain")
237 appDomain := strings.Split(cfg.Domain, ":")[0]
238
239 if !strings.Contains(hostDomain, appDomain) {
240 subdomain := shared.GetCustomDomain(hostDomain, cfg.TxtPrefix)
241 props, err := shared.GetProjectFromSubdomain(subdomain)
242 if err != nil {
243 logger.Error(
244 "could not get project from subdomain",
245 "subdomain", subdomain,
246 "err", err.Error(),
247 )
248 w.WriteHeader(http.StatusNotFound)
249 return
250 }
251
252 u, err := dbpool.FindUserByName(props.Username)
253 if err != nil {
254 logger.Error("could not find user", "err", err.Error())
255 w.WriteHeader(http.StatusNotFound)
256 return
257 }
258
259 logger = logger.With(
260 "user", u.Name,
261 "project", props.ProjectName,
262 )
263 p, err := dbpool.FindProjectByName(u.ID, props.ProjectName)
264 if err != nil {
265 logger.Error(
266 "could not find project for user",
267 "user", u.Name,
268 "project", props.ProjectName,
269 "err", err.Error(),
270 )
271 w.WriteHeader(http.StatusNotFound)
272 return
273 }
274
275 if u != nil && p != nil {
276 w.WriteHeader(http.StatusOK)
277 return
278 }
279 }
280
281 w.WriteHeader(http.StatusNotFound)
282}
283
284func (web *WebRouter) CacheMgmt(ctx context.Context, httpCache *middleware.SouinBaseHandler, notify chan string) {
285 storer := httpCache.Storers[0]
286 drain := createSubCacheDrain(ctx, web.Cfg.Logger)
287
288 for {
289 scanner := bufio.NewScanner(drain)
290 scanner.Buffer(make([]byte, 32*1024), 32*1024)
291 for scanner.Scan() {
292 surrogateKey := strings.TrimSpace(scanner.Text())
293 web.Cfg.Logger.Info("received cache-drain item", "surrogateKey", surrogateKey)
294 notify <- surrogateKey
295
296 if surrogateKey == "*" {
297 storer.DeleteMany(".+")
298 err := httpCache.SurrogateKeyStorer.Destruct()
299 if err != nil {
300 web.Cfg.Logger.Error("could not clear cache and surrogate key store", "err", err)
301 } else {
302 web.Cfg.Logger.Info("successfully cleared cache and surrogate keys store")
303 }
304 continue
305 }
306
307 var header http.Header = map[string][]string{}
308 header.Add("Surrogate-Key", surrogateKey)
309
310 ck, _ := httpCache.SurrogateKeyStorer.Purge(header)
311 for _, key := range ck {
312 key, _ = strings.CutPrefix(key, core.MappingKeyPrefix)
313 if b := storer.Get(core.MappingKeyPrefix + key); len(b) > 0 {
314 var mapping core.StorageMapper
315 if e := proto.Unmarshal(b, &mapping); e == nil {
316 for k := range mapping.GetMapping() {
317 qkey, _ := url.QueryUnescape(k)
318 web.Cfg.Logger.Info(
319 "deleting key from surrogate cache",
320 "surrogateKey", surrogateKey,
321 "key", qkey,
322 )
323 storer.Delete(qkey)
324 }
325 }
326 }
327
328 qkey, _ := url.QueryUnescape(key)
329 web.Cfg.Logger.Info(
330 "deleting from cache",
331 "surrogateKey", surrogateKey,
332 "key", core.MappingKeyPrefix+qkey,
333 )
334 storer.Delete(core.MappingKeyPrefix + qkey)
335 }
336 }
337 }
338}
339
340func (web *WebRouter) createRssHandler(by string) http.HandlerFunc {
341 return func(w http.ResponseWriter, r *http.Request) {
342 dbpool := web.Cfg.DB
343 logger := web.Cfg.Logger
344 cfg := web.Cfg
345
346 projects, err := dbpool.FindProjects(by)
347 if err != nil {
348 logger.Error("could not find projects", "err", err.Error())
349 http.Error(w, err.Error(), http.StatusInternalServerError)
350 return
351 }
352
353 feed := &feeds.Feed{
354 Title: fmt.Sprintf("%s discovery feed %s", cfg.Domain, by),
355 Link: &feeds.Link{Href: "https://pgs.sh"},
356 Description: fmt.Sprintf("%s projects %s", cfg.Domain, by),
357 Author: &feeds.Author{Name: cfg.Domain},
358 Created: time.Now(),
359 }
360
361 var feedItems []*feeds.Item
362 for _, project := range projects {
363 realUrl := strings.TrimSuffix(
364 cfg.AssetURL(project.Username, project.Name, ""),
365 "/",
366 )
367 uat := project.UpdatedAt.Unix()
368 id := realUrl
369 title := fmt.Sprintf("%s-%s", project.Username, project.Name)
370 if by == "updated_at" {
371 id = fmt.Sprintf("%s:%d", realUrl, uat)
372 title = fmt.Sprintf("%s - %d", title, uat)
373 }
374
375 item := &feeds.Item{
376 Id: id,
377 Title: title,
378 Link: &feeds.Link{Href: realUrl},
379 Content: fmt.Sprintf(`<a href="%s">%s</a>`, realUrl, realUrl),
380 Created: *project.CreatedAt,
381 Updated: *project.CreatedAt,
382 Description: "",
383 Author: &feeds.Author{Name: project.Username},
384 }
385
386 feedItems = append(feedItems, item)
387 }
388 feed.Items = feedItems
389
390 rss, err := feed.ToAtom()
391 if err != nil {
392 logger.Error("could not convert feed to atom", "err", err.Error())
393 http.Error(w, "Could not generate atom rss feed", http.StatusInternalServerError)
394 }
395
396 w.Header().Add("Content-Type", "application/atom+xml")
397 _, err = w.Write([]byte(rss))
398 if err != nil {
399 logger.Error("http write failed", "err", err.Error())
400 }
401 }
402}
403
404func WebPerm(proj *db.Project) bool {
405 return proj.Acl.Type == "public" || proj.Acl.Type == ""
406}
407
408var imgRegex = regexp.MustCompile("(.+.(?:jpg|jpeg|png|gif|webp|svg))(/.+)")
409
410func (web *WebRouter) AssetRequest(perm func(proj *db.Project) bool) http.HandlerFunc {
411 return func(w http.ResponseWriter, r *http.Request) {
412 fname := r.PathValue("fname")
413 if imgRegex.MatchString(fname) {
414 web.ImageRequest(perm)(w, r)
415 return
416 }
417 web.ServeAsset(fname, nil, perm, w, r)
418 }
419}
420
421func (web *WebRouter) ImageRequest(perm func(proj *db.Project) bool) http.HandlerFunc {
422 return func(w http.ResponseWriter, r *http.Request) {
423 rawname := r.PathValue("fname")
424 matches := imgRegex.FindStringSubmatch(rawname)
425 fname := rawname
426 imgOpts := ""
427 if len(matches) >= 2 {
428 fname = matches[1]
429 }
430 if len(matches) >= 3 {
431 imgOpts = matches[2]
432 }
433
434 opts, err := storage.UriToImgProcessOpts(imgOpts)
435 if err != nil {
436 errMsg := fmt.Sprintf("error processing img options: %s", err.Error())
437 web.Cfg.Logger.Error("error processing img options", "err", errMsg)
438 http.Error(w, errMsg, http.StatusUnprocessableEntity)
439 return
440 }
441
442 web.ServeAsset(fname, opts, perm, w, r)
443 }
444}
445
446func (web *WebRouter) ServeAsset(fname string, opts *storage.ImgProcessOpts, hasPerm HasPerm, w http.ResponseWriter, r *http.Request) {
447 subdomain := shared.GetSubdomain(r)
448
449 logger := web.Cfg.Logger.With(
450 "subdomain", subdomain,
451 "filename", fname,
452 "url", fmt.Sprintf("%s%s", r.Host, r.URL.Path),
453 "host", r.Host,
454 )
455
456 props, err := shared.GetProjectFromSubdomain(subdomain)
457 if err != nil {
458 logger.Info(
459 "could not determine project from subdomain",
460 "err", err,
461 )
462 http.Error(w, err.Error(), http.StatusNotFound)
463 return
464 }
465
466 logger = logger.With(
467 "project", props.ProjectName,
468 "user", props.Username,
469 )
470
471 user, err := web.Cfg.DB.FindUserByName(props.Username)
472 if err != nil {
473 logger.Info("user not found")
474 http.Error(w, "user not found", http.StatusNotFound)
475 return
476 }
477
478 logger = logger.With(
479 "userId", user.ID,
480 )
481
482 var bucket sst.Bucket
483 bucket, err = web.Cfg.Storage.GetBucket(shared.GetAssetBucketName(user.ID))
484 project, perr := web.Cfg.DB.FindProjectByName(user.ID, props.ProjectName)
485 if perr != nil {
486 logger.Info("project not found")
487 http.Error(w, "project not found", http.StatusNotFound)
488 return
489 }
490
491 logger = logger.With(
492 "projectId", project.ID,
493 "project", project.Name,
494 )
495
496 if project.Blocked != "" {
497 logger.Error("project has been blocked")
498 http.Error(w, project.Blocked, http.StatusForbidden)
499 return
500 }
501
502 if !hasPerm(project) {
503 logger.Error("You do not have access to this site")
504 http.Error(w, "You do not have access to this site", http.StatusUnauthorized)
505 return
506 }
507
508 if err != nil {
509 logger.Error("bucket not found", "err", err)
510 http.Error(w, "bucket not found", http.StatusNotFound)
511 return
512 }
513
514 hasPicoPlus := false
515 ff, _ := web.Cfg.DB.FindFeature(user.ID, "plus")
516 if ff != nil {
517 if ff.ExpiresAt.Before(time.Now()) {
518 hasPicoPlus = true
519 }
520 }
521
522 asset := &ApiAssetHandler{
523 WebRouter: web,
524 Logger: logger,
525
526 Username: props.Username,
527 UserID: user.ID,
528 Subdomain: subdomain,
529 ProjectID: project.ID,
530 ProjectDir: project.ProjectDir,
531 Filepath: fname,
532 Bucket: bucket,
533 ImgProcessOpts: opts,
534 HasPicoPlus: hasPicoPlus,
535 }
536
537 asset.ServeHTTP(w, r)
538}
539
540func (web *WebRouter) ServeHTTP(w http.ResponseWriter, r *http.Request) {
541 subdomain := shared.GetSubdomainFromRequest(r, web.Cfg.Domain, web.Cfg.TxtPrefix)
542 if web.RootRouter == nil || web.UserRouter == nil {
543 web.Cfg.Logger.Error("routers not initialized")
544 http.Error(w, "routers not initialized", http.StatusInternalServerError)
545 return
546 }
547
548 var router *http.ServeMux
549 if subdomain == "" {
550 router = web.RootRouter
551 } else {
552 router = web.UserRouter
553 }
554
555 ctx := r.Context()
556 ctx = context.WithValue(ctx, shared.CtxSubdomainKey{}, subdomain)
557 router.ServeHTTP(w, r.WithContext(ctx))
558}
559
560type CompatLogger struct {
561 Logger *slog.Logger
562}
563
564func (cl *CompatLogger) marshall(int ...interface{}) string {
565 res := ""
566 for _, val := range int {
567 switch r := val.(type) {
568 case string:
569 res += " " + r
570 }
571 }
572 return res
573}
574func (cl *CompatLogger) DPanic(int ...interface{}) {
575 cl.Logger.Error("panic", "output", cl.marshall(int))
576}
577func (cl *CompatLogger) DPanicf(st string, int ...interface{}) {
578 cl.Logger.Error(fmt.Sprintf(st, int...))
579}
580func (cl *CompatLogger) Debug(int ...interface{}) {
581 cl.Logger.Debug("debug", "output", cl.marshall(int))
582}
583func (cl *CompatLogger) Debugf(st string, int ...interface{}) {
584 cl.Logger.Debug(fmt.Sprintf(st, int...))
585}
586func (cl *CompatLogger) Error(int ...interface{}) {
587 cl.Logger.Error("error", "output", cl.marshall(int))
588}
589func (cl *CompatLogger) Errorf(st string, int ...interface{}) {
590 cl.Logger.Error(fmt.Sprintf(st, int...))
591}
592func (cl *CompatLogger) Fatal(int ...interface{}) {
593 cl.Logger.Error("fatal", "outpu", cl.marshall(int))
594}
595func (cl *CompatLogger) Fatalf(st string, int ...interface{}) {
596 cl.Logger.Error(fmt.Sprintf(st, int...))
597}
598func (cl *CompatLogger) Info(int ...interface{}) {
599 cl.Logger.Info("info", "output", cl.marshall(int))
600}
601func (cl *CompatLogger) Infof(st string, int ...interface{}) {
602 cl.Logger.Info(fmt.Sprintf(st, int...))
603}
604func (cl *CompatLogger) Panic(int ...interface{}) {
605 cl.Logger.Error("panic", "output", cl.marshall(int))
606}
607func (cl *CompatLogger) Panicf(st string, int ...interface{}) {
608 cl.Logger.Error(fmt.Sprintf(st, int...))
609}
610func (cl *CompatLogger) Warn(int ...interface{}) {
611 cl.Logger.Warn("warn", "output", cl.marshall(int))
612}
613func (cl *CompatLogger) Warnf(st string, int ...interface{}) {
614 cl.Logger.Warn(fmt.Sprintf(st, int...))
615}