Eric Bower
·
2025-07-04
web.go
1package pgs
2
3import (
4 "bufio"
5 "context"
6 "fmt"
7 "html/template"
8 "log/slog"
9 "net/http"
10 "net/url"
11 "os"
12 "path/filepath"
13 "regexp"
14 "strings"
15 "time"
16
17 _ "net/http/pprof"
18
19 "github.com/darkweak/souin/configurationtypes"
20 "github.com/darkweak/souin/pkg/middleware"
21 "github.com/darkweak/souin/plugins/souin/storages"
22 "github.com/darkweak/storages/core"
23 "github.com/gorilla/feeds"
24 "github.com/hashicorp/golang-lru/v2/expirable"
25 "github.com/picosh/pico/pkg/cache"
26 "github.com/picosh/pico/pkg/db"
27 sst "github.com/picosh/pico/pkg/pobj/storage"
28 "github.com/picosh/pico/pkg/shared"
29 "github.com/picosh/pico/pkg/shared/storage"
30 "github.com/prometheus/client_golang/prometheus/promhttp"
31 "google.golang.org/protobuf/proto"
32)
33
34type CachedHttp struct {
35 handler *middleware.SouinBaseHandler
36 routes *WebRouter
37}
38
39func (c *CachedHttp) ServeHTTP(writer http.ResponseWriter, req *http.Request) {
40 _ = c.handler.ServeHTTP(writer, req, func(w http.ResponseWriter, r *http.Request) error {
41 c.routes.ServeHTTP(w, r)
42 return nil
43 })
44}
45
46func SetupCache(cfg *PgsConfig) *middleware.SouinBaseHandler {
47 ttl := configurationtypes.Duration{Duration: cfg.CacheTTL}
48 stale := configurationtypes.Duration{Duration: cfg.CacheTTL * 2}
49 c := &middleware.BaseConfiguration{
50 API: configurationtypes.API{
51 Prometheus: configurationtypes.APIEndpoint{
52 Enable: true,
53 },
54 },
55 DefaultCache: &configurationtypes.DefaultCache{
56 TTL: ttl,
57 Stale: stale,
58 Otter: configurationtypes.CacheProvider{
59 Uuid: fmt.Sprintf("OTTER-%s", stale),
60 Configuration: map[string]interface{}{},
61 },
62 Regex: configurationtypes.Regex{
63 Exclude: "/check|/_metrics",
64 },
65 MaxBodyBytes: uint64(cfg.MaxAssetSize),
66 DefaultCacheControl: cfg.CacheControl,
67 },
68 }
69 c.SetLogger(&CompatLogger{Logger: cfg.Logger})
70 storages.InitFromConfiguration(c)
71 return middleware.NewHTTPCacheHandler(c)
72}
73
74func StartApiServer(cfg *PgsConfig) {
75 ctx := context.Background()
76
77 httpCache := SetupCache(cfg)
78 routes := NewWebRouter(cfg)
79 cacher := &CachedHttp{
80 handler: httpCache,
81 routes: routes,
82 }
83
84 go routes.CacheMgmt(ctx, httpCache, cfg.CacheClearingQueue)
85
86 portStr := fmt.Sprintf(":%s", cfg.WebPort)
87 cfg.Logger.Info(
88 "starting server on port",
89 "port", cfg.WebPort,
90 "domain", cfg.Domain,
91 )
92 err := http.ListenAndServe(portStr, cacher)
93 cfg.Logger.Error(
94 "listen and serve",
95 "err", err.Error(),
96 )
97}
98
99type HasPerm = func(proj *db.Project) bool
100
101type WebRouter struct {
102 Cfg *PgsConfig
103 RootRouter *http.ServeMux
104 UserRouter *http.ServeMux
105 RedirectsCache *expirable.LRU[string, []*RedirectRule]
106 HeadersCache *expirable.LRU[string, []*HeaderRule]
107}
108
109func NewWebRouter(cfg *PgsConfig) *WebRouter {
110 router := &WebRouter{
111 Cfg: cfg,
112 RedirectsCache: expirable.NewLRU[string, []*RedirectRule](2048, nil, cache.CacheTimeout),
113 HeadersCache: expirable.NewLRU[string, []*HeaderRule](2048, nil, cache.CacheTimeout),
114 }
115 router.initRouters()
116 go router.WatchCacheClear()
117 return router
118}
119
120func (web *WebRouter) WatchCacheClear() {
121 for key := range web.Cfg.CacheClearingQueue {
122 web.Cfg.Logger.Info("lru cache clear request", "key", key)
123 rKey := filepath.Join(key, "_redirects")
124 web.RedirectsCache.Remove(rKey)
125 hKey := filepath.Join(key, "_headers")
126 web.HeadersCache.Remove(hKey)
127 }
128}
129
130func (web *WebRouter) initRouters() {
131 // ensure legacy router is disabled
132 // GODEBUG=httpmuxgo121=0
133
134 // root domain
135 rootRouter := http.NewServeMux()
136 rootRouter.HandleFunc("GET /check", web.checkHandler)
137 rootRouter.HandleFunc("GET /_metrics", promhttp.Handler().ServeHTTP)
138 rootRouter.Handle("GET /main.css", web.serveFile("main.css", "text/css"))
139 rootRouter.Handle("GET /favicon-16x16.png", web.serveFile("favicon-16x16.png", "image/png"))
140 rootRouter.Handle("GET /favicon.ico", web.serveFile("favicon.ico", "image/x-icon"))
141 rootRouter.Handle("GET /robots.txt", web.serveFile("robots.txt", "text/plain"))
142
143 rootRouter.Handle("GET /rss/updated", web.createRssHandler("updated_at"))
144 rootRouter.Handle("GET /rss", web.createRssHandler("created_at"))
145 rootRouter.Handle("GET /{$}", web.createPageHandler("html/marketing.page.tmpl"))
146 web.RootRouter = rootRouter
147
148 // subdomain or custom domains
149 userRouter := http.NewServeMux()
150 userRouter.HandleFunc("GET /{fname...}", web.AssetRequest(WebPerm))
151 userRouter.HandleFunc("GET /{$}", web.AssetRequest(WebPerm))
152 web.UserRouter = userRouter
153}
154
155func (web *WebRouter) serveFile(file string, contentType string) http.HandlerFunc {
156 return func(w http.ResponseWriter, r *http.Request) {
157 logger := web.Cfg.Logger
158 cfg := web.Cfg
159
160 contents, err := os.ReadFile(cfg.StaticPath(fmt.Sprintf("public/%s", file)))
161 if err != nil {
162 logger.Error(
163 "could not read file",
164 "fname", file,
165 "err", err.Error(),
166 )
167 http.Error(w, "file not found", 404)
168 }
169
170 w.Header().Add("Content-Type", contentType)
171
172 _, err = w.Write(contents)
173 if err != nil {
174 logger.Error(
175 "could not write http response",
176 "file", file,
177 "err", err.Error(),
178 )
179 }
180 }
181}
182
183func renderTemplate(cfg *PgsConfig, templates []string) (*template.Template, error) {
184 files := make([]string, len(templates))
185 copy(files, templates)
186 files = append(
187 files,
188 cfg.StaticPath("html/footer.partial.tmpl"),
189 cfg.StaticPath("html/marketing-footer.partial.tmpl"),
190 cfg.StaticPath("html/base.layout.tmpl"),
191 )
192
193 ts, err := template.New("base").ParseFiles(files...)
194 if err != nil {
195 return nil, err
196 }
197 return ts, nil
198}
199
200func (web *WebRouter) createPageHandler(fname string) http.HandlerFunc {
201 return func(w http.ResponseWriter, r *http.Request) {
202 logger := web.Cfg.Logger
203 cfg := web.Cfg
204 ts, err := renderTemplate(cfg, []string{cfg.StaticPath(fname)})
205
206 if err != nil {
207 logger.Error(
208 "could not render template",
209 "fname", fname,
210 "err", err.Error(),
211 )
212 http.Error(w, err.Error(), http.StatusInternalServerError)
213 return
214 }
215
216 data := shared.PageData{
217 Site: shared.SitePageData{Domain: template.URL(cfg.Domain), HomeURL: "/"},
218 }
219 err = ts.Execute(w, data)
220 if err != nil {
221 logger.Error(
222 "could not execute template",
223 "fname", fname,
224 "err", err.Error(),
225 )
226 http.Error(w, err.Error(), http.StatusInternalServerError)
227 }
228 }
229}
230
231func (web *WebRouter) checkHandler(w http.ResponseWriter, r *http.Request) {
232 dbpool := web.Cfg.DB
233 cfg := web.Cfg
234 logger := web.Cfg.Logger
235
236 hostDomain := r.URL.Query().Get("domain")
237 appDomain := strings.Split(cfg.Domain, ":")[0]
238
239 if !strings.Contains(hostDomain, appDomain) {
240 subdomain := shared.GetCustomDomain(hostDomain, cfg.TxtPrefix)
241 props, err := shared.GetProjectFromSubdomain(subdomain)
242 if err != nil {
243 logger.Error(
244 "could not get project from subdomain",
245 "subdomain", subdomain,
246 "err", err.Error(),
247 )
248 w.WriteHeader(http.StatusNotFound)
249 return
250 }
251
252 u, err := dbpool.FindUserByName(props.Username)
253 if err != nil {
254 logger.Error("could not find user", "err", err.Error())
255 w.WriteHeader(http.StatusNotFound)
256 return
257 }
258
259 logger = logger.With(
260 "user", u.Name,
261 "project", props.ProjectName,
262 )
263 p, err := dbpool.FindProjectByName(u.ID, props.ProjectName)
264 if err != nil {
265 logger.Error(
266 "could not find project for user",
267 "user", u.Name,
268 "project", props.ProjectName,
269 "err", err.Error(),
270 )
271 w.WriteHeader(http.StatusNotFound)
272 return
273 }
274
275 if u != nil && p != nil {
276 w.WriteHeader(http.StatusOK)
277 return
278 }
279 }
280
281 w.WriteHeader(http.StatusNotFound)
282}
283
284func (web *WebRouter) CacheMgmt(ctx context.Context, httpCache *middleware.SouinBaseHandler, notify chan string) {
285 storer := httpCache.Storers[0]
286
287 for {
288 scanner := bufio.NewScanner(web.Cfg.Pubsub)
289 scanner.Buffer(make([]byte, 32*1024), 32*1024)
290 for scanner.Scan() {
291 surrogateKey := strings.TrimSpace(scanner.Text())
292 web.Cfg.Logger.Info("received cache-drain item", "surrogateKey", surrogateKey)
293 notify <- surrogateKey
294
295 if surrogateKey == "*" {
296 storer.DeleteMany(".+")
297 err := httpCache.SurrogateKeyStorer.Destruct()
298 if err != nil {
299 web.Cfg.Logger.Error("could not clear cache and surrogate key store", "err", err)
300 } else {
301 web.Cfg.Logger.Info("successfully cleared cache and surrogate keys store")
302 }
303 continue
304 }
305
306 var header http.Header = map[string][]string{}
307 header.Add("Surrogate-Key", surrogateKey)
308
309 ck, _ := httpCache.SurrogateKeyStorer.Purge(header)
310 for _, key := range ck {
311 key, _ = strings.CutPrefix(key, core.MappingKeyPrefix)
312 if b := storer.Get(core.MappingKeyPrefix + key); len(b) > 0 {
313 var mapping core.StorageMapper
314 if e := proto.Unmarshal(b, &mapping); e == nil {
315 for k := range mapping.GetMapping() {
316 qkey, _ := url.QueryUnescape(k)
317 web.Cfg.Logger.Info(
318 "deleting key from surrogate cache",
319 "surrogateKey", surrogateKey,
320 "key", qkey,
321 )
322 storer.Delete(qkey)
323 }
324 }
325 }
326
327 qkey, _ := url.QueryUnescape(key)
328 web.Cfg.Logger.Info(
329 "deleting from cache",
330 "surrogateKey", surrogateKey,
331 "key", core.MappingKeyPrefix+qkey,
332 )
333 storer.Delete(core.MappingKeyPrefix + qkey)
334 }
335 }
336 }
337}
338
339func (web *WebRouter) createRssHandler(by string) http.HandlerFunc {
340 return func(w http.ResponseWriter, r *http.Request) {
341 dbpool := web.Cfg.DB
342 logger := web.Cfg.Logger
343 cfg := web.Cfg
344
345 projects, err := dbpool.FindProjects(by)
346 if err != nil {
347 logger.Error("could not find projects", "err", err.Error())
348 http.Error(w, err.Error(), http.StatusInternalServerError)
349 return
350 }
351
352 feed := &feeds.Feed{
353 Title: fmt.Sprintf("%s discovery feed %s", cfg.Domain, by),
354 Link: &feeds.Link{Href: "https://pgs.sh"},
355 Description: fmt.Sprintf("%s projects %s", cfg.Domain, by),
356 Author: &feeds.Author{Name: cfg.Domain},
357 Created: time.Now(),
358 }
359
360 var feedItems []*feeds.Item
361 for _, project := range projects {
362 realUrl := strings.TrimSuffix(
363 cfg.AssetURL(project.Username, project.Name, ""),
364 "/",
365 )
366 uat := project.UpdatedAt.Unix()
367 id := realUrl
368 title := fmt.Sprintf("%s-%s", project.Username, project.Name)
369 if by == "updated_at" {
370 id = fmt.Sprintf("%s:%d", realUrl, uat)
371 title = fmt.Sprintf("%s - %d", title, uat)
372 }
373
374 item := &feeds.Item{
375 Id: id,
376 Title: title,
377 Link: &feeds.Link{Href: realUrl},
378 Content: fmt.Sprintf(`<a href="%s">%s</a>`, realUrl, realUrl),
379 Created: *project.CreatedAt,
380 Updated: *project.CreatedAt,
381 Description: "",
382 Author: &feeds.Author{Name: project.Username},
383 }
384
385 feedItems = append(feedItems, item)
386 }
387 feed.Items = feedItems
388
389 rss, err := feed.ToAtom()
390 if err != nil {
391 logger.Error("could not convert feed to atom", "err", err.Error())
392 http.Error(w, "Could not generate atom rss feed", http.StatusInternalServerError)
393 }
394
395 w.Header().Add("Content-Type", "application/atom+xml")
396 _, err = w.Write([]byte(rss))
397 if err != nil {
398 logger.Error("http write failed", "err", err.Error())
399 }
400 }
401}
402
403func WebPerm(proj *db.Project) bool {
404 return proj.Acl.Type == "public" || proj.Acl.Type == ""
405}
406
407var imgRegex = regexp.MustCompile("(.+.(?:jpg|jpeg|png|gif|webp|svg))(/.+)")
408
409func (web *WebRouter) AssetRequest(perm func(proj *db.Project) bool) http.HandlerFunc {
410 return func(w http.ResponseWriter, r *http.Request) {
411 fname := r.PathValue("fname")
412 if imgRegex.MatchString(fname) {
413 web.ImageRequest(perm)(w, r)
414 return
415 }
416 web.ServeAsset(fname, nil, perm, w, r)
417 }
418}
419
420func (web *WebRouter) ImageRequest(perm func(proj *db.Project) bool) http.HandlerFunc {
421 return func(w http.ResponseWriter, r *http.Request) {
422 rawname := r.PathValue("fname")
423 matches := imgRegex.FindStringSubmatch(rawname)
424 fname := rawname
425 imgOpts := ""
426 if len(matches) >= 2 {
427 fname = matches[1]
428 }
429 if len(matches) >= 3 {
430 imgOpts = matches[2]
431 }
432
433 opts, err := storage.UriToImgProcessOpts(imgOpts)
434 if err != nil {
435 errMsg := fmt.Sprintf("error processing img options: %s", err.Error())
436 web.Cfg.Logger.Error("error processing img options", "err", errMsg)
437 http.Error(w, errMsg, http.StatusUnprocessableEntity)
438 return
439 }
440
441 web.ServeAsset(fname, opts, perm, w, r)
442 }
443}
444
445func (web *WebRouter) ServeAsset(fname string, opts *storage.ImgProcessOpts, hasPerm HasPerm, w http.ResponseWriter, r *http.Request) {
446 subdomain := shared.GetSubdomain(r)
447
448 logger := web.Cfg.Logger.With(
449 "subdomain", subdomain,
450 "filename", fname,
451 "url", fmt.Sprintf("%s%s", r.Host, r.URL.Path),
452 "host", r.Host,
453 )
454
455 props, err := shared.GetProjectFromSubdomain(subdomain)
456 if err != nil {
457 logger.Info(
458 "could not determine project from subdomain",
459 "err", err,
460 )
461 http.Error(w, err.Error(), http.StatusNotFound)
462 return
463 }
464
465 logger = logger.With(
466 "project", props.ProjectName,
467 "user", props.Username,
468 )
469
470 user, err := web.Cfg.DB.FindUserByName(props.Username)
471 if err != nil {
472 logger.Info("user not found")
473 http.Error(w, "user not found", http.StatusNotFound)
474 return
475 }
476
477 logger = logger.With(
478 "userId", user.ID,
479 )
480
481 var bucket sst.Bucket
482 bucket, err = web.Cfg.Storage.GetBucket(shared.GetAssetBucketName(user.ID))
483 project, perr := web.Cfg.DB.FindProjectByName(user.ID, props.ProjectName)
484 if perr != nil {
485 logger.Info("project not found")
486 http.Error(w, "project not found", http.StatusNotFound)
487 return
488 }
489
490 logger = logger.With(
491 "projectId", project.ID,
492 "project", project.Name,
493 )
494
495 if project.Blocked != "" {
496 logger.Error("project has been blocked")
497 http.Error(w, project.Blocked, http.StatusForbidden)
498 return
499 }
500
501 if !hasPerm(project) {
502 logger.Error("You do not have access to this site")
503 http.Error(w, "You do not have access to this site", http.StatusUnauthorized)
504 return
505 }
506
507 if err != nil {
508 logger.Error("bucket not found", "err", err)
509 http.Error(w, "bucket not found", http.StatusNotFound)
510 return
511 }
512
513 hasPicoPlus := false
514 ff, _ := web.Cfg.DB.FindFeature(user.ID, "plus")
515 if ff != nil {
516 if ff.ExpiresAt.Before(time.Now()) {
517 hasPicoPlus = true
518 }
519 }
520
521 asset := &ApiAssetHandler{
522 WebRouter: web,
523 Logger: logger,
524
525 Username: props.Username,
526 UserID: user.ID,
527 Subdomain: subdomain,
528 ProjectID: project.ID,
529 ProjectDir: project.ProjectDir,
530 Filepath: fname,
531 Bucket: bucket,
532 ImgProcessOpts: opts,
533 HasPicoPlus: hasPicoPlus,
534 }
535
536 asset.ServeHTTP(w, r)
537}
538
539func (web *WebRouter) ServeHTTP(w http.ResponseWriter, r *http.Request) {
540 subdomain := shared.GetSubdomainFromRequest(r, web.Cfg.Domain, web.Cfg.TxtPrefix)
541 if web.RootRouter == nil || web.UserRouter == nil {
542 web.Cfg.Logger.Error("routers not initialized")
543 http.Error(w, "routers not initialized", http.StatusInternalServerError)
544 return
545 }
546
547 var router *http.ServeMux
548 if subdomain == "" {
549 router = web.RootRouter
550 } else {
551 router = web.UserRouter
552 }
553
554 ctx := r.Context()
555 ctx = context.WithValue(ctx, shared.CtxSubdomainKey{}, subdomain)
556 router.ServeHTTP(w, r.WithContext(ctx))
557}
558
559type CompatLogger struct {
560 Logger *slog.Logger
561}
562
563func (cl *CompatLogger) marshall(int ...interface{}) string {
564 res := ""
565 for _, val := range int {
566 switch r := val.(type) {
567 case string:
568 res += " " + r
569 }
570 }
571 return res
572}
573func (cl *CompatLogger) DPanic(int ...interface{}) {
574 cl.Logger.Error("panic", "output", cl.marshall(int))
575}
576func (cl *CompatLogger) DPanicf(st string, int ...interface{}) {
577 cl.Logger.Error(fmt.Sprintf(st, int...))
578}
579func (cl *CompatLogger) Debug(int ...interface{}) {
580 cl.Logger.Debug("debug", "output", cl.marshall(int))
581}
582func (cl *CompatLogger) Debugf(st string, int ...interface{}) {
583 cl.Logger.Debug(fmt.Sprintf(st, int...))
584}
585func (cl *CompatLogger) Error(int ...interface{}) {
586 cl.Logger.Error("error", "output", cl.marshall(int))
587}
588func (cl *CompatLogger) Errorf(st string, int ...interface{}) {
589 cl.Logger.Error(fmt.Sprintf(st, int...))
590}
591func (cl *CompatLogger) Fatal(int ...interface{}) {
592 cl.Logger.Error("fatal", "outpu", cl.marshall(int))
593}
594func (cl *CompatLogger) Fatalf(st string, int ...interface{}) {
595 cl.Logger.Error(fmt.Sprintf(st, int...))
596}
597func (cl *CompatLogger) Info(int ...interface{}) {
598 cl.Logger.Info("info", "output", cl.marshall(int))
599}
600func (cl *CompatLogger) Infof(st string, int ...interface{}) {
601 cl.Logger.Info(fmt.Sprintf(st, int...))
602}
603func (cl *CompatLogger) Panic(int ...interface{}) {
604 cl.Logger.Error("panic", "output", cl.marshall(int))
605}
606func (cl *CompatLogger) Panicf(st string, int ...interface{}) {
607 cl.Logger.Error(fmt.Sprintf(st, int...))
608}
609func (cl *CompatLogger) Warn(int ...interface{}) {
610 cl.Logger.Warn("warn", "output", cl.marshall(int))
611}
612func (cl *CompatLogger) Warnf(st string, int ...interface{}) {
613 cl.Logger.Warn(fmt.Sprintf(st, int...))
614}