Eric Bower
·
2025-04-10
web.go
1package pgs
2
3import (
4 "bufio"
5 "context"
6 "fmt"
7 "html/template"
8 "log/slog"
9 "net/http"
10 "net/url"
11 "os"
12 "path/filepath"
13 "regexp"
14 "strings"
15 "time"
16
17 _ "net/http/pprof"
18
19 "github.com/darkweak/souin/configurationtypes"
20 "github.com/darkweak/souin/pkg/middleware"
21 "github.com/darkweak/souin/plugins/souin/storages"
22 "github.com/darkweak/storages/core"
23 "github.com/gorilla/feeds"
24 "github.com/hashicorp/golang-lru/v2/expirable"
25 "github.com/picosh/pico/pkg/cache"
26 "github.com/picosh/pico/pkg/db"
27 sst "github.com/picosh/pico/pkg/pobj/storage"
28 "github.com/picosh/pico/pkg/shared"
29 "github.com/picosh/pico/pkg/shared/storage"
30 "github.com/prometheus/client_golang/prometheus/promhttp"
31 "google.golang.org/protobuf/proto"
32)
33
34type CachedHttp struct {
35 handler *middleware.SouinBaseHandler
36 routes *WebRouter
37}
38
39func (c *CachedHttp) ServeHTTP(writer http.ResponseWriter, req *http.Request) {
40 _ = c.handler.ServeHTTP(writer, req, func(w http.ResponseWriter, r *http.Request) error {
41 c.routes.ServeHTTP(w, r)
42 return nil
43 })
44}
45
46func StartApiServer(cfg *PgsConfig) {
47 ctx := context.Background()
48
49 ttl := configurationtypes.Duration{Duration: cfg.CacheTTL}
50 stale := configurationtypes.Duration{Duration: cfg.CacheTTL * 2}
51 c := &middleware.BaseConfiguration{
52 API: configurationtypes.API{
53 Prometheus: configurationtypes.APIEndpoint{
54 Enable: true,
55 },
56 },
57 DefaultCache: &configurationtypes.DefaultCache{
58 TTL: ttl,
59 Stale: stale,
60 Otter: configurationtypes.CacheProvider{
61 Uuid: fmt.Sprintf("OTTER-%s", stale),
62 Configuration: map[string]interface{}{},
63 },
64 Regex: configurationtypes.Regex{
65 Exclude: "/check|/_metrics",
66 },
67 MaxBodyBytes: uint64(cfg.MaxAssetSize),
68 DefaultCacheControl: cfg.CacheControl,
69 },
70 }
71 c.SetLogger(&CompatLogger{cfg.Logger})
72 storages.InitFromConfiguration(c)
73 httpCache := middleware.NewHTTPCacheHandler(c)
74 routes := NewWebRouter(cfg)
75 cacher := &CachedHttp{
76 handler: httpCache,
77 routes: routes,
78 }
79
80 go routes.cacheMgmt(ctx, httpCache, cfg.CacheClearingQueue)
81
82 portStr := fmt.Sprintf(":%s", cfg.WebPort)
83 cfg.Logger.Info(
84 "starting server on port",
85 "port", cfg.WebPort,
86 "domain", cfg.Domain,
87 )
88 err := http.ListenAndServe(portStr, cacher)
89 cfg.Logger.Error(
90 "listen and serve",
91 "err", err.Error(),
92 )
93}
94
95type HasPerm = func(proj *db.Project) bool
96
97type WebRouter struct {
98 Cfg *PgsConfig
99 RootRouter *http.ServeMux
100 UserRouter *http.ServeMux
101 RedirectsCache *expirable.LRU[string, []*RedirectRule]
102 HeadersCache *expirable.LRU[string, []*HeaderRule]
103}
104
105func NewWebRouter(cfg *PgsConfig) *WebRouter {
106 router := &WebRouter{
107 Cfg: cfg,
108 RedirectsCache: expirable.NewLRU[string, []*RedirectRule](2048, nil, cache.CacheTimeout),
109 HeadersCache: expirable.NewLRU[string, []*HeaderRule](2048, nil, cache.CacheTimeout),
110 }
111 router.initRouters()
112 go router.watchCacheClear()
113 return router
114}
115
116func (web *WebRouter) watchCacheClear() {
117 for key := range web.Cfg.CacheClearingQueue {
118 web.Cfg.Logger.Info("lru cache clear request", "key", key)
119 rKey := filepath.Join(key, "_redirects")
120 web.RedirectsCache.Remove(rKey)
121 hKey := filepath.Join(key, "_headers")
122 web.HeadersCache.Remove(hKey)
123 }
124}
125
126func (web *WebRouter) initRouters() {
127 // ensure legacy router is disabled
128 // GODEBUG=httpmuxgo121=0
129
130 // root domain
131 rootRouter := http.NewServeMux()
132 rootRouter.HandleFunc("GET /check", web.checkHandler)
133 rootRouter.HandleFunc("GET /_metrics", promhttp.Handler().ServeHTTP)
134 rootRouter.Handle("GET /main.css", web.serveFile("main.css", "text/css"))
135 rootRouter.Handle("GET /favicon-16x16.png", web.serveFile("favicon-16x16.png", "image/png"))
136 rootRouter.Handle("GET /favicon.ico", web.serveFile("favicon.ico", "image/x-icon"))
137 rootRouter.Handle("GET /robots.txt", web.serveFile("robots.txt", "text/plain"))
138
139 rootRouter.Handle("GET /rss/updated", web.createRssHandler("updated_at"))
140 rootRouter.Handle("GET /rss", web.createRssHandler("created_at"))
141 rootRouter.Handle("GET /{$}", web.createPageHandler("html/marketing.page.tmpl"))
142 web.RootRouter = rootRouter
143
144 // subdomain or custom domains
145 userRouter := http.NewServeMux()
146 userRouter.HandleFunc("GET /{fname...}", web.AssetRequest)
147 userRouter.HandleFunc("GET /{$}", web.AssetRequest)
148 web.UserRouter = userRouter
149}
150
151func (web *WebRouter) serveFile(file string, contentType string) http.HandlerFunc {
152 return func(w http.ResponseWriter, r *http.Request) {
153 logger := web.Cfg.Logger
154 cfg := web.Cfg
155
156 contents, err := os.ReadFile(cfg.StaticPath(fmt.Sprintf("public/%s", file)))
157 if err != nil {
158 logger.Error(
159 "could not read file",
160 "fname", file,
161 "err", err.Error(),
162 )
163 http.Error(w, "file not found", 404)
164 }
165
166 w.Header().Add("Content-Type", contentType)
167
168 _, err = w.Write(contents)
169 if err != nil {
170 logger.Error(
171 "could not write http response",
172 "file", file,
173 "err", err.Error(),
174 )
175 }
176 }
177}
178
179func renderTemplate(cfg *PgsConfig, templates []string) (*template.Template, error) {
180 files := make([]string, len(templates))
181 copy(files, templates)
182 files = append(
183 files,
184 cfg.StaticPath("html/footer.partial.tmpl"),
185 cfg.StaticPath("html/marketing-footer.partial.tmpl"),
186 cfg.StaticPath("html/base.layout.tmpl"),
187 )
188
189 ts, err := template.New("base").ParseFiles(files...)
190 if err != nil {
191 return nil, err
192 }
193 return ts, nil
194}
195
196func (web *WebRouter) createPageHandler(fname string) http.HandlerFunc {
197 return func(w http.ResponseWriter, r *http.Request) {
198 logger := web.Cfg.Logger
199 cfg := web.Cfg
200 ts, err := renderTemplate(cfg, []string{cfg.StaticPath(fname)})
201
202 if err != nil {
203 logger.Error(
204 "could not render template",
205 "fname", fname,
206 "err", err.Error(),
207 )
208 http.Error(w, err.Error(), http.StatusInternalServerError)
209 return
210 }
211
212 data := shared.PageData{
213 Site: shared.SitePageData{Domain: template.URL(cfg.Domain), HomeURL: "/"},
214 }
215 err = ts.Execute(w, data)
216 if err != nil {
217 logger.Error(
218 "could not execute template",
219 "fname", fname,
220 "err", err.Error(),
221 )
222 http.Error(w, err.Error(), http.StatusInternalServerError)
223 }
224 }
225}
226
227func (web *WebRouter) checkHandler(w http.ResponseWriter, r *http.Request) {
228 dbpool := web.Cfg.DB
229 cfg := web.Cfg
230 logger := web.Cfg.Logger
231
232 hostDomain := r.URL.Query().Get("domain")
233 appDomain := strings.Split(cfg.Domain, ":")[0]
234
235 if !strings.Contains(hostDomain, appDomain) {
236 subdomain := shared.GetCustomDomain(hostDomain, cfg.TxtPrefix)
237 props, err := shared.GetProjectFromSubdomain(subdomain)
238 if err != nil {
239 logger.Error(
240 "could not get project from subdomain",
241 "subdomain", subdomain,
242 "err", err.Error(),
243 )
244 w.WriteHeader(http.StatusNotFound)
245 return
246 }
247
248 u, err := dbpool.FindUserByName(props.Username)
249 if err != nil {
250 logger.Error("could not find user", "err", err.Error())
251 w.WriteHeader(http.StatusNotFound)
252 return
253 }
254
255 logger = logger.With(
256 "user", u.Name,
257 "project", props.ProjectName,
258 )
259 p, err := dbpool.FindProjectByName(u.ID, props.ProjectName)
260 if err != nil {
261 logger.Error(
262 "could not find project for user",
263 "user", u.Name,
264 "project", props.ProjectName,
265 "err", err.Error(),
266 )
267 w.WriteHeader(http.StatusNotFound)
268 return
269 }
270
271 if u != nil && p != nil {
272 w.WriteHeader(http.StatusOK)
273 return
274 }
275 }
276
277 w.WriteHeader(http.StatusNotFound)
278}
279
280func (web *WebRouter) cacheMgmt(ctx context.Context, httpCache *middleware.SouinBaseHandler, notify chan string) {
281 storer := httpCache.Storers[0]
282 drain := createSubCacheDrain(ctx, web.Cfg.Logger)
283
284 for {
285 scanner := bufio.NewScanner(drain)
286 scanner.Buffer(make([]byte, 32*1024), 32*1024)
287 for scanner.Scan() {
288 surrogateKey := strings.TrimSpace(scanner.Text())
289 web.Cfg.Logger.Info("received cache-drain item", "surrogateKey", surrogateKey)
290 notify <- surrogateKey
291
292 if surrogateKey == "*" {
293 storer.DeleteMany(".+")
294 err := httpCache.SurrogateKeyStorer.Destruct()
295 if err != nil {
296 web.Cfg.Logger.Error("could not clear cache and surrogate key store", "err", err)
297 } else {
298 web.Cfg.Logger.Info("successfully cleared cache and surrogate keys store")
299 }
300 continue
301 }
302
303 var header http.Header = map[string][]string{}
304 header.Add("Surrogate-Key", surrogateKey)
305
306 ck, _ := httpCache.SurrogateKeyStorer.Purge(header)
307 for _, key := range ck {
308 key, _ = strings.CutPrefix(key, core.MappingKeyPrefix)
309 if b := storer.Get(core.MappingKeyPrefix + key); len(b) > 0 {
310 var mapping core.StorageMapper
311 if e := proto.Unmarshal(b, &mapping); e == nil {
312 for k := range mapping.GetMapping() {
313 qkey, _ := url.QueryUnescape(k)
314 web.Cfg.Logger.Info(
315 "deleting key from surrogate cache",
316 "surrogateKey", surrogateKey,
317 "key", qkey,
318 )
319 storer.Delete(qkey)
320 }
321 }
322 }
323
324 qkey, _ := url.QueryUnescape(key)
325 web.Cfg.Logger.Info(
326 "deleting from cache",
327 "surrogateKey", surrogateKey,
328 "key", core.MappingKeyPrefix+qkey,
329 )
330 storer.Delete(core.MappingKeyPrefix + qkey)
331 }
332 }
333 }
334}
335
336func (web *WebRouter) createRssHandler(by string) http.HandlerFunc {
337 return func(w http.ResponseWriter, r *http.Request) {
338 dbpool := web.Cfg.DB
339 logger := web.Cfg.Logger
340 cfg := web.Cfg
341
342 projects, err := dbpool.FindProjects(by)
343 if err != nil {
344 logger.Error("could not find projects", "err", err.Error())
345 http.Error(w, err.Error(), http.StatusInternalServerError)
346 return
347 }
348
349 feed := &feeds.Feed{
350 Title: fmt.Sprintf("%s discovery feed %s", cfg.Domain, by),
351 Link: &feeds.Link{Href: "https://pgs.sh"},
352 Description: fmt.Sprintf("%s projects %s", cfg.Domain, by),
353 Author: &feeds.Author{Name: cfg.Domain},
354 Created: time.Now(),
355 }
356
357 var feedItems []*feeds.Item
358 for _, project := range projects {
359 realUrl := strings.TrimSuffix(
360 cfg.AssetURL(project.Username, project.Name, ""),
361 "/",
362 )
363 uat := project.UpdatedAt.Unix()
364 id := realUrl
365 title := fmt.Sprintf("%s-%s", project.Username, project.Name)
366 if by == "updated_at" {
367 id = fmt.Sprintf("%s:%d", realUrl, uat)
368 title = fmt.Sprintf("%s - %d", title, uat)
369 }
370
371 item := &feeds.Item{
372 Id: id,
373 Title: title,
374 Link: &feeds.Link{Href: realUrl},
375 Content: fmt.Sprintf(`<a href="%s">%s</a>`, realUrl, realUrl),
376 Created: *project.CreatedAt,
377 Updated: *project.CreatedAt,
378 Description: "",
379 Author: &feeds.Author{Name: project.Username},
380 }
381
382 feedItems = append(feedItems, item)
383 }
384 feed.Items = feedItems
385
386 rss, err := feed.ToAtom()
387 if err != nil {
388 logger.Error("could not convert feed to atom", "err", err.Error())
389 http.Error(w, "Could not generate atom rss feed", http.StatusInternalServerError)
390 }
391
392 w.Header().Add("Content-Type", "application/atom+xml")
393 _, err = w.Write([]byte(rss))
394 if err != nil {
395 logger.Error("http write failed", "err", err.Error())
396 }
397 }
398}
399
400func (web *WebRouter) Perm(proj *db.Project) bool {
401 return proj.Acl.Type == "public" || proj.Acl.Type == ""
402}
403
404var imgRegex = regexp.MustCompile("(.+.(?:jpg|jpeg|png|gif|webp|svg))(/.+)")
405
406func (web *WebRouter) AssetRequest(w http.ResponseWriter, r *http.Request) {
407 fname := r.PathValue("fname")
408 if imgRegex.MatchString(fname) {
409 web.ImageRequest(w, r)
410 return
411 }
412 web.ServeAsset(fname, nil, false, web.Perm, w, r)
413}
414
415func (web *WebRouter) ImageRequest(w http.ResponseWriter, r *http.Request) {
416 rawname := r.PathValue("fname")
417 matches := imgRegex.FindStringSubmatch(rawname)
418 fname := rawname
419 imgOpts := ""
420 if len(matches) >= 2 {
421 fname = matches[1]
422 }
423 if len(matches) >= 3 {
424 imgOpts = matches[2]
425 }
426
427 opts, err := storage.UriToImgProcessOpts(imgOpts)
428 if err != nil {
429 errMsg := fmt.Sprintf("error processing img options: %s", err.Error())
430 web.Cfg.Logger.Error("error processing img options", "err", errMsg)
431 http.Error(w, errMsg, http.StatusUnprocessableEntity)
432 return
433 }
434
435 web.ServeAsset(fname, opts, false, web.Perm, w, r)
436}
437
438func (web *WebRouter) ServeAsset(fname string, opts *storage.ImgProcessOpts, fromImgs bool, hasPerm HasPerm, w http.ResponseWriter, r *http.Request) {
439 subdomain := shared.GetSubdomain(r)
440
441 logger := web.Cfg.Logger.With(
442 "subdomain", subdomain,
443 "filename", fname,
444 "url", fmt.Sprintf("%s%s", r.Host, r.URL.Path),
445 "host", r.Host,
446 )
447
448 props, err := shared.GetProjectFromSubdomain(subdomain)
449 if err != nil {
450 logger.Info(
451 "could not determine project from subdomain",
452 "err", err,
453 )
454 http.Error(w, err.Error(), http.StatusNotFound)
455 return
456 }
457
458 logger = logger.With(
459 "project", props.ProjectName,
460 "user", props.Username,
461 )
462
463 user, err := web.Cfg.DB.FindUserByName(props.Username)
464 if err != nil {
465 logger.Info("user not found")
466 http.Error(w, "user not found", http.StatusNotFound)
467 return
468 }
469
470 logger = logger.With(
471 "userId", user.ID,
472 )
473
474 projectID := ""
475 // TODO: this could probably be cleaned up more
476 // imgs wont have a project directory
477 projectDir := ""
478 var bucket sst.Bucket
479 // imgs has a different bucket directory
480 if fromImgs {
481 bucket, err = web.Cfg.Storage.GetBucket(shared.GetImgsBucketName(user.ID))
482 } else {
483 bucket, err = web.Cfg.Storage.GetBucket(shared.GetAssetBucketName(user.ID))
484 project, perr := web.Cfg.DB.FindProjectByName(user.ID, props.ProjectName)
485 if perr != nil {
486 logger.Info("project not found")
487 http.Error(w, "project not found", http.StatusNotFound)
488 return
489 }
490
491 logger = logger.With(
492 "projectId", project.ID,
493 "project", project.Name,
494 )
495
496 if project.Blocked != "" {
497 logger.Error("project has been blocked")
498 http.Error(w, project.Blocked, http.StatusForbidden)
499 return
500 }
501
502 projectID = project.ID
503 projectDir = project.ProjectDir
504 if !hasPerm(project) {
505 http.Error(w, "You do not have access to this site", http.StatusUnauthorized)
506 return
507 }
508 }
509
510 if err != nil {
511 logger.Error("bucket not found", "err", err)
512 http.Error(w, "bucket not found", http.StatusNotFound)
513 return
514 }
515
516 hasPicoPlus := false
517 ff, _ := web.Cfg.DB.FindFeature(user.ID, "plus")
518 if ff != nil {
519 if ff.ExpiresAt.Before(time.Now()) {
520 hasPicoPlus = true
521 }
522 }
523
524 asset := &ApiAssetHandler{
525 WebRouter: web,
526 Logger: logger,
527
528 Username: props.Username,
529 UserID: user.ID,
530 Subdomain: subdomain,
531 ProjectDir: projectDir,
532 Filepath: fname,
533 Bucket: bucket,
534 ImgProcessOpts: opts,
535 ProjectID: projectID,
536 HasPicoPlus: hasPicoPlus,
537 }
538
539 asset.ServeHTTP(w, r)
540}
541
542func (web *WebRouter) ServeHTTP(w http.ResponseWriter, r *http.Request) {
543 subdomain := shared.GetSubdomainFromRequest(r, web.Cfg.Domain, web.Cfg.TxtPrefix)
544 if web.RootRouter == nil || web.UserRouter == nil {
545 web.Cfg.Logger.Error("routers not initialized")
546 http.Error(w, "routers not initialized", http.StatusInternalServerError)
547 return
548 }
549
550 var router *http.ServeMux
551 if subdomain == "" {
552 router = web.RootRouter
553 } else {
554 router = web.UserRouter
555 }
556
557 ctx := r.Context()
558 ctx = context.WithValue(ctx, shared.CtxSubdomainKey{}, subdomain)
559 router.ServeHTTP(w, r.WithContext(ctx))
560}
561
562type CompatLogger struct {
563 logger *slog.Logger
564}
565
566func (cl *CompatLogger) marshall(int ...interface{}) string {
567 res := ""
568 for _, val := range int {
569 switch r := val.(type) {
570 case string:
571 res += " " + r
572 }
573 }
574 return res
575}
576func (cl *CompatLogger) DPanic(int ...interface{}) {
577 cl.logger.Error("panic", "output", cl.marshall(int))
578}
579func (cl *CompatLogger) DPanicf(st string, int ...interface{}) {
580 cl.logger.Error(fmt.Sprintf(st, int...))
581}
582func (cl *CompatLogger) Debug(int ...interface{}) {
583 cl.logger.Debug("debug", "output", cl.marshall(int))
584}
585func (cl *CompatLogger) Debugf(st string, int ...interface{}) {
586 cl.logger.Debug(fmt.Sprintf(st, int...))
587}
588func (cl *CompatLogger) Error(int ...interface{}) {
589 cl.logger.Error("error", "output", cl.marshall(int))
590}
591func (cl *CompatLogger) Errorf(st string, int ...interface{}) {
592 cl.logger.Error(fmt.Sprintf(st, int...))
593}
594func (cl *CompatLogger) Fatal(int ...interface{}) {
595 cl.logger.Error("fatal", "outpu", cl.marshall(int))
596}
597func (cl *CompatLogger) Fatalf(st string, int ...interface{}) {
598 cl.logger.Error(fmt.Sprintf(st, int...))
599}
600func (cl *CompatLogger) Info(int ...interface{}) {
601 cl.logger.Info("info", "output", cl.marshall(int))
602}
603func (cl *CompatLogger) Infof(st string, int ...interface{}) {
604 cl.logger.Info(fmt.Sprintf(st, int...))
605}
606func (cl *CompatLogger) Panic(int ...interface{}) {
607 cl.logger.Error("panic", "output", cl.marshall(int))
608}
609func (cl *CompatLogger) Panicf(st string, int ...interface{}) {
610 cl.logger.Error(fmt.Sprintf(st, int...))
611}
612func (cl *CompatLogger) Warn(int ...interface{}) {
613 cl.logger.Warn("warn", "output", cl.marshall(int))
614}
615func (cl *CompatLogger) Warnf(st string, int ...interface{}) {
616 cl.logger.Warn(fmt.Sprintf(st, int...))
617}