- commit
- ab80ef0
- parent
- f22cdb1
- author
- Eric Bower
- date
- 2025-01-18 09:27:41 -0500 EST
chore(prose): migrate images to pgs
8 files changed,
+135,
-231
+99,
-0
1@@ -0,0 +1,99 @@
2+package main
3+
4+import (
5+ "bytes"
6+ "io"
7+ "log/slog"
8+ "path/filepath"
9+ "time"
10+
11+ "github.com/picosh/pico/db"
12+ "github.com/picosh/pico/db/postgres"
13+ "github.com/picosh/pico/prose"
14+ "github.com/picosh/pico/shared"
15+ "github.com/picosh/pico/shared/storage"
16+ sst "github.com/picosh/pobj/storage"
17+ sendUtils "github.com/picosh/send/utils"
18+)
19+
20+func bail(err error) {
21+ if err != nil {
22+ panic(err)
23+ }
24+}
25+
26+func upload(logger *slog.Logger, st storage.StorageServe, bucket sst.Bucket, fpath string, rdr io.Reader) error {
27+ toSite := filepath.Join("prose", fpath)
28+ logger.Info("uploading object", "bucket", bucket.Name, "object", toSite)
29+ buf := &bytes.Buffer{}
30+ size, err := io.Copy(buf, rdr)
31+ if err != nil {
32+ return err
33+ }
34+
35+ _, _, err = st.PutObject(bucket, toSite, buf, &sendUtils.FileEntry{
36+ Mtime: time.Now().Unix(),
37+ Size: size,
38+ })
39+ return err
40+}
41+
42+func images(logger *slog.Logger, dbh db.DB, st storage.StorageServe, bucket sst.Bucket, user *db.User) error {
43+ posts, err := dbh.FindPostsForUser(&db.Pager{Num: 2000, Page: 0}, user.ID, "imgs")
44+ if err != nil {
45+ return err
46+ }
47+
48+ if len(posts.Data) == 0 {
49+ logger.Info("user does not have any images, skipping")
50+ return nil
51+ }
52+
53+ imgBucket, err := st.GetBucket(shared.GetImgsBucketName(user.ID))
54+ if err != nil {
55+ logger.Info("user does not have an images dir, skipping")
56+ return nil
57+ }
58+
59+ /* imgs, err := st.ListObjects(imgBucket, "/", false)
60+ if err != nil {
61+ return err
62+ } */
63+
64+ for _, posts := range posts.Data {
65+ rdr, _, err := st.GetObject(imgBucket, posts.Filename)
66+ if err != nil {
67+ logger.Error("get object", "err", err)
68+ return err
69+ }
70+ err = upload(logger, st, bucket, posts.Filename, rdr)
71+ if err != nil {
72+ return err
73+ }
74+ }
75+
76+ return nil
77+}
78+
79+func main() {
80+ cfg := prose.NewConfigSite()
81+ logger := cfg.Logger
82+ picoDb := postgres.NewDB(cfg.DbURL, logger)
83+ st, err := storage.NewStorageMinio(logger, cfg.MinioURL, cfg.MinioUser, cfg.MinioPass)
84+ bail(err)
85+
86+ users, err := picoDb.FindUsers()
87+ bail(err)
88+
89+ for _, user := range users {
90+ logger.Info("migrating user images", "user", user.Name)
91+
92+ bucket, err := st.UpsertBucket(shared.GetAssetBucketName(user.ID))
93+ bail(err)
94+ _, _ = picoDb.InsertProject(user.ID, "prose", "prose")
95+ err = images(logger, picoDb, st, bucket, user)
96+ if err != nil {
97+ logger.Error("image uploader", "err", err)
98+ }
99+ }
100+}
+8,
-4
1@@ -47,6 +47,10 @@ func NewUploadImgHandler(dbpool db.DB, cfg *shared.ConfigSite, storage storage.S
2 }
3 }
4
5+func (h *UploadImgHandler) getObjectPath(fpath string) string {
6+ return filepath.Join("prose", fpath)
7+}
8+
9 func (h *UploadImgHandler) Read(s ssh.Session, entry *sendutils.FileEntry) (os.FileInfo, sendutils.ReaderAtCloser, error) {
10 user, err := h.DBPool.FindUser(s.Permissions().Extensions["user_id"])
11 if err != nil {
12@@ -71,12 +75,12 @@ func (h *UploadImgHandler) Read(s ssh.Session, entry *sendutils.FileEntry) (os.F
13 FModTime: *post.UpdatedAt,
14 }
15
16- bucket, err := h.Storage.GetBucket(user.ID)
17+ bucket, err := h.Storage.GetBucket(shared.GetAssetBucketName(user.ID))
18 if err != nil {
19 return nil, nil, err
20 }
21
22- contents, _, err := h.Storage.GetObject(bucket, post.Filename)
23+ contents, _, err := h.Storage.GetObject(bucket, h.getObjectPath(post.Filename))
24 if err != nil {
25 return nil, nil, err
26 }
27@@ -218,13 +222,13 @@ func (h *UploadImgHandler) Delete(s ssh.Session, entry *sendutils.FileEntry) err
28 return fmt.Errorf("error for %s: %v", filename, err)
29 }
30
31- bucket, err := h.Storage.UpsertBucket(user.ID)
32+ bucket, err := h.Storage.UpsertBucket(shared.GetAssetBucketName(user.ID))
33 if err != nil {
34 return err
35 }
36
37 logger.Info("deleting image")
38- err = h.Storage.DeleteObject(bucket, filename)
39+ err = h.Storage.DeleteObject(bucket, h.getObjectPath(filename))
40 if err != nil {
41 return err
42 }
+2,
-24
1@@ -49,7 +49,7 @@ func (h *UploadImgHandler) metaImg(data *PostMetaData) error {
2 return nil
3 }
4
5- bucket, err := h.Storage.UpsertBucket(data.User.ID)
6+ bucket, err := h.Storage.UpsertBucket(shared.GetAssetBucketName(data.User.ID))
7 if err != nil {
8 return err
9 }
10@@ -58,7 +58,7 @@ func (h *UploadImgHandler) metaImg(data *PostMetaData) error {
11
12 fname, _, err := h.Storage.PutObject(
13 bucket,
14- data.Filename,
15+ h.getObjectPath(data.Filename),
16 sendutils.NopReaderAtCloser(reader),
17 &sendutils.FileEntry{},
18 )
19@@ -128,18 +128,6 @@ func (h *UploadImgHandler) writeImg(s ssh.Session, data *PostMetaData) error {
20 logger.Error("post could not create", "err", err.Error())
21 return fmt.Errorf("error for %s: %v", data.Filename, err)
22 }
23-
24- if len(data.Tags) > 0 {
25- logger.Info(
26- "found post tags, replacing with old tags",
27- "tags", strings.Join(data.Tags, ","),
28- )
29- err = h.DBPool.ReplaceTagsForPost(data.Tags, data.Post.ID)
30- if err != nil {
31- logger.Error("post could not replace tags", "err", err.Error())
32- return fmt.Errorf("error for %s: %v", data.Filename, err)
33- }
34- }
35 } else {
36 if data.Shasum == data.Cur.Shasum && modTime.Equal(*data.Cur.UpdatedAt) {
37 logger.Info("image found, but image is identical, skipping")
38@@ -167,16 +155,6 @@ func (h *UploadImgHandler) writeImg(s ssh.Session, data *PostMetaData) error {
39 logger.Error("post could not update", "err", err.Error())
40 return fmt.Errorf("error for %s: %v", data.Filename, err)
41 }
42-
43- logger.Info(
44- "found post tags, replacing with old tags",
45- "tags", strings.Join(data.Tags, ","),
46- )
47- err = h.DBPool.ReplaceTagsForPost(data.Tags, data.Cur.ID)
48- if err != nil {
49- logger.Error("post could not replace tags", "err", err.Error())
50- return fmt.Errorf("error for %s: %v", data.Filename, err)
51- }
52 }
53
54 return nil
+0,
-168
1@@ -1,168 +0,0 @@
2-package imgs
3-
4-import (
5- "fmt"
6- "html/template"
7- "net/http"
8- "net/url"
9- "path/filepath"
10-
11- "github.com/picosh/pico/db"
12- "github.com/picosh/pico/pgs"
13- "github.com/picosh/pico/shared"
14- "github.com/picosh/pico/shared/storage"
15- "github.com/picosh/utils"
16-)
17-
18-type PostPageData struct {
19- ImgURL template.URL
20-}
21-
22-type BlogPageData struct {
23- Site *shared.SitePageData
24- PageTitle string
25- URL template.URL
26- Username string
27- Posts []template.URL
28-}
29-
30-var Space = "imgs"
31-
32-func ImgsListHandler(w http.ResponseWriter, r *http.Request) {
33- username := shared.GetUsernameFromRequest(r)
34- dbpool := shared.GetDB(r)
35- logger := shared.GetLogger(r)
36- cfg := shared.GetCfg(r)
37-
38- user, err := dbpool.FindUserForName(username)
39- if err != nil {
40- logger.Info("blog not found", "username", username)
41- http.Error(w, "blog not found", http.StatusNotFound)
42- return
43- }
44-
45- var posts []*db.Post
46- pager := &db.Pager{Num: 1000, Page: 0}
47- p, err := dbpool.FindPostsForUser(pager, user.ID, Space)
48- posts = p.Data
49-
50- if err != nil {
51- logger.Error(err.Error())
52- http.Error(w, "could not fetch posts for blog", http.StatusInternalServerError)
53- return
54- }
55-
56- ts, err := shared.RenderTemplate(cfg, []string{
57- cfg.StaticPath("html/imgs.page.tmpl"),
58- })
59-
60- if err != nil {
61- logger.Error(err.Error())
62- http.Error(w, err.Error(), http.StatusInternalServerError)
63- return
64- }
65-
66- curl := shared.CreateURLFromRequest(cfg, r)
67- postCollection := make([]template.URL, 0, len(posts))
68- for _, post := range posts {
69- url := cfg.ImgURL(curl, post.Username, post.Slug)
70- postCollection = append(postCollection, template.URL(url))
71- }
72-
73- data := BlogPageData{
74- Site: cfg.GetSiteData(),
75- PageTitle: fmt.Sprintf("%s imgs", username),
76- URL: template.URL(cfg.FullBlogURL(curl, username)),
77- Username: username,
78- Posts: postCollection,
79- }
80-
81- err = ts.Execute(w, data)
82- if err != nil {
83- logger.Error(err.Error())
84- http.Error(w, err.Error(), http.StatusInternalServerError)
85- }
86-}
87-
88-func anyPerm(proj *db.Project) bool {
89- return true
90-}
91-
92-func ImgRequest(w http.ResponseWriter, r *http.Request) {
93- subdomain := shared.GetSubdomain(r)
94- cfg := shared.GetCfg(r)
95- st := shared.GetStorage(r)
96- dbpool := shared.GetDB(r)
97- logger := shared.GetLogger(r)
98- username := shared.GetUsernameFromRequest(r)
99-
100- user, err := dbpool.FindUserForName(username)
101- if err != nil {
102- logger.Info("user not found", "user", username)
103- http.Error(w, "user not found", http.StatusNotFound)
104- return
105- }
106-
107- var imgOpts string
108- var slug string
109- if !cfg.IsSubdomains() || subdomain == "" {
110- slug, _ = url.PathUnescape(shared.GetField(r, 1))
111- imgOpts, _ = url.PathUnescape(shared.GetField(r, 2))
112- } else {
113- slug, _ = url.PathUnescape(shared.GetField(r, 0))
114- imgOpts, _ = url.PathUnescape(shared.GetField(r, 1))
115- }
116-
117- opts, err := storage.UriToImgProcessOpts(imgOpts)
118- if err != nil {
119- errMsg := fmt.Sprintf("error processing img options: %s", err.Error())
120- logger.Info(errMsg)
121- http.Error(w, errMsg, http.StatusUnprocessableEntity)
122- return
123- }
124-
125- // set default quality for web optimization
126- if opts.Quality == 0 {
127- opts.Quality = 80
128- }
129-
130- ext := filepath.Ext(slug)
131- // set default format to be webp
132- if opts.Ext == "" && ext == "" {
133- opts.Ext = "webp"
134- }
135-
136- // Files can contain periods. `filepath.Ext` is greedy and will clip the last period in the slug
137- // and call that a file extension so we want to be explicit about what
138- // file extensions we clip here
139- for _, fext := range cfg.AllowedExt {
140- if ext == fext {
141- // users might add the file extension when requesting an image
142- // but we want to remove that
143- slug = utils.SanitizeFileExt(slug)
144- break
145- }
146- }
147-
148- post, err := FindImgPost(r, user, slug)
149- if err != nil {
150- errMsg := fmt.Sprintf("image not found %s/%s", user.Name, slug)
151- logger.Info(errMsg)
152- http.Error(w, errMsg, http.StatusNotFound)
153- return
154- }
155-
156- fname := post.Filename
157- router := pgs.NewWebRouter(
158- cfg,
159- logger,
160- dbpool,
161- st,
162- )
163- router.ServeAsset(fname, opts, true, anyPerm, w, r)
164-}
165-
166-func FindImgPost(r *http.Request, user *db.User, slug string) (*db.Post, error) {
167- dbpool := shared.GetDB(r)
168- return dbpool.FindPostWithSlug(slug, user.ID, Space)
169-}
+0,
-1
1@@ -1 +0,0 @@
2-<img src="{{.ImgURL}}" />
+0,
-0
+0,
-2
1@@ -397,7 +397,6 @@ var imgRegex = regexp.MustCompile("(.+.(?:jpg|jpeg|png|gif|webp|svg))(/.+)")
2 func (web *WebRouter) AssetRequest(w http.ResponseWriter, r *http.Request) {
3 fname := r.PathValue("fname")
4 if imgRegex.MatchString(fname) {
5- fmt.Println("HIT")
6 web.ImageRequest(w, r)
7 return
8 }
9@@ -415,7 +414,6 @@ func (web *WebRouter) ImageRequest(w http.ResponseWriter, r *http.Request) {
10 if len(matches) >= 3 {
11 imgOpts = matches[2]
12 }
13- fmt.Println("ZZZ", fname, imgOpts)
14
15 opts, err := storage.UriToImgProcessOpts(imgOpts)
16 if err != nil {
+26,
-32
1@@ -5,6 +5,7 @@ import (
2 "fmt"
3 "html/template"
4 "net/http"
5+ "net/http/httputil"
6 "net/url"
7 "os"
8 "strconv"
9@@ -16,7 +17,6 @@ import (
10 "github.com/gorilla/feeds"
11 "github.com/picosh/pico/db"
12 "github.com/picosh/pico/db/postgres"
13- "github.com/picosh/pico/imgs"
14 "github.com/picosh/pico/shared"
15 "github.com/picosh/pico/shared/storage"
16 "github.com/picosh/utils"
17@@ -170,13 +170,6 @@ func blogHandler(w http.ResponseWriter, r *http.Request) {
18 }
19 posts = p.Data
20
21- byUpdated := strings.Contains(r.URL.Path, "live")
22- if byUpdated {
23- slices.SortFunc(posts, func(a *db.Post, b *db.Post) int {
24- return b.UpdatedAt.Compare(*a.UpdatedAt)
25- })
26- }
27-
28 if err != nil {
29 logger.Error(err.Error())
30 http.Error(w, "could not fetch posts for blog", http.StatusInternalServerError)
31@@ -438,14 +431,6 @@ func postHandler(w http.ResponseWriter, r *http.Request) {
32 WithStyles: withStyles,
33 }
34 } else {
35- // TODO: HACK to support imgs slugs inside prose
36- // We definitely want to kill this feature in time
37- imgPost, err := imgs.FindImgPost(r, user, slug)
38- if err == nil && imgPost != nil {
39- imgs.ImgRequest(w, r)
40- return
41- }
42-
43 notFound, err := dbpool.FindPostWithFilename("_404.md", user.ID, cfg.Space)
44 contents := template.HTML("Oops! we can't seem to find this post.")
45 title := "Post not found"
46@@ -645,13 +630,6 @@ func rssBlogHandler(w http.ResponseWriter, r *http.Request) {
47 curl := shared.CreateURLFromRequest(cfg, r)
48 blogUrl := cfg.FullBlogURL(curl, username)
49
50- byUpdated := strings.Contains(r.URL.Path, "live")
51- if byUpdated {
52- slices.SortFunc(posts, func(a *db.Post, b *db.Post) int {
53- return b.UpdatedAt.Compare(*a.UpdatedAt)
54- })
55- }
56-
57 feed := &feeds.Feed{
58 Id: blogUrl,
59 Title: headerTxt.Title,
60@@ -692,10 +670,6 @@ func rssBlogHandler(w http.ResponseWriter, r *http.Request) {
61 realUrl := cfg.FullPostURL(curl, post.Username, post.Slug)
62 feedId := realUrl
63
64- if byUpdated {
65- feedId = fmt.Sprintf("%s:%s", realUrl, post.UpdatedAt.Format(time.RFC3339))
66- }
67-
68 item := &feeds.Item{
69 Id: feedId,
70 Title: utils.FilenameToTitle(post.Filename, post.Title),
71@@ -859,13 +833,32 @@ func createMainRoutes(staticRoutes []shared.Route) []shared.Route {
72 return routes
73 }
74
75+func imgRequest(w http.ResponseWriter, r *http.Request) {
76+ logger := shared.GetLogger(r)
77+ username := shared.GetUsernameFromRequest(r)
78+ destUrl, err := url.Parse(fmt.Sprintf("https://%s-prose.pgs.sh%s", username, r.URL.Path))
79+ if err != nil {
80+ logger.Error("could not parse image proxy url", "username", username)
81+ http.Error(w, "could not parse image proxy url", http.StatusInternalServerError)
82+ return
83+ }
84+ logger.Info("proxy image request", "url", destUrl.String())
85+
86+ proxy := httputil.NewSingleHostReverseProxy(destUrl)
87+ oldDirector := proxy.Director
88+ proxy.Director = func(r *http.Request) {
89+ oldDirector(r)
90+ r.Host = destUrl.Host
91+ r.URL = destUrl
92+ }
93+ proxy.ServeHTTP(w, r)
94+}
95+
96 func createSubdomainRoutes(staticRoutes []shared.Route) []shared.Route {
97 routes := []shared.Route{
98 shared.NewRoute("GET", "/", blogHandler),
99- shared.NewRoute("GET", "/live", blogHandler),
100 shared.NewRoute("GET", "/_styles.css", blogStyleHandler),
101 shared.NewRoute("GET", "/rss", rssBlogHandler),
102- shared.NewRoute("GET", "/live/rss", rssBlogHandler),
103 shared.NewRoute("GET", "/rss.xml", rssBlogHandler),
104 shared.NewRoute("GET", "/atom.xml", rssBlogHandler),
105 shared.NewRoute("GET", "/feed.xml", rssBlogHandler),
106@@ -881,9 +874,10 @@ func createSubdomainRoutes(staticRoutes []shared.Route) []shared.Route {
107 routes = append(
108 routes,
109 shared.NewRoute("GET", "/raw/(.+)", postRawHandler),
110- shared.NewRoute("GET", "/([^/]+)/(.+)", imgs.ImgRequest),
111- shared.NewRoute("GET", "/(.+.(?:jpg|jpeg|png|gif|webp|svg))$", imgs.ImgRequest),
112- shared.NewRoute("GET", "/i", imgs.ImgsListHandler),
113+ shared.NewRoute("GET", "/(.+).md", postRawHandler),
114+ shared.NewRoute("GET", "/([^/]+)/(.+)", imgRequest),
115+ shared.NewRoute("GET", "/(.+.(?:jpg|jpeg|png|gif|webp|svg))$", imgRequest),
116+ shared.NewRoute("GET", "/(.+).html", postHandler),
117 shared.NewRoute("GET", "/(.+)", postHandler),
118 )
119