Commit bcb252d
Eric Bower
·
2025-08-06 15:42:44 -0400 EDT
parent e1bec7f
fix(feeds): sanitize feed content to be utf8 Our postgres db expects utf8
4 files changed,
+98,
-6
+5,
-5
| ... | ... | @@ -106,10 +106,10 @@ func DigestOptionToTime(lastDigest time.Time, interval string) time.Time { | |
| 106 | 106 | } | |
| 107 | 107 | ||
| 108 | 108 | func getFeedItemID(logger *slog.Logger, item *gofeed.Item) string { | |
| 109 | - | guid := item.GUID | |
| 109 | + | guid := strings.ToValidUTF8(item.GUID, "") | |
| 110 | 110 | if item.GUID == "" { | |
| 111 | 111 | logger.Info("no <guid> found for feed item, using <link> instead for its unique id") | |
| 112 | - | return item.Link | |
| 112 | + | return strings.ToValidUTF8(item.Link, "") | |
| 113 | 113 | } | |
| 114 | 114 | return guid | |
| 115 | 115 | } |
| ... | ... | @@ -487,9 +487,9 @@ func (f *Fetcher) FetchAll(logger *slog.Logger, urls []string, inlineContent boo | |
| 487 | 487 | PostID: post.ID, | |
| 488 | 488 | GUID: uid, | |
| 489 | 489 | Data: db.FeedItemData{ | |
| 490 | - | Title: item.Title, | |
| 491 | - | Description: item.Description, | |
| 492 | - | Content: item.Content, | |
| 490 | + | Title: strings.ToValidUTF8(item.Title, ""), | |
| 491 | + | Description: strings.ToValidUTF8(item.Description, ""), | |
| 492 | + | Content: strings.ToValidUTF8(item.Content, ""), | |
| 493 | 493 | Link: item.Link, | |
| 494 | 494 | PublishedAt: item.PublishedParsed, | |
| 495 | 495 | }, |
+1,
-1
| ... | ... | @@ -1561,7 +1561,7 @@ func (me *PsqlDB) InsertFeedItems(postID string, items []*db.FeedItem) error { | |
| 1561 | 1561 | item.Data, | |
| 1562 | 1562 | ) | |
| 1563 | 1563 | if err != nil { | |
| 1564 | - | return err | |
| 1564 | + | return fmt.Errorf("post id:%s, guid:%s, err:%w", item.PostID, item.GUID, err) | |
| 1565 | 1565 | } | |
| 1566 | 1566 | } | |
| 1567 | 1567 |
A
test.txt
+47,
-0
| ... | ... | @@ -0,0 +1,47 @@ | |
| 1 | + | =: email tmp@erock.io | |
| 2 | + | =: digest_interval 1day | |
| 3 | + | =: inline_content false | |
| 4 | + | => https://blog.pico.sh/rss | |
| 5 | + | => https://uncenter.dev/feed.xml | |
| 6 | + | => https://ryanccn.dev/feed/rss.xml | |
| 7 | + | => https://shivjm.blog/feed.xml | |
| 8 | + | => https://binyam.in/feeds/blog.xml | |
| 9 | + | => https://tonsky.me/blog/atom.xml | |
| 10 | + | => https://tty1.blog/feed/ | |
| 11 | + | => https://sindresorhus.com/rss.xml | |
| 12 | + | => https://bower.sh/rss | |
| 13 | + | => https://fasterthanli.me/index.xml | |
| 14 | + | => https://antfu.me/feed.xml | |
| 15 | + | => https://zackoverflow.dev/rss.xml | |
| 16 | + | => https://humanwhocodes.com/feeds/blog.xml | |
| 17 | + | => https://mitchellh.com/feed.xml | |
| 18 | + | => https://chrisdone.com/rss.xml | |
| 19 | + | => https://www.hoeser.dev/feed.xml | |
| 20 | + | => https://daverupert.com/atom.xml | |
| 21 | + | => https://blog.orhun.dev/rss.xml | |
| 22 | + | => https://evanhahn.com/blog/index.xml | |
| 23 | + | => https://www.11ty.dev/blog/feed.xml | |
| 24 | + | => https://thorstenball.com/atom.xml | |
| 25 | + | => https://registerspill.thorstenball.com/feed | |
| 26 | + | => https://isabelroses.com/rss.xml | |
| 27 | + | => https://boehs.org/in/blog.xml | |
| 28 | + | => https://huonw.github.io/blog/atom.xml | |
| 29 | + | => https://nixpkgs.news/rss.xml | |
| 30 | + | => https://rbluethl.com/rss | |
| 31 | + | => https://lucumr.pocoo.org/feed.atom | |
| 32 | + | => https://yorickpeterse.com/feed.xml | |
| 33 | + | => https://ardislu.dev/atom.xml | |
| 34 | + | => https://kettanaito.com/blog/rss.xml | |
| 35 | + | => https://kilo.bytesize.xyz/feed/ | |
| 36 | + | => https://bytesize.xyz/feed/ | |
| 37 | + | => https://zed.dev/blog.rss | |
| 38 | + | => https://www.sophiajt.com/atom.xml | |
| 39 | + | => https://robinmalfait.com/feed.xml | |
| 40 | + | => https://twobithistory.org/feed.xml | |
| 41 | + | => https://ruudvanasseldonk.com/feed.xml | |
| 42 | + | => https://blog.rust-lang.org/feed.xml | |
| 43 | + | => https://nolanlawson.com/feed/ | |
| 44 | + | => https://matklad.github.io/feed.xml | |
| 45 | + | => https://www.ntietz.com/atom.xml | |
| 46 | + | => https://2ality.com/feeds/posts.atom | |
| 47 | + | => https://dbushell.com/rss.xml |
+45,
-0
| ... | ... | @@ -0,0 +1,45 @@ | |
| 1 | + | =: email tmp@erock.io | |
| 2 | + | =: digest_interval 1day | |
| 3 | + | =: inline_content false | |
| 4 | + | => https://blog.pico.sh/rss | |
| 5 | + | => https://uncenter.dev/feed.xml | |
| 6 | + | => https://ryanccn.dev/feed/rss.xml | |
| 7 | + | => https://shivjm.blog/feed.xml | |
| 8 | + | => https://binyam.in/feeds/blog.xml | |
| 9 | + | => https://tonsky.me/blog/atom.xml | |
| 10 | + | => https://tty1.blog/feed/ | |
| 11 | + | => https://sindresorhus.com/rss.xml | |
| 12 | + | => https://bower.sh/rss | |
| 13 | + | => https://fasterthanli.me/index.xml | |
| 14 | + | => https://antfu.me/feed.xml | |
| 15 | + | => https://zackoverflow.dev/rss.xml | |
| 16 | + | => https://humanwhocodes.com/feeds/blog.xml | |
| 17 | + | => https://mitchellh.com/feed.xml | |
| 18 | + | => https://chrisdone.com/rss.xml | |
| 19 | + | => https://www.hoeser.dev/feed.xml | |
| 20 | + | => https://daverupert.com/atom.xml | |
| 21 | + | => https://blog.orhun.dev/rss.xml | |
| 22 | + | => https://evanhahn.com/blog/index.xml | |
| 23 | + | => https://www.11ty.dev/blog/feed.xml | |
| 24 | + | => https://thorstenball.com/atom.xml | |
| 25 | + | => https://registerspill.thorstenball.com/feed | |
| 26 | + | => https://isabelroses.com/rss.xml | |
| 27 | + | => https://boehs.org/in/blog.xml | |
| 28 | + | => https://huonw.github.io/blog/atom.xml | |
| 29 | + | => https://nixpkgs.news/rss.xml | |
| 30 | + | => https://rbluethl.com/rss | |
| 31 | + | => https://lucumr.pocoo.org/feed.atom | |
| 32 | + | => https://yorickpeterse.com/feed.xml | |
| 33 | + | => https://ardislu.dev/atom.xml | |
| 34 | + | => https://kettanaito.com/blog/rss.xml | |
| 35 | + | => https://kilo.bytesize.xyz/feed/ | |
| 36 | + | => https://bytesize.xyz/feed/ | |
| 37 | + | => https://zed.dev/blog.rss | |
| 38 | + | => https://www.sophiajt.com/atom.xml | |
| 39 | + | => https://robinmalfait.com/feed.xml | |
| 40 | + | => https://twobithistory.org/feed.xml | |
| 41 | + | => https://ruudvanasseldonk.com/feed.xml | |
| 42 | + | => https://blog.rust-lang.org/feed.xml | |
| 43 | + | => https://nolanlawson.com/feed/ | |
| 44 | + | => https://matklad.github.io/feed.xml | |
| 45 | + | => https://www.ntietz.com/atom.xml |