repos / pico

commit

4a540a6

parent

3eda1b9

author

Eric Bower

date

2025-01-09 21:48:45 -0500 EST

chore(feeds): better error handling for invalid feed files

When a user uploads an invalid feed file we do not have any mechanism
currently to notify that user.  Further will continue to run the file
through our rss-to-email cron.

This commit makes it so we perform validation on the feed files as they
are uploaded and refuse to save files that we know will eventually
fail.

Further, we don't want to continuously try files that we know will not
succeed so we are pushing those known issues into our retry-then-delete
mechanism inside our cron.

 1@@ -9,6 +9,7 @@ import (
 2 	"log/slog"
 3 	"math"
 4 	"net/http"
 5+	"net/url"
 6 	"strings"
 7 	"text/template"
 8 	"time"
 9@@ -161,23 +162,29 @@ func (f *Fetcher) RunPost(logger *slog.Logger, user *db.User, post *db.Post) err
10 
11 	urls := []string{}
12 	for _, item := range parsed.Items {
13-		url := ""
14-		if item.IsText {
15-			url = item.Value
16+		u := ""
17+		if item.IsText || item.IsURL {
18+			u = item.Value
19 		} else if item.IsURL {
20-			url = string(item.URL)
21+			u = string(item.Value)
22 		}
23 
24-		if url == "" {
25+		if u == "" {
26 			continue
27 		}
28 
29-		urls = append(urls, url)
30+		_, err := url.Parse(string(item.URL))
31+		if err != nil {
32+			logger.Info("invalid url", "url", string(item.URL))
33+			continue
34+		}
35+
36+		urls = append(urls, u)
37 	}
38 
39 	now := time.Now().UTC()
40 	if post.ExpiresAt == nil {
41-		expiresAt := time.Now().AddDate(0, 6, 0)
42+		expiresAt := time.Now().AddDate(0, 12, 0)
43 		post.ExpiresAt = &expiresAt
44 	}
45 	_, err = f.db.UpdatePost(post)
46@@ -199,7 +206,7 @@ func (f *Fetcher) RunPost(logger *slog.Logger, user *db.User, post *db.Post) err
47 		post.Data.Attempts += 1
48 		logger.Error("could not fetch urls", "err", err, "attempts", post.Data.Attempts)
49 
50-		errBody := fmt.Sprintf(`There was an error attempting to fetch your feeds (%d) times.  After (3) attempts we remove the file from our system.  Please check all the URLs and re-upload.
51+		errBody := fmt.Sprintf(`There was an error attempting to fetch your feeds (%d) times.  After (5) attempts we remove the file from our system.  Please check all the URLs and re-upload.
52 Also, we have centralized logs in our pico.sh TUI that will display realtime feed errors so you can debug.
53 
54 
55@@ -217,7 +224,7 @@ Also, we have centralized logs in our pico.sh TUI that will display realtime fee
56 			return err
57 		}
58 
59-		if post.Data.Attempts >= 3 {
60+		if post.Data.Attempts >= 5 {
61 			err = f.db.RemovePosts([]string{post.ID})
62 			if err != nil {
63 				return err
64@@ -410,6 +417,10 @@ func (f *Fetcher) FetchAll(logger *slog.Logger, urls []string, inlineContent boo
65 		return nil, err
66 	}
67 
68+	if len(urls) == 0 {
69+		return nil, fmt.Errorf("feed file does not contain any urls")
70+	}
71+
72 	var allErrors error
73 	for _, url := range urls {
74 		feedTmpl, err := f.Fetch(logger, fp, url, username, feedItems)

 1@@ -1,12 +1,13 @@
 2 package feeds
 3 
 4 import (
 5+	"errors"
 6 	"fmt"
 7+	"net/url"
 8+
 9 	"strings"
10 	"time"
11 
12-	"slices"
13-
14 	"github.com/charmbracelet/ssh"
15 	"github.com/picosh/pico/db"
16 	"github.com/picosh/pico/filehandlers"
17@@ -38,23 +39,41 @@ func (p *FeedHooks) FileValidate(s ssh.Session, data *filehandlers.PostMetaData)
18 		return false, err
19 	}
20 
21-	return true, nil
22-}
23-
24-func (p *FeedHooks) FileMeta(s ssh.Session, data *filehandlers.PostMetaData) error {
25-	parsedText := shared.ListParseText(string(data.Text))
26+	// Because we need to support sshfs, sftp runs our Write handler twice
27+	// and on the first pass we do not have access to the file data.
28+	// In that case we should skip the parsing validation
29+	if data.Text == "" {
30+		return true, nil
31+	}
32 
33-	if parsedText.Title == "" {
34-		data.Title = utils.ToUpper(data.Slug)
35-	} else {
36-		data.Title = parsedText.Title
37+	parsed := shared.ListParseText(string(data.Text))
38+	if parsed.Email == "" {
39+		return false, fmt.Errorf("ERROR: no email variable detected for %s, check the format of your file, skipping", data.Filename)
40 	}
41 
42-	data.Description = parsedText.Description
43-	data.Tags = parsedText.Tags
44+	var allErr error
45+	for _, txt := range parsed.Items {
46+		u := ""
47+		if txt.IsText {
48+			u = txt.Value
49+		} else if txt.IsURL {
50+			u = string(txt.URL)
51+		}
52 
53-	data.Hidden = slices.Contains(p.Cfg.HiddenPosts, data.Filename)
54+		_, err := url.Parse(u)
55+		if err != nil {
56+			allErr = errors.Join(allErr, fmt.Errorf("%s: %w", u, err))
57+			continue
58+		}
59+	}
60+	if allErr != nil {
61+		return false, fmt.Errorf("ERROR: some urls provided were invalid check the format of your file, skipping: %w", allErr)
62+	}
63+
64+	return true, nil
65+}
66 
67+func (p *FeedHooks) FileMeta(s ssh.Session, data *filehandlers.PostMetaData) error {
68 	if data.Data.LastDigest == nil {
69 		now := time.Now()
70 		data.Data.LastDigest = &now