Commit d32eeb3
Eric Bower
·
2025-08-08 21:43:58 -0400 EDT
parent 92e3a32
refactor(feeds): deprecate `digest_interval`; use `cron` instead Reference: https://pico.sh/feeds#cron
5 files changed,
+64,
-83
M
Makefile
+10,
-10
| ... | ... | @@ -6,7 +6,7 @@ DB_CONTAINER?=pico-postgres-1 | |
| 6 | 6 | DOCKER_TAG?=$(shell git log --format="%H" -n 1) | |
| 7 | 7 | DOCKER_PLATFORM?=linux/amd64,linux/arm64 | |
| 8 | 8 | DOCKER_CMD?=docker | |
| 9 | - | DOCKER_BUILDX_BUILD?=$(DOCKER_CMD) buildx build --push --platform $(DOCKER_PLATFORM) | |
| 9 | + | DOCKER_BUILDX_BUILD?=$(DOCKER_CMD) buildx build --push --platform $(DOCKER_PLATFORM) -t | |
| 10 | 10 | WRITE?=0 | |
| 11 | 11 | ||
| 12 | 12 | smol: |
| ... | ... | @@ -42,36 +42,36 @@ endif | |
| 42 | 42 | .PHONY: bp-setup | |
| 43 | 43 | ||
| 44 | 44 | bp-caddy: bp-setup | |
| 45 | - | $(DOCKER_BUILDX_BUILD) -t ghcr.io/picosh/pico/caddy:$(DOCKER_TAG) ./caddy | |
| 45 | + | $(DOCKER_BUILDX_BUILD) ghcr.io/picosh/pico/caddy:$(DOCKER_TAG) ./caddy | |
| 46 | 46 | .PHONY: bp-caddy | |
| 47 | 47 | ||
| 48 | 48 | bp-auth: bp-setup | |
| 49 | - | $(DOCKER_BUILDX_BUILD) -t ghcr.io/picosh/pico/auth-web:$(DOCKER_TAG) --build-arg APP=auth --target release-web . | |
| 49 | + | $(DOCKER_BUILDX_BUILD) ghcr.io/picosh/pico/auth-web:$(DOCKER_TAG) --build-arg APP=auth --target release-web . | |
| 50 | 50 | .PHONY: bp-auth | |
| 51 | 51 | ||
| 52 | 52 | bp-pgs-cdn: bp-setup | |
| 53 | - | $(DOCKER_BUILDX_BUILD) -t ghcr.io/picosh/pico/pgs-cdn:$(DOCKER_TAG) --target release-web -f Dockerfile.cdn . | |
| 53 | + | $(DOCKER_BUILDX_BUILD) ghcr.io/picosh/pico/pgs-cdn:$(DOCKER_TAG) --target release-web -f Dockerfile.cdn . | |
| 54 | 54 | .PHONY: bp-pgs-cdn | |
| 55 | 55 | ||
| 56 | 56 | bp-pgs-standalone: bp-setup | |
| 57 | - | $(DOCKER_BUILDX_BUILD) --manifest ghcr.io/picosh/pgs:$(DOCKER_TAG) --target release -f Dockerfile.standalone . | |
| 57 | + | $(DOCKER_BUILDX_BUILD) ghcr.io/picosh/pgs:$(DOCKER_TAG) --target release -f Dockerfile.standalone . | |
| 58 | 58 | .PHONY: bp-pgs-standalone | |
| 59 | 59 | ||
| 60 | 60 | bp-pico: bp-setup | |
| 61 | - | $(DOCKER_BUILDX_BUILD) -t ghcr.io/picosh/pico/pico-ssh:$(DOCKER_TAG) --build-arg APP=pico --target release-ssh . | |
| 61 | + | $(DOCKER_BUILDX_BUILD) ghcr.io/picosh/pico/pico-ssh:$(DOCKER_TAG) --build-arg APP=pico --target release-ssh . | |
| 62 | 62 | .PHONY: bp-auth | |
| 63 | 63 | ||
| 64 | 64 | bp-bouncer: bp-setup | |
| 65 | - | $(DOCKER_BUILDX_BUILD) -t ghcr.io/picosh/pico/bouncer:$(DOCKER_TAG) ./bouncer | |
| 65 | + | $(DOCKER_BUILDX_BUILD) ghcr.io/picosh/pico/bouncer:$(DOCKER_TAG) ./bouncer | |
| 66 | 66 | .PHONY: bp-bouncer | |
| 67 | 67 | ||
| 68 | 68 | bp-ssh-%: bp-setup | |
| 69 | - | $(DOCKER_BUILDX_BUILD) --build-arg "APP=$*" -t "ghcr.io/picosh/pico/$*-ssh:$(DOCKER_TAG)" --target release-ssh . | |
| 69 | + | $(DOCKER_BUILDX_BUILD) "ghcr.io/picosh/pico/$*-ssh:$(DOCKER_TAG)" --build-arg "APP=$*" --target release-ssh . | |
| 70 | 70 | .PHONY: pgs-ssh | |
| 71 | 71 | ||
| 72 | 72 | bp-%: bp-setup | |
| 73 | - | $(DOCKER_BUILDX_BUILD) --build-arg "APP=$*" -t "ghcr.io/picosh/pico/$*-ssh:$(DOCKER_TAG)" --target release-ssh . | |
| 74 | - | $(DOCKER_BUILDX_BUILD) --build-arg "APP=$*" -t "ghcr.io/picosh/pico/$*-web:$(DOCKER_TAG)" --target release-web . | |
| 73 | + | $(DOCKER_BUILDX_BUILD) "ghcr.io/picosh/pico/$*-ssh:$(DOCKER_TAG)" --build-arg "APP=$*" --target release-ssh . | |
| 74 | + | $(DOCKER_BUILDX_BUILD) "ghcr.io/picosh/pico/$*-web:$(DOCKER_TAG)" --build-arg "APP=$*" --target release-web . | |
| 75 | 75 | .PHONY: bp-% | |
| 76 | 76 | ||
| 77 | 77 | bp-all: bp-prose bp-pastes bp-feeds bp-pgs bp-auth bp-bouncer bp-pipe bp-pgs-cdn |
+8,
-7
| ... | ... | @@ -5,6 +5,7 @@ import ( | |
| 5 | 5 | "text/tabwriter" | |
| 6 | 6 | "time" | |
| 7 | 7 | ||
| 8 | + | "github.com/adhocore/gronx" | |
| 8 | 9 | "github.com/picosh/pico/pkg/db" | |
| 9 | 10 | "github.com/picosh/pico/pkg/pssh" | |
| 10 | 11 | "github.com/picosh/pico/pkg/shared" |
| ... | ... | @@ -67,24 +68,24 @@ func Middleware(dbpool db.DB, cfg *shared.ConfigSite) pssh.SSHServerMiddleware { | |
| 67 | 68 | } | |
| 68 | 69 | ||
| 69 | 70 | writer := tabwriter.NewWriter(sesh, 0, 0, 1, ' ', tabwriter.TabIndent) | |
| 70 | - | _, _ = fmt.Fprintln(writer, "Filename\tLast Digest\tNext Digest\tInterval\tFailed Attempts") | |
| 71 | + | _, _ = fmt.Fprintln(writer, "Filename\tLast Digest\tNext Digest\tCron\tFailed Attempts") | |
| 71 | 72 | for _, post := range posts.Data { | |
| 72 | 73 | parsed := shared.ListParseText(post.Text) | |
| 73 | 74 | ||
| 74 | 75 | nextDigest := "" | |
| 75 | - | if parsed.Cron != "" { | |
| 76 | - | nextDigest = parsed.Cron | |
| 77 | - | } else { | |
| 78 | - | digestOption := DigestOptionToTime(*post.Data.LastDigest, parsed.DigestInterval) | |
| 79 | - | nextDigest = digestOption.Format(time.RFC3339) | |
| 76 | + | cron := parsed.Cron | |
| 77 | + | if parsed.DigestInterval != "" { | |
| 78 | + | cron = DigestIntervalToCron(parsed.DigestInterval) | |
| 80 | 79 | } | |
| 80 | + | nd, _ := gronx.NextTickAfter(cron, DateToMin(time.Now()), true) | |
| 81 | + | nextDigest = nd.Format(time.RFC3339) | |
| 81 | 82 | _, _ = fmt.Fprintf( | |
| 82 | 83 | writer, | |
| 83 | 84 | "%s\t%s\t%s\t%s\t%d/10\r\n", | |
| 84 | 85 | post.Filename, | |
| 85 | 86 | post.Data.LastDigest.Format(time.RFC3339), | |
| 86 | 87 | nextDigest, | |
| 87 | - | parsed.DigestInterval, | |
| 88 | + | cron, | |
| 88 | 89 | post.Data.Attempts, | |
| 89 | 90 | ) | |
| 90 | 91 | } |
+35,
-35
| ... | ... | @@ -84,25 +84,24 @@ func itemToTemplate(item *gofeed.Item) *FeedItemTmpl { | |
| 84 | 84 | } | |
| 85 | 85 | } | |
| 86 | 86 | ||
| 87 | - | func DigestOptionToTime(lastDigest time.Time, interval string) time.Time { | |
| 88 | - | day := 24 * time.Hour | |
| 87 | + | func DigestIntervalToCron(interval string) string { | |
| 89 | 88 | switch interval { | |
| 90 | 89 | case "10min": | |
| 91 | - | return lastDigest.Add(10 * time.Minute) | |
| 90 | + | return "*/10 * * * *" | |
| 92 | 91 | case "1hour": | |
| 93 | - | return lastDigest.Add(1 * time.Hour) | |
| 92 | + | return "0 * * * *" | |
| 94 | 93 | case "6hour": | |
| 95 | - | return lastDigest.Add(6 * time.Hour) | |
| 94 | + | return "0 */6 * * *" | |
| 96 | 95 | case "12hour": | |
| 97 | - | return lastDigest.Add(12 * time.Hour) | |
| 96 | + | return "0 */12 * * *" | |
| 98 | 97 | case "1day", "": | |
| 99 | - | return lastDigest.Add(1 * day) | |
| 98 | + | return "0 13 * * *" | |
| 100 | 99 | case "7day": | |
| 101 | - | return lastDigest.Add(7 * day) | |
| 100 | + | return "0 13 * * 0" | |
| 102 | 101 | case "30day": | |
| 103 | - | return lastDigest.Add(30 * day) | |
| 102 | + | return "0 13 1 * *" | |
| 104 | 103 | default: | |
| 105 | - | return lastDigest | |
| 104 | + | return "0 13 * * *" | |
| 106 | 105 | } | |
| 107 | 106 | } | |
| 108 | 107 |
| ... | ... | @@ -146,19 +145,16 @@ func NewFetcher(dbpool db.DB, cfg *shared.ConfigSite) *Fetcher { | |
| 146 | 145 | } | |
| 147 | 146 | } | |
| 148 | 147 | ||
| 149 | - | func (f *Fetcher) Validate(post *db.Post, parsed *shared.ListParsedText, now time.Time) error { | |
| 150 | - | lastDigest := post.Data.LastDigest | |
| 151 | - | if lastDigest == nil { | |
| 152 | - | return nil | |
| 153 | - | } | |
| 154 | - | ||
| 155 | - | toTheMin := time.Date( | |
| 148 | + | func DateToMin(now time.Time) time.Time { | |
| 149 | + | return time.Date( | |
| 156 | 150 | now.Year(), now.Month(), now.Day(), | |
| 157 | 151 | now.Hour(), now.Minute(), | |
| 158 | 152 | 0, 0, // zero out second and nano-second for cron | |
| 159 | 153 | now.Location(), | |
| 160 | 154 | ) | |
| 155 | + | } | |
| 161 | 156 | ||
| 157 | + | func (f *Fetcher) Validate(post *db.Post, parsed *shared.ListParsedText, now time.Time) error { | |
| 162 | 158 | expiresAt := post.ExpiresAt | |
| 163 | 159 | if expiresAt != nil { | |
| 164 | 160 | if post.ExpiresAt.Before(now) { |
| ... | ... | @@ -166,24 +162,28 @@ func (f *Fetcher) Validate(post *db.Post, parsed *shared.ListParsedText, now tim | |
| 166 | 162 | } | |
| 167 | 163 | } | |
| 168 | 164 | ||
| 169 | - | if parsed.Cron != "" { | |
| 170 | - | isDue, err := f.gron.IsDue(parsed.Cron, toTheMin) | |
| 171 | - | if err != nil { | |
| 172 | - | return fmt.Errorf("cron error, skipping; err: %w", err) | |
| 173 | - | } | |
| 174 | - | if !isDue { | |
| 175 | - | nextTime, _ := gronx.NextTick(parsed.Cron, true) | |
| 176 | - | return fmt.Errorf( | |
| 177 | - | "cron not time to digest, skipping; cur run: %s, next run: %s", | |
| 178 | - | f.gron.C.GetRef(), | |
| 179 | - | nextTime, | |
| 180 | - | ) | |
| 181 | - | } | |
| 182 | - | } else if parsed.DigestInterval != "" { | |
| 183 | - | digestAt := DigestOptionToTime(*lastDigest, parsed.DigestInterval) | |
| 184 | - | if digestAt.After(now) { | |
| 185 | - | return fmt.Errorf("(%s) not time to digest, skipping", digestAt.Format(time.RFC3339)) | |
| 186 | - | } | |
| 165 | + | cron := parsed.Cron | |
| 166 | + | // support for posts with deprecated `digest_interval` property | |
| 167 | + | if parsed.DigestInterval != "" { | |
| 168 | + | cron = DigestIntervalToCron(parsed.DigestInterval) | |
| 169 | + | } | |
| 170 | + | ||
| 171 | + | if !f.gron.IsValid(cron) { | |
| 172 | + | return fmt.Errorf("(%s) is invalid `cron`, skipping", cron) | |
| 173 | + | } | |
| 174 | + | ||
| 175 | + | dt := DateToMin(now) | |
| 176 | + | isDue, err := f.gron.IsDue(cron, dt) | |
| 177 | + | if err != nil { | |
| 178 | + | return fmt.Errorf("cron error, skipping; err: %w", err) | |
| 179 | + | } | |
| 180 | + | if !isDue { | |
| 181 | + | nextTime, _ := gronx.NextTickAfter(cron, dt, true) | |
| 182 | + | return fmt.Errorf( | |
| 183 | + | "cron not time to digest, skipping; cur run: %s, next run: %s", | |
| 184 | + | f.gron.C.GetRef(), | |
| 185 | + | nextTime, | |
| 186 | + | ) | |
| 187 | 187 | } | |
| 188 | 188 | return nil | |
| 189 | 189 | } |
+11,
-9
| ... | ... | @@ -4,10 +4,9 @@ import ( | |
| 4 | 4 | "errors" | |
| 5 | 5 | "fmt" | |
| 6 | 6 | "net/url" | |
| 7 | - | ||
| 8 | 7 | "strings" | |
| 9 | - | "time" | |
| 10 | 8 | ||
| 9 | + | "github.com/adhocore/gronx" | |
| 11 | 10 | "github.com/picosh/pico/pkg/db" | |
| 12 | 11 | "github.com/picosh/pico/pkg/filehandlers" | |
| 13 | 12 | "github.com/picosh/pico/pkg/pssh" |
| ... | ... | @@ -51,6 +50,16 @@ func (p *FeedHooks) FileValidate(s *pssh.SSHServerConnSession, data *filehandler | |
| 51 | 50 | return false, fmt.Errorf("ERROR: no email variable detected for %s, check the format of your file, skipping", data.Filename) | |
| 52 | 51 | } | |
| 53 | 52 | ||
| 53 | + | if parsed.DigestInterval != "" { | |
| 54 | + | return false, fmt.Errorf("ERROR: `digest_interval` is deprecated; use `cron`: https://pico.sh/feeds#cron") | |
| 55 | + | } | |
| 56 | + | ||
| 57 | + | if parsed.Cron != "" { | |
| 58 | + | if !gronx.IsValid(parsed.Cron) { | |
| 59 | + | return false, fmt.Errorf("ERROR: `cron` is invalid, reference: https://github.com/adhocore/gronx?tab=readme-ov-file#cron-expression") | |
| 60 | + | } | |
| 61 | + | } | |
| 62 | + | ||
| 54 | 63 | var allErr error | |
| 55 | 64 | for _, txt := range parsed.Items { | |
| 56 | 65 | u := "" |
| ... | ... | @@ -74,12 +83,5 @@ func (p *FeedHooks) FileValidate(s *pssh.SSHServerConnSession, data *filehandler | |
| 74 | 83 | } | |
| 75 | 84 | ||
| 76 | 85 | func (p *FeedHooks) FileMeta(s *pssh.SSHServerConnSession, data *filehandlers.PostMetaData) error { | |
| 77 | - | if data.Data.LastDigest == nil { | |
| 78 | - | now := time.Now() | |
| 79 | - | // let it run on the next loop | |
| 80 | - | dd := now.AddDate(0, 0, -31) | |
| 81 | - | data.Data.LastDigest = &dd | |
| 82 | - | } | |
| 83 | - | ||
| 84 | 86 | return nil | |
| 85 | 87 | } |
| ... | ... | @@ -9,22 +9,10 @@ import ( | |
| 9 | 9 | "strings" | |
| 10 | 10 | "time" | |
| 11 | 11 | ||
| 12 | - | "slices" | |
| 13 | - | ||
| 14 | - | "github.com/adhocore/gronx" | |
| 15 | 12 | "github.com/araddon/dateparse" | |
| 16 | 13 | ) | |
| 17 | 14 | ||
| 18 | 15 | var reIndent = regexp.MustCompile(`^[[:blank:]]+`) | |
| 19 | - | var DigestIntervalOpts = []string{ | |
| 20 | - | "10min", | |
| 21 | - | "1hour", | |
| 22 | - | "6hour", | |
| 23 | - | "12hour", | |
| 24 | - | "1day", | |
| 25 | - | "7day", | |
| 26 | - | "30day", | |
| 27 | - | } | |
| 28 | 16 | ||
| 29 | 17 | type ListParsedText struct { | |
| 30 | 18 | Items []*ListItem |
| ... | ... | @@ -124,18 +112,8 @@ func TokenToMetaField(meta *ListMetaData, token *SplitToken) error { | |
| 124 | 112 | case "layout": | |
| 125 | 113 | meta.Layout = token.Value | |
| 126 | 114 | case "digest_interval": | |
| 127 | - | if !slices.Contains(DigestIntervalOpts, token.Value) { | |
| 128 | - | return fmt.Errorf( | |
| 129 | - | "(%s) is not a valid option, choose from [%s]", | |
| 130 | - | token.Value, | |
| 131 | - | strings.Join(DigestIntervalOpts, ","), | |
| 132 | - | ) | |
| 133 | - | } | |
| 134 | 115 | meta.DigestInterval = token.Value | |
| 135 | 116 | case "cron": | |
| 136 | - | if !gronx.IsValid(token.Value) { | |
| 137 | - | return fmt.Errorf("(%s) is not in a valid cron format: https://github.com/adhocore/gronx?tab=readme-ov-file#cron-expression", token.Value) | |
| 138 | - | } | |
| 139 | 117 | meta.Cron = token.Value | |
| 140 | 118 | case "email": | |
| 141 | 119 | meta.Email = token.Value |