- commit
- b3098e1
- parent
- e924593
- author
- Mac Chaffee
- date
- 2024-12-31 14:50:46 -0500 EST
feat(pgs): initial support for conditional requests (#178)
5 files changed,
+414,
-3
M
dev.md
+24,
-0
1@@ -7,6 +7,8 @@
2 cp ./.env.example .env
3 ```
4
5+If you are running apps outside of docker, remember to change the postgres, minio, and imgproxy hostnames to "localhost" in `.env`.
6+
7 Initialize local env variables using direnv
8
9 ```bash
10@@ -32,6 +34,28 @@ go run ./cmd/pgs/ssh
11 go run ./cmd/pgs/web
12 ```
13
14+## sign up and upload files
15+
16+The initial database has no users, you need to sign up via pico/ssh:
17+
18+```bash
19+go run ./cmd/pico/ssh
20+# in a separate terminal, complete the signup flow, set your username to "picouser"
21+ssh localhost -p 2222
22+```
23+
24+Stop the pico SSH server, then you can upload files:
25+
26+```bash
27+go run ./cmd/pgs/ssh
28+# in a separate terminal
29+go run ./cmd/pgs/web
30+# in a third terminal
31+echo 'Hello, World!' > file.txt
32+scp -P 2222 file.txt localhost:/test/file.txt
33+curl -iH "Host: picouser-test.pgs.dev.pico.sh" localhost:3000/file.txt
34+```
35+
36 ## deployment
37
38 We use an image based deployment, so all of our images are uploaded to
+282,
-0
1@@ -0,0 +1,282 @@
2+// Copyright 2009 The Go Authors.
3+
4+// Redistribution and use in source and binary forms, with or without
5+// modification, are permitted provided that the following conditions are
6+// met:
7+
8+// * Redistributions of source code must retain the above copyright
9+// notice, this list of conditions and the following disclaimer.
10+// * Redistributions in binary form must reproduce the above
11+// copyright notice, this list of conditions and the following disclaimer
12+// in the documentation and/or other materials provided with the
13+// distribution.
14+// * Neither the name of Google LLC nor the names of its
15+// contributors may be used to endorse or promote products derived from
16+// this software without specific prior written permission.
17+
18+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
21+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29+
30+// HTTP file system request handler
31+//
32+// Upstream: https://cs.opensource.google/go/go/+/refs/tags/go1.23.4:src/net/http/fs.go
33+// Modifications from upstream:
34+// * Deleted everything except checkPreconditions and dependent functions
35+// * Added "http" package prefixes
36+
37+package pgs
38+
39+import (
40+ "net/http"
41+ "net/textproto"
42+ "strings"
43+ "time"
44+)
45+
46+// scanETag determines if a syntactically valid ETag is present at s. If so,
47+// the ETag and remaining text after consuming ETag is returned. Otherwise,
48+// it returns "", "".
49+func scanETag(s string) (etag string, remain string) {
50+ s = textproto.TrimString(s)
51+ start := 0
52+ if strings.HasPrefix(s, "W/") {
53+ start = 2
54+ }
55+ if len(s[start:]) < 2 || s[start] != '"' {
56+ return "", ""
57+ }
58+ // ETag is either W/"text" or "text".
59+ // See RFC 7232 2.3.
60+ for i := start + 1; i < len(s); i++ {
61+ c := s[i]
62+ switch {
63+ // Character values allowed in ETags.
64+ case c == 0x21 || c >= 0x23 && c <= 0x7E || c >= 0x80:
65+ case c == '"':
66+ return s[:i+1], s[i+1:]
67+ default:
68+ return "", ""
69+ }
70+ }
71+ return "", ""
72+}
73+
74+// etagStrongMatch reports whether a and b match using strong ETag comparison.
75+// Assumes a and b are valid ETags.
76+func etagStrongMatch(a, b string) bool {
77+ return a == b && a != "" && a[0] == '"'
78+}
79+
80+// etagWeakMatch reports whether a and b match using weak ETag comparison.
81+// Assumes a and b are valid ETags.
82+func etagWeakMatch(a, b string) bool {
83+ return strings.TrimPrefix(a, "W/") == strings.TrimPrefix(b, "W/")
84+}
85+
86+// condResult is the result of an HTTP request precondition check.
87+// See https://tools.ietf.org/html/rfc7232 section 3.
88+type condResult int
89+
90+const (
91+ condNone condResult = iota
92+ condTrue
93+ condFalse
94+)
95+
96+func checkIfMatch(w http.ResponseWriter, r *http.Request) condResult {
97+ im := r.Header.Get("If-Match")
98+ if im == "" {
99+ return condNone
100+ }
101+ for {
102+ im = textproto.TrimString(im)
103+ if len(im) == 0 {
104+ break
105+ }
106+ if im[0] == ',' {
107+ im = im[1:]
108+ continue
109+ }
110+ if im[0] == '*' {
111+ return condTrue
112+ }
113+ etag, remain := scanETag(im)
114+ if etag == "" {
115+ break
116+ }
117+ if etagStrongMatch(etag, w.Header().Get("Etag")) {
118+ return condTrue
119+ }
120+ im = remain
121+ }
122+
123+ return condFalse
124+}
125+
126+func checkIfUnmodifiedSince(r *http.Request, modtime time.Time) condResult {
127+ ius := r.Header.Get("If-Unmodified-Since")
128+ if ius == "" || isZeroTime(modtime) {
129+ return condNone
130+ }
131+ t, err := http.ParseTime(ius)
132+ if err != nil {
133+ return condNone
134+ }
135+
136+ // The Last-Modified header truncates sub-second precision so
137+ // the modtime needs to be truncated too.
138+ modtime = modtime.Truncate(time.Second)
139+ if ret := modtime.Compare(t); ret <= 0 {
140+ return condTrue
141+ }
142+ return condFalse
143+}
144+
145+func checkIfNoneMatch(w http.ResponseWriter, r *http.Request) condResult {
146+ inm := r.Header.Get("If-None-Match")
147+ if inm == "" {
148+ return condNone
149+ }
150+ buf := inm
151+ for {
152+ buf = textproto.TrimString(buf)
153+ if len(buf) == 0 {
154+ break
155+ }
156+ if buf[0] == ',' {
157+ buf = buf[1:]
158+ continue
159+ }
160+ if buf[0] == '*' {
161+ return condFalse
162+ }
163+ etag, remain := scanETag(buf)
164+ if etag == "" {
165+ break
166+ }
167+ if etagWeakMatch(etag, w.Header().Get("Etag")) {
168+ return condFalse
169+ }
170+ buf = remain
171+ }
172+ return condTrue
173+}
174+
175+func checkIfModifiedSince(r *http.Request, modtime time.Time) condResult {
176+ if r.Method != "GET" && r.Method != "HEAD" {
177+ return condNone
178+ }
179+ ims := r.Header.Get("If-Modified-Since")
180+ if ims == "" || isZeroTime(modtime) {
181+ return condNone
182+ }
183+ t, err := http.ParseTime(ims)
184+ if err != nil {
185+ return condNone
186+ }
187+ // The Last-Modified header truncates sub-second precision so
188+ // the modtime needs to be truncated too.
189+ modtime = modtime.Truncate(time.Second)
190+ if ret := modtime.Compare(t); ret <= 0 {
191+ return condFalse
192+ }
193+ return condTrue
194+}
195+
196+func checkIfRange(w http.ResponseWriter, r *http.Request, modtime time.Time) condResult {
197+ if r.Method != "GET" && r.Method != "HEAD" {
198+ return condNone
199+ }
200+ ir := r.Header.Get("If-Range")
201+ if ir == "" {
202+ return condNone
203+ }
204+ etag, _ := scanETag(ir)
205+ if etag != "" {
206+ if etagStrongMatch(etag, w.Header().Get("Etag")) {
207+ return condTrue
208+ } else {
209+ return condFalse
210+ }
211+ }
212+ // The If-Range value is typically the ETag value, but it may also be
213+ // the modtime date. See golang.org/issue/8367.
214+ if modtime.IsZero() {
215+ return condFalse
216+ }
217+ t, err := http.ParseTime(ir)
218+ if err != nil {
219+ return condFalse
220+ }
221+ if t.Unix() == modtime.Unix() {
222+ return condTrue
223+ }
224+ return condFalse
225+}
226+
227+var unixEpochTime = time.Unix(0, 0)
228+
229+// isZeroTime reports whether t is obviously unspecified (either zero or Unix()=0).
230+func isZeroTime(t time.Time) bool {
231+ return t.IsZero() || t.Equal(unixEpochTime)
232+}
233+
234+func writeNotModified(w http.ResponseWriter) {
235+ // RFC 7232 section 4.1:
236+ // a sender SHOULD NOT generate representation metadata other than the
237+ // above listed fields unless said metadata exists for the purpose of
238+ // guiding cache updates (e.g., Last-Modified might be useful if the
239+ // response does not have an ETag field).
240+ h := w.Header()
241+ delete(h, "Content-Type")
242+ delete(h, "Content-Length")
243+ delete(h, "Content-Encoding")
244+ if h.Get("Etag") != "" {
245+ delete(h, "Last-Modified")
246+ }
247+ w.WriteHeader(http.StatusNotModified)
248+}
249+
250+// checkPreconditions evaluates request preconditions and reports whether a precondition
251+// resulted in sending http.StatusNotModified or http.StatusPreconditionFailed.
252+func checkPreconditions(w http.ResponseWriter, r *http.Request, modtime time.Time) (done bool, rangeHeader string) {
253+ // This function carefully follows RFC 7232 section 6.
254+ ch := checkIfMatch(w, r)
255+ if ch == condNone {
256+ ch = checkIfUnmodifiedSince(r, modtime)
257+ }
258+ if ch == condFalse {
259+ w.WriteHeader(http.StatusPreconditionFailed)
260+ return true, ""
261+ }
262+ switch checkIfNoneMatch(w, r) {
263+ case condFalse:
264+ if r.Method == "GET" || r.Method == "HEAD" {
265+ writeNotModified(w)
266+ return true, ""
267+ } else {
268+ w.WriteHeader(http.StatusPreconditionFailed)
269+ return true, ""
270+ }
271+ case condNone:
272+ if checkIfModifiedSince(r, modtime) == condFalse {
273+ writeNotModified(w)
274+ return true, ""
275+ }
276+ }
277+
278+ rangeHeader = r.Header.Get("Range")
279+ if rangeHeader != "" && checkIfRange(w, r, modtime) == condFalse {
280+ rangeHeader = ""
281+ }
282+ return false, rangeHeader
283+}
+8,
-3
1@@ -199,11 +199,12 @@ func (h *ApiAssetHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
2 w.Header().Add("content-length", strconv.Itoa(int(info.Size)))
3 }
4 if info.ETag != "" {
5- w.Header().Add("etag", info.ETag)
6+ // Minio SDK trims off the mandatory quotes (RFC 7232 ยง 2.3)
7+ w.Header().Add("etag", fmt.Sprintf("\"%s\"", info.ETag))
8 }
9
10 if !info.LastModified.IsZero() {
11- w.Header().Add("last-modified", info.LastModified.Format(http.TimeFormat))
12+ w.Header().Add("last-modified", info.LastModified.UTC().Format(http.TimeFormat))
13 }
14 }
15
16@@ -225,7 +226,11 @@ func (h *ApiAssetHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
17 "status", status,
18 "contentType", finContentType,
19 )
20-
21+ done, _ := checkPreconditions(w, r, info.LastModified.UTC())
22+ if done {
23+ // A conditional request was detected, status and headers are set, no body required (either 412 or 304)
24+ return
25+ }
26 w.WriteHeader(status)
27 _, err = io.Copy(w, contents)
28
+92,
-0
1@@ -8,6 +8,7 @@ import (
2 "net/http/httptest"
3 "strings"
4 "testing"
5+ "time"
6
7 "github.com/picosh/pico/db"
8 "github.com/picosh/pico/db/stub"
9@@ -22,6 +23,7 @@ var testUsername = "user"
10 type ApiExample struct {
11 name string
12 path string
13+ reqHeaders map[string]string
14 want string
15 wantUrl string
16 status int
17@@ -227,11 +229,101 @@ func TestApiBasic(t *testing.T) {
18 },
19 },
20 },
21+ {
22+ name: "conditional-if-modified-since-future",
23+ path: "/test.html",
24+ reqHeaders: map[string]string{
25+ "If-Modified-Since": time.Now().UTC().Add(time.Hour).Format(http.TimeFormat),
26+ },
27+ want: "",
28+ status: http.StatusNotModified,
29+ contentType: "",
30+
31+ dbpool: NewPgsDb(cfg.Logger),
32+ storage: map[string]map[string]string{
33+ bucketName: {
34+ "test/test.html": "hello world!",
35+ },
36+ },
37+ },
38+ {
39+ name: "conditional-if-modified-since-past",
40+ path: "/test.html",
41+ reqHeaders: map[string]string{
42+ "If-Modified-Since": time.Now().UTC().Add(-time.Hour).Format(http.TimeFormat),
43+ },
44+ want: "hello world!",
45+ status: http.StatusOK,
46+ contentType: "text/html",
47+
48+ dbpool: NewPgsDb(cfg.Logger),
49+ storage: map[string]map[string]string{
50+ bucketName: {
51+ "test/test.html": "hello world!",
52+ },
53+ },
54+ },
55+ {
56+ name: "conditional-if-none-match-pass",
57+ path: "/test.html",
58+ reqHeaders: map[string]string{
59+ "If-None-Match": "\"static-etag-for-testing-purposes\"",
60+ },
61+ want: "",
62+ status: http.StatusNotModified,
63+ contentType: "",
64+
65+ dbpool: NewPgsDb(cfg.Logger),
66+ storage: map[string]map[string]string{
67+ bucketName: {
68+ "test/test.html": "hello world!",
69+ },
70+ },
71+ },
72+ {
73+ name: "conditional-if-none-match-fail",
74+ path: "/test.html",
75+ reqHeaders: map[string]string{
76+ "If-None-Match": "\"non-matching-etag\"",
77+ },
78+ want: "hello world!",
79+ status: http.StatusOK,
80+ contentType: "text/html",
81+
82+ dbpool: NewPgsDb(cfg.Logger),
83+ storage: map[string]map[string]string{
84+ bucketName: {
85+ "test/test.html": "hello world!",
86+ },
87+ },
88+ },
89+ {
90+ name: "conditional-if-none-match-and-if-modified-since",
91+ path: "/test.html",
92+ reqHeaders: map[string]string{
93+ // The matching etag should take precedence over the past mod time
94+ "If-None-Match": "\"static-etag-for-testing-purposes\"",
95+ "If-Modified-Since": time.Now().UTC().Add(-time.Hour).Format(http.TimeFormat),
96+ },
97+ want: "",
98+ status: http.StatusNotModified,
99+ contentType: "",
100+
101+ dbpool: NewPgsDb(cfg.Logger),
102+ storage: map[string]map[string]string{
103+ bucketName: {
104+ "test/test.html": "hello world!",
105+ },
106+ },
107+ },
108 }
109
110 for _, tc := range tt {
111 t.Run(tc.name, func(t *testing.T) {
112 request := httptest.NewRequest("GET", mkpath(tc.path), strings.NewReader(""))
113+ for key, val := range tc.reqHeaders {
114+ request.Header.Set(key, val)
115+ }
116 responseRecorder := httptest.NewRecorder()
117
118 st, _ := storage.NewStorageMemory(tc.storage)
1@@ -3,6 +3,7 @@ package storage
2 import (
3 "io"
4 "net/http"
5+ "time"
6
7 sst "github.com/picosh/pobj/storage"
8 )
9@@ -24,6 +25,13 @@ func (s *StorageMemory) ServeObject(bucket sst.Bucket, fpath string, opts *ImgPr
10 if info.Metadata == nil {
11 info.Metadata = make(http.Header)
12 }
13+ // Make tests work by supplying non-null Last-Modified and Etag values
14+ if info.LastModified.IsZero() {
15+ info.LastModified = time.Now().UTC()
16+ }
17+ if info.ETag == "" {
18+ info.ETag = "static-etag-for-testing-purposes"
19+ }
20 mimeType := GetMimeType(fpath)
21 info.Metadata.Set("content-type", mimeType)
22 return obj, info, err