Skip to content

Commit ca2767a

Browse files
StebalienStebalien
authored
Merge pull request #6680 from ipfs/fix/symlink-size
improve gateway symlink handling
2 parents 2b9a2d5 + c64eb11 commit ca2767a

File tree

6 files changed

+279
-49
lines changed

6 files changed

+279
-49
lines changed

‎core/corehttp/gateway_handler.go‎

Lines changed: 38 additions & 44 deletions
Original file line numberDiff line numberDiff line change
@@ -289,7 +289,7 @@ func (i *gatewayHandler) getOrHeadHandler(w http.ResponseWriter, r *http.Request
289289
}
290290

291291
// write to request
292-
http.ServeContent(w, r, "index.html", modtime, f)
292+
i.serveFile(w, r, "index.html", modtime, f)
293293
return
294294
case resolver.ErrNoLink:
295295
// no index.html; noop
@@ -306,14 +306,14 @@ func (i *gatewayHandler) getOrHeadHandler(w http.ResponseWriter, r *http.Request
306306
var dirListing []directoryItem
307307
dirit := dir.Entries()
308308
for dirit.Next() {
309-
// See comment above where originalUrlPath is declared.
310-
s, err := dirit.Node().Size()
311-
if err != nil {
312-
internalWebError(w, err)
313-
return
309+
size := "?"
310+
if s, err := dirit.Node().Size(); err == nil {
311+
// Size may not be defined/supported. Continue anyways.
312+
size = humanize.Bytes(uint64(s))
314313
}
315314

316-
di := directoryItem{humanize.Bytes(uint64(s)), dirit.Name(), gopath.Join(originalUrlPath, dirit.Name())}
315+
// See comment above where originalUrlPath is declared.
316+
di := directoryItem{size, dirit.Name(), gopath.Join(originalUrlPath, dirit.Name())}
317317
dirListing = append(dirListing, di)
318318
}
319319
if dirit.Err() != nil {
@@ -372,48 +372,42 @@ func (i *gatewayHandler) getOrHeadHandler(w http.ResponseWriter, r *http.Request
372372
}
373373
}
374374

375-
type sizeReadSeeker interface {
376-
Size() (int64, error)
377-
378-
io.ReadSeeker
379-
}
380-
381-
type sizeSeeker struct {
382-
sizeReadSeeker
383-
}
384-
385-
func (s *sizeSeeker) Seek(offset int64, whence int) (int64, error) {
386-
if whence == io.SeekEnd && offset == 0 {
387-
return s.Size()
375+
func (i *gatewayHandler) serveFile(w http.ResponseWriter, req *http.Request, name string, modtime time.Time, file files.File) {
376+
size, err := file.Size()
377+
if err != nil {
378+
http.Error(w, "cannot serve files with unknown sizes", http.StatusBadGateway)
379+
return
388380
}
389381

390-
return s.sizeReadSeeker.Seek(offset, whence)
391-
}
392-
393-
func (i *gatewayHandler) serveFile(w http.ResponseWriter, req *http.Request, name string, modtime time.Time, content io.ReadSeeker) {
394-
if sp, ok := content.(sizeReadSeeker); ok {
395-
content = &sizeSeeker{
396-
sizeReadSeeker: sp,
397-
}
382+
content := &lazySeeker{
383+
size: size,
384+
reader: file,
398385
}
399386

400-
ctype := mime.TypeByExtension(gopath.Ext(name))
401-
if ctype == "" {
402-
buf := make([]byte, 512)
403-
n, _ := io.ReadFull(content, buf[:])
404-
ctype = http.DetectContentType(buf[:n])
405-
_, err := content.Seek(0, io.SeekStart)
406-
if err != nil {
407-
http.Error(w, "seeker can't seek", http.StatusInternalServerError)
408-
return
387+
var ctype string
388+
if _, isSymlink := file.(*files.Symlink); isSymlink {
389+
// We should be smarter about resolving symlinks but this is the
390+
// "most correct" we can be without doing that.
391+
ctype = "inode/symlink"
392+
} else {
393+
ctype = mime.TypeByExtension(gopath.Ext(name))
394+
if ctype == "" {
395+
buf := make([]byte, 512)
396+
n, _ := io.ReadFull(content, buf[:])
397+
ctype = http.DetectContentType(buf[:n])
398+
_, err := content.Seek(0, io.SeekStart)
399+
if err != nil {
400+
http.Error(w, "seeker can't seek", http.StatusInternalServerError)
401+
return
402+
}
403+
}
404+
// Strip the encoding from the HTML Content-Type header and let the
405+
// browser figure it out.
406+
//
407+
// Fixes https://github.com/ipfs/go-ipfs/issues/2203
408+
if strings.HasPrefix(ctype, "text/html;") {
409+
ctype = "text/html"
409410
}
410-
}
411-
// Strip the encoding from the HTML Content-Type header and let the
412-
// browser figure it out.
413-
//
414-
// Fixes https://github.com/ipfs/go-ipfs/issues/2203
415-
if strings.HasPrefix(ctype, "text/html;") {
416-
ctype = "text/html"
417411
}
418412
w.Header().Set("Content-Type", ctype)
419413

‎core/corehttp/lazyseek.go‎

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
package corehttp
2+
3+
import (
4+
"fmt"
5+
"io"
6+
)
7+
8+
// The HTTP server uses seek to determine the file size. Actually _seeking_ can
9+
// be slow so we wrap the seeker in a _lazy_ seeker.
10+
type lazySeeker struct {
11+
reader io.ReadSeeker
12+
13+
size int64
14+
offset int64
15+
realOffset int64
16+
}
17+
18+
func (s *lazySeeker) Seek(offset int64, whence int) (int64, error) {
19+
switch whence {
20+
case io.SeekEnd:
21+
return s.Seek(s.size+offset, io.SeekStart)
22+
case io.SeekCurrent:
23+
return s.Seek(s.offset+offset, io.SeekStart)
24+
case io.SeekStart:
25+
if offset < 0 {
26+
return s.offset, fmt.Errorf("invalid seek offset")
27+
}
28+
s.offset = offset
29+
return s.offset, nil
30+
default:
31+
return s.offset, fmt.Errorf("invalid whence: %d", whence)
32+
}
33+
}
34+
35+
func (s *lazySeeker) Read(b []byte) (int, error) {
36+
// If we're past the end, EOF.
37+
if s.offset >= s.size {
38+
return 0, io.EOF
39+
}
40+
41+
// actually seek
42+
for s.offset != s.realOffset {
43+
off, err := s.reader.Seek(s.offset, io.SeekStart)
44+
if err != nil {
45+
return 0, err
46+
}
47+
s.realOffset = off
48+
}
49+
off, err := s.reader.Read(b)
50+
s.realOffset += int64(off)
51+
s.offset += int64(off)
52+
return off, err
53+
}
54+
55+
func (s *lazySeeker) Close() error {
56+
if closer, ok := s.reader.(io.Closer); ok {
57+
return closer.Close()
58+
}
59+
return nil
60+
}

‎core/corehttp/lazyseek_test.go‎

Lines changed: 137 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,137 @@
1+
package corehttp
2+
3+
import (
4+
"fmt"
5+
"io"
6+
"io/ioutil"
7+
"strings"
8+
"testing"
9+
)
10+
11+
type badSeeker struct {
12+
io.ReadSeeker
13+
}
14+
15+
var badSeekErr = fmt.Errorf("I'm a bad seeker")
16+
17+
func (bs badSeeker) Seek(offset int64, whence int) (int64, error) {
18+
off, err := bs.ReadSeeker.Seek(0, io.SeekCurrent)
19+
if err != nil {
20+
panic(err)
21+
}
22+
return off, badSeekErr
23+
}
24+
25+
func TestLazySeekerError(t *testing.T) {
26+
underlyingBuffer := strings.NewReader("fubar")
27+
s := &lazySeeker{
28+
reader: badSeeker{underlyingBuffer},
29+
size: underlyingBuffer.Size(),
30+
}
31+
off, err := s.Seek(0, io.SeekEnd)
32+
if err != nil {
33+
t.Fatal(err)
34+
}
35+
if off != s.size {
36+
t.Fatal("expected to seek to the end")
37+
}
38+
39+
// shouldn't have actually seeked.
40+
b, err := ioutil.ReadAll(s)
41+
if err != nil {
42+
t.Fatal(err)
43+
}
44+
if len(b) != 0 {
45+
t.Fatal("expected to read nothing")
46+
}
47+
48+
// shouldn't need to actually seek.
49+
off, err = s.Seek(0, io.SeekStart)
50+
if err != nil {
51+
t.Fatal(err)
52+
}
53+
if off != 0 {
54+
t.Fatal("expected to seek to the start")
55+
}
56+
b, err = ioutil.ReadAll(s)
57+
if err != nil {
58+
t.Fatal(err)
59+
}
60+
if string(b) != "fubar" {
61+
t.Fatal("expected to read string")
62+
}
63+
64+
// should fail the second time.
65+
off, err = s.Seek(0, io.SeekStart)
66+
if err != nil {
67+
t.Fatal(err)
68+
}
69+
if off != 0 {
70+
t.Fatal("expected to seek to the start")
71+
}
72+
// right here...
73+
b, err = ioutil.ReadAll(s)
74+
if err == nil {
75+
t.Fatalf("expected an error, got output %s", string(b))
76+
}
77+
if err != badSeekErr {
78+
t.Fatalf("expected a bad seek error, got %s", err)
79+
}
80+
if len(b) != 0 {
81+
t.Fatalf("expected to read nothing")
82+
}
83+
}
84+
85+
func TestLazySeeker(t *testing.T) {
86+
underlyingBuffer := strings.NewReader("fubar")
87+
s := &lazySeeker{
88+
reader: underlyingBuffer,
89+
size: underlyingBuffer.Size(),
90+
}
91+
expectByte := func(b byte) {
92+
t.Helper()
93+
var buf [1]byte
94+
n, err := io.ReadFull(s, buf[:])
95+
if err != nil {
96+
t.Fatal(err)
97+
}
98+
if n != 1 {
99+
t.Fatalf("expected to read one byte, read %d", n)
100+
}
101+
if buf[0] != b {
102+
t.Fatalf("expected %b, got %b", b, buf[0])
103+
}
104+
}
105+
expectSeek := func(whence int, off, expOff int64, expErr string) {
106+
t.Helper()
107+
n, err := s.Seek(off, whence)
108+
if expErr == "" {
109+
if err != nil {
110+
t.Fatal("unexpected seek error: ", err)
111+
}
112+
} else {
113+
if err == nil || err.Error() != expErr {
114+
t.Fatalf("expected %s, got %s", err, expErr)
115+
}
116+
}
117+
if n != expOff {
118+
t.Fatalf("expected offset %d, got, %d", expOff, n)
119+
}
120+
}
121+
122+
expectSeek(io.SeekEnd, 0, s.size, "")
123+
b, err := ioutil.ReadAll(s)
124+
if err != nil {
125+
t.Fatal(err)
126+
}
127+
if len(b) != 0 {
128+
t.Fatal("expected to read nothing")
129+
}
130+
expectSeek(io.SeekEnd, -1, s.size-1, "")
131+
expectByte('r')
132+
expectSeek(io.SeekStart, 0, 0, "")
133+
expectByte('f')
134+
expectSeek(io.SeekCurrent, 1, 2, "")
135+
expectByte('b')
136+
expectSeek(io.SeekCurrent, -100, 3, "invalid seek offset")
137+
}

‎go.mod‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,7 @@ require (
3434
github.com/ipfs/go-ipfs-ds-help v0.0.1
3535
github.com/ipfs/go-ipfs-exchange-interface v0.0.1
3636
github.com/ipfs/go-ipfs-exchange-offline v0.0.1
37-
github.com/ipfs/go-ipfs-files v0.0.4
37+
github.com/ipfs/go-ipfs-files v0.0.6
3838
github.com/ipfs/go-ipfs-pinner v0.0.3
3939
github.com/ipfs/go-ipfs-posinfo v0.0.1
4040
github.com/ipfs/go-ipfs-provider v0.3.0
@@ -50,7 +50,7 @@ require (
5050
github.com/ipfs/go-metrics-prometheus v0.0.2
5151
github.com/ipfs/go-mfs v0.1.1
5252
github.com/ipfs/go-path v0.0.7
53-
github.com/ipfs/go-unixfs v0.2.1
53+
github.com/ipfs/go-unixfs v0.2.3
5454
github.com/ipfs/go-verifcid v0.0.1
5555
github.com/ipfs/interface-go-ipfs-core v0.2.5
5656
github.com/jbenet/go-is-domain v1.0.3

‎go.sum‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -223,8 +223,9 @@ github.com/ipfs/go-ipfs-exchange-offline v0.0.1 h1:P56jYKZF7lDDOLx5SotVh5KFxoY6C
223223
github.com/ipfs/go-ipfs-exchange-offline v0.0.1/go.mod h1:WhHSFCVYX36H/anEKQboAzpUws3x7UeEGkzQc3iNkM0=
224224
github.com/ipfs/go-ipfs-files v0.0.2/go.mod h1:INEFm0LL2LWXBhNJ2PMIIb2w45hpXgPjNoE7yA8Y1d4=
225225
github.com/ipfs/go-ipfs-files v0.0.3/go.mod h1:INEFm0LL2LWXBhNJ2PMIIb2w45hpXgPjNoE7yA8Y1d4=
226-
github.com/ipfs/go-ipfs-files v0.0.4 h1:WzRCivcybUQch/Qh6v8LBRhKtRsjnwyiuOV09mK7mrE=
227226
github.com/ipfs/go-ipfs-files v0.0.4/go.mod h1:INEFm0LL2LWXBhNJ2PMIIb2w45hpXgPjNoE7yA8Y1d4=
227+
github.com/ipfs/go-ipfs-files v0.0.6 h1:sMRtPiSmDrTA2FEiFTtk1vWgO2Dkg7bxXKJ+s8/cDAc=
228+
github.com/ipfs/go-ipfs-files v0.0.6/go.mod h1:lVYE6sgAdtZN5825beJjSAHibw7WOBNPDWz5LaJeukg=
228229
github.com/ipfs/go-ipfs-flags v0.0.1/go.mod h1:RnXBb9WV53GSfTrSDVK61NLTFKvWc60n+K9EgCDh+rA=
229230
github.com/ipfs/go-ipfs-pinner v0.0.3 h1:ez/yNYYyH1W7DiCF/L29tmp6L7lBO8eqbJtPi2pHicA=
230231
github.com/ipfs/go-ipfs-pinner v0.0.3/go.mod h1:s4kFZWLWGDudN8Jyd/GTpt222A12C2snA2+OTdy/7p8=
@@ -278,8 +279,8 @@ github.com/ipfs/go-todocounter v0.0.2 h1:9UBngSQhylg2UDcxSAtpkT+rEWFr26hDPXVStE8
278279
github.com/ipfs/go-todocounter v0.0.2/go.mod h1:l5aErvQc8qKE2r7NDMjmq5UNAvuZy0rC8BHOplkWvZ4=
279280
github.com/ipfs/go-unixfs v0.0.4/go.mod h1:eIo/p9ADu/MFOuyxzwU+Th8D6xoxU//r590vUpWyfz8=
280281
github.com/ipfs/go-unixfs v0.1.0/go.mod h1:lysk5ELhOso8+Fed9U1QTGey2ocsfaZ18h0NCO2Fj9s=
281-
github.com/ipfs/go-unixfs v0.2.1 h1:g51t9ODICFZ3F51FPivm8dE7NzYcdAQNUL9wGP5AYa0=
282-
github.com/ipfs/go-unixfs v0.2.1/go.mod h1:IwAAgul1UQIcNZzKPYZWOCijryFBeCV79cNubPzol+k=
282+
github.com/ipfs/go-unixfs v0.2.3 h1:VsZwK3Z6+rjFxha87GBrp3kZHDsztSIuKlsScr3Iw4s=
283+
github.com/ipfs/go-unixfs v0.2.3/go.mod h1:SUdisfUjNoSDzzhGVxvCL9QO/nKdwXdr+gbMUdqcbYw=
283284
github.com/ipfs/go-verifcid v0.0.1 h1:m2HI7zIuR5TFyQ1b79Da5N9dnnCP1vcu2QqawmWlK2E=
284285
github.com/ipfs/go-verifcid v0.0.1/go.mod h1:5Hrva5KBeIog4A+UpqlaIU+DEstipcJYQQZc0g37pY0=
285286
github.com/ipfs/interface-go-ipfs-core v0.2.5 h1:/rspOe8RbIxwtssEXHB+X9JXhOBDCQt8x50d2kFPXL8=

‎test/sharness/t0113-gateway-symlink.sh‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
#!/usr/bin/env bash
2+
#
3+
# Copyright (c) Protocol Labs
4+
5+
test_description="Test symlink support on the HTTP gateway"
6+
7+
. lib/test-lib.sh
8+
9+
test_init_ipfs
10+
test_launch_ipfs_daemon
11+
12+
13+
test_expect_success "Create a test directory with symlinks" '
14+
mkdir testfiles &&
15+
echo "content" > testfiles/foo &&
16+
ln -s foo testfiles/bar &&
17+
test_cmp testfiles/foo testfiles/bar
18+
'
19+
20+
test_expect_success "Add the test directory" '
21+
HASH=$(ipfs add -Qr testfiles)
22+
'
23+
24+
test_expect_success "Test the directory listing" '
25+
curl "$GWAY_ADDR/ipfs/$HASH" > list_response &&
26+
test_should_contain ">foo<" list_response &&
27+
test_should_contain ">bar<" list_response
28+
'
29+
30+
test_expect_success "Test the symlink" '
31+
curl "$GWAY_ADDR/ipfs/$HASH/bar" > bar_actual &&
32+
echo -n "foo" > bar_expected &&
33+
test_cmp bar_expected bar_actual
34+
'
35+
36+
test_kill_ipfs_daemon
37+
38+
test_done

0 commit comments

Comments
 (0)