Skip to content

Commit ec5061a

Browse files
aschmahmannhacdiaslidel
committed
feat: refactor gateway api to operate on higher level semantics (#176)
* refactor gateway api to operate on semantics to closer match user requests * rename gateway api interface to IPFSBackend * add blocks gateway implementation to the gateway package * refactored gateway examples * add default DNS resolvers * add default IPLD codecs --------- Co-authored-by: Henrique Dias <hacdias@gmail.com> Co-authored-by: Marcin Rataj <lidel@lidel.org>
1 parent 5bbb21b commit ec5061a

28 files changed

Lines changed: 1399 additions & 784 deletions

‎examples/gateway/car/main.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@ func main() {
2929
}
3030
defer f.Close()
3131

32-
gwAPI, err := common.NewBlocksGateway(blockService, nil)
32+
gwAPI, err := gateway.NewBlocksGateway(blockService)
3333
if err != nil {
3434
log.Fatal(err)
3535
}

‎examples/gateway/car/main_test.go‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ import (
77
"testing"
88

99
"github.com/ipfs/boxo/examples/gateway/common"
10+
"github.com/ipfs/boxo/gateway"
1011
"github.com/ipld/go-ipld-prime/codec/dagjson"
1112
"github.com/ipld/go-ipld-prime/node/basicnode"
1213
"github.com/stretchr/testify/assert"
@@ -22,7 +23,7 @@ func newTestServer() (*httptest.Server, io.Closer, error) {
2223
return nil, nil, err
2324
}
2425

25-
gateway, err := common.NewBlocksGateway(blockService, nil)
26+
gateway, err := gateway.NewBlocksGateway(blockService)
2627
if err != nil {
2728
_ = f.Close()
2829
return nil, nil, err

‎examples/gateway/common/blocks.go‎

Lines changed: 1 addition & 230 deletions
Original file line numberDiff line numberDiff line change
@@ -1,42 +1,12 @@
11
package common
22

33
import (
4-
"context"
5-
"errors"
6-
"fmt"
74
"net/http"
8-
gopath "path"
95

10-
"github.com/ipfs/boxo/blockservice"
11-
blockstore "github.com/ipfs/boxo/blockstore"
12-
iface "github.com/ipfs/boxo/coreiface"
13-
nsopts "github.com/ipfs/boxo/coreiface/options/namesys"
14-
ifacepath "github.com/ipfs/boxo/coreiface/path"
15-
bsfetcher "github.com/ipfs/boxo/fetcher/impl/blockservice"
16-
"github.com/ipfs/boxo/files"
176
"github.com/ipfs/boxo/gateway"
18-
"github.com/ipfs/boxo/ipld/merkledag"
19-
"github.com/ipfs/boxo/ipld/unixfs"
20-
ufile "github.com/ipfs/boxo/ipld/unixfs/file"
21-
uio "github.com/ipfs/boxo/ipld/unixfs/io"
22-
"github.com/ipfs/boxo/namesys"
23-
"github.com/ipfs/boxo/namesys/resolve"
24-
ipfspath "github.com/ipfs/boxo/path"
25-
"github.com/ipfs/boxo/path/resolver"
26-
"github.com/ipfs/go-block-format"
27-
"github.com/ipfs/go-cid"
28-
format "github.com/ipfs/go-ipld-format"
29-
"github.com/ipfs/go-unixfsnode"
30-
dagpb "github.com/ipld/go-codec-dagpb"
31-
"github.com/ipld/go-ipld-prime"
32-
"github.com/ipld/go-ipld-prime/node/basicnode"
33-
"github.com/ipld/go-ipld-prime/schema"
34-
"github.com/libp2p/go-libp2p/core/peer"
35-
"github.com/libp2p/go-libp2p/core/routing"
36-
mc "github.com/multiformats/go-multicodec"
377
)
388

39-
func NewBlocksHandler(gw *BlocksGateway, port int) http.Handler {
9+
func NewBlocksHandler(gw gateway.IPFSBackend, port int) http.Handler {
4010
headers := map[string][]string{}
4111
gateway.AddAccessControlHeaders(headers)
4212

@@ -50,202 +20,3 @@ func NewBlocksHandler(gw *BlocksGateway, port int) http.Handler {
5020
mux.Handle("/ipns/", gwHandler)
5121
return mux
5222
}
53-
54-
type BlocksGateway struct {
55-
blockStore blockstore.Blockstore
56-
blockService blockservice.BlockService
57-
dagService format.DAGService
58-
resolver resolver.Resolver
59-
60-
// Optional routing system to handle /ipns addresses.
61-
namesys namesys.NameSystem
62-
routing routing.ValueStore
63-
}
64-
65-
func NewBlocksGateway(blockService blockservice.BlockService, routing routing.ValueStore) (*BlocksGateway, error) {
66-
// Setup the DAG services, which use the CAR block store.
67-
dagService := merkledag.NewDAGService(blockService)
68-
69-
// Setup the UnixFS resolver.
70-
fetcherConfig := bsfetcher.NewFetcherConfig(blockService)
71-
fetcherConfig.PrototypeChooser = dagpb.AddSupportToChooser(func(lnk ipld.Link, lnkCtx ipld.LinkContext) (ipld.NodePrototype, error) {
72-
if tlnkNd, ok := lnkCtx.LinkNode.(schema.TypedLinkNode); ok {
73-
return tlnkNd.LinkTargetNodePrototype(), nil
74-
}
75-
return basicnode.Prototype.Any, nil
76-
})
77-
fetcher := fetcherConfig.WithReifier(unixfsnode.Reify)
78-
resolver := resolver.NewBasicResolver(fetcher)
79-
80-
// Setup a name system so that we are able to resolve /ipns links.
81-
var (
82-
ns namesys.NameSystem
83-
err error
84-
)
85-
if routing != nil {
86-
ns, err = namesys.NewNameSystem(routing)
87-
if err != nil {
88-
return nil, err
89-
}
90-
}
91-
92-
return &BlocksGateway{
93-
blockStore: blockService.Blockstore(),
94-
blockService: blockService,
95-
dagService: dagService,
96-
resolver: resolver,
97-
routing: routing,
98-
namesys: ns,
99-
}, nil
100-
}
101-
102-
func (api *BlocksGateway) GetUnixFsNode(ctx context.Context, p ifacepath.Resolved) (files.Node, error) {
103-
nd, err := api.resolveNode(ctx, p)
104-
if err != nil {
105-
return nil, err
106-
}
107-
108-
return ufile.NewUnixfsFile(ctx, api.dagService, nd)
109-
}
110-
111-
func (api *BlocksGateway) LsUnixFsDir(ctx context.Context, p ifacepath.Resolved) (<-chan iface.DirEntry, error) {
112-
node, err := api.resolveNode(ctx, p)
113-
if err != nil {
114-
return nil, err
115-
}
116-
117-
dir, err := uio.NewDirectoryFromNode(api.dagService, node)
118-
if err != nil {
119-
return nil, err
120-
}
121-
122-
out := make(chan iface.DirEntry, uio.DefaultShardWidth)
123-
124-
go func() {
125-
defer close(out)
126-
for l := range dir.EnumLinksAsync(ctx) {
127-
select {
128-
case out <- api.processLink(ctx, l):
129-
case <-ctx.Done():
130-
return
131-
}
132-
}
133-
}()
134-
135-
return out, nil
136-
}
137-
138-
func (api *BlocksGateway) GetBlock(ctx context.Context, c cid.Cid) (blocks.Block, error) {
139-
return api.blockService.GetBlock(ctx, c)
140-
}
141-
142-
func (api *BlocksGateway) GetIPNSRecord(ctx context.Context, c cid.Cid) ([]byte, error) {
143-
if api.routing == nil {
144-
return nil, routing.ErrNotSupported
145-
}
146-
147-
// Fails fast if the CID is not an encoded Libp2p Key, avoids wasteful
148-
// round trips to the remote routing provider.
149-
if mc.Code(c.Type()) != mc.Libp2pKey {
150-
return nil, errors.New("provided cid is not an encoded libp2p key")
151-
}
152-
153-
// The value store expects the key itself to be encoded as a multihash.
154-
id, err := peer.FromCid(c)
155-
if err != nil {
156-
return nil, err
157-
}
158-
159-
return api.routing.GetValue(ctx, "/ipns/"+string(id))
160-
}
161-
162-
func (api *BlocksGateway) GetDNSLinkRecord(ctx context.Context, hostname string) (ifacepath.Path, error) {
163-
if api.namesys != nil {
164-
p, err := api.namesys.Resolve(ctx, "/ipns/"+hostname, nsopts.Depth(1))
165-
if err == namesys.ErrResolveRecursion {
166-
err = nil
167-
}
168-
return ifacepath.New(p.String()), err
169-
}
170-
171-
return nil, errors.New("not implemented")
172-
}
173-
174-
func (api *BlocksGateway) IsCached(ctx context.Context, p ifacepath.Path) bool {
175-
rp, err := api.ResolvePath(ctx, p)
176-
if err != nil {
177-
return false
178-
}
179-
180-
has, _ := api.blockStore.Has(ctx, rp.Cid())
181-
return has
182-
}
183-
184-
func (api *BlocksGateway) ResolvePath(ctx context.Context, p ifacepath.Path) (ifacepath.Resolved, error) {
185-
if _, ok := p.(ifacepath.Resolved); ok {
186-
return p.(ifacepath.Resolved), nil
187-
}
188-
189-
err := p.IsValid()
190-
if err != nil {
191-
return nil, err
192-
}
193-
194-
ipath := ipfspath.Path(p.String())
195-
if ipath.Segments()[0] == "ipns" {
196-
ipath, err = resolve.ResolveIPNS(ctx, api.namesys, ipath)
197-
if err != nil {
198-
return nil, err
199-
}
200-
}
201-
202-
if ipath.Segments()[0] != "ipfs" {
203-
return nil, fmt.Errorf("unsupported path namespace: %s", p.Namespace())
204-
}
205-
206-
node, rest, err := api.resolver.ResolveToLastNode(ctx, ipath)
207-
if err != nil {
208-
return nil, err
209-
}
210-
211-
root, err := cid.Parse(ipath.Segments()[1])
212-
if err != nil {
213-
return nil, err
214-
}
215-
216-
return ifacepath.NewResolvedPath(ipath, node, root, gopath.Join(rest...)), nil
217-
}
218-
219-
func (api *BlocksGateway) resolveNode(ctx context.Context, p ifacepath.Path) (format.Node, error) {
220-
rp, err := api.ResolvePath(ctx, p)
221-
if err != nil {
222-
return nil, err
223-
}
224-
225-
node, err := api.dagService.Get(ctx, rp.Cid())
226-
if err != nil {
227-
return nil, fmt.Errorf("get node: %w", err)
228-
}
229-
return node, nil
230-
}
231-
232-
func (api *BlocksGateway) processLink(ctx context.Context, result unixfs.LinkResult) iface.DirEntry {
233-
if result.Err != nil {
234-
return iface.DirEntry{Err: result.Err}
235-
}
236-
237-
link := iface.DirEntry{
238-
Name: result.Link.Name,
239-
Cid: result.Link.Cid,
240-
}
241-
242-
switch link.Cid.Type() {
243-
case cid.Raw:
244-
link.Type = iface.TFile
245-
link.Size = result.Link.Size
246-
case cid.DagProtobuf:
247-
link.Size = result.Link.Size
248-
}
249-
250-
return link
251-
}

‎examples/gateway/proxy/main.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,7 @@ func main() {
2727
routing := newProxyRouting(*gatewayUrlPtr, nil)
2828

2929
// Creates the gateway with the block service and the routing.
30-
gwAPI, err := common.NewBlocksGateway(blockService, routing)
30+
gwAPI, err := gateway.NewBlocksGateway(blockService, gateway.WithValueStore(routing))
3131
if err != nil {
3232
log.Fatal(err)
3333
}

‎examples/gateway/proxy/main_test.go‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ import (
99
"github.com/ipfs/boxo/blockservice"
1010
"github.com/ipfs/boxo/examples/gateway/common"
1111
offline "github.com/ipfs/boxo/exchange/offline"
12+
"github.com/ipfs/boxo/gateway"
1213
"github.com/ipfs/go-block-format"
1314
"github.com/stretchr/testify/assert"
1415
)
@@ -22,12 +23,12 @@ func newProxyGateway(t *testing.T, rs *httptest.Server) *httptest.Server {
2223
blockService := blockservice.New(blockStore, offline.Exchange(blockStore))
2324
routing := newProxyRouting(rs.URL, nil)
2425

25-
gateway, err := common.NewBlocksGateway(blockService, routing)
26+
gw, err := gateway.NewBlocksGateway(blockService, gateway.WithValueStore(routing))
2627
if err != nil {
2728
t.Error(err)
2829
}
2930

30-
handler := common.NewBlocksHandler(gateway, 0)
31+
handler := common.NewBlocksHandler(gw, 0)
3132
ts := httptest.NewServer(handler)
3233
t.Cleanup(ts.Close)
3334

‎examples/go.mod‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -8,9 +8,6 @@ require (
88
github.com/ipfs/go-block-format v0.1.2
99
github.com/ipfs/go-cid v0.4.0
1010
github.com/ipfs/go-datastore v0.6.0
11-
github.com/ipfs/go-ipld-format v0.4.0
12-
github.com/ipfs/go-unixfsnode v1.6.0
13-
github.com/ipld/go-codec-dagpb v1.6.0
1411
github.com/ipld/go-ipld-prime v0.20.0
1512
github.com/libp2p/go-libp2p v0.26.3
1613
github.com/libp2p/go-libp2p-routing-helpers v0.6.0
@@ -59,12 +56,15 @@ require (
5956
github.com/ipfs/go-ipfs-redirects-file v0.1.1 // indirect
6057
github.com/ipfs/go-ipfs-util v0.0.2 // indirect
6158
github.com/ipfs/go-ipld-cbor v0.0.6 // indirect
59+
github.com/ipfs/go-ipld-format v0.4.0 // indirect
6260
github.com/ipfs/go-ipld-legacy v0.1.1 // indirect
6361
github.com/ipfs/go-ipns v0.3.0 // indirect
6462
github.com/ipfs/go-log v1.0.5 // indirect
6563
github.com/ipfs/go-log/v2 v2.5.1 // indirect
6664
github.com/ipfs/go-metrics-interface v0.0.1 // indirect
6765
github.com/ipfs/go-peertaskqueue v0.8.1 // indirect
66+
github.com/ipfs/go-unixfsnode v1.6.0 // indirect
67+
github.com/ipld/go-codec-dagpb v1.6.0 // indirect
6868
github.com/jackpal/go-nat-pmp v1.0.2 // indirect
6969
github.com/jbenet/go-temp-err-catcher v0.1.0 // indirect
7070
github.com/jbenet/goprocess v0.1.4 // indirect
@@ -73,6 +73,7 @@ require (
7373
github.com/koron/go-ssdp v0.0.3 // indirect
7474
github.com/libp2p/go-buffer-pool v0.1.0 // indirect
7575
github.com/libp2p/go-cidranger v1.1.0 // indirect
76+
github.com/libp2p/go-doh-resolver v0.4.0 // indirect
7677
github.com/libp2p/go-flow-metrics v0.1.0 // indirect
7778
github.com/libp2p/go-libp2p-asn-util v0.2.0 // indirect
7879
github.com/libp2p/go-libp2p-kad-dht v0.21.1 // indirect

‎examples/go.sum‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -375,6 +375,8 @@ github.com/libp2p/go-buffer-pool v0.1.0 h1:oK4mSFcQz7cTQIfqbe4MIj9gLW+mnanjyFtc6
375375
github.com/libp2p/go-buffer-pool v0.1.0/go.mod h1:N+vh8gMqimBzdKkSMVuydVDq+UV5QTWy5HSiZacSbPg=
376376
github.com/libp2p/go-cidranger v1.1.0 h1:ewPN8EZ0dd1LSnrtuwd4709PXVcITVeuwbag38yPW7c=
377377
github.com/libp2p/go-cidranger v1.1.0/go.mod h1:KWZTfSr+r9qEo9OkI9/SIEeAtw+NNoU0dXIXt15Okic=
378+
github.com/libp2p/go-doh-resolver v0.4.0 h1:gUBa1f1XsPwtpE1du0O+nnZCUqtG7oYi7Bb+0S7FQqw=
379+
github.com/libp2p/go-doh-resolver v0.4.0/go.mod h1:v1/jwsFusgsWIGX/c6vCRrnJ60x7bhTiq/fs2qt0cAg=
378380
github.com/libp2p/go-flow-metrics v0.1.0 h1:0iPhMI8PskQwzh57jB9WxIuIOQ0r+15PChFGkx3Q3WM=
379381
github.com/libp2p/go-flow-metrics v0.1.0/go.mod h1:4Xi8MX8wj5aWNDAZttg6UPmc0ZrnFNsMtpsYUClFtro=
380382
github.com/libp2p/go-libp2p v0.26.3 h1:6g/psubqwdaBqNNoidbRKSTBEYgaOuKBhHl8Q5tO+PM=
@@ -450,6 +452,7 @@ github.com/multiformats/go-multiaddr v0.1.1/go.mod h1:aMKBKNEYmzmDmxfX88/vz+J5IU
450452
github.com/multiformats/go-multiaddr v0.2.0/go.mod h1:0nO36NvPpyV4QzvTLi/lafl2y95ncPj0vFwVF6k6wJ4=
451453
github.com/multiformats/go-multiaddr v0.8.0 h1:aqjksEcqK+iD/Foe1RRFsGZh8+XFiGo7FgUCZlpv3LU=
452454
github.com/multiformats/go-multiaddr v0.8.0/go.mod h1:Fs50eBDWvZu+l3/9S6xAE7ZYj6yhxlvaVZjakWN7xRs=
455+
github.com/multiformats/go-multiaddr-dns v0.3.0/go.mod h1:mNzQ4eTGDg0ll1N9jKPOUogZPoJ30W8a7zk66FQPpdQ=
453456
github.com/multiformats/go-multiaddr-dns v0.3.1 h1:QgQgR+LQVt3NPTjbrLLpsaT2ufAA2y0Mkk+QRVJbW3A=
454457
github.com/multiformats/go-multiaddr-dns v0.3.1/go.mod h1:G/245BRQ6FJGmryJCrOuTdB37AMA5AMOVuO6NY3JwTk=
455458
github.com/multiformats/go-multiaddr-fmt v0.1.0 h1:WLEFClPycPkp4fnIzoFoV9FVd49/eQsuaL3/CWe167E=

‎files/readerfile.go‎

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,18 @@ type ReaderFile struct {
1818
}
1919

2020
func NewBytesFile(b []byte) File {
21-
return &ReaderFile{"", NewReaderFile(bytes.NewReader(b)), nil, int64(len(b))}
21+
return &ReaderFile{"", bytesReaderCloser{bytes.NewReader(b)}, nil, int64(len(b))}
22+
}
23+
24+
// TODO: Is this the best way to fix this bug?
25+
// The bug is we want to be an io.ReadSeekCloser, but bytes.NewReader only gives a io.ReadSeeker and io.NopCloser
26+
// effectively removes the io.Seeker ability from the passed in io.Reader
27+
type bytesReaderCloser struct {
28+
*bytes.Reader
29+
}
30+
31+
func (b bytesReaderCloser) Close() error {
32+
return nil
2233
}
2334

2435
func NewReaderFile(reader io.Reader) File {

0 commit comments

Comments
 (0)