From 61b5355499c410f9cc41ce266238a0f6a8d9e4bb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Kripalani?= Date: Thu, 4 Oct 2018 10:50:45 +0100 Subject: [PATCH 1/5] run tests against multiple datastores. leveldb disabled for now, until TTL shim is implemented. --- pstoreds/ds_test.go | 209 ++++++++++++++++---------------------------- 1 file changed, 76 insertions(+), 133 deletions(-) diff --git a/pstoreds/ds_test.go b/pstoreds/ds_test.go index 2eb3f99..29ac9eb 100644 --- a/pstoreds/ds_test.go +++ b/pstoreds/ds_test.go @@ -2,170 +2,92 @@ package pstoreds import ( "context" - "fmt" "io/ioutil" "os" "testing" "time" - bd "github.com/dgraph-io/badger" - ds "github.com/ipfs/go-datastore" badger "github.com/ipfs/go-ds-badger" + leveldb "github.com/ipfs/go-ds-leveldb" pstore "github.com/libp2p/go-libp2p-peerstore" pt "github.com/libp2p/go-libp2p-peerstore/test" ) -func BenchmarkBaselineBadgerDatastorePutEntry(b *testing.B) { - bds, closer := badgerStore(b) - defer closer() +type datastoreFactory func(tb testing.TB) (ds.TxnDatastore, func()) - b.ResetTimer() - for i := 0; i < b.N; i++ { - txn, _ := bds.NewTransaction(false) +var dstores = map[string]datastoreFactory{ + "Badger": badgerStore, + // TODO: Enable once go-ds-leveldb supports TTL via a shim. + // "Leveldb": leveldbStore, +} - key := ds.RawKey(fmt.Sprintf("/key/%d", i)) - txn.Put(key, []byte(fmt.Sprintf("/value/%d", i))) - - txn.Commit() - txn.Discard() +func TestDsPeerstore(t *testing.T) { + for name, dsFactory := range dstores { + t.Run(name, func(t *testing.T) { + pt.TestPeerstore(t, peerstoreFactory(t, dsFactory, DefaultOpts())) + }) } } -func BenchmarkBaselineBadgerDatastoreGetEntry(b *testing.B) { - bds, closer := badgerStore(b) - defer closer() +func TestDsAddrBook(t *testing.T) { + for name, dsFactory := range dstores { + t.Run(name, func(t *testing.T) { + t.Run("Cacheful", func(t *testing.T) { + t.Parallel() - txn, _ := bds.NewTransaction(false) - keys := make([]ds.Key, 1000) - for i := 0; i < 1000; i++ { - key := ds.RawKey(fmt.Sprintf("/key/%d", i)) - txn.Put(key, []byte(fmt.Sprintf("/value/%d", i))) - keys[i] = key - } - if err := txn.Commit(); err != nil { - b.Fatal(err) - } + opts := DefaultOpts() + opts.TTLInterval = 100 * time.Microsecond + opts.CacheSize = 1024 - b.ResetTimer() - for i := 0; i < b.N; i++ { - txn, _ := bds.NewTransaction(true) - if _, err := txn.Get(keys[i%1000]); err != nil { - b.Fatal(err) - } - txn.Discard() + pt.TestAddrBook(t, addressBookFactory(t, dsFactory, opts)) + }) + + t.Run("Cacheless", func(t *testing.T) { + t.Parallel() + + opts := DefaultOpts() + opts.TTLInterval = 100 * time.Microsecond + opts.CacheSize = 0 + + pt.TestAddrBook(t, addressBookFactory(t, dsFactory, opts)) + }) + }) } } -func BenchmarkBaselineBadgerDirectPutEntry(b *testing.B) { - opts := bd.DefaultOptions - - dataPath, err := ioutil.TempDir(os.TempDir(), "badger") - if err != nil { - b.Fatal(err) - } - - opts.Dir = dataPath - opts.ValueDir = dataPath - opts.SyncWrites = false - - db, err := bd.Open(opts) - if err != nil { - b.Fatal(err) - } - - defer db.Close() - - b.ResetTimer() - for i := 0; i < b.N; i++ { - txn := db.NewTransaction(true) - txn.Set([]byte(fmt.Sprintf("/key/%d", i)), []byte(fmt.Sprintf("/value/%d", i))) - txn.Commit(nil) +func TestDsKeyBook(t *testing.T) { + for name, dsFactory := range dstores { + t.Run(name, func(t *testing.T) { + pt.TestKeyBook(t, keyBookFactory(t, dsFactory, DefaultOpts())) + }) } } -func BenchmarkBaselineBadgerDirectGetEntry(b *testing.B) { - opts := bd.DefaultOptions - - dataPath, err := ioutil.TempDir(os.TempDir(), "badger") - if err != nil { - b.Fatal(err) - } - - opts.Dir = dataPath - opts.ValueDir = dataPath - - db, err := bd.Open(opts) - if err != nil { - b.Fatal(err) - } - - defer db.Close() - - txn := db.NewTransaction(true) - for i := 0; i < 1000; i++ { - txn.Set([]byte(fmt.Sprintf("/key/%d", i)), []byte(fmt.Sprintf("/value/%d", i))) - } - txn.Commit(nil) - - b.ResetTimer() - for i := 0; i < b.N; i++ { - txn := db.NewTransaction(false) - txn.Get([]byte(fmt.Sprintf("/key/%d", i%1000))) - txn.Discard() - } -} - -func TestBadgerDsPeerstore(t *testing.T) { - pt.TestPeerstore(t, peerstoreFactory(t, DefaultOpts())) -} - -func TestBadgerDsAddrBook(t *testing.T) { - t.Run("Cacheful", func(t *testing.T) { - t.Parallel() - - opts := DefaultOpts() - opts.TTLInterval = 100 * time.Microsecond - opts.CacheSize = 1024 - - pt.TestAddrBook(t, addressBookFactory(t, opts)) - }) - - t.Run("Cacheless", func(t *testing.T) { - t.Parallel() - - opts := DefaultOpts() - opts.TTLInterval = 100 * time.Microsecond - opts.CacheSize = 0 - - pt.TestAddrBook(t, addressBookFactory(t, opts)) - }) -} - -func TestBadgerDsKeyBook(t *testing.T) { - pt.TestKeyBook(t, keyBookFactory(t, DefaultOpts())) -} - -func BenchmarkBadgerDsPeerstore(b *testing.B) { +func BenchmarkDsPeerstore(b *testing.B) { caching := DefaultOpts() caching.CacheSize = 1024 cacheless := DefaultOpts() cacheless.CacheSize = 0 - pt.BenchmarkPeerstore(b, peerstoreFactory(b, caching), "Caching") - pt.BenchmarkPeerstore(b, peerstoreFactory(b, cacheless), "Cacheless") + for name, dsFactory := range dstores { + b.Run(name, func(b *testing.B) { + pt.BenchmarkPeerstore(b, peerstoreFactory(b, dsFactory, caching), "Caching") + pt.BenchmarkPeerstore(b, peerstoreFactory(b, dsFactory, cacheless), "Cacheless") + }) + } } -func badgerStore(t testing.TB) (ds.TxnDatastore, func()) { +func badgerStore(tb testing.TB) (ds.TxnDatastore, func()) { dataPath, err := ioutil.TempDir(os.TempDir(), "badger") if err != nil { - t.Fatal(err) + tb.Fatal(err) } store, err := badger.NewDatastore(dataPath, nil) if err != nil { - t.Fatal(err) + tb.Fatal(err) } closer := func() { store.Close() @@ -174,9 +96,25 @@ func badgerStore(t testing.TB) (ds.TxnDatastore, func()) { return store, closer } -func peerstoreFactory(tb testing.TB, opts Options) pt.PeerstoreFactory { +func leveldbStore(tb testing.TB) (ds.TxnDatastore, func()) { + dataPath, err := ioutil.TempDir(os.TempDir(), "leveldb") + if err != nil { + tb.Fatal(err) + } + store, err := leveldb.NewDatastore(dataPath, nil) + if err != nil { + tb.Fatal(err) + } + closer := func() { + store.Close() + os.RemoveAll(dataPath) + } + return store, closer +} + +func peerstoreFactory(tb testing.TB, storeFactory datastoreFactory, opts Options) pt.PeerstoreFactory { return func() (pstore.Peerstore, func()) { - store, closeFunc := badgerStore(tb) + store, closeFunc := storeFactory(tb) ps, err := NewPeerstore(context.Background(), store, opts) if err != nil { @@ -187,9 +125,9 @@ func peerstoreFactory(tb testing.TB, opts Options) pt.PeerstoreFactory { } } -func addressBookFactory(tb testing.TB, opts Options) pt.AddrBookFactory { +func addressBookFactory(tb testing.TB, storeFactory datastoreFactory, opts Options) pt.AddrBookFactory { return func() (pstore.AddrBook, func()) { - store, closeFunc := badgerStore(tb) + store, closeFunc := storeFactory(tb) ab, err := NewAddrBook(context.Background(), store, opts) if err != nil { @@ -200,10 +138,15 @@ func addressBookFactory(tb testing.TB, opts Options) pt.AddrBookFactory { } } -func keyBookFactory(tb testing.TB, opts Options) pt.KeyBookFactory { +func keyBookFactory(tb testing.TB, storeFactory datastoreFactory, opts Options) pt.KeyBookFactory { return func() (pstore.KeyBook, func()) { - store, closeFunc := badgerStore(tb) - kb, _ := NewKeyBook(context.Background(), store, opts) + store, closeFunc := storeFactory(tb) + + kb, err := NewKeyBook(context.Background(), store, opts) + if err != nil { + tb.Fatal(err) + } + return kb, closeFunc } } From 4e2e2de65f92b3fcbcf5b8bffb58766a31c8df75 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Kripalani?= Date: Thu, 4 Oct 2018 10:51:57 +0100 Subject: [PATCH 2/5] ds key encoding: b58 => b32 (no padding). --- package.json | 6 ++++++ pstoreds/addr_book.go | 13 +++++++------ pstoreds/keybook.go | 12 +++++++----- pstoreds/metadata.go | 8 +++++--- pstoreds/peerstore.go | 6 ++++-- 5 files changed, 29 insertions(+), 16 deletions(-) diff --git a/package.json b/package.json index 1a97f2d..40205ea 100644 --- a/package.json +++ b/package.json @@ -76,6 +76,12 @@ "hash": "QmUQy76yspPa3fRyY3GzXFTg9n8JVwFru6ue3KFRt4MeTw", "name": "go-buffer-pool", "version": "0.1.1" + }, + { + "author": "whyrusleeping", + "hash": "QmfVj3x4D6Jkq9SEoi5n2NmoUomLwoeiwnYz2KQa15wRw6", + "name": "base32", + "version": "0.0.2" } ], "gxVersion": "0.4.0", diff --git a/pstoreds/addr_book.go b/pstoreds/addr_book.go index ad37529..9e92336 100644 --- a/pstoreds/addr_book.go +++ b/pstoreds/addr_book.go @@ -7,6 +7,7 @@ import ( "time" lru "github.com/hashicorp/golang-lru" + base32 "github.com/whyrusleeping/base32" ds "github.com/ipfs/go-datastore" query "github.com/ipfs/go-datastore/query" @@ -31,7 +32,7 @@ var ( ) // Peer addresses are stored under the following db key pattern: -// /peers/addr// +// /peers/addr// var abBase = ds.NewKey("/peers/addrs") var _ pstore.AddrBook = (*dsAddrBook)(nil) @@ -107,7 +108,7 @@ func keysAndAddrs(p peer.ID, addrs []ma.Multiaddr) ([]ds.Key, []ma.Multiaddr, er var ( keys = make([]ds.Key, len(addrs)) clean = make([]ma.Multiaddr, len(addrs)) - parentKey = abBase.ChildString(peer.IDB58Encode(p)) + parentKey = abBase.ChildString(base32.RawStdEncoding.EncodeToString([]byte(p))) i = 0 ) @@ -120,7 +121,7 @@ func keysAndAddrs(p peer.ID, addrs []ma.Multiaddr) ([]ds.Key, []ma.Multiaddr, er if err != nil { return nil, nil, err } - keys[i] = parentKey.ChildString(hash.B58String()) + keys[i] = parentKey.ChildString(base32.RawStdEncoding.EncodeToString(hash)) clean[i] = addr i++ } @@ -293,7 +294,7 @@ func (mgr *dsAddrBook) UpdateAddrs(p peer.ID, oldTTL time.Duration, newTTL time. func (mgr *dsAddrBook) dbUpdateTTL(p peer.ID, oldTTL time.Duration, newTTL time.Duration) error { var ( - prefix = abBase.ChildString(peer.IDB58Encode(p)) + prefix = abBase.ChildString(base32.RawStdEncoding.EncodeToString([]byte(p))) q = query.Query{Prefix: prefix.String(), KeysOnly: false} results query.Results err error @@ -336,7 +337,7 @@ func (mgr *dsAddrBook) dbUpdateTTL(p peer.ID, oldTTL time.Duration, newTTL time. // Addrs returns all of the non-expired addresses for a given peer. func (mgr *dsAddrBook) Addrs(p peer.ID) []ma.Multiaddr { var ( - prefix = abBase.ChildString(peer.IDB58Encode(p)) + prefix = abBase.ChildString(base32.RawStdEncoding.EncodeToString([]byte(p))) q = query.Query{Prefix: prefix.String(), KeysOnly: false, ReturnExpirations: true} results query.Results err error @@ -411,7 +412,7 @@ func (mgr *dsAddrBook) AddrStream(ctx context.Context, p peer.ID) <-chan ma.Mult func (mgr *dsAddrBook) ClearAddrs(p peer.ID) { var ( err error - prefix = abBase.ChildString(peer.IDB58Encode(p)) + prefix = abBase.ChildString(base32.RawStdEncoding.EncodeToString([]byte(p))) deleteFn func() error ) diff --git a/pstoreds/keybook.go b/pstoreds/keybook.go index 4801e6b..307f7df 100644 --- a/pstoreds/keybook.go +++ b/pstoreds/keybook.go @@ -4,6 +4,8 @@ import ( "context" "errors" + base32 "github.com/whyrusleeping/base32" + ds "github.com/ipfs/go-datastore" query "github.com/ipfs/go-datastore/query" @@ -13,7 +15,7 @@ import ( ) // Public and private keys are stored under the following db key pattern: -// /peers/keys//{pub, priv} +// /peers/keys//{pub, priv} var ( kbBase = ds.NewKey("/peers/keys") pubSuffix = ds.NewKey("/pub") @@ -31,7 +33,7 @@ func NewKeyBook(_ context.Context, store ds.TxnDatastore, _ Options) (pstore.Key } func (kb *dsKeyBook) PubKey(p peer.ID) ic.PubKey { - key := kbBase.ChildString(peer.IDB58Encode(p)).Child(pubSuffix) + key := kbBase.ChildString(base32.RawStdEncoding.EncodeToString([]byte(p))).Child(pubSuffix) var pk ic.PubKey if value, err := kb.ds.Get(key); err == nil { @@ -68,7 +70,7 @@ func (kb *dsKeyBook) AddPubKey(p peer.ID, pk ic.PubKey) error { return errors.New("peer ID does not match public key") } - key := kbBase.ChildString(peer.IDB58Encode(p)).Child(pubSuffix) + key := kbBase.ChildString(base32.RawStdEncoding.EncodeToString([]byte(p))).Child(pubSuffix) val, err := pk.Bytes() if err != nil { log.Errorf("error while converting pubkey byte string for peer %s: %s\n", p.Pretty(), err) @@ -82,7 +84,7 @@ func (kb *dsKeyBook) AddPubKey(p peer.ID, pk ic.PubKey) error { } func (kb *dsKeyBook) PrivKey(p peer.ID) ic.PrivKey { - key := kbBase.ChildString(peer.IDB58Encode(p)).Child(privSuffix) + key := kbBase.ChildString(base32.RawStdEncoding.EncodeToString([]byte(p))).Child(privSuffix) value, err := kb.ds.Get(key) if err != nil { log.Errorf("error while fetching privkey from datastore for peer %s: %s\n", p.Pretty(), err) @@ -104,7 +106,7 @@ func (kb *dsKeyBook) AddPrivKey(p peer.ID, sk ic.PrivKey) error { return errors.New("peer ID does not match private key") } - key := kbBase.ChildString(peer.IDB58Encode(p)).Child(privSuffix) + key := kbBase.ChildString(base32.RawStdEncoding.EncodeToString([]byte(p))).Child(privSuffix) val, err := sk.Bytes() if err != nil { log.Errorf("error while converting privkey byte string for peer %s: %s\n", p.Pretty(), err) diff --git a/pstoreds/metadata.go b/pstoreds/metadata.go index de21434..7c1d124 100644 --- a/pstoreds/metadata.go +++ b/pstoreds/metadata.go @@ -5,6 +5,8 @@ import ( "context" "encoding/gob" + base32 "github.com/whyrusleeping/base32" + ds "github.com/ipfs/go-datastore" pool "github.com/libp2p/go-buffer-pool" @@ -13,7 +15,7 @@ import ( ) // Metadata is stored under the following db key pattern: -// /peers/metadata// +// /peers/metadata// var pmBase = ds.NewKey("/peers/metadata") type dsPeerMetadata struct { @@ -39,7 +41,7 @@ func NewPeerMetadata(_ context.Context, store ds.Datastore, _ Options) (pstore.P } func (pm *dsPeerMetadata) Get(p peer.ID, key string) (interface{}, error) { - k := pmBase.ChildString(peer.IDB58Encode(p)).ChildString(key) + k := pmBase.ChildString(base32.RawStdEncoding.EncodeToString([]byte(p))).ChildString(key) value, err := pm.ds.Get(k) if err != nil { if err == ds.ErrNotFound { @@ -56,7 +58,7 @@ func (pm *dsPeerMetadata) Get(p peer.ID, key string) (interface{}, error) { } func (pm *dsPeerMetadata) Put(p peer.ID, key string, val interface{}) error { - k := pmBase.ChildString(peer.IDB58Encode(p)).ChildString(key) + k := pmBase.ChildString(base32.RawStdEncoding.EncodeToString([]byte(p))).ChildString(key) var buf pool.Buffer if err := gob.NewEncoder(&buf).Encode(&val); err != nil { return err diff --git a/pstoreds/peerstore.go b/pstoreds/peerstore.go index cc3c6f8..b1a64f0 100644 --- a/pstoreds/peerstore.go +++ b/pstoreds/peerstore.go @@ -4,6 +4,8 @@ import ( "context" "time" + base32 "github.com/whyrusleeping/base32" + ds "github.com/ipfs/go-datastore" query "github.com/ipfs/go-datastore/query" @@ -91,8 +93,8 @@ func uniquePeerIds(ds ds.TxnDatastore, prefix ds.Key, extractor func(result quer ids := make(peer.IDSlice, len(idset)) i := 0 for id := range idset { - pid, _ := peer.IDB58Decode(id) - ids[i] = pid + pid, _ := base32.RawStdEncoding.DecodeString(id) + ids[i], _ = peer.IDFromBytes(pid) i++ } return ids, nil From e18b1285389d3521c26b367e540857f002f3a8fd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Kripalani?= Date: Thu, 4 Oct 2018 22:11:51 +0100 Subject: [PATCH 3/5] add go-ds-leveldb gx dep for tests. --- package.json | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/package.json b/package.json index 40205ea..f01b1e7 100644 --- a/package.json +++ b/package.json @@ -82,6 +82,12 @@ "hash": "QmfVj3x4D6Jkq9SEoi5n2NmoUomLwoeiwnYz2KQa15wRw6", "name": "base32", "version": "0.0.2" + }, + { + "author": "whyrusleeping", + "hash": "QmcxDvw8NnJsfdEcfrypwHkLeVxZY2rT8iiWsUuBnw93gb", + "name": "go-ds-leveldb", + "version": "1.2.0" } ], "gxVersion": "0.4.0", From 43dbbcfb18e4b637a24cff513516d3aa5313df37 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Kripalani?= Date: Thu, 11 Oct 2018 17:50:35 +0100 Subject: [PATCH 4/5] gx upgrade go-ds-leveldb. --- package.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/package.json b/package.json index 0fa0ffd..a324673 100644 --- a/package.json +++ b/package.json @@ -85,9 +85,9 @@ }, { "author": "whyrusleeping", - "hash": "QmcxDvw8NnJsfdEcfrypwHkLeVxZY2rT8iiWsUuBnw93gb", + "hash": "QmccqjKZUTqp4ikWNyAbjBuP5HEdqSqRuAr9mcEhYab54a", "name": "go-ds-leveldb", - "version": "1.2.0" + "version": "1.2.1" } ], "gxVersion": "0.4.0", From 1e98679c50f8de546cbd95abeaf75ba6895d4372 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Kripalani?= Date: Thu, 11 Oct 2018 17:51:04 +0100 Subject: [PATCH 5/5] gx publish 2.0.4 --- .gx/lastpubver | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gx/lastpubver b/.gx/lastpubver index 73748e7..9c4b5d3 100644 --- a/.gx/lastpubver +++ b/.gx/lastpubver @@ -1 +1 @@ -2.0.4: QmWtCpWB39Rzc2xTB75MKorsxNpo3TyecTEN24CJ3KVohE +2.0.4: QmbCj7DgMufitEmChV74DuBBic45jv1gL5aZYtv5BQLdr4