libp2p/go-libp2p-peerstore
1
0
Fork 0
mirror of https://github.com/libp2p/go-libp2p-peerstore.git synced 2025-01-01 00:20:11 +08:00
Code Issues Projects Releases Wiki Activity

Merge pull request #40 from raulk/b32

Browse Source 
Replace b58-encoded keys with b32 (no padding) + test against multiple ds. Resolves #38.
This commit is contained in:
Raúl Kripalani 2018-10-11 17:55:56 +01:00 committed by GitHub
commit b87bbd4e21
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 112 additions and 150 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.gx/lastpubver
View File

@ -1 +1 @@
2.0.4: QmWtCpWB39Rzc2xTB75MKorsxNpo3TyecTEN24CJ3KVohE
2.0.4: QmbCj7DgMufitEmChV74DuBBic45jv1gL5aZYtv5BQLdr4

12
package.json
View File

@ -76,6 +76,18 @@
"hash": "QmQDvJoB6aJWN3sjr3xsgXqKCXf4jU5zdMXpDMsBkYVNqa",
"name": "go-buffer-pool",
"version": "0.1.3"
},
{
"author": "whyrusleeping",
"hash": "QmfVj3x4D6Jkq9SEoi5n2NmoUomLwoeiwnYz2KQa15wRw6",
"name": "base32",
"version": "0.0.2"
},
{
"author": "whyrusleeping",
"hash": "QmccqjKZUTqp4ikWNyAbjBuP5HEdqSqRuAr9mcEhYab54a",
"name": "go-ds-leveldb",
"version": "1.2.1"
}
],
"gxVersion": "0.4.0",

13
pstoreds/addr_book.go
View File

@ -7,6 +7,7 @@ import (
"time"
lru "github.com/hashicorp/golang-lru"
base32 "github.com/whyrusleeping/base32"
ds "github.com/ipfs/go-datastore"
query "github.com/ipfs/go-datastore/query"
@ -31,7 +32,7 @@ var (
)
// Peer addresses are stored under the following db key pattern:
// /peers/addr/<b58 of peer id>/<hash of maddr>
// /peers/addr/<b32 peer id no padding>/<hash of maddr>
var abBase = ds.NewKey("/peers/addrs")
var _ pstore.AddrBook = (*dsAddrBook)(nil)
@ -107,7 +108,7 @@ func keysAndAddrs(p peer.ID, addrs []ma.Multiaddr) ([]ds.Key, []ma.Multiaddr, er
var (
keys = make([]ds.Key, len(addrs))
clean = make([]ma.Multiaddr, len(addrs))
parentKey = abBase.ChildString(peer.IDB58Encode(p))
parentKey = abBase.ChildString(base32.RawStdEncoding.EncodeToString([]byte(p)))
i = 0
)
@ -120,7 +121,7 @@ func keysAndAddrs(p peer.ID, addrs []ma.Multiaddr) ([]ds.Key, []ma.Multiaddr, er
if err != nil {
return nil, nil, err
}
keys[i] = parentKey.ChildString(hash.B58String())
keys[i] = parentKey.ChildString(base32.RawStdEncoding.EncodeToString(hash))
clean[i] = addr
i++
}
@ -293,7 +294,7 @@ func (mgr *dsAddrBook) UpdateAddrs(p peer.ID, oldTTL time.Duration, newTTL time.
func (mgr *dsAddrBook) dbUpdateTTL(p peer.ID, oldTTL time.Duration, newTTL time.Duration) error {
var (
prefix = abBase.ChildString(peer.IDB58Encode(p))
prefix = abBase.ChildString(base32.RawStdEncoding.EncodeToString([]byte(p)))
q = query.Query{Prefix: prefix.String(), KeysOnly: false}
results query.Results
err error
@ -336,7 +337,7 @@ func (mgr *dsAddrBook) dbUpdateTTL(p peer.ID, oldTTL time.Duration, newTTL time.
// Addrs returns all of the non-expired addresses for a given peer.
func (mgr *dsAddrBook) Addrs(p peer.ID) []ma.Multiaddr {
var (
prefix = abBase.ChildString(peer.IDB58Encode(p))
prefix = abBase.ChildString(base32.RawStdEncoding.EncodeToString([]byte(p)))
q = query.Query{Prefix: prefix.String(), KeysOnly: false, ReturnExpirations: true}
results query.Results
err error
@ -411,7 +412,7 @@ func (mgr *dsAddrBook) AddrStream(ctx context.Context, p peer.ID) <-chan ma.Mult
func (mgr *dsAddrBook) ClearAddrs(p peer.ID) {
var (
err error
prefix = abBase.ChildString(peer.IDB58Encode(p))
prefix = abBase.ChildString(base32.RawStdEncoding.EncodeToString([]byte(p)))
deleteFn func() error
)

209
pstoreds/ds_test.go
View File

@ -2,170 +2,92 @@ package pstoreds
import (
"context"
"fmt"
"io/ioutil"
"os"
"testing"
"time"
bd "github.com/dgraph-io/badger"
ds "github.com/ipfs/go-datastore"
badger "github.com/ipfs/go-ds-badger"
leveldb "github.com/ipfs/go-ds-leveldb"
pstore "github.com/libp2p/go-libp2p-peerstore"
pt "github.com/libp2p/go-libp2p-peerstore/test"
)
func BenchmarkBaselineBadgerDatastorePutEntry(b *testing.B) {
bds, closer := badgerStore(b)
defer closer()
type datastoreFactory func(tb testing.TB) (ds.TxnDatastore, func())
b.ResetTimer()
for i := 0; i < b.N; i++ {
txn, _ := bds.NewTransaction(false)
var dstores = map[string]datastoreFactory{
"Badger": badgerStore,
// TODO: Enable once go-ds-leveldb supports TTL via a shim.
// "Leveldb": leveldbStore,
}
key := ds.RawKey(fmt.Sprintf("/key/%d", i))
txn.Put(key, []byte(fmt.Sprintf("/value/%d", i)))
txn.Commit()
txn.Discard()
func TestDsPeerstore(t *testing.T) {
for name, dsFactory := range dstores {
t.Run(name, func(t *testing.T) {
pt.TestPeerstore(t, peerstoreFactory(t, dsFactory, DefaultOpts()))
})
}
}
func BenchmarkBaselineBadgerDatastoreGetEntry(b *testing.B) {
bds, closer := badgerStore(b)
defer closer()
func TestDsAddrBook(t *testing.T) {
for name, dsFactory := range dstores {
t.Run(name, func(t *testing.T) {
t.Run("Cacheful", func(t *testing.T) {
t.Parallel()
txn, _ := bds.NewTransaction(false)
keys := make([]ds.Key, 1000)
for i := 0; i < 1000; i++ {
key := ds.RawKey(fmt.Sprintf("/key/%d", i))
txn.Put(key, []byte(fmt.Sprintf("/value/%d", i)))
keys[i] = key
}
if err := txn.Commit(); err != nil {
b.Fatal(err)
}
opts := DefaultOpts()
opts.TTLInterval = 100 * time.Microsecond
opts.CacheSize = 1024
b.ResetTimer()
for i := 0; i < b.N; i++ {
txn, _ := bds.NewTransaction(true)
if _, err := txn.Get(keys[i%1000]); err != nil {
b.Fatal(err)
}
txn.Discard()
pt.TestAddrBook(t, addressBookFactory(t, dsFactory, opts))
})
t.Run("Cacheless", func(t *testing.T) {
t.Parallel()
opts := DefaultOpts()
opts.TTLInterval = 100 * time.Microsecond
opts.CacheSize = 0
pt.TestAddrBook(t, addressBookFactory(t, dsFactory, opts))
})
})
}
}
func BenchmarkBaselineBadgerDirectPutEntry(b *testing.B) {
opts := bd.DefaultOptions
dataPath, err := ioutil.TempDir(os.TempDir(), "badger")
if err != nil {
b.Fatal(err)
}
opts.Dir = dataPath
opts.ValueDir = dataPath
opts.SyncWrites = false
db, err := bd.Open(opts)
if err != nil {
b.Fatal(err)
}
defer db.Close()
b.ResetTimer()
for i := 0; i < b.N; i++ {
txn := db.NewTransaction(true)
txn.Set([]byte(fmt.Sprintf("/key/%d", i)), []byte(fmt.Sprintf("/value/%d", i)))
txn.Commit(nil)
func TestDsKeyBook(t *testing.T) {
for name, dsFactory := range dstores {
t.Run(name, func(t *testing.T) {
pt.TestKeyBook(t, keyBookFactory(t, dsFactory, DefaultOpts()))
})
}
}
func BenchmarkBaselineBadgerDirectGetEntry(b *testing.B) {
opts := bd.DefaultOptions
dataPath, err := ioutil.TempDir(os.TempDir(), "badger")
if err != nil {
b.Fatal(err)
}
opts.Dir = dataPath
opts.ValueDir = dataPath
db, err := bd.Open(opts)
if err != nil {
b.Fatal(err)
}
defer db.Close()
txn := db.NewTransaction(true)
for i := 0; i < 1000; i++ {
txn.Set([]byte(fmt.Sprintf("/key/%d", i)), []byte(fmt.Sprintf("/value/%d", i)))
}
txn.Commit(nil)
b.ResetTimer()
for i := 0; i < b.N; i++ {
txn := db.NewTransaction(false)
txn.Get([]byte(fmt.Sprintf("/key/%d", i%1000)))
txn.Discard()
}
}
func TestBadgerDsPeerstore(t *testing.T) {
pt.TestPeerstore(t, peerstoreFactory(t, DefaultOpts()))
}
func TestBadgerDsAddrBook(t *testing.T) {
t.Run("Cacheful", func(t *testing.T) {
t.Parallel()
opts := DefaultOpts()
opts.TTLInterval = 100 * time.Microsecond
opts.CacheSize = 1024
pt.TestAddrBook(t, addressBookFactory(t, opts))
})
t.Run("Cacheless", func(t *testing.T) {
t.Parallel()
opts := DefaultOpts()
opts.TTLInterval = 100 * time.Microsecond
opts.CacheSize = 0
pt.TestAddrBook(t, addressBookFactory(t, opts))
})
}
func TestBadgerDsKeyBook(t *testing.T) {
pt.TestKeyBook(t, keyBookFactory(t, DefaultOpts()))
}
func BenchmarkBadgerDsPeerstore(b *testing.B) {
func BenchmarkDsPeerstore(b *testing.B) {
caching := DefaultOpts()
caching.CacheSize = 1024
cacheless := DefaultOpts()
cacheless.CacheSize = 0
pt.BenchmarkPeerstore(b, peerstoreFactory(b, caching), "Caching")
pt.BenchmarkPeerstore(b, peerstoreFactory(b, cacheless), "Cacheless")
for name, dsFactory := range dstores {
b.Run(name, func(b *testing.B) {
pt.BenchmarkPeerstore(b, peerstoreFactory(b, dsFactory, caching), "Caching")
pt.BenchmarkPeerstore(b, peerstoreFactory(b, dsFactory, cacheless), "Cacheless")
})
}
}
func badgerStore(t testing.TB) (ds.TxnDatastore, func()) {
func badgerStore(tb testing.TB) (ds.TxnDatastore, func()) {
dataPath, err := ioutil.TempDir(os.TempDir(), "badger")
if err != nil {
t.Fatal(err)
tb.Fatal(err)
}
store, err := badger.NewDatastore(dataPath, nil)
if err != nil {
t.Fatal(err)
tb.Fatal(err)
}
closer := func() {
store.Close()
@ -174,9 +96,25 @@ func badgerStore(t testing.TB) (ds.TxnDatastore, func()) {
return store, closer
}
func peerstoreFactory(tb testing.TB, opts Options) pt.PeerstoreFactory {
func leveldbStore(tb testing.TB) (ds.TxnDatastore, func()) {
dataPath, err := ioutil.TempDir(os.TempDir(), "leveldb")
if err != nil {
tb.Fatal(err)
}
store, err := leveldb.NewDatastore(dataPath, nil)
if err != nil {
tb.Fatal(err)
}
closer := func() {
store.Close()
os.RemoveAll(dataPath)
}
return store, closer
}
func peerstoreFactory(tb testing.TB, storeFactory datastoreFactory, opts Options) pt.PeerstoreFactory {
return func() (pstore.Peerstore, func()) {
store, closeFunc := badgerStore(tb)
store, closeFunc := storeFactory(tb)
ps, err := NewPeerstore(context.Background(), store, opts)
if err != nil {
@ -187,9 +125,9 @@ func peerstoreFactory(tb testing.TB, opts Options) pt.PeerstoreFactory {
}
}
func addressBookFactory(tb testing.TB, opts Options) pt.AddrBookFactory {
func addressBookFactory(tb testing.TB, storeFactory datastoreFactory, opts Options) pt.AddrBookFactory {
return func() (pstore.AddrBook, func()) {
store, closeFunc := badgerStore(tb)
store, closeFunc := storeFactory(tb)
ab, err := NewAddrBook(context.Background(), store, opts)
if err != nil {
@ -200,10 +138,15 @@ func addressBookFactory(tb testing.TB, opts Options) pt.AddrBookFactory {
}
}
func keyBookFactory(tb testing.TB, opts Options) pt.KeyBookFactory {
func keyBookFactory(tb testing.TB, storeFactory datastoreFactory, opts Options) pt.KeyBookFactory {
return func() (pstore.KeyBook, func()) {
store, closeFunc := badgerStore(tb)
kb, _ := NewKeyBook(context.Background(), store, opts)
store, closeFunc := storeFactory(tb)
kb, err := NewKeyBook(context.Background(), store, opts)
if err != nil {
tb.Fatal(err)
}
return kb, closeFunc
}
}

12
pstoreds/keybook.go
View File

@ -4,6 +4,8 @@ import (
"context"
"errors"
base32 "github.com/whyrusleeping/base32"
ds "github.com/ipfs/go-datastore"
query "github.com/ipfs/go-datastore/query"
@ -13,7 +15,7 @@ import (
)
// Public and private keys are stored under the following db key pattern:
// /peers/keys/<b58 of peer id>/{pub, priv}
// /peers/keys/<b32 peer id no padding>/{pub, priv}
var (
kbBase = ds.NewKey("/peers/keys")
pubSuffix = ds.NewKey("/pub")
@ -31,7 +33,7 @@ func NewKeyBook(_ context.Context, store ds.TxnDatastore, _ Options) (pstore.Key
}
func (kb *dsKeyBook) PubKey(p peer.ID) ic.PubKey {
key := kbBase.ChildString(peer.IDB58Encode(p)).Child(pubSuffix)
key := kbBase.ChildString(base32.RawStdEncoding.EncodeToString([]byte(p))).Child(pubSuffix)
var pk ic.PubKey
if value, err := kb.ds.Get(key); err == nil {
@ -68,7 +70,7 @@ func (kb *dsKeyBook) AddPubKey(p peer.ID, pk ic.PubKey) error {
return errors.New("peer ID does not match public key")
}
key := kbBase.ChildString(peer.IDB58Encode(p)).Child(pubSuffix)
key := kbBase.ChildString(base32.RawStdEncoding.EncodeToString([]byte(p))).Child(pubSuffix)
val, err := pk.Bytes()
if err != nil {
log.Errorf("error while converting pubkey byte string for peer %s: %s\n", p.Pretty(), err)
@ -82,7 +84,7 @@ func (kb *dsKeyBook) AddPubKey(p peer.ID, pk ic.PubKey) error {
}
func (kb *dsKeyBook) PrivKey(p peer.ID) ic.PrivKey {
key := kbBase.ChildString(peer.IDB58Encode(p)).Child(privSuffix)
key := kbBase.ChildString(base32.RawStdEncoding.EncodeToString([]byte(p))).Child(privSuffix)
value, err := kb.ds.Get(key)
if err != nil {
log.Errorf("error while fetching privkey from datastore for peer %s: %s\n", p.Pretty(), err)
@ -104,7 +106,7 @@ func (kb *dsKeyBook) AddPrivKey(p peer.ID, sk ic.PrivKey) error {
return errors.New("peer ID does not match private key")
}
key := kbBase.ChildString(peer.IDB58Encode(p)).Child(privSuffix)
key := kbBase.ChildString(base32.RawStdEncoding.EncodeToString([]byte(p))).Child(privSuffix)
val, err := sk.Bytes()
if err != nil {
log.Errorf("error while converting privkey byte string for peer %s: %s\n", p.Pretty(), err)

8
pstoreds/metadata.go
View File

@ -5,6 +5,8 @@ import (
"context"
"encoding/gob"
base32 "github.com/whyrusleeping/base32"
ds "github.com/ipfs/go-datastore"
pool "github.com/libp2p/go-buffer-pool"
@ -13,7 +15,7 @@ import (
)
// Metadata is stored under the following db key pattern:
// /peers/metadata/<b58 peer id>/<key>
// /peers/metadata/<b32 peer id no padding>/<key>
var pmBase = ds.NewKey("/peers/metadata")
type dsPeerMetadata struct {
@ -39,7 +41,7 @@ func NewPeerMetadata(_ context.Context, store ds.Datastore, _ Options) (pstore.P
}
func (pm *dsPeerMetadata) Get(p peer.ID, key string) (interface{}, error) {
k := pmBase.ChildString(peer.IDB58Encode(p)).ChildString(key)
k := pmBase.ChildString(base32.RawStdEncoding.EncodeToString([]byte(p))).ChildString(key)
value, err := pm.ds.Get(k)
if err != nil {
if err == ds.ErrNotFound {
@ -56,7 +58,7 @@ func (pm *dsPeerMetadata) Get(p peer.ID, key string) (interface{}, error) {
}
func (pm *dsPeerMetadata) Put(p peer.ID, key string, val interface{}) error {
k := pmBase.ChildString(peer.IDB58Encode(p)).ChildString(key)
k := pmBase.ChildString(base32.RawStdEncoding.EncodeToString([]byte(p))).ChildString(key)
var buf pool.Buffer
if err := gob.NewEncoder(&buf).Encode(&val); err != nil {
return err

6
pstoreds/peerstore.go
View File

@ -4,6 +4,8 @@ import (
"context"
"time"
base32 "github.com/whyrusleeping/base32"
ds "github.com/ipfs/go-datastore"
query "github.com/ipfs/go-datastore/query"
@ -91,8 +93,8 @@ func uniquePeerIds(ds ds.TxnDatastore, prefix ds.Key, extractor func(result quer
ids := make(peer.IDSlice, len(idset))
i := 0
for id := range idset {
pid, _ := peer.IDB58Decode(id)
ids[i] = pid
pid, _ := base32.RawStdEncoding.DecodeString(id)
ids[i], _ = peer.IDFromBytes(pid)
i++
}
return ids, nil