Merge pull request #71 from libp2p/feat/optimize-mem

optimize peerstore memory

peerstore.go

@@ -10,6 +10,9 @@ import (
 
 var _ Peerstore = (*peerstore)(nil)
 
+const maxInternedProtocols = 512
+const maxInternedProtocolSize = 256
+
 type peerstore struct {
 	Metrics
 
@@ -18,17 +21,19 @@ type peerstore struct {
 	PeerMetadata
 
 	// lock for protocol information, separate from datastore lock
-	protolock sync.Mutex
+	protolock         sync.Mutex
+	internedProtocols map[string]string
 }
 
 // NewPeerstore creates a data structure that stores peer data, backed by the
 // supplied implementations of KeyBook, AddrBook and PeerMetadata.
 func NewPeerstore(kb KeyBook, ab AddrBook, md PeerMetadata) Peerstore {
 	return &peerstore{
-		KeyBook:      kb,
-		AddrBook:     ab,
-		PeerMetadata: md,
-		Metrics:      NewMetrics(),
+		KeyBook:           kb,
+		AddrBook:          ab,
+		PeerMetadata:      md,
+		Metrics:           NewMetrics(),
+		internedProtocols: make(map[string]string),
 	}
 }
 
@@ -75,13 +80,30 @@ func (ps *peerstore) PeerInfo(p peer.ID) PeerInfo {
 	}
 }
 
+func (ps *peerstore) internProtocol(s string) string {
+	if len(s) > maxInternedProtocolSize {
+		return s
+	}
+
+	if interned, ok := ps.internedProtocols[s]; ok {
+		return interned
+	}
+
+	if len(ps.internedProtocols) >= maxInternedProtocols {
+		ps.internedProtocols = make(map[string]string, maxInternedProtocols)
+	}
+
+	ps.internedProtocols[s] = s
+	return s
+}
+
 func (ps *peerstore) SetProtocols(p peer.ID, protos ...string) error {
 	ps.protolock.Lock()
 	defer ps.protolock.Unlock()
 
-	protomap := make(map[string]struct{})
+	protomap := make(map[string]struct{}, len(protos))
 	for _, proto := range protos {
-		protomap[proto] = struct{}{}
+		protomap[ps.internProtocol(proto)] = struct{}{}
 	}
 
 	return ps.Put(p, "protocols", protomap)
@@ -96,7 +118,7 @@ func (ps *peerstore) AddProtocols(p peer.ID, protos ...string) error {
 	}
 
 	for _, proto := range protos {
-		protomap[proto] = struct{}{}
+		protomap[ps.internProtocol(proto)] = struct{}{}
 	}
 
 	return ps.Put(p, "protocols", protomap)
@@ -127,8 +149,8 @@ func (ps *peerstore) GetProtocols(p peer.ID) ([]string, error) {
 		return nil, err
 	}
 
-	var out []string
-	for k, _ := range pmap {
+	out := make([]string, 0, len(pmap))
+	for k := range pmap {
 		out = append(out, k)
 	}
 
@@ -143,7 +165,7 @@ func (ps *peerstore) SupportsProtocols(p peer.ID, protos ...string) ([]string, e
 		return nil, err
 	}
 
-	var out []string
+	out := make([]string, 0, len(protos))
 	for _, proto := range protos {
 		if _, ok := pmap[proto]; ok {
 			out = append(out, proto)

pstoremem/metadata.go

@@ -7,10 +7,15 @@ import (
 	pstore "github.com/libp2p/go-libp2p-peerstore"
 )
 
+type metakey struct {
+	id  peer.ID
+	key string
+}
+
 type memoryPeerMetadata struct {
 	// store other data, like versions
 	//ds ds.ThreadSafeDatastore
-	ds     map[string]interface{}
+	ds     map[metakey]interface{}
 	dslock sync.Mutex
 }
 
@@ -18,26 +23,21 @@ var _ pstore.PeerMetadata = (*memoryPeerMetadata)(nil)
 
 func NewPeerMetadata() pstore.PeerMetadata {
 	return &memoryPeerMetadata{
-		ds: make(map[string]interface{}),
+		ds: make(map[metakey]interface{}),
 	}
 }
 
 func (ps *memoryPeerMetadata) Put(p peer.ID, key string, val interface{}) error {
-	//dsk := ds.NewKey(string(p) + "/" + key)
-	//return ps.ds.Put(dsk, val)
 	ps.dslock.Lock()
 	defer ps.dslock.Unlock()
-	ps.ds[string(p)+"/"+key] = val
+	ps.ds[metakey{p, key}] = val
 	return nil
 }
 
 func (ps *memoryPeerMetadata) Get(p peer.ID, key string) (interface{}, error) {
-	//dsk := ds.NewKey(string(p) + "/" + key)
-	//return ps.ds.Get(dsk)
-
 	ps.dslock.Lock()
 	defer ps.dslock.Unlock()
-	i, ok := ps.ds[string(p)+"/"+key]
+	i, ok := ps.ds[metakey{p, key}]
 	if !ok {
 		return nil, pstore.ErrNotFound
 	}