diff --git a/.travis.yml b/.travis.yml index a2df56d..a5dfefb 100644 --- a/.travis.yml +++ b/.travis.yml @@ -13,7 +13,7 @@ install: script: - pytest --cov=./libp2p tests/ - - pylint --rcfile=.pylintrc libp2p/!(kademlia) tests + - pylint --rcfile=.pylintrc libp2p tests after_success: - codecov diff --git a/libp2p/kademlia/crawling.py b/libp2p/kademlia/crawling.py index 6dabd10..6006bef 100644 --- a/libp2p/kademlia/crawling.py +++ b/libp2p/kademlia/crawling.py @@ -15,6 +15,7 @@ class SpiderCrawl: """ def __init__(self, protocol, node, peers, ksize, alpha): + # pylint: disable=too-many-arguments """ Create a new C{SpiderCrawl}er. @@ -71,6 +72,7 @@ class SpiderCrawl: class ValueSpiderCrawl(SpiderCrawl): def __init__(self, protocol, node, peers, ksize, alpha): + # pylint: disable=too-many-arguments SpiderCrawl.__init__(self, protocol, node, peers, ksize, alpha) # keep track of the single nearest node without value - per # section 2.3 so we can set the key there if found diff --git a/libp2p/kademlia/protocol.py b/libp2p/kademlia/protocol.py index 3573983..5273230 100644 --- a/libp2p/kademlia/protocol.py +++ b/libp2p/kademlia/protocol.py @@ -12,6 +12,15 @@ log = logging.getLogger(__name__) # pylint: disable=invalid-name class KademliaProtocol(RPCProtocol): + """ + There are four main RPCs in the Kademlia protocol + PING, STORE, FIND_NODE, FIND_VALUE + PING probes if a node is still online + STORE instructs a node to store (key, value) + FIND_NODE takes a 160-bit ID and gets back + (ip, udp_port, node_id) for k closest nodes to target + FIND_VALUE behaves like FIND_NODE unless a value is stored + """ def __init__(self, source_node, storage, ksize): RPCProtocol.__init__(self) self.router = RoutingTable(self, ksize, source_node) @@ -28,6 +37,12 @@ class KademliaProtocol(RPCProtocol): ids.append(rid) return ids + def rpc_add_provider(self, sender, nodeid, key): + pass + + def rpc_get_providers(self, sender, nodeid, key): + pass + def rpc_stun(self, sender): # pylint: disable=no-self-use return sender diff --git a/libp2p/kademlia/routing.py b/libp2p/kademlia/routing.py index 2ffd525..3f88e8b 100644 --- a/libp2p/kademlia/routing.py +++ b/libp2p/kademlia/routing.py @@ -8,6 +8,13 @@ from .utils import OrderedSet, shared_prefix, bytes_to_bit_string class KBucket: + """ + each node keeps a list of (ip, udp_port, node_id) + for nodes of distance between 2^i and 2^(i+1) + this list that every node keeps is a k-bucket + each k-bucket implements a last seen eviction + policy except that live nodes are never removed + """ def __init__(self, rangeLower, rangeUpper, ksize): self.range = (rangeLower, rangeUpper) self.nodes = OrderedDict() diff --git a/libp2p/kademlia/rpc.proto b/libp2p/kademlia/rpc.proto new file mode 100644 index 0000000..96c14a8 --- /dev/null +++ b/libp2p/kademlia/rpc.proto @@ -0,0 +1,78 @@ +// Record represents a dht record that contains a value +// for a key value pair +message Record { + // The key that references this record + bytes key = 1; + + // The actual value this record is storing + bytes value = 2; + + // Note: These fields were removed from the Record message + // hash of the authors public key + //optional string author = 3; + // A PKI signature for the key+value+author + //optional bytes signature = 4; + + // Time the record was received, set by receiver + string timeReceived = 5; +}; + +message Message { + enum MessageType { + PUT_VALUE = 0; + GET_VALUE = 1; + ADD_PROVIDER = 2; + GET_PROVIDERS = 3; + FIND_NODE = 4; + PING = 5; + } + + enum ConnectionType { + // sender does not have a connection to peer, and no extra information (default) + NOT_CONNECTED = 0; + + // sender has a live connection to peer + CONNECTED = 1; + + // sender recently connected to peer + CAN_CONNECT = 2; + + // sender recently tried to connect to peer repeatedly but failed to connect + // ("try" here is loose, but this should signal "made strong effort, failed") + CANNOT_CONNECT = 3; + } + + message Peer { + // ID of a given peer. + bytes id = 1; + + // multiaddrs for a given peer + repeated bytes addrs = 2; + + // used to signal the sender's connection capabilities to the peer + ConnectionType connection = 3; + } + + // defines what type of message it is. + MessageType type = 1; + + // defines what coral cluster level this query/response belongs to. + // in case we want to implement coral's cluster rings in the future. + int32 clusterLevelRaw = 10; // NOT USED + + // Used to specify the key associated with this message. + // PUT_VALUE, GET_VALUE, ADD_PROVIDER, GET_PROVIDERS + bytes key = 2; + + // Used to return a value + // PUT_VALUE, GET_VALUE + Record record = 3; + + // Used to return peers closer to a key in a query + // GET_VALUE, GET_PROVIDERS, FIND_NODE + repeated Peer closerPeers = 8; + + // Used to return Providers + // GET_VALUE, ADD_PROVIDER, GET_PROVIDERS + repeated Peer providerPeers = 9; +} \ No newline at end of file diff --git a/libp2p/kademlia/storage.py b/libp2p/kademlia/storage.py index 1aa1ac7..bcbf625 100644 --- a/libp2p/kademlia/storage.py +++ b/libp2p/kademlia/storage.py @@ -33,7 +33,7 @@ class IStorage(ABC): def iter_older_than(self, seconds_old): """ Return the an iterator over (key, value) tuples for items older - than the given secondsOld. + than the given seconds_old. """ @abstractmethod diff --git a/libp2p/routing/kadmelia/kadmelia_content_router.py b/libp2p/routing/kadmelia/kadmelia_content_router.py index d96269a..60cf55b 100644 --- a/libp2p/routing/kadmelia/kadmelia_content_router.py +++ b/libp2p/routing/kadmelia/kadmelia_content_router.py @@ -9,6 +9,8 @@ class KadmeliaContentRouter(IContentRouting): it also announces it, otherwise it is just kept in the local accounting of which objects are being provided. """ + # the DHT finds the closest peers to `key` using the `FIND_NODE` RPC + # then sends a `ADD_PROVIDER` RPC with its own `PeerInfo` to each of these peers. pass def find_provider_iter(self, cid, count): diff --git a/libp2p/routing/kadmelia/kadmelia_peer_router.py b/libp2p/routing/kadmelia/kadmelia_peer_router.py index 0201b7d..27be67c 100644 --- a/libp2p/routing/kadmelia/kadmelia_peer_router.py +++ b/libp2p/routing/kadmelia/kadmelia_peer_router.py @@ -5,6 +5,7 @@ from libp2p.peer.peerdata import PeerData class KadmeliaPeerRouter(IPeerRouting): + # pylint: disable=too-few-public-methods def __init__(self, dht_server): self.server = dht_server diff --git a/tests/kademlia/test_basic.py b/tests/kademlia/test_basic.py index e35154b..82014a9 100644 --- a/tests/kademlia/test_basic.py +++ b/tests/kademlia/test_basic.py @@ -1,5 +1,6 @@ import pytest from libp2p.kademlia.network import Server +import math @pytest.mark.asyncio @@ -28,16 +29,15 @@ async def test_example(): @pytest.mark.parametrize("nodes_nr", [(2**i) for i in range(2, 5)]) @pytest.mark.asyncio async def test_multiple_nodes_bootstrap_set_get(nodes_nr): - nodes_nr = 25 node_bootstrap = Server() - await node_bootstrap.listen(5678) + await node_bootstrap.listen(1000 + nodes_nr * 2) nodes = [] for i in range(nodes_nr): node = Server() - addrs = [("127.0.0.1", 5678)] - await node.listen(5679 + i) + addrs = [("127.0.0.1", 1000 + nodes_nr * 2)] + await node.listen(1001 + i + nodes_nr * 2) await node.bootstrap(addrs) nodes.append(node) @@ -57,16 +57,14 @@ async def test_multiple_nodes_bootstrap_set_get(nodes_nr): @pytest.mark.parametrize("nodes_nr", [(2**i) for i in range(2, 5)]) @pytest.mark.asyncio async def test_multiple_nodes_set_bootstrap_get(nodes_nr): - nodes_nr = 25 - node_bootstrap = Server() - await node_bootstrap.listen(5678) - + await node_bootstrap.listen(2000 + nodes_nr * 2) + nodes = [] for i in range(nodes_nr): node = Server() - addrs = [("127.0.0.1", 5678)] - await node.listen(5679 + i) + addrs = [("127.0.0.1", 2000 + nodes_nr * 2)] + await node.listen(2001 + i + nodes_nr * 2) await node.bootstrap(addrs) value = "my awesome value %d" % i