From 8eb6a230ff817499a53d0ede64559cc9dc3be4aa Mon Sep 17 00:00:00 2001 From: NIC619 Date: Wed, 24 Jul 2019 16:29:02 +0800 Subject: [PATCH 01/20] Fix and add type hints to pubsub.py --- libp2p/pubsub/pubsub.py | 75 +++++++++++++++++++++++------------------ 1 file changed, 42 insertions(+), 33 deletions(-) diff --git a/libp2p/pubsub/pubsub.py b/libp2p/pubsub/pubsub.py index 2af4e24..7643bef 100644 --- a/libp2p/pubsub/pubsub.py +++ b/libp2p/pubsub/pubsub.py @@ -5,12 +5,17 @@ from typing import ( Any, Dict, List, - Sequence, Tuple, ) from lru import LRU + +from .pb import rpc_pb2 +from .pubsub_notifee import PubsubNotifee +from .pubsub_router_interface import ( + IPubsubRouter, +) from libp2p.host.host_interface import ( IHost, ) @@ -21,12 +26,6 @@ from libp2p.network.stream.net_stream_interface import ( INetStream, ) -from .pb import rpc_pb2 -from .pubsub_notifee import PubsubNotifee -from .pubsub_router_interface import ( - IPubsubRouter, -) - def get_msg_id(msg: rpc_pb2.Message) -> Tuple[bytes, bytes]: # NOTE: `string(from, seqno)` in Go @@ -38,26 +37,32 @@ class Pubsub: host: IHost my_id: ID + router: IPubsubRouter - peer_queue: asyncio.Queue - protocols: Sequence[str] - incoming_msgs_from_peers: asyncio.Queue() - outgoing_messages: asyncio.Queue() + + # FIXME: Should be changed to `asyncio.Queue[ID]` + peer_queue: asyncio.Queue[str] + + protocols: List[str] + + incoming_msgs_from_peers: asyncio.Queue[rpc_pb2.Message] + outgoing_messages: asyncio.Queue[rpc_pb2.Message] + seen_messages: LRU my_topics: Dict[str, asyncio.Queue] # FIXME: Should be changed to `Dict[str, List[ID]]` peer_topics: Dict[str, List[str]] # FIXME: Should be changed to `Dict[ID, INetStream]` peers: Dict[str, INetStream] + # NOTE: Be sure it is increased atomically everytime. counter: int # uint64 - def __init__( - self, - host: IHost, - router: IPubsubRouter, - my_id: ID, - cache_size: int = None) -> None: + def __init__(self, + host: IHost, + router: IPubsubRouter, + my_id: ID, + cache_size: int = None) -> None: """ Construct a new Pubsub object, which is responsible for handling all Pubsub-related messages and relaying messages as appropriate to the @@ -73,6 +78,7 @@ class Pubsub: self.router.attach(self) # Register a notifee + # FIXME: Should be changed to `asyncio.Queue[ID]` self.peer_queue = asyncio.Queue() self.host.get_network().notify(PubsubNotifee(self.peer_queue)) @@ -99,9 +105,11 @@ class Pubsub: self.my_topics = {} # Map of topic to peers to keep track of what peers are subscribed to + # FIXME: Should be changed to `Dict[str, ID]` self.peer_topics = {} # Create peers map, which maps peer_id (as string) to stream (to a given peer) + # FIXME: Should be changed to `Dict[ID, INetStream]` self.peers = {} self.counter = time.time_ns() @@ -114,7 +122,7 @@ class Pubsub: Generate subscription message with all topics we are subscribed to only send hello packet if we have subscribed topics """ - packet = rpc_pb2.RPC() + packet: rpc_pb2.RPC = rpc_pb2.RPC() if self.my_topics: for topic_id in self.my_topics: packet.subscriptions.extend([rpc_pb2.RPC.SubOpts( @@ -131,8 +139,8 @@ class Pubsub: peer_id = stream.mplex_conn.peer_id while True: - incoming = (await stream.read()) - rpc_incoming = rpc_pb2.RPC() + incoming: bytes = (await stream.read()) + rpc_incoming: rpc_pb2.RPC = rpc_pb2.RPC() rpc_incoming.ParseFromString(incoming) if rpc_incoming.publish: @@ -168,12 +176,12 @@ class Pubsub: """ # Add peer # Map peer to stream - peer_id = stream.mplex_conn.peer_id + peer_id: ID = stream.mplex_conn.peer_id self.peers[str(peer_id)] = stream self.router.add_peer(peer_id, stream.get_protocol()) # Send hello packet - hello = self.get_hello_packet() + hello: bytes = self.get_hello_packet() await stream.write(hello) # Pass stream off to stream reader @@ -188,12 +196,12 @@ class Pubsub: """ while True: - peer_id = await self.peer_queue.get() + peer_id: ID = await self.peer_queue.get() # Open a stream to peer on existing connection # (we know connection exists since that's the only way # an element gets added to peer_queue) - stream = await self.host.new_stream(peer_id, self.protocols) + stream: INetStream = await self.host.new_stream(peer_id, self.protocols) # Add Peer # Map peer to stream @@ -201,7 +209,7 @@ class Pubsub: self.router.add_peer(peer_id, stream.get_protocol()) # Send hello packet - hello = self.get_hello_packet() + hello: bytes = self.get_hello_packet() await stream.write(hello) # Pass stream off to stream reader @@ -210,8 +218,9 @@ class Pubsub: # Force context switch await asyncio.sleep(0) + # FIXME: type of `origin_id` should be changed to `ID` # FIXME: `sub_message` can be further type hinted with mypy_protobuf - def handle_subscription(self, origin_id: ID, sub_message: Any) -> None: + def handle_subscription(self, origin_id: str, sub_message: Any) -> None: """ Handle an incoming subscription message from a peer. Update internal mapping to mark the peer as subscribed or unsubscribed to topics as @@ -236,7 +245,7 @@ class Pubsub: async def handle_talk(self, publish_message: Any) -> None: """ Put incoming message from a peer onto my blocking queue - :param talk: RPC.Message format + :param publish_message: RPC.Message format """ # Check if this message has any topics that we are subscribed to @@ -247,7 +256,7 @@ class Pubsub: # for each topic await self.my_topics[topic].put(publish_message) - async def subscribe(self, topic_id: str) -> asyncio.Queue: + async def subscribe(self, topic_id: str) -> asyncio.Queue[rpc_pb2.Message]: """ Subscribe ourself to a topic :param topic_id: topic_id to subscribe to @@ -261,7 +270,7 @@ class Pubsub: self.my_topics[topic_id] = asyncio.Queue() # Create subscribe message - packet = rpc_pb2.RPC() + packet: rpc_pb2.RPC = rpc_pb2.RPC() packet.subscriptions.extend([rpc_pb2.RPC.SubOpts( subscribe=True, topicid=topic_id.encode('utf-8') @@ -289,7 +298,7 @@ class Pubsub: del self.my_topics[topic_id] # Create unsubscribe message - packet = rpc_pb2.RPC() + packet: rpc_pb2.RPC = rpc_pb2.RPC() packet.subscriptions.extend([rpc_pb2.RPC.SubOpts( subscribe=False, topicid=topic_id.encode('utf-8') @@ -301,8 +310,8 @@ class Pubsub: # Tell router we are leaving this topic await self.router.leave(topic_id) - # FIXME: `rpc_msg` can be further type hinted with mypy_protobuf - async def message_all_peers(self, rpc_msg: Any) -> None: + # FIXME: `raw_msg` can be further type hinted with mypy_protobuf + async def message_all_peers(self, raw_msg: Any) -> None: """ Broadcast a message to peers :param raw_msg: raw contents of the message to broadcast @@ -311,7 +320,7 @@ class Pubsub: # Broadcast message for _, stream in self.peers.items(): # Write message to stream - await stream.write(rpc_msg) + await stream.write(raw_msg) async def publish(self, topic_id: str, data: bytes) -> None: """ From b920955db6fb50bc42f33f48849fae54c5bb5299 Mon Sep 17 00:00:00 2001 From: NIC619 Date: Wed, 24 Jul 2019 16:29:14 +0800 Subject: [PATCH 02/20] Add type hints to gossipsub.py --- libp2p/pubsub/gossipsub.py | 179 ++++++++++++++++++++++++------------- 1 file changed, 118 insertions(+), 61 deletions(-) diff --git a/libp2p/pubsub/gossipsub.py b/libp2p/pubsub/gossipsub.py index 0af5e5f..0c9bde9 100644 --- a/libp2p/pubsub/gossipsub.py +++ b/libp2p/pubsub/gossipsub.py @@ -1,6 +1,8 @@ import asyncio import random from typing import ( + Any, + Dict, Iterable, List, MutableSet, @@ -16,6 +18,7 @@ from libp2p.peer.id import ( from .mcache import MessageCache from .pb import rpc_pb2 +from .pubsub import Pubsub from .pubsub_router_interface import IPubsubRouter @@ -24,11 +27,43 @@ class GossipSub(IPubsubRouter): # pylint: disable=too-many-instance-attributes # pylint: disable=too-many-public-methods - def __init__(self, protocols, degree, degree_low, degree_high, time_to_live, gossip_window=3, - gossip_history=5, heartbeat_interval=120): + protocols: Sequence[str] + pubsub: Pubsub + + degree: int + degree_high: int + degree_low: int + + time_to_live: int + + # FIXME: Should be changed to `Dict[str, List[ID]]` + mesh: Dict[str, List[str]] + # FIXME: Should be changed to `Dict[str, List[ID]]` + fanout: Dict[str, List[str]] + + time_since_last_publish: Dict[str, int] + + #FIXME: Should be changed to List[ID] + peers_gossipsub: List[str] + #FIXME: Should be changed to List[ID] + peers_floodsub: List[str] + + mcache: MessageCache + + heartbeat_interval: int + + def __init__(self, + protocols: Sequence[str], + degree: int, + degree_low: int, + degree_high: int, + time_to_live: int, + gossip_window: int=3, + gossip_history: int=5, + heartbeat_interval: int=120) -> None: # pylint: disable=too-many-arguments - self.protocols = protocols - self.pubsub = None + self.protocols: List[str] = protocols + self.pubsub: Pubsub = None # Store target degree, upper degree bound, and lower degree bound self.degree = degree @@ -36,7 +71,7 @@ class GossipSub(IPubsubRouter): self.degree_high = degree_high # Store time to live (for topics in fanout) - self.time_to_live = time_to_live + self.time_to_live: int = time_to_live # Create topic --> list of peers mappings self.mesh = {} @@ -56,13 +91,13 @@ class GossipSub(IPubsubRouter): # Interface functions - def get_protocols(self): + def get_protocols(self) -> List: """ :return: the list of protocols supported by the router """ return self.protocols - def attach(self, pubsub): + def attach(self, pubsub: Pubsub) -> None: """ Attach is invoked by the PubSub constructor to attach the router to a freshly initialized PubSub instance. @@ -74,10 +109,11 @@ class GossipSub(IPubsubRouter): # TODO: Start after delay asyncio.ensure_future(self.heartbeat()) - def add_peer(self, peer_id, protocol_id): + def add_peer(self, peer_id: ID, protocol_id: str): """ Notifies the router that a new peer has been connected :param peer_id: id of peer to add + :param protocol_id: router protocol the peer speaks, e.g., floodsub, gossipsub """ # Add peer to the correct peer list @@ -88,7 +124,7 @@ class GossipSub(IPubsubRouter): elif peer_type == "flood": self.peers_floodsub.append(peer_id_str) - def remove_peer(self, peer_id): + def remove_peer(self, peer_id: ID) -> None: """ Notifies the router that a peer has been disconnected :param peer_id: id of peer to remove @@ -96,16 +132,18 @@ class GossipSub(IPubsubRouter): peer_id_str = str(peer_id) self.peers_to_protocol.remove(peer_id_str) - async def handle_rpc(self, rpc, sender_peer_id): + # FIXME: type of `sender_peer_id` should be changed to `ID` + async def handle_rpc(self, rpc: rpc_pb2.Message, sender_peer_id: str): """ Invoked to process control messages in the RPC envelope. It is invoked after subscriptions and payload messages have been processed - :param rpc: rpc message + :param rpc: RPC message + :param sender_peer_id: id of the peer who sent the message """ control_message = rpc.control sender_peer_id = str(sender_peer_id) - # Relay each rpc control to the appropriate handler + # Relay each rpc control message to the appropriate handler if control_message.ihave: for ihave in control_message.ihave: await self.handle_ihave(ihave, sender_peer_id) @@ -191,7 +229,7 @@ class GossipSub(IPubsubRouter): # Excludes `msg_forwarder` and `origin` yield from send_to.difference([msg_forwarder, origin]) - async def join(self, topic): + async def join(self, topic: str) -> None: # Note: the comments here are the near-exact algorithm description from the spec """ Join notifies the router that we want to receive and @@ -204,8 +242,9 @@ class GossipSub(IPubsubRouter): # Create mesh[topic] if it does not yet exist self.mesh[topic] = [] - topic_in_fanout = topic in self.fanout - fanout_peers = self.fanout[topic] if topic_in_fanout else [] + topic_in_fanout: bool = topic in self.fanout + # FIXME: Should be changed to `List[ID]` + fanout_peers: List[str] = self.fanout[topic] if topic_in_fanout else [] fanout_size = len(fanout_peers) if not topic_in_fanout or (topic_in_fanout and fanout_size < self.degree): # There are less than D peers (let this number be x) @@ -229,7 +268,7 @@ class GossipSub(IPubsubRouter): if topic_in_fanout: del self.fanout[topic] - async def leave(self, topic): + async def leave(self, topic: str) -> None: # Note: the comments here are the near-exact algorithm description from the spec """ Leave notifies the router that we are no longer interested in a topic. @@ -247,7 +286,7 @@ class GossipSub(IPubsubRouter): # Interface Helper Functions @staticmethod - def get_peer_type(protocol_id): + def get_peer_type(protocol_id: str) -> str: # TODO: Do this in a better, more efficient way if "gossipsub" in protocol_id: return "gossip" @@ -255,7 +294,13 @@ class GossipSub(IPubsubRouter): return "flood" return "unknown" - async def deliver_messages_to_peers(self, peers, msg_sender, origin_id, serialized_packet): + # FIXME: type of `peers` should be changed to `List[ID]` + # FIXME: type of `msg_sender` and `origin_id` should be changed to `ID` + async def deliver_messages_to_peers(self, + peers: List[str], + msg_sender: str, + origin_id: str, + serialized_packet: bytes): for peer_id_in_topic in peers: # Forward to all peers that are not the # message sender and are not the message origin @@ -267,7 +312,7 @@ class GossipSub(IPubsubRouter): await stream.write(serialized_packet) # Heartbeat - async def heartbeat(self): + async def heartbeat(self) -> None: """ Call individual heartbeats. Note: the heartbeats are called with awaits because each heartbeat depends on the @@ -281,7 +326,7 @@ class GossipSub(IPubsubRouter): await asyncio.sleep(self.heartbeat_interval) - async def mesh_heartbeat(self): + async def mesh_heartbeat(self) -> None: # Note: the comments here are the exact pseudocode from the spec for topic in self.mesh: # Skip if no peers have subscribed to the topic @@ -297,7 +342,8 @@ class GossipSub(IPubsubRouter): self.mesh[topic], ) - fanout_peers_not_in_mesh = [ + # FIXME: Should be changed to `List[ID]` + fanout_peers_not_in_mesh: List[str] = [ peer for peer in selected_peers if peer not in self.mesh[topic] @@ -311,8 +357,12 @@ class GossipSub(IPubsubRouter): if num_mesh_peers_in_topic > self.degree_high: # Select |mesh[topic]| - D peers from mesh[topic] - selected_peers = GossipSub.select_from_minus(num_mesh_peers_in_topic - self.degree, - self.mesh[topic], []) + # FIXME: Should be changed to `List[ID]` + selected_peers: List[str] = GossipSub.select_from_minus( + num_mesh_peers_in_topic - self.degree, + self.mesh[topic], + [], + ) for peer in selected_peers: # Remove peer from mesh[topic] self.mesh[topic].remove(peer) @@ -320,7 +370,7 @@ class GossipSub(IPubsubRouter): # Emit PRUNE(topic) control message to peer await self.emit_prune(topic, peer) - async def fanout_heartbeat(self): + async def fanout_heartbeat(self) -> None: # Note: the comments here are the exact pseudocode from the spec for topic in self.fanout: # If time since last published > ttl @@ -362,14 +412,14 @@ class GossipSub(IPubsubRouter): # TODO: this line is a monster, can hopefully be simplified if (topic not in self.mesh or (peer not in self.mesh[topic]))\ and (topic not in self.fanout or (peer not in self.fanout[topic])): - msg_ids = [str(msg) for msg in msg_ids] + msg_ids: List[str] = [str(msg) for msg in msg_ids] await self.emit_ihave(topic, msg_ids, peer) # TODO: Refactor and Dedup. This section is the roughly the same as the above. # Do the same for fanout, for all topics not already hit in mesh for topic in self.fanout: if topic not in self.mesh: - msg_ids = self.mcache.window(topic) + msg_ids: List[str] = self.mcache.window(topic) if msg_ids: # TODO: Make more efficient, possibly using a generator? # Get all pubsub peers in topic and only add if they are gossipsub peers also @@ -383,13 +433,13 @@ class GossipSub(IPubsubRouter): for peer in peers_to_emit_ihave_to: if peer not in self.mesh[topic] and peer not in self.fanout[topic]: - msg_ids = [str(msg) for msg in msg_ids] + msg_ids: List[str] = [str(msg) for msg in msg_ids] await self.emit_ihave(topic, msg_ids, peer) self.mcache.shift() @staticmethod - def select_from_minus(num_to_select, pool, minus): + def select_from_minus(num_to_select: int, pool: Sequence[Any], minus: Sequence[Any]) -> List[Any]: """ Select at most num_to_select subset of elements from the set (pool - minus) randomly. :param num_to_select: number of elements to randomly select @@ -400,10 +450,10 @@ class GossipSub(IPubsubRouter): # Create selection pool, which is selection_pool = pool - minus if minus: # Create a new selection pool by removing elements of minus - selection_pool = [x for x in pool if x not in minus] + selection_pool: List[Any] = [x for x in pool if x not in minus] else: # Don't create a new selection_pool if we are not subbing anything - selection_pool = pool + selection_pool: List[Any] = pool # If num_to_select > size(selection_pool), then return selection_pool (which has the most # possible elements s.t. the number of elements is less than num_to_select) @@ -411,7 +461,7 @@ class GossipSub(IPubsubRouter): return selection_pool # Random selection - selection = random.sample(selection_pool, num_to_select) + selection: List[Any] = random.sample(selection_pool, num_to_select) return selection @@ -433,7 +483,7 @@ class GossipSub(IPubsubRouter): # RPC handlers - async def handle_ihave(self, ihave_msg, sender_peer_id): + async def handle_ihave(self, ihave_msg: rpc_pb2.Message, sender_peer_id: str) -> None: """ Checks the seen set and requests unknown messages with an IWANT message. """ @@ -442,29 +492,36 @@ class GossipSub(IPubsubRouter): from_id_str = sender_peer_id # Get list of all seen (seqnos, from) from the (seqno, from) tuples in seen_messages cache - seen_seqnos_and_peers = [seqno_and_from - for seqno_and_from in self.pubsub.seen_messages.keys()] + seen_seqnos_and_peers = [ + seqno_and_from + for seqno_and_from in self.pubsub.seen_messages.keys() + ] # Add all unknown message ids (ids that appear in ihave_msg but not in seen_seqnos) to list # of messages we want to request - msg_ids_wanted = [msg_id for msg_id in ihave_msg.messageIDs - if literal_eval(msg_id) not in seen_seqnos_and_peers] + # FIXME: Update type of message ID + msg_ids_wanted = [ + msg_id + for msg_id in ihave_msg.messageIDs + if literal_eval(msg_id) not in seen_seqnos_and_peers + ] # Request messages with IWANT message if msg_ids_wanted: await self.emit_iwant(msg_ids_wanted, from_id_str) - async def handle_iwant(self, iwant_msg, sender_peer_id): + async def handle_iwant(self, iwant_msg: rpc_pb2.Message, sender_peer_id: str) -> None: """ Forwards all request messages that are present in mcache to the requesting peer. """ from_id_str = sender_peer_id - msg_ids = [literal_eval(msg) for msg in iwant_msg.messageIDs] - msgs_to_forward = [] + # FIXME: Update type of message ID + msg_ids: List[Any] = [literal_eval(msg) for msg in iwant_msg.messageIDs] + msgs_to_forward: List = [] for msg_id_iwant in msg_ids: # Check if the wanted message ID is present in mcache - msg = self.mcache.get(msg_id_iwant) + msg: rpc_pb2.Message = self.mcache.get(msg_id_iwant) # Cache hit if msg: @@ -476,12 +533,12 @@ class GossipSub(IPubsubRouter): # because then the message will forwarded to peers in the topics contained in the messages. # We should # 1) Package these messages into a single packet - packet = rpc_pb2.RPC() + packet: rpc_pb2.RPC = rpc_pb2.RPC() packet.publish.extend(msgs_to_forward) # 2) Serialize that packet - rpc_msg = packet.SerializeToString() + rpc_msg: bytes = packet.SerializeToString() # 3) Get the stream to this peer # TODO: Should we pass in from_id or from_id_str here? @@ -490,8 +547,8 @@ class GossipSub(IPubsubRouter): # 4) And write the packet to the stream await peer_stream.write(rpc_msg) - async def handle_graft(self, graft_msg, sender_peer_id): - topic = graft_msg.topicID + async def handle_graft(self, graft_msg: rpc_pb2.Message, sender_peer_id: str) -> None: + topic: str = graft_msg.topicID from_id_str = sender_peer_id @@ -503,8 +560,8 @@ class GossipSub(IPubsubRouter): # Respond with PRUNE if not subscribed to the topic await self.emit_prune(topic, sender_peer_id) - async def handle_prune(self, prune_msg, sender_peer_id): - topic = prune_msg.topicID + async def handle_prune(self, prune_msg: rpc_pb2.Message, sender_peer_id: str) -> None: + topic: str = prune_msg.topicID from_id_str = sender_peer_id @@ -514,65 +571,65 @@ class GossipSub(IPubsubRouter): # RPC emitters - async def emit_ihave(self, topic, msg_ids, to_peer): + async def emit_ihave(self, topic: str, msg_ids: Any, to_peer: str) -> None: """ Emit ihave message, sent to to_peer, for topic and msg_ids """ - ihave_msg = rpc_pb2.ControlIHave() + ihave_msg: rpc_pb2.ControlIHave = rpc_pb2.ControlIHave() ihave_msg.messageIDs.extend(msg_ids) ihave_msg.topicID = topic - control_msg = rpc_pb2.ControlMessage() + control_msg: rpc_pb2.ControlMessage = rpc_pb2.ControlMessage() control_msg.ihave.extend([ihave_msg]) await self.emit_control_message(control_msg, to_peer) - async def emit_iwant(self, msg_ids, to_peer): + async def emit_iwant(self, msg_ids: Any, to_peer: str) -> None: """ Emit iwant message, sent to to_peer, for msg_ids """ - iwant_msg = rpc_pb2.ControlIWant() + iwant_msg: rpc_pb2.ControlIWant = rpc_pb2.ControlIWant() iwant_msg.messageIDs.extend(msg_ids) - control_msg = rpc_pb2.ControlMessage() + control_msg: rpc_pb2.ControlMessage = rpc_pb2.ControlMessage() control_msg.iwant.extend([iwant_msg]) await self.emit_control_message(control_msg, to_peer) - async def emit_graft(self, topic, to_peer): + async def emit_graft(self, topic: str, to_peer: str) -> None: """ Emit graft message, sent to to_peer, for topic """ - graft_msg = rpc_pb2.ControlGraft() + graft_msg: rpc_pb2.ControlGraft = rpc_pb2.ControlGraft() graft_msg.topicID = topic - control_msg = rpc_pb2.ControlMessage() + control_msg: rpc_pb2.ControlMessage = rpc_pb2.ControlMessage() control_msg.graft.extend([graft_msg]) await self.emit_control_message(control_msg, to_peer) - async def emit_prune(self, topic, to_peer): + async def emit_prune(self, topic: str, to_peer: str) -> None: """ Emit graft message, sent to to_peer, for topic """ - prune_msg = rpc_pb2.ControlPrune() + prune_msg: rpc_pb2.ControlPrune = rpc_pb2.ControlPrune() prune_msg.topicID = topic - control_msg = rpc_pb2.ControlMessage() + control_msg: rpc_pb2.ControlMessage = rpc_pb2.ControlMessage() control_msg.prune.extend([prune_msg]) await self.emit_control_message(control_msg, to_peer) - async def emit_control_message(self, control_msg, to_peer): + async def emit_control_message(self, control_msg: rpc_pb2.ControlMessage, to_peer: str) -> None: # Add control message to packet - packet = rpc_pb2.RPC() + packet: rpc_pb2.RPC = rpc_pb2.RPC() packet.control.CopyFrom(control_msg) - rpc_msg = packet.SerializeToString() + rpc_msg: bytes = packet.SerializeToString() # Get stream for peer from pubsub peer_stream = self.pubsub.peers[to_peer] From 63014eeaae4b71d5f480187ad310ea48c53f84ff Mon Sep 17 00:00:00 2001 From: NIC619 Date: Fri, 26 Jul 2019 16:25:15 +0800 Subject: [PATCH 03/20] Add type hints to floodsub.py --- libp2p/pubsub/floodsub.py | 26 ++++++++++++++++++-------- 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/libp2p/pubsub/floodsub.py b/libp2p/pubsub/floodsub.py index 0404021..126e4c2 100644 --- a/libp2p/pubsub/floodsub.py +++ b/libp2p/pubsub/floodsub.py @@ -1,5 +1,7 @@ from typing import ( Iterable, + List, + Sequence, ) from libp2p.peer.id import ( @@ -8,23 +10,31 @@ from libp2p.peer.id import ( ) from .pb import rpc_pb2 +from .pubsub import Pubsub from .pubsub_router_interface import IPubsubRouter +from libp2p.network.stream.net_stream_interface import ( + INetStream, +) class FloodSub(IPubsubRouter): # pylint: disable=no-member - def __init__(self, protocols): + protocols: List[str] + + pubsub: Pubsub + + def __init__(self, protocols: Sequence[str]) -> None: self.protocols = protocols self.pubsub = None - def get_protocols(self): + def get_protocols(self) -> List[str]: """ :return: the list of protocols supported by the router """ return self.protocols - def attach(self, pubsub): + def attach(self, pubsub: Pubsub) -> None: """ Attach is invoked by the PubSub constructor to attach the router to a freshly initialized PubSub instance. @@ -32,19 +42,19 @@ class FloodSub(IPubsubRouter): """ self.pubsub = pubsub - def add_peer(self, peer_id, protocol_id): + def add_peer(self, peer_id: ID, protocol_id: str): """ Notifies the router that a new peer has been connected :param peer_id: id of peer to add """ - def remove_peer(self, peer_id): + def remove_peer(self, peer_id: ID): """ Notifies the router that a peer has been disconnected :param peer_id: id of peer to remove """ - async def handle_rpc(self, rpc, sender_peer_id): + async def handle_rpc(self, rpc: rpc_pb2.ControlMessage, sender_peer_id: ID): """ Invoked to process control messages in the RPC envelope. It is invoked after subscriptions and payload messages have been processed @@ -80,7 +90,7 @@ class FloodSub(IPubsubRouter): # Ref: https://github.com/libp2p/go-libp2p-pubsub/blob/master/comm.go#L107 await stream.write(rpc_msg.SerializeToString()) - async def join(self, topic): + async def join(self, topic: str): """ Join notifies the router that we want to receive and forward messages in a topic. It is invoked after the @@ -88,7 +98,7 @@ class FloodSub(IPubsubRouter): :param topic: topic to join """ - async def leave(self, topic): + async def leave(self, topic: str): """ Leave notifies the router that we are no longer interested in a topic. It is invoked after the unsubscription announcement. From 3549f2ff8bad75b5acddedfaa150fcd478e0564c Mon Sep 17 00:00:00 2001 From: NIC619 Date: Fri, 26 Jul 2019 16:44:33 +0800 Subject: [PATCH 04/20] Add type hints to mcache.py --- libp2p/pubsub/mcache.py | 41 +++++++++++++++++++++++++++++++---------- 1 file changed, 31 insertions(+), 10 deletions(-) diff --git a/libp2p/pubsub/mcache.py b/libp2p/pubsub/mcache.py index 071945a..42d954c 100644 --- a/libp2p/pubsub/mcache.py +++ b/libp2p/pubsub/mcache.py @@ -1,11 +1,25 @@ +from typing import ( + Dict, + List, + Optional, + Tuple, +) + +from .pb import rpc_pb2 + + class MessageCache: class CacheEntry: # pylint: disable=too-few-public-methods + + mid: Tuple[bytes, bytes] + topics: List[str] + """ A logical representation of an entry in the mcache's _history_. """ - def __init__(self, mid, topics): + def __init__(self, mid: Tuple[bytes, bytes], topics: List[str]) -> None: """ Constructor. :param mid: (seqno, from_id) of the msg @@ -14,7 +28,14 @@ class MessageCache: self.mid = mid self.topics = topics - def __init__(self, window_size, history_size): + window_size: int + history_size: int + + msgs: Dict[Tuple[bytes, bytes], rpc_pb2.Message] + + history = List[List[CacheEntry]] + + def __init__(self, window_size: int, history_size: int) -> None: """ Constructor. :param window_size: Size of the window desired. @@ -34,12 +55,12 @@ class MessageCache: for _ in range(history_size): self.history.append([]) - def put(self, msg): + def put(self, msg: rpc_pb2.Message) -> None: """ Put a message into the mcache. :param msg: The rpc message to put in. Should contain seqno and from_id """ - mid = (msg.seqno, msg.from_id) + mid: Tuple[bytes, bytes] = (msg.seqno, msg.from_id) self.msgs[mid] = msg if not self.history[0]: @@ -47,7 +68,7 @@ class MessageCache: self.history[0].append(self.CacheEntry(mid, msg.topicIDs)) - def get(self, mid): + def get(self, mid: Tuple[bytes, bytes]) -> Optional[rpc_pb2.Message]: """ Get a message from the mcache. :param mid: (seqno, from_id) of the message to get. @@ -58,13 +79,13 @@ class MessageCache: return None - def window(self, topic): + def window(self, topic: str) -> List[Tuple[bytes, bytes]]: """ Get the window for this topic. :param topic: Topic whose message ids we desire. :return: List of mids in the current window. """ - mids = [] + mids: List[Tuple[bytes, bytes]] = [] for entries_list in self.history[: self.window_size]: for entry in entries_list: @@ -74,16 +95,16 @@ class MessageCache: return mids - def shift(self): + def shift(self) -> None: """ Shift the window over by 1 position, dropping the last element of the history. """ - last_entries = self.history[len(self.history) - 1] + last_entries: List[CacheEntry] = self.history[len(self.history) - 1] for entry in last_entries: del self.msgs[entry.mid] - i = len(self.history) - 2 + i: int = len(self.history) - 2 while i >= 0: self.history[i + 1] = self.history[i] From a0aa105867cb85c5d6957ee762cac64087bbec3b Mon Sep 17 00:00:00 2001 From: NIC619 Date: Fri, 26 Jul 2019 17:30:51 +0800 Subject: [PATCH 05/20] Add type hint to pubsub notifee/interface --- libp2p/pubsub/pubsub_notifee.py | 40 ++++++++++++++++++------ libp2p/pubsub/pubsub_router_interface.py | 25 ++++++++++----- 2 files changed, 48 insertions(+), 17 deletions(-) diff --git a/libp2p/pubsub/pubsub_notifee.py b/libp2p/pubsub/pubsub_notifee.py index 4173bd8..1fa3b41 100644 --- a/libp2p/pubsub/pubsub_notifee.py +++ b/libp2p/pubsub/pubsub_notifee.py @@ -1,23 +1,45 @@ -from libp2p.network.notifee_interface import INotifee +from typing import ( + List, + Sequence, +) +from multiaddr import Multiaddr + +from libp2p.network.connection.raw_connection import ( + RawConnection, +) +from libp2p.network.notifee_interface import ( + INotifee, +) +from libp2p.network.network_interface import ( + INetwork, +) +from libp2p.network.stream.net_stream_interface import ( + INetStream, +) +from libp2p.peer.id import ( + ID, +) class PubsubNotifee(INotifee): # pylint: disable=too-many-instance-attributes, cell-var-from-loop - def __init__(self, initiator_peers_queue): + initiator_peers_queue: List[ID] + + def __init__(self, initiator_peers_queue: Sequence[ID]) -> None: """ :param initiator_peers_queue: queue to add new peers to so that pubsub can process new peers after we connect to them """ - self.initiator_peers_queue = initiator_peers_queue + self.initiator_peers_queue = List(initiator_peers_queue) - async def opened_stream(self, network, stream): + async def opened_stream(self, network: INetwork, stream: INetStream) -> None: pass - async def closed_stream(self, network, stream): + async def closed_stream(self, network: INetwork, stream: INetStream) -> None: pass - async def connected(self, network, conn): + async def connected(self, network: INetwork, conn: RawConnection) -> None: """ Add peer_id to initiator_peers_queue, so that this peer_id can be used to create a stream and we only want to have one pubsub stream with each peer. @@ -30,11 +52,11 @@ class PubsubNotifee(INotifee): if conn.initiator: await self.initiator_peers_queue.put(conn.peer_id) - async def disconnected(self, network, conn): + async def disconnected(self, network: INetwork, conn: RawConnection) -> None: pass - async def listen(self, network, multiaddr): + async def listen(self, network: INetwork, multiaddr: Multiaddr) -> None: pass - async def listen_close(self, network, multiaddr): + async def listen_close(self, network: INetwork, multiaddr: Multiaddr) -> None: pass diff --git a/libp2p/pubsub/pubsub_router_interface.py b/libp2p/pubsub/pubsub_router_interface.py index 8819e5f..e0787e2 100644 --- a/libp2p/pubsub/pubsub_router_interface.py +++ b/libp2p/pubsub/pubsub_router_interface.py @@ -1,15 +1,24 @@ from abc import ABC, abstractmethod +from typing import ( + List, +) + +from .pb import rpc_pb2 +from .pubsub import Pubsub +from libp2p.peer.id import ( + ID, +) class IPubsubRouter(ABC): @abstractmethod - def get_protocols(self): + def get_protocols(self) -> List[str]: """ :return: the list of protocols supported by the router """ @abstractmethod - def attach(self, pubsub): + def attach(self, pubsub: Pubsub) -> None: """ Attach is invoked by the PubSub constructor to attach the router to a freshly initialized PubSub instance. @@ -17,21 +26,21 @@ class IPubsubRouter(ABC): """ @abstractmethod - def add_peer(self, peer_id, protocol_id): + def add_peer(self, peer_id: ID, protocol_id: str) -> None: """ Notifies the router that a new peer has been connected :param peer_id: id of peer to add """ @abstractmethod - def remove_peer(self, peer_id): + def remove_peer(self, peer_id: ID) -> None: """ Notifies the router that a peer has been disconnected :param peer_id: id of peer to remove """ @abstractmethod - def handle_rpc(self, rpc, sender_peer_id): + def handle_rpc(self, rpc: rpc_pb2.ControlMessage, sender_peer_id: ID) -> None: """ Invoked to process control messages in the RPC envelope. It is invoked after subscriptions and payload messages have been processed @@ -42,7 +51,7 @@ class IPubsubRouter(ABC): """ @abstractmethod - async def publish(self, msg_forwarder, pubsub_msg): + async def publish(self, msg_forwarder: ID, pubsub_msg: rpc_pb2.Message): """ Invoked to forward a new message that has been validated :param msg_forwarder: peer_id of message sender @@ -50,7 +59,7 @@ class IPubsubRouter(ABC): """ @abstractmethod - def join(self, topic): + def join(self, topic: str) -> None: """ Join notifies the router that we want to receive and forward messages in a topic. It is invoked after the @@ -59,7 +68,7 @@ class IPubsubRouter(ABC): """ @abstractmethod - def leave(self, topic): + def leave(self, topic: str) -> None: """ Leave notifies the router that we are no longer interested in a topic. It is invoked after the unsubscription announcement. From b2f496d081db5176636449d585dbbdba7f71061d Mon Sep 17 00:00:00 2001 From: NIC619 Date: Sat, 27 Jul 2019 11:27:47 +0800 Subject: [PATCH 06/20] Fix type hints except pb msg in pubsub folder --- libp2p/pubsub/floodsub.py | 14 ++++--- libp2p/pubsub/gossipsub.py | 37 +++++++++-------- libp2p/pubsub/mcache.py | 52 ++++++++++++------------ libp2p/pubsub/pubsub.py | 15 +++---- libp2p/pubsub/pubsub_notifee.py | 21 +++++----- libp2p/pubsub/pubsub_router_interface.py | 13 +++--- 6 files changed, 79 insertions(+), 73 deletions(-) diff --git a/libp2p/pubsub/floodsub.py b/libp2p/pubsub/floodsub.py index 126e4c2..8b10eb4 100644 --- a/libp2p/pubsub/floodsub.py +++ b/libp2p/pubsub/floodsub.py @@ -25,7 +25,7 @@ class FloodSub(IPubsubRouter): pubsub: Pubsub def __init__(self, protocols: Sequence[str]) -> None: - self.protocols = protocols + self.protocols = list(protocols) self.pubsub = None def get_protocols(self) -> List[str]: @@ -42,19 +42,21 @@ class FloodSub(IPubsubRouter): """ self.pubsub = pubsub - def add_peer(self, peer_id: ID, protocol_id: str): + # FIXME: Should be changed to type 'peer.ID' + def add_peer(self, peer_id: str, protocol_id: str) -> None: """ Notifies the router that a new peer has been connected :param peer_id: id of peer to add """ - def remove_peer(self, peer_id: ID): + def remove_peer(self, peer_id: ID) -> None: """ Notifies the router that a peer has been disconnected :param peer_id: id of peer to remove """ - async def handle_rpc(self, rpc: rpc_pb2.ControlMessage, sender_peer_id: ID): + # FIXME: Should be changed to type 'peer.ID' + async def handle_rpc(self, rpc: rpc_pb2.ControlMessage, sender_peer_id: str) -> None: """ Invoked to process control messages in the RPC envelope. It is invoked after subscriptions and payload messages have been processed @@ -90,7 +92,7 @@ class FloodSub(IPubsubRouter): # Ref: https://github.com/libp2p/go-libp2p-pubsub/blob/master/comm.go#L107 await stream.write(rpc_msg.SerializeToString()) - async def join(self, topic: str): + async def join(self, topic: str) -> None: """ Join notifies the router that we want to receive and forward messages in a topic. It is invoked after the @@ -98,7 +100,7 @@ class FloodSub(IPubsubRouter): :param topic: topic to join """ - async def leave(self, topic: str): + async def leave(self, topic: str) -> None: """ Leave notifies the router that we are no longer interested in a topic. It is invoked after the unsubscription announcement. diff --git a/libp2p/pubsub/gossipsub.py b/libp2p/pubsub/gossipsub.py index 0c9bde9..c10dc76 100644 --- a/libp2p/pubsub/gossipsub.py +++ b/libp2p/pubsub/gossipsub.py @@ -27,7 +27,7 @@ class GossipSub(IPubsubRouter): # pylint: disable=too-many-instance-attributes # pylint: disable=too-many-public-methods - protocols: Sequence[str] + protocols: List[str] pubsub: Pubsub degree: int @@ -62,8 +62,8 @@ class GossipSub(IPubsubRouter): gossip_history: int=5, heartbeat_interval: int=120) -> None: # pylint: disable=too-many-arguments - self.protocols: List[str] = protocols - self.pubsub: Pubsub = None + self.protocols = list(protocols) + self.pubsub = None # Store target degree, upper degree bound, and lower degree bound self.degree = degree @@ -71,7 +71,7 @@ class GossipSub(IPubsubRouter): self.degree_high = degree_high # Store time to live (for topics in fanout) - self.time_to_live: int = time_to_live + self.time_to_live = time_to_live # Create topic --> list of peers mappings self.mesh = {} @@ -91,7 +91,7 @@ class GossipSub(IPubsubRouter): # Interface functions - def get_protocols(self) -> List: + def get_protocols(self) -> List[str]: """ :return: the list of protocols supported by the router """ @@ -109,7 +109,8 @@ class GossipSub(IPubsubRouter): # TODO: Start after delay asyncio.ensure_future(self.heartbeat()) - def add_peer(self, peer_id: ID, protocol_id: str): + # FIXME: Shoudl be changed to type 'peer.ID' + def add_peer(self, peer_id: str, protocol_id: str) -> None: """ Notifies the router that a new peer has been connected :param peer_id: id of peer to add @@ -133,7 +134,7 @@ class GossipSub(IPubsubRouter): self.peers_to_protocol.remove(peer_id_str) # FIXME: type of `sender_peer_id` should be changed to `ID` - async def handle_rpc(self, rpc: rpc_pb2.Message, sender_peer_id: str): + async def handle_rpc(self, rpc: rpc_pb2.Message, sender_peer_id: str) -> None: """ Invoked to process control messages in the RPC envelope. It is invoked after subscriptions and payload messages have been processed @@ -300,7 +301,7 @@ class GossipSub(IPubsubRouter): peers: List[str], msg_sender: str, origin_id: str, - serialized_packet: bytes): + serialized_packet: bytes) -> None: for peer_id_in_topic in peers: # Forward to all peers that are not the # message sender and are not the message origin @@ -358,7 +359,7 @@ class GossipSub(IPubsubRouter): if num_mesh_peers_in_topic > self.degree_high: # Select |mesh[topic]| - D peers from mesh[topic] # FIXME: Should be changed to `List[ID]` - selected_peers: List[str] = GossipSub.select_from_minus( + selected_peers = GossipSub.select_from_minus( num_mesh_peers_in_topic - self.degree, self.mesh[topic], [], @@ -378,7 +379,7 @@ class GossipSub(IPubsubRouter): if self.time_since_last_publish[topic] > self.time_to_live: # Remove topic from fanout del self.fanout[topic] - self.time_since_last_publish.remove(topic) + del self.time_since_last_publish[topic] else: num_fanout_peers_in_topic = len(self.fanout[topic]) @@ -393,7 +394,7 @@ class GossipSub(IPubsubRouter): # Add the peers to fanout[topic] self.fanout[topic].extend(selected_peers) - async def gossip_heartbeat(self): + async def gossip_heartbeat(self) -> None: # pylint: disable=too-many-nested-blocks for topic in self.mesh: msg_ids = self.mcache.window(topic) @@ -412,14 +413,14 @@ class GossipSub(IPubsubRouter): # TODO: this line is a monster, can hopefully be simplified if (topic not in self.mesh or (peer not in self.mesh[topic]))\ and (topic not in self.fanout or (peer not in self.fanout[topic])): - msg_ids: List[str] = [str(msg) for msg in msg_ids] - await self.emit_ihave(topic, msg_ids, peer) + msg_id_strs = [str(msg_id) for msg_id in msg_ids] + await self.emit_ihave(topic, msg_id_strs, peer) # TODO: Refactor and Dedup. This section is the roughly the same as the above. # Do the same for fanout, for all topics not already hit in mesh for topic in self.fanout: if topic not in self.mesh: - msg_ids: List[str] = self.mcache.window(topic) + msg_ids = self.mcache.window(topic) if msg_ids: # TODO: Make more efficient, possibly using a generator? # Get all pubsub peers in topic and only add if they are gossipsub peers also @@ -433,8 +434,8 @@ class GossipSub(IPubsubRouter): for peer in peers_to_emit_ihave_to: if peer not in self.mesh[topic] and peer not in self.fanout[topic]: - msg_ids: List[str] = [str(msg) for msg in msg_ids] - await self.emit_ihave(topic, msg_ids, peer) + msg_id_strs = [str(msg) for msg in msg_ids] + await self.emit_ihave(topic, msg_id_strs, peer) self.mcache.shift() @@ -453,7 +454,7 @@ class GossipSub(IPubsubRouter): selection_pool: List[Any] = [x for x in pool if x not in minus] else: # Don't create a new selection_pool if we are not subbing anything - selection_pool: List[Any] = pool + selection_pool = list(pool) # If num_to_select > size(selection_pool), then return selection_pool (which has the most # possible elements s.t. the number of elements is less than num_to_select) @@ -518,7 +519,7 @@ class GossipSub(IPubsubRouter): # FIXME: Update type of message ID msg_ids: List[Any] = [literal_eval(msg) for msg in iwant_msg.messageIDs] - msgs_to_forward: List = [] + msgs_to_forward: List[rpc_pb2.Message] = [] for msg_id_iwant in msg_ids: # Check if the wanted message ID is present in mcache msg: rpc_pb2.Message = self.mcache.get(msg_id_iwant) diff --git a/libp2p/pubsub/mcache.py b/libp2p/pubsub/mcache.py index 42d954c..ef140c5 100644 --- a/libp2p/pubsub/mcache.py +++ b/libp2p/pubsub/mcache.py @@ -2,38 +2,41 @@ from typing import ( Dict, List, Optional, + Sequence, Tuple, ) from .pb import rpc_pb2 +class CacheEntry: + # pylint: disable=too-few-public-methods + + mid: Tuple[bytes, bytes] + topics: List[str] + + """ + A logical representation of an entry in the mcache's _history_. + """ + def __init__(self, mid: Tuple[bytes, bytes], topics: Sequence[str]) -> None: + """ + Constructor. + :param mid: (seqno, from_id) of the msg + :param topics: list of topics this message was sent on + """ + self.mid = mid + self.topics = list(topics) + + class MessageCache: - class CacheEntry: - # pylint: disable=too-few-public-methods - - mid: Tuple[bytes, bytes] - topics: List[str] - - """ - A logical representation of an entry in the mcache's _history_. - """ - def __init__(self, mid: Tuple[bytes, bytes], topics: List[str]) -> None: - """ - Constructor. - :param mid: (seqno, from_id) of the msg - :param topics: list of topics this message was sent on - """ - self.mid = mid - self.topics = topics window_size: int history_size: int msgs: Dict[Tuple[bytes, bytes], rpc_pb2.Message] - history = List[List[CacheEntry]] + history: List[List[CacheEntry]] def __init__(self, window_size: int, history_size: int) -> None: """ @@ -50,10 +53,10 @@ class MessageCache: # max length of history_size. each item is a list of CacheEntry. # messages lost upon shift(). - self.history = [] - - for _ in range(history_size): - self.history.append([]) + self.history = [ + [] + for _ in range(history_size) + ] def put(self, msg: rpc_pb2.Message) -> None: """ @@ -63,10 +66,7 @@ class MessageCache: mid: Tuple[bytes, bytes] = (msg.seqno, msg.from_id) self.msgs[mid] = msg - if not self.history[0]: - self.history[0] = [] - - self.history[0].append(self.CacheEntry(mid, msg.topicIDs)) + self.history[0].append(CacheEntry(mid, msg.topicIDs)) def get(self, mid: Tuple[bytes, bytes]) -> Optional[rpc_pb2.Message]: """ diff --git a/libp2p/pubsub/pubsub.py b/libp2p/pubsub/pubsub.py index 7643bef..f63fa6a 100644 --- a/libp2p/pubsub/pubsub.py +++ b/libp2p/pubsub/pubsub.py @@ -22,8 +22,8 @@ from libp2p.host.host_interface import ( from libp2p.peer.id import ( ID, ) -from libp2p.network.stream.net_stream_interface import ( - INetStream, +from libp2p.network.stream.net_stream import ( + NetStream, ) @@ -109,7 +109,7 @@ class Pubsub: self.peer_topics = {} # Create peers map, which maps peer_id (as string) to stream (to a given peer) - # FIXME: Should be changed to `Dict[ID, INetStream]` + # FIXME: Should be changed to `Dict[ID, NetStream]` self.peers = {} self.counter = time.time_ns() @@ -130,7 +130,7 @@ class Pubsub: return packet.SerializeToString() - async def continuously_read_stream(self, stream: INetStream) -> None: + async def continuously_read_stream(self, stream: NetStream) -> None: """ Read from input stream in an infinite loop. Process messages from other nodes @@ -168,7 +168,7 @@ class Pubsub: # Force context switch await asyncio.sleep(0) - async def stream_handler(self, stream: INetStream) -> None: + async def stream_handler(self, stream: NetStream) -> None: """ Stream handler for pubsub. Gets invoked whenever a new stream is created on one of the supported pubsub protocols. @@ -196,12 +196,13 @@ class Pubsub: """ while True: - peer_id: ID = await self.peer_queue.get() + # FIXME: Should be changed to type 'ID' + peer_id: str = await self.peer_queue.get() # Open a stream to peer on existing connection # (we know connection exists since that's the only way # an element gets added to peer_queue) - stream: INetStream = await self.host.new_stream(peer_id, self.protocols) + stream: NetStream = await self.host.new_stream(peer_id, self.protocols) # Add Peer # Map peer to stream diff --git a/libp2p/pubsub/pubsub_notifee.py b/libp2p/pubsub/pubsub_notifee.py index 1fa3b41..2798bb1 100644 --- a/libp2p/pubsub/pubsub_notifee.py +++ b/libp2p/pubsub/pubsub_notifee.py @@ -1,12 +1,12 @@ +import asyncio from typing import ( - List, Sequence, ) from multiaddr import Multiaddr -from libp2p.network.connection.raw_connection import ( - RawConnection, +from libp2p.stream_muxer.mplex.mplex import ( + Mplex, ) from libp2p.network.notifee_interface import ( INotifee, @@ -17,21 +17,20 @@ from libp2p.network.network_interface import ( from libp2p.network.stream.net_stream_interface import ( INetStream, ) -from libp2p.peer.id import ( - ID, -) + class PubsubNotifee(INotifee): # pylint: disable=too-many-instance-attributes, cell-var-from-loop - initiator_peers_queue: List[ID] + # FIXME: Should be changed to type 'peer.ID' + initiator_peers_queue: asyncio.Queue[str] - def __init__(self, initiator_peers_queue: Sequence[ID]) -> None: + def __init__(self, initiator_peers_queue: asyncio.Queue[str]) -> None: """ :param initiator_peers_queue: queue to add new peers to so that pubsub can process new peers after we connect to them """ - self.initiator_peers_queue = List(initiator_peers_queue) + self.initiator_peers_queue = initiator_peers_queue async def opened_stream(self, network: INetwork, stream: INetStream) -> None: pass @@ -39,7 +38,7 @@ class PubsubNotifee(INotifee): async def closed_stream(self, network: INetwork, stream: INetStream) -> None: pass - async def connected(self, network: INetwork, conn: RawConnection) -> None: + async def connected(self, network: INetwork, conn: Mplex) -> None: """ Add peer_id to initiator_peers_queue, so that this peer_id can be used to create a stream and we only want to have one pubsub stream with each peer. @@ -52,7 +51,7 @@ class PubsubNotifee(INotifee): if conn.initiator: await self.initiator_peers_queue.put(conn.peer_id) - async def disconnected(self, network: INetwork, conn: RawConnection) -> None: + async def disconnected(self, network: INetwork, conn: Mplex) -> None: pass async def listen(self, network: INetwork, multiaddr: Multiaddr) -> None: diff --git a/libp2p/pubsub/pubsub_router_interface.py b/libp2p/pubsub/pubsub_router_interface.py index e0787e2..14ad2f0 100644 --- a/libp2p/pubsub/pubsub_router_interface.py +++ b/libp2p/pubsub/pubsub_router_interface.py @@ -25,8 +25,9 @@ class IPubsubRouter(ABC): :param pubsub: pubsub instance to attach to """ + # FIXME: Should be changed to type 'peer.ID' @abstractmethod - def add_peer(self, peer_id: ID, protocol_id: str) -> None: + def add_peer(self, peer_id: str, protocol_id: str) -> None: """ Notifies the router that a new peer has been connected :param peer_id: id of peer to add @@ -39,8 +40,9 @@ class IPubsubRouter(ABC): :param peer_id: id of peer to remove """ + # FIXME: Should be changed to type 'peer.ID' @abstractmethod - def handle_rpc(self, rpc: rpc_pb2.ControlMessage, sender_peer_id: ID) -> None: + async def handle_rpc(self, rpc: rpc_pb2.ControlMessage, sender_peer_id: str) -> None: """ Invoked to process control messages in the RPC envelope. It is invoked after subscriptions and payload messages have been processed @@ -50,8 +52,9 @@ class IPubsubRouter(ABC): :param rpc: rpc message """ + # FIXME: Should be changed to type 'peer.ID' @abstractmethod - async def publish(self, msg_forwarder: ID, pubsub_msg: rpc_pb2.Message): + async def publish(self, msg_forwarder: ID, pubsub_msg: rpc_pb2.Message) -> None: """ Invoked to forward a new message that has been validated :param msg_forwarder: peer_id of message sender @@ -59,7 +62,7 @@ class IPubsubRouter(ABC): """ @abstractmethod - def join(self, topic: str) -> None: + async def join(self, topic: str) -> None: """ Join notifies the router that we want to receive and forward messages in a topic. It is invoked after the @@ -68,7 +71,7 @@ class IPubsubRouter(ABC): """ @abstractmethod - def leave(self, topic: str) -> None: + async def leave(self, topic: str) -> None: """ Leave notifies the router that we are no longer interested in a topic. It is invoked after the unsubscription announcement. From b695b0e1ec30f433f5473fd087e511216466f9bc Mon Sep 17 00:00:00 2001 From: NIC619 Date: Sat, 27 Jul 2019 15:15:42 +0800 Subject: [PATCH 07/20] Add type hint to host folder --- libp2p/host/basic_host.py | 59 ++++++++++++++++++++++++++--------- libp2p/host/host_interface.py | 45 +++++++++++++++++++++----- 2 files changed, 82 insertions(+), 22 deletions(-) diff --git a/libp2p/host/basic_host.py b/libp2p/host/basic_host.py index 40292df..ac4fe26 100644 --- a/libp2p/host/basic_host.py +++ b/libp2p/host/basic_host.py @@ -1,6 +1,33 @@ +from typing import ( + Any, + Callable, + List, + Sequence, +) + import multiaddr -from .host_interface import IHost +from .host_interface import ( + IHost, +) +from libp2p.network.swarm import ( + Swarm +) +from libp2p.peer.id import ( + ID, +) +from libp2p.peer.peerinfo import ( + PeerInfo, +) +from libp2p.peer.peerstore import ( + PeerStore, +) +from libp2p.network.stream.net_stream_interface import ( + INetStream, +) +from libp2p.routing.kademlia.kademlia_peer_router import ( + KadmeliaPeerRouter, +) # Upon host creation, host takes in options, # including the list of addresses on which to listen. @@ -10,48 +37,53 @@ from .host_interface import IHost class BasicHost(IHost): + _network: Swarm + router: KadmeliaPeerRouter + peerstore: PeerStore + # default options constructor - def __init__(self, network, router=None): + def __init__(self, network: Swarm, router: KadmeliaPeerRouter=None) -> None: self._network = network self._router = router self.peerstore = self._network.peerstore - def get_id(self): + def get_id(self) -> ID: """ :return: peer_id of host """ return self._network.get_peer_id() - def get_network(self): + def get_network(self) -> Swarm: """ :return: network instance of host """ return self._network - def get_peerstore(self): + def get_peerstore(self) -> PeerStore: """ :return: peerstore of the host (same one as in its network instance) """ return self.peerstore - def get_mux(self): + # FIXME: Replace with correct return type + def get_mux(self) -> Any: """ :return: mux instance of host """ - def get_addrs(self): + def get_addrs(self) -> List[multiaddr.Multiaddr]: """ :return: all the multiaddr addresses this host is listening too """ p2p_part = multiaddr.Multiaddr('/p2p/{}'.format(self.get_id().pretty())) - addrs = [] + addrs: List[multiaddr.Multiaddr] = [] for transport in self._network.listeners.values(): for addr in transport.get_addrs(): addrs.append(addr.encapsulate(p2p_part)) return addrs - def set_stream_handler(self, protocol_id, stream_handler): + def set_stream_handler(self, protocol_id: str, stream_handler: Callable[[INetStream], None]) -> bool: """ set stream handler for host :param protocol_id: protocol id used on stream @@ -62,16 +94,15 @@ class BasicHost(IHost): # protocol_id can be a list of protocol_ids # stream will decide which protocol_id to run on - async def new_stream(self, peer_id, protocol_ids): + async def new_stream(self, peer_id: ID, protocol_ids: Sequence[str]) -> INetStream: """ :param peer_id: peer_id that host is connecting :param protocol_id: protocol id that stream runs on - :return: true if successful + :return: stream: new stream created """ - stream = await self._network.new_stream(peer_id, protocol_ids) - return stream + return await self._network.new_stream(peer_id, protocol_ids) - async def connect(self, peer_info): + async def connect(self, peer_info: PeerInfo) -> None: """ connect ensures there is a connection between this host and the peer with given peer_info.peer_id. connect will absorb the addresses in peer_info into its internal diff --git a/libp2p/host/host_interface.py b/libp2p/host/host_interface.py index 57847c8..0ee7457 100644 --- a/libp2p/host/host_interface.py +++ b/libp2p/host/host_interface.py @@ -1,34 +1,63 @@ from abc import ABC, abstractmethod +from typing import ( + Any, + Callable, + Coroutine, + List, + Sequence, +) + +import multiaddr + +from libp2p.network.swarm import ( + Swarm +) +from libp2p.peer.id import ( + ID, +) +from libp2p.peer.peerinfo import ( + PeerInfo, +) +from libp2p.peer.peerstore import ( + PeerStore, +) +from libp2p.network.stream.net_stream_interface import ( + INetStream, +) +from libp2p.routing.kademlia.kademlia_peer_router import ( + KadmeliaPeerRouter, +) class IHost(ABC): @abstractmethod - def get_id(self): + def get_id(self) -> ID: """ :return: peer_id of host """ @abstractmethod - def get_network(self): + def get_network(self) -> Swarm: """ :return: network instance of host """ + # FIXME: Replace with correct return type @abstractmethod - def get_mux(self): + def get_mux(self) -> Any: """ :return: mux instance of host """ @abstractmethod - def get_addrs(self): + def get_addrs(self) -> List[multiaddr.Multiaddr]: """ :return: all the multiaddr addresses this host is listening too """ @abstractmethod - def set_stream_handler(self, protocol_id, stream_handler): + def set_stream_handler(self, protocol_id: str, stream_handler: Callable[[INetStream], None]) -> bool: """ set stream handler for host :param protocol_id: protocol id used on stream @@ -39,15 +68,15 @@ class IHost(ABC): # protocol_id can be a list of protocol_ids # stream will decide which protocol_id to run on @abstractmethod - def new_stream(self, peer_id, protocol_ids): + def new_stream(self, peer_id: ID, protocol_ids: Sequence[str]) -> Coroutine[Any, Any, INetStream]: """ :param peer_id: peer_id that host is connecting :param protocol_ids: protocol ids that stream can run on - :return: true if successful + :return: stream: new stream created """ @abstractmethod - def connect(self, peer_info): + def connect(self, peer_info: PeerInfo) -> Coroutine[Any, Any, None]: """ connect ensures there is a connection between this host and the peer with given peer_info.peer_id. connect will absorb the addresses in peer_info into its internal From f2de986c744dafbdf9b6d5e619d82259b870a6ca Mon Sep 17 00:00:00 2001 From: NIC619 Date: Sat, 27 Jul 2019 16:27:01 +0800 Subject: [PATCH 08/20] Add type hints to peer folder --- libp2p/peer/addrbook_interface.py | 21 +++++++--- libp2p/peer/id.py | 47 +++++++++++++-------- libp2p/peer/peerdata.py | 37 +++++++++++------ libp2p/peer/peerdata_interface.py | 23 +++++++---- libp2p/peer/peerinfo.py | 23 ++++++++--- libp2p/peer/peermetadata_interface.py | 13 ++++-- libp2p/peer/peerstore.py | 59 ++++++++++++++++----------- libp2p/peer/peerstore_interface.py | 19 ++++++--- tests/libp2p/test_libp2p.py | 8 ++-- tests/peer/test_peerstore.py | 2 +- 10 files changed, 165 insertions(+), 87 deletions(-) diff --git a/libp2p/peer/addrbook_interface.py b/libp2p/peer/addrbook_interface.py index 914d937..5ac34f0 100644 --- a/libp2p/peer/addrbook_interface.py +++ b/libp2p/peer/addrbook_interface.py @@ -1,13 +1,22 @@ from abc import ABC, abstractmethod +from typing import ( + List, + Sequence, +) + + +from multiaddr import Multiaddr + +from .id import ID class IAddrBook(ABC): - def __init__(self): + def __init__(self) -> None: pass @abstractmethod - def add_addr(self, peer_id, addr, ttl): + def add_addr(self, peer_id: ID, addr: Multiaddr, ttl: int) -> None: """ Calls add_addrs(peer_id, [addr], ttl) :param peer_id: the peer to add address for @@ -16,7 +25,7 @@ class IAddrBook(ABC): """ @abstractmethod - def add_addrs(self, peer_id, addrs, ttl): + def add_addrs(self, peer_id: ID, addrs: Sequence[Multiaddr], ttl: int) -> None: """ Adds addresses for a given peer all with the same time-to-live. If one of the addresses already exists for the peer and has a longer TTL, no operation should take place. @@ -27,21 +36,21 @@ class IAddrBook(ABC): """ @abstractmethod - def addrs(self, peer_id): + def addrs(self, peer_id: ID) -> List[Multiaddr]: """ :param peer_id: peer to get addresses of :return: all known (and valid) addresses for the given peer """ @abstractmethod - def clear_addrs(self, peer_id): + def clear_addrs(self, peer_id: ID) -> None: """ Removes all previously stored addresses :param peer_id: peer to remove addresses of """ @abstractmethod - def peers_with_addrs(self): + def peers_with_addrs(self) -> List[ID]: """ :return: all of the peer IDs stored with addresses """ diff --git a/libp2p/peer/id.py b/libp2p/peer/id.py index 77e2f87..578f67f 100644 --- a/libp2p/peer/id.py +++ b/libp2p/peer/id.py @@ -1,3 +1,10 @@ +from typing import ( + Union, +) + +from Crypto.PublicKey.RSA import ( + RsaKey, +) import hashlib import base58 import multihash @@ -13,67 +20,71 @@ MAX_INLINE_KEY_LENGTH = 42 class ID: - def __init__(self, id_str): + _id_str: str + + def __init__(self, id_str: str) -> None: self._id_str = id_str def to_bytes(self) -> bytes: return self._id_str - def get_raw_id(self): + def get_raw_id(self) -> str: return self._id_str - def pretty(self): + def pretty(self) -> str: return base58.b58encode(self._id_str).decode() - def get_xor_id(self): + def get_xor_id(self) -> int: return int(digest(self.get_raw_id()).hex(), 16) - def __str__(self): + def __str__(self) -> str: pid = self.pretty() return pid __repr__ = __str__ - def __eq__(self, other): + def __eq__(self, other: object) -> bool: #pylint: disable=protected-access + if not isinstance(other, ID): + return NotImplemented return self._id_str == other._id_str - def __hash__(self): + def __hash__(self) -> int: return hash(self._id_str) -def id_b58_encode(peer_id): +def id_b58_encode(peer_id: ID) -> str: """ return a b58-encoded string """ #pylint: disable=protected-access - return base58.b58encode(peer_id._id_str).decode() + return base58.b58encode(peer_id.get_raw_id()).decode() -def id_b58_decode(peer_id_str): +def id_b58_decode(peer_id_str: str) -> ID: """ return a base58-decoded peer ID """ return ID(base58.b58decode(peer_id_str)) -def id_from_public_key(key): +def id_from_public_key(key: RsaKey) -> ID: # export into binary format key_bin = key.exportKey("DER") - algo = multihash.Func.sha2_256 + algo: int = multihash.Func.sha2_256 # TODO: seems identity is not yet supported in pymultihash # if len(b) <= MAX_INLINE_KEY_LENGTH: # algo multihash.func.identity - mh_digest = multihash.digest(key_bin, algo) + mh_digest: multihash.Multihash = multihash.digest(key_bin, algo) return ID(mh_digest.encode()) -def id_from_private_key(key): +def id_from_private_key(key: RsaKey) -> ID: return id_from_public_key(key.publickey()) -def digest(string): - if not isinstance(string, bytes): - string = str(string).encode('utf8') - return hashlib.sha1(string).digest() +def digest(data: Union[str, bytes]) -> bytes: + if not isinstance(data, bytes): + data_bytes = str(data).encode('utf8') + return hashlib.sha1(data_bytes).digest() diff --git a/libp2p/peer/peerdata.py b/libp2p/peer/peerdata.py index c505bad..696e736 100644 --- a/libp2p/peer/peerdata.py +++ b/libp2p/peer/peerdata.py @@ -1,35 +1,48 @@ +from typing import ( + Any, + Dict, + List, + Sequence, +) + +from multiaddr import Multiaddr + from .peerdata_interface import IPeerData class PeerData(IPeerData): - def __init__(self): + metadata: Dict[Any, Any] + protocols: List[str] + addrs: List[Multiaddr] + + def __init__(self) -> None: self.metadata = {} self.protocols = [] self.addrs = [] - def get_protocols(self): + def get_protocols(self) -> List[str]: return self.protocols - def add_protocols(self, protocols): - self.protocols.extend(protocols) + def add_protocols(self, protocols: Sequence[str]) -> None: + self.protocols.extend(list(protocols)) - def set_protocols(self, protocols): - self.protocols = protocols + def set_protocols(self, protocols: Sequence[str]) -> None: + self.protocols = list(protocols) - def add_addrs(self, addrs): - self.addrs.extend(addrs) + def add_addrs(self, addrs: Sequence[Multiaddr]) -> None: + self.addrs.extend(list(addrs)) - def get_addrs(self): + def get_addrs(self) -> List[Multiaddr]: return self.addrs - def clear_addrs(self): + def clear_addrs(self) -> None: self.addrs = [] - def put_metadata(self, key, val): + def put_metadata(self, key: Any, val: Any) -> None: self.metadata[key] = val - def get_metadata(self, key): + def get_metadata(self, key: Any) -> Any: if key in self.metadata: return self.metadata[key] raise PeerDataError("key not found") diff --git a/libp2p/peer/peerdata_interface.py b/libp2p/peer/peerdata_interface.py index cf4a292..e2f150e 100644 --- a/libp2p/peer/peerdata_interface.py +++ b/libp2p/peer/peerdata_interface.py @@ -1,46 +1,53 @@ from abc import ABC, abstractmethod +from typing import ( + Any, + List, + Sequence, +) + +from multiaddr import Multiaddr class IPeerData(ABC): @abstractmethod - def get_protocols(self): + def get_protocols(self) -> List[str]: """ :return: all protocols associated with given peer """ @abstractmethod - def add_protocols(self, protocols): + def add_protocols(self, protocols: Sequence[str]) -> None: """ :param protocols: protocols to add """ @abstractmethod - def set_protocols(self, protocols): + def set_protocols(self, protocols: Sequence[str]) -> None: """ :param protocols: protocols to add """ @abstractmethod - def add_addrs(self, addrs): + def add_addrs(self, addrs: Sequence[Multiaddr]) -> None: """ :param addrs: multiaddresses to add """ @abstractmethod - def get_addrs(self): + def get_addrs(self) -> List[Multiaddr]: """ :return: all multiaddresses """ @abstractmethod - def clear_addrs(self): + def clear_addrs(self) -> None: """ Clear all addresses """ @abstractmethod - def put_metadata(self, key, val): + def put_metadata(self, key: Any, val: Any) -> None: """ :param key: key in KV pair :param val: val to associate with key @@ -48,7 +55,7 @@ class IPeerData(ABC): """ @abstractmethod - def get_metadata(self, key): + def get_metadata(self, key: Any) -> Any: """ :param key: key in KV pair :return: val for key diff --git a/libp2p/peer/peerinfo.py b/libp2p/peer/peerinfo.py index e173066..6271f18 100644 --- a/libp2p/peer/peerinfo.py +++ b/libp2p/peer/peerinfo.py @@ -1,12 +1,23 @@ +from typing import ( + List, +) + import multiaddr -from .id import id_b58_decode +from .id import ( + ID, + id_b58_decode, +) from .peerdata import PeerData class PeerInfo: # pylint: disable=too-few-public-methods - def __init__(self, peer_id, peer_data=None): + + peer_id: ID + addrs: List[multiaddr.Multiaddr] + + def __init__(self, peer_id: ID, peer_data: PeerData=None) -> None: self.peer_id = peer_id self.addrs = peer_data.get_addrs() if peer_data else None @@ -30,16 +41,16 @@ def info_from_p2p_addr(addr: multiaddr.Multiaddr) -> PeerInfo: ) # make sure the /p2p value parses as a peer.ID - peer_id_str = p2p_part.value_for_protocol(multiaddr.protocols.P_P2P) - peer_id = id_b58_decode(peer_id_str) + peer_id_str: str = p2p_part.value_for_protocol(multiaddr.protocols.P_P2P) + peer_id: ID = id_b58_decode(peer_id_str) # we might have received just an / p2p part, which means there's no addr. if len(parts) > 1: addr = multiaddr.Multiaddr.join(*parts[:-1]) peer_data = PeerData() - peer_data.addrs = [addr] - peer_data.protocols = [p.code for p in addr.protocols()] + peer_data.add_addrs(addr) + peer_data.set_protocols([p.code for p in addr.protocols()]) return PeerInfo(peer_id, peer_data) diff --git a/libp2p/peer/peermetadata_interface.py b/libp2p/peer/peermetadata_interface.py index 1badabd..6fcf8de 100644 --- a/libp2p/peer/peermetadata_interface.py +++ b/libp2p/peer/peermetadata_interface.py @@ -1,13 +1,20 @@ from abc import ABC, abstractmethod +from typing import ( + Any, +) + +from .id import ( + ID, +) class IPeerMetadata(ABC): - def __init__(self): + def __init__(self) -> None: pass @abstractmethod - def get(self, peer_id, key): + def get(self, peer_id: ID, key: Any) -> Any: """ :param peer_id: peer ID to lookup key for :param key: key to look up @@ -16,7 +23,7 @@ class IPeerMetadata(ABC): """ @abstractmethod - def put(self, peer_id, key, val): + def put(self, peer_id: ID, key: Any, val: Any) -> None: """ :param peer_id: peer ID to lookup key for :param key: key to associate with peer diff --git a/libp2p/peer/peerstore.py b/libp2p/peer/peerstore.py index 5a6a77b..5d32db1 100644 --- a/libp2p/peer/peerstore.py +++ b/libp2p/peer/peerstore.py @@ -1,3 +1,14 @@ +from typing import ( + Any, + Dict, + List, + Optional, + Sequence, +) + +from multiaddr import Multiaddr + +from .id import ID from .peerstore_interface import IPeerStore from .peerdata import PeerData from .peerinfo import PeerInfo @@ -5,11 +16,13 @@ from .peerinfo import PeerInfo class PeerStore(IPeerStore): - def __init__(self): + peer_map: Dict[ID, PeerData] + + def __init__(self) -> None: IPeerStore.__init__(self) self.peer_map = {} - def __create_or_get_peer(self, peer_id): + def __create_or_get_peer(self, peer_id: ID) -> PeerData: """ Returns the peer data for peer_id or creates a new peer data (and stores it in peer_map) if peer @@ -23,65 +36,65 @@ class PeerStore(IPeerStore): self.peer_map[peer_id] = data return self.peer_map[peer_id] - def peer_info(self, peer_id): + def peer_info(self, peer_id: ID) -> Optional[PeerInfo]: if peer_id in self.peer_map: - peer = self.peer_map[peer_id] - return PeerInfo(peer_id, peer) + peer_data = self.peer_map[peer_id] + return PeerInfo(peer_id, peer_data) return None - def get_protocols(self, peer_id): + def get_protocols(self, peer_id: ID) -> List[str]: if peer_id in self.peer_map: return self.peer_map[peer_id].get_protocols() raise PeerStoreError("peer ID not found") - def add_protocols(self, peer_id, protocols): + def add_protocols(self, peer_id: ID, protocols: Sequence[str]) -> None: peer = self.__create_or_get_peer(peer_id) - peer.add_protocols(protocols) + peer.add_protocols(list(protocols)) - def set_protocols(self, peer_id, protocols): + def set_protocols(self, peer_id: ID, protocols: Sequence[str]) -> None: peer = self.__create_or_get_peer(peer_id) - peer.set_protocols(protocols) + peer.set_protocols(list(protocols)) - def peers(self): + def peer_ids(self) -> List[ID]: return list(self.peer_map.keys()) - def get(self, peer_id, key): + def get(self, peer_id: ID, key: Any) -> Any: if peer_id in self.peer_map: val = self.peer_map[peer_id].get_metadata(key) return val raise PeerStoreError("peer ID not found") - def put(self, peer_id, key, val): + def put(self, peer_id: ID, key: Any, val: Any) -> None: # <> # This can output an error, not sure what the possible errors are peer = self.__create_or_get_peer(peer_id) peer.put_metadata(key, val) - def add_addr(self, peer_id, addr, ttl): + def add_addr(self, peer_id: ID, addr: Multiaddr, ttl: int) -> None: self.add_addrs(peer_id, [addr], ttl) - def add_addrs(self, peer_id, addrs, ttl): + def add_addrs(self, peer_id: ID, addrs: Sequence[Multiaddr], ttl: int) -> None: # Ignore ttl for now peer = self.__create_or_get_peer(peer_id) - peer.add_addrs(addrs) + peer.add_addrs(list(addrs)) - def addrs(self, peer_id): + def addrs(self, peer_id: ID) -> List[Multiaddr]: if peer_id in self.peer_map: return self.peer_map[peer_id].get_addrs() raise PeerStoreError("peer ID not found") - def clear_addrs(self, peer_id): + def clear_addrs(self, peer_id: ID) -> None: # Only clear addresses if the peer is in peer map if peer_id in self.peer_map: self.peer_map[peer_id].clear_addrs() - def peers_with_addrs(self): + def peers_with_addrs(self) -> List[ID]: # Add all peers with addrs at least 1 to output - output = [] + output: List[ID] = [] - for key in self.peer_map: - if len(self.peer_map[key].get_addrs()) >= 1: - output.append(key) + for peer_id in self.peer_map: + if len(self.peer_map[peer_id].get_addrs()) >= 1: + output.append(peer_id) return output diff --git a/libp2p/peer/peerstore_interface.py b/libp2p/peer/peerstore_interface.py index b368d8f..01b5b83 100644 --- a/libp2p/peer/peerstore_interface.py +++ b/libp2p/peer/peerstore_interface.py @@ -1,24 +1,31 @@ from abc import abstractmethod +from typing import ( + List, + Sequence, +) + +from .id import ID +from .peerinfo import PeerInfo from .addrbook_interface import IAddrBook from .peermetadata_interface import IPeerMetadata class IPeerStore(IAddrBook, IPeerMetadata): - def __init__(self): + def __init__(self) -> None: IPeerMetadata.__init__(self) IAddrBook.__init__(self) @abstractmethod - def peer_info(self, peer_id): + def peer_info(self, peer_id: ID) -> PeerInfo: """ :param peer_id: peer ID to get info for :return: peer info object """ @abstractmethod - def get_protocols(self, peer_id): + def get_protocols(self, peer_id: ID) -> List[str]: """ :param peer_id: peer ID to get protocols for :return: protocols (as strings) @@ -26,7 +33,7 @@ class IPeerStore(IAddrBook, IPeerMetadata): """ @abstractmethod - def add_protocols(self, peer_id, protocols): + def add_protocols(self, peer_id: ID, protocols: Sequence[str]) -> None: """ :param peer_id: peer ID to add protocols for :param protocols: protocols to add @@ -34,7 +41,7 @@ class IPeerStore(IAddrBook, IPeerMetadata): """ @abstractmethod - def set_protocols(self, peer_id, protocols): + def set_protocols(self, peer_id: ID, protocols: Sequence[str]) -> None: """ :param peer_id: peer ID to set protocols for :param protocols: protocols to set @@ -42,7 +49,7 @@ class IPeerStore(IAddrBook, IPeerMetadata): """ @abstractmethod - def peers(self): + def peer_ids(self) -> List[ID]: """ :return: all of the peer IDs stored in peer store """ diff --git a/tests/libp2p/test_libp2p.py b/tests/libp2p/test_libp2p.py index e9bfb83..a746503 100644 --- a/tests/libp2p/test_libp2p.py +++ b/tests/libp2p/test_libp2p.py @@ -316,20 +316,20 @@ async def test_host_connect(): transport_opt_list = [["/ip4/127.0.0.1/tcp/0"], ["/ip4/127.0.0.1/tcp/0"]] (node_a, node_b) = await set_up_nodes_by_transport_opt(transport_opt_list) - assert not node_a.get_peerstore().peers() + assert not node_a.get_peerstore().peer_ids() addr = node_b.get_addrs()[0] info = info_from_p2p_addr(addr) await node_a.connect(info) - assert len(node_a.get_peerstore().peers()) == 1 + assert len(node_a.get_peerstore().peer_ids()) == 1 await node_a.connect(info) # make sure we don't do double connection - assert len(node_a.get_peerstore().peers()) == 1 + assert len(node_a.get_peerstore().peer_ids()) == 1 - assert node_b.get_id() in node_a.get_peerstore().peers() + assert node_b.get_id() in node_a.get_peerstore().peer_ids() ma_node_b = multiaddr.Multiaddr('/p2p/%s' % node_b.get_id().pretty()) for addr in node_a.get_peerstore().addrs(node_b.get_id()): assert addr.encapsulate(ma_node_b) in node_b.get_addrs() diff --git a/tests/peer/test_peerstore.py b/tests/peer/test_peerstore.py index 52029ba..ffb7b2d 100644 --- a/tests/peer/test_peerstore.py +++ b/tests/peer/test_peerstore.py @@ -55,4 +55,4 @@ def test_peers(): store.put("peer2", "key", "val") store.add_addr("peer3", "/foo", 10) - assert set(store.peers()) == set(["peer1", "peer2", "peer3"]) + assert set(store.peer_ids()) == set(["peer1", "peer2", "peer3"]) From e1592997a88cb0858ba0b4a1633b058ddb57e0ce Mon Sep 17 00:00:00 2001 From: NIC619 Date: Sat, 27 Jul 2019 17:10:03 +0800 Subject: [PATCH 09/20] Add type hints to routing folder --- libp2p/routing/interfaces.py | 18 +++++++++++++++--- .../kademlia/kademlia_content_router.py | 9 +++++++-- .../routing/kademlia/kademlia_peer_router.py | 18 ++++++++++++++---- 3 files changed, 36 insertions(+), 9 deletions(-) diff --git a/libp2p/routing/interfaces.py b/libp2p/routing/interfaces.py index 1f29d48..8f3f841 100644 --- a/libp2p/routing/interfaces.py +++ b/libp2p/routing/interfaces.py @@ -1,11 +1,23 @@ from abc import ABC, abstractmethod +from typing import ( + Any, + Coroutine, + Iterable, +) + +from libp2p.peer.id import ( + ID, +) +from libp2p.peer.peerinfo import ( + PeerInfo, +) # pylint: disable=too-few-public-methods class IContentRouting(ABC): @abstractmethod - def provide(self, cid, announce=True): + def provide(self, cid: bytes, announce: bool=True) -> None: """ Provide adds the given cid to the content routing system. If announce is True, it also announces it, otherwise it is just kept in the local @@ -13,7 +25,7 @@ class IContentRouting(ABC): """ @abstractmethod - def find_provider_iter(self, cid, count): + def find_provider_iter(self, cid: bytes, count: int) -> Iterable[PeerInfo]: """ Search for peers who are able to provide a given key returns an iterator of peer.PeerInfo @@ -23,7 +35,7 @@ class IContentRouting(ABC): class IPeerRouting(ABC): @abstractmethod - def find_peer(self, peer_id): + def find_peer(self, peer_id: ID) -> Coroutine[Any, Any, PeerInfo]: """ Find specific Peer FindPeer searches for a peer with given peer_id, returns a peer.PeerInfo diff --git a/libp2p/routing/kademlia/kademlia_content_router.py b/libp2p/routing/kademlia/kademlia_content_router.py index ac49be8..e5f31d8 100644 --- a/libp2p/routing/kademlia/kademlia_content_router.py +++ b/libp2p/routing/kademlia/kademlia_content_router.py @@ -1,9 +1,14 @@ +from typing import ( + Iterable, +) + +from libp2p.peer.peerinfo import PeerInfo from libp2p.routing.interfaces import IContentRouting class KadmeliaContentRouter(IContentRouting): - def provide(self, cid, announce=True): + def provide(self, cid: bytes, announce: bool=True) -> None: """ Provide adds the given cid to the content routing system. If announce is True, it also announces it, otherwise it is just kept in the local @@ -12,7 +17,7 @@ class KadmeliaContentRouter(IContentRouting): # the DHT finds the closest peers to `key` using the `FIND_NODE` RPC # then sends a `ADD_PROVIDER` RPC with its own `PeerInfo` to each of these peers. - def find_provider_iter(self, cid, count): + def find_provider_iter(self, cid: bytes, count: int) -> Iterable[PeerInfo]: """ Search for peers who are able to provide a given key returns an iterator of peer.PeerInfo diff --git a/libp2p/routing/kademlia/kademlia_peer_router.py b/libp2p/routing/kademlia/kademlia_peer_router.py index 45c43c8..5e426fe 100644 --- a/libp2p/routing/kademlia/kademlia_peer_router.py +++ b/libp2p/routing/kademlia/kademlia_peer_router.py @@ -1,16 +1,26 @@ import ast +from typing import ( + Union, +) +from libp2p.kademlia.kad_peerinfo import ( + KadPeerInfo, + create_kad_peerinfo, +) +from libp2p.kademlia.network import KademliaServer +from libp2p.peer.id import ID from libp2p.routing.interfaces import IPeerRouting -from libp2p.kademlia.kad_peerinfo import create_kad_peerinfo class KadmeliaPeerRouter(IPeerRouting): # pylint: disable=too-few-public-methods - def __init__(self, dht_server): + server: KademliaServer + + def __init__(self, dht_server: KademliaServer) -> None: self.server = dht_server - async def find_peer(self, peer_id): + async def find_peer(self, peer_id: ID) -> KadPeerInfo: """ Find a specific peer :param peer_id: peer to search for @@ -21,7 +31,7 @@ class KadmeliaPeerRouter(IPeerRouting): value = await self.server.get(xor_id) return decode_peerinfo(value) -def decode_peerinfo(encoded): +def decode_peerinfo(encoded: Union[bytes, str]) -> KadPeerInfo: if isinstance(encoded, bytes): encoded = encoded.decode() try: From edd164c878d7814a0367d8e5e29fbffb239159dd Mon Sep 17 00:00:00 2001 From: NIC619 Date: Sun, 28 Jul 2019 14:06:29 +0800 Subject: [PATCH 10/20] Add type hints to network folder --- libp2p/network/connection/raw_connection.py | 24 ++++-- libp2p/network/network_interface.py | 27 +++++-- libp2p/network/notifee_interface.py | 19 +++-- libp2p/network/stream/net_stream.py | 18 +++-- libp2p/network/stream/net_stream_interface.py | 14 ++-- libp2p/network/swarm.py | 75 +++++++++++++++---- 6 files changed, 133 insertions(+), 44 deletions(-) diff --git a/libp2p/network/connection/raw_connection.py b/libp2p/network/connection/raw_connection.py index 23cb551..3e5716b 100644 --- a/libp2p/network/connection/raw_connection.py +++ b/libp2p/network/connection/raw_connection.py @@ -1,8 +1,22 @@ +import asyncio + from .raw_connection_interface import IRawConnection class RawConnection(IRawConnection): - def __init__(self, ip, port, reader, writer, initiator): + conn_ip: str + conn_port: str + reader: asyncio.StreamReader + writer: asyncio.StreamWriter + _next_id: int + initiator: bool + + def __init__(self, + ip: str, + port: str, + reader: asyncio.StreamReader, + writer: asyncio.StreamWriter, + initiator: bool) -> None: # pylint: disable=too-many-arguments self.conn_ip = ip self.conn_port = port @@ -11,12 +25,12 @@ class RawConnection(IRawConnection): self._next_id = 0 if initiator else 1 self.initiator = initiator - async def write(self, data): + async def write(self, data: bytes) -> None: self.writer.write(data) self.writer.write("\n".encode()) await self.writer.drain() - async def read(self): + async def read(self) -> bytes: line = await self.reader.readline() adjusted_line = line.decode().rstrip('\n') @@ -24,10 +38,10 @@ class RawConnection(IRawConnection): # encoding and decoding return adjusted_line.encode() - def close(self): + def close(self) -> None: self.writer.close() - def next_stream_id(self): + def next_stream_id(self) -> int: """ Get next available stream id :return: next available stream id for the connection diff --git a/libp2p/network/network_interface.py b/libp2p/network/network_interface.py index 71e813c..6bffc9c 100644 --- a/libp2p/network/network_interface.py +++ b/libp2p/network/network_interface.py @@ -1,16 +1,31 @@ from abc import ABC, abstractmethod +from typing import ( + Any, + Callable, + Coroutine, + Sequence, +) + +from multiaddr import Multiaddr + +from .notifee_interface import INotifee +from .stream.net_stream import NetStream +from libp2p.peer.id import ( + ID, +) +from libp2p.stream_muxer.muxed_connection_interface import IMuxedConn class INetwork(ABC): @abstractmethod - def get_peer_id(self): + def get_peer_id(self) -> ID: """ :return: the peer id """ @abstractmethod - def dial_peer(self, peer_id): + def dial_peer(self, peer_id: ID) -> Coroutine[Any, Any, IMuxedConn]: """ dial_peer try to create a connection to peer_id @@ -20,7 +35,7 @@ class INetwork(ABC): """ @abstractmethod - def set_stream_handler(self, protocol_id, stream_handler): + def set_stream_handler(self, protocol_id: str, stream_handler: Callable[[NetStream], None]) -> bool: """ :param protocol_id: protocol id used on stream :param stream_handler: a stream handler instance @@ -28,7 +43,7 @@ class INetwork(ABC): """ @abstractmethod - def new_stream(self, peer_id, protocol_ids): + def new_stream(self, peer_id: ID, protocol_ids: Sequence[str]) -> Coroutine[Any, Any, NetStream]: """ :param peer_id: peer_id of destination :param protocol_ids: available protocol ids to use for stream @@ -36,14 +51,14 @@ class INetwork(ABC): """ @abstractmethod - def listen(self, *args): + def listen(self, *args: Multiaddr) -> Coroutine[Any, Any, bool]: """ :param *args: one or many multiaddrs to start listening on :return: True if at least one success """ @abstractmethod - def notify(self, notifee): + def notify(self, notifee: INotifee) -> bool: """ :param notifee: object implementing Notifee interface :return: true if notifee registered successfully, false otherwise diff --git a/libp2p/network/notifee_interface.py b/libp2p/network/notifee_interface.py index 17ba1fe..168de42 100644 --- a/libp2p/network/notifee_interface.py +++ b/libp2p/network/notifee_interface.py @@ -1,44 +1,51 @@ from abc import ABC, abstractmethod +from multiaddr import Multiaddr + +from libp2p.network.network_interface import INetwork +from libp2p.network.stream.net_stream_interface import INetStream +from libp2p.stream_muxer.muxed_connection_interface import IMuxedConn + + class INotifee(ABC): @abstractmethod - async def opened_stream(self, network, stream): + async def opened_stream(self, network: INetwork, stream: INetStream) -> None: """ :param network: network the stream was opened on :param stream: stream that was opened """ @abstractmethod - async def closed_stream(self, network, stream): + async def closed_stream(self, network: INetwork, stream: INetStream) -> None: """ :param network: network the stream was closed on :param stream: stream that was closed """ @abstractmethod - async def connected(self, network, conn): + async def connected(self, network: INetwork, conn: IMuxedConn) -> None: """ :param network: network the connection was opened on :param conn: connection that was opened """ @abstractmethod - async def disconnected(self, network, conn): + async def disconnected(self, network: INetwork, conn: IMuxedConn) -> None: """ :param network: network the connection was closed on :param conn: connection that was closed """ @abstractmethod - async def listen(self, network, multiaddr): + async def listen(self, network: INetwork, multiaddr: Multiaddr) -> None: """ :param network: network the listener is listening on :param multiaddr: multiaddress listener is listening on """ @abstractmethod - async def listen_close(self, network, multiaddr): + async def listen_close(self, network: INetwork, multiaddr: Multiaddr) -> None: """ :param network: network the connection was opened on :param multiaddr: multiaddress listener is no longer listening on diff --git a/libp2p/network/stream/net_stream.py b/libp2p/network/stream/net_stream.py index f708962..c1be2e9 100644 --- a/libp2p/network/stream/net_stream.py +++ b/libp2p/network/stream/net_stream.py @@ -1,41 +1,47 @@ from .net_stream_interface import INetStream +from libp2p.stream_muxer.mplex.mplex_stream import MplexStream +from libp2p.stream_muxer.mplex.mplex import Mplex class NetStream(INetStream): - def __init__(self, muxed_stream): + muxed_stream: MplexStream + mplex_conn: Mplex + protocol_id: str + + def __init__(self, muxed_stream: MplexStream) -> None: self.muxed_stream = muxed_stream self.mplex_conn = muxed_stream.mplex_conn self.protocol_id = None - def get_protocol(self): + def get_protocol(self) -> str: """ :return: protocol id that stream runs on """ return self.protocol_id - def set_protocol(self, protocol_id): + def set_protocol(self, protocol_id: str) -> None: """ :param protocol_id: protocol id that stream runs on :return: true if successful """ self.protocol_id = protocol_id - async def read(self): + async def read(self) -> bytes: """ read from stream :return: bytes of input until EOF """ return await self.muxed_stream.read() - async def write(self, data): + async def write(self, data: bytes) -> int: """ write to stream :return: number of bytes written """ return await self.muxed_stream.write(data) - async def close(self): + async def close(self) -> bool: """ close stream :return: true if successful diff --git a/libp2p/network/stream/net_stream_interface.py b/libp2p/network/stream/net_stream_interface.py index 056f924..2221741 100644 --- a/libp2p/network/stream/net_stream_interface.py +++ b/libp2p/network/stream/net_stream_interface.py @@ -1,37 +1,41 @@ from abc import ABC, abstractmethod +from typing import ( + Any, + Coroutine, +) class INetStream(ABC): @abstractmethod - def get_protocol(self): + def get_protocol(self) -> str: """ :return: protocol id that stream runs on """ @abstractmethod - def set_protocol(self, protocol_id): + def set_protocol(self, protocol_id: str) -> bool: """ :param protocol_id: protocol id that stream runs on :return: true if successful """ @abstractmethod - def read(self): + def read(self) -> Coroutine[Any, Any, bytes]: """ reads from the underlying muxed_stream :return: bytes of input """ @abstractmethod - def write(self, _bytes): + def write(self, data: bytes) -> Coroutine[Any, Any, int]: """ write to the underlying muxed_stream :return: number of bytes written """ @abstractmethod - def close(self): + def close(self) -> Coroutine[Any, Any, bool]: """ close the underlying muxed stream :return: true if successful diff --git a/libp2p/network/swarm.py b/libp2p/network/swarm.py index e3d037e..7a6aa2b 100644 --- a/libp2p/network/swarm.py +++ b/libp2p/network/swarm.py @@ -1,18 +1,57 @@ import asyncio +from typing import ( + Any, + Callable, + Coroutine, + Dict, + List, + Sequence, +) -from libp2p.protocol_muxer.multiselect_client import MultiselectClient -from libp2p.protocol_muxer.multiselect import Multiselect -from libp2p.peer.id import id_b58_decode +from multiaddr import Multiaddr from .network_interface import INetwork from .notifee_interface import INotifee -from .stream.net_stream import NetStream from .connection.raw_connection import RawConnection +from .stream.net_stream import NetStream +from libp2p.peer.id import ( + ID, + id_b58_decode, +) +from libp2p.peer.peerstore import PeerStore +from libp2p.protocol_muxer.multiselect import Multiselect +from libp2p.protocol_muxer.multiselect_client import MultiselectClient +from libp2p.routing.interfaces import IPeerRouting +from libp2p.transport.upgrader import TransportUpgrader +from libp2p.transport.transport_interface import ITransport +from libp2p.transport.listener_interface import IListener +from libp2p.stream_muxer.mplex.mplex_stream import MplexStream +from libp2p.stream_muxer.muxed_connection_interface import IMuxedConn + class Swarm(INetwork): # pylint: disable=too-many-instance-attributes,cell-var-from-loop,too-many-arguments - def __init__(self, peer_id, peerstore, upgrader, transport, router): + self_id: ID + peerstore: PeerStore + upgrader: TransportUpgrader + transport: ITransport + router: IPeerRouting + connections: Dict[ID, IMuxedConn] + listeners: Dict[str, IListener] + stream_handlers: Dict[NetStream, Callable[[NetStream], None]] + + multiselect: Multiselect + multiselect_client: MultiselectClient + + notifees: List[INotifee] + + def __init__(self, + peer_id: ID, + peerstore: PeerStore, + upgrader: TransportUpgrader, + transport: ITransport, + router: IPeerRouting): self.self_id = peer_id self.peerstore = peerstore self.upgrader = upgrader @@ -32,10 +71,10 @@ class Swarm(INetwork): # Create generic protocol handler self.generic_protocol_handler = create_generic_protocol_handler(self) - def get_peer_id(self): + def get_peer_id(self) -> ID: return self.self_id - def set_stream_handler(self, protocol_id, stream_handler): + def set_stream_handler(self, protocol_id: str, stream_handler: Callable[[NetStream], None]) -> bool: """ :param protocol_id: protocol id used on stream :param stream_handler: a stream handler instance @@ -44,7 +83,7 @@ class Swarm(INetwork): self.multiselect.add_handler(protocol_id, stream_handler) return True - async def dial_peer(self, peer_id): + async def dial_peer(self, peer_id: ID) -> IMuxedConn: """ dial_peer try to create a connection to peer_id :param peer_id: peer if we want to dial @@ -87,7 +126,7 @@ class Swarm(INetwork): return muxed_conn - async def new_stream(self, peer_id, protocol_ids): + async def new_stream(self, peer_id: ID, protocol_ids: Sequence[str]) -> NetStream: """ :param peer_id: peer_id of destination :param protocol_id: protocol id @@ -109,7 +148,7 @@ class Swarm(INetwork): muxed_stream = await muxed_conn.open_stream(protocol_ids[0], multiaddr) # Perform protocol muxing to determine protocol to use - selected_protocol = await self.multiselect_client.select_one_of(protocol_ids, muxed_stream) + selected_protocol = await self.multiselect_client.select_one_of(list(protocol_ids), muxed_stream) # Create a net stream with the selected protocol net_stream = NetStream(muxed_stream) @@ -121,7 +160,7 @@ class Swarm(INetwork): return net_stream - async def listen(self, *args): + async def listen(self, *args: Multiaddr) -> bool: """ :param *args: one or many multiaddrs to start listening on :return: true if at least one success @@ -139,7 +178,7 @@ class Swarm(INetwork): if str(multiaddr) in self.listeners: return True - async def conn_handler(reader, writer): + async def conn_handler(reader: asyncio.StreamReader, writer: asyncio.StreamWriter) -> None: # Read in first message (should be peer_id of initiator) and ack peer_id = id_b58_decode((await reader.read(1024)).decode()) @@ -182,7 +221,7 @@ class Swarm(INetwork): # No multiaddr succeeded return False - def notify(self, notifee): + def notify(self, notifee: INotifee) -> bool: """ :param notifee: object implementing Notifee interface :return: true if notifee registered successfully, false otherwise @@ -192,7 +231,7 @@ class Swarm(INetwork): return True return False - def add_router(self, router): + def add_router(self, router: IPeerRouting) -> None: self.router = router # TODO: `tear_down` @@ -204,7 +243,10 @@ class Swarm(INetwork): # TODO: `disconnect`? -def create_generic_protocol_handler(swarm): +GenericProtocolHandlerFn = Callable[[MplexStream], Coroutine[Any, Any, None]] + + +def create_generic_protocol_handler(swarm: Swarm) -> GenericProtocolHandlerFn: """ Create a generic protocol handler from the given swarm. We use swarm to extract the multiselect module so that generic_protocol_handler @@ -213,7 +255,7 @@ def create_generic_protocol_handler(swarm): """ multiselect = swarm.multiselect - async def generic_protocol_handler(muxed_stream): + async def generic_protocol_handler(muxed_stream: MplexStream) -> None: # Perform protocol muxing to determine protocol to use protocol, handler = await multiselect.negotiate(muxed_stream) @@ -229,5 +271,6 @@ def create_generic_protocol_handler(swarm): return generic_protocol_handler + class SwarmException(Exception): pass From d716e90e174e8532a956c42ce9fe8472c151ec54 Mon Sep 17 00:00:00 2001 From: NIC619 Date: Sun, 28 Jul 2019 14:30:15 +0800 Subject: [PATCH 11/20] Fix on type hints --- libp2p/host/basic_host.py | 3 ++- libp2p/host/host_interface.py | 2 +- libp2p/network/network_interface.py | 2 +- libp2p/network/stream/net_stream_interface.py | 4 ++++ libp2p/network/swarm.py | 2 +- libp2p/pubsub/floodsub.py | 3 +-- libp2p/pubsub/gossipsub.py | 3 +-- libp2p/pubsub/pubsub.py | 19 ++++++++----------- libp2p/pubsub/pubsub_notifee.py | 16 +++++++++------- libp2p/pubsub/pubsub_router_interface.py | 3 +-- .../muxed_connection_interface.py | 5 +++++ 11 files changed, 34 insertions(+), 28 deletions(-) diff --git a/libp2p/host/basic_host.py b/libp2p/host/basic_host.py index ac4fe26..5e83f5c 100644 --- a/libp2p/host/basic_host.py +++ b/libp2p/host/basic_host.py @@ -1,6 +1,7 @@ from typing import ( Any, Callable, + Coroutine, List, Sequence, ) @@ -83,7 +84,7 @@ class BasicHost(IHost): addrs.append(addr.encapsulate(p2p_part)) return addrs - def set_stream_handler(self, protocol_id: str, stream_handler: Callable[[INetStream], None]) -> bool: + def set_stream_handler(self, protocol_id: str, stream_handler: Callable[[INetStream], Coroutine[Any, Any, None]]) -> bool: """ set stream handler for host :param protocol_id: protocol id used on stream diff --git a/libp2p/host/host_interface.py b/libp2p/host/host_interface.py index 0ee7457..b14c11d 100644 --- a/libp2p/host/host_interface.py +++ b/libp2p/host/host_interface.py @@ -57,7 +57,7 @@ class IHost(ABC): """ @abstractmethod - def set_stream_handler(self, protocol_id: str, stream_handler: Callable[[INetStream], None]) -> bool: + def set_stream_handler(self, protocol_id: str, stream_handler: Callable[[INetStream], Coroutine[Any, Any, None]]) -> bool: """ set stream handler for host :param protocol_id: protocol id used on stream diff --git a/libp2p/network/network_interface.py b/libp2p/network/network_interface.py index 6bffc9c..41ab900 100644 --- a/libp2p/network/network_interface.py +++ b/libp2p/network/network_interface.py @@ -35,7 +35,7 @@ class INetwork(ABC): """ @abstractmethod - def set_stream_handler(self, protocol_id: str, stream_handler: Callable[[NetStream], None]) -> bool: + def set_stream_handler(self, protocol_id: str, stream_handler: Callable[[NetStream], Coroutine[Any, Any, None]]) -> bool: """ :param protocol_id: protocol id used on stream :param stream_handler: a stream handler instance diff --git a/libp2p/network/stream/net_stream_interface.py b/libp2p/network/stream/net_stream_interface.py index 2221741..30d7ac7 100644 --- a/libp2p/network/stream/net_stream_interface.py +++ b/libp2p/network/stream/net_stream_interface.py @@ -4,9 +4,13 @@ from typing import ( Coroutine, ) +from libp2p.stream_muxer.mplex.mplex import Mplex + class INetStream(ABC): + mplex_conn: Mplex + @abstractmethod def get_protocol(self) -> str: """ diff --git a/libp2p/network/swarm.py b/libp2p/network/swarm.py index 7a6aa2b..50fd603 100644 --- a/libp2p/network/swarm.py +++ b/libp2p/network/swarm.py @@ -74,7 +74,7 @@ class Swarm(INetwork): def get_peer_id(self) -> ID: return self.self_id - def set_stream_handler(self, protocol_id: str, stream_handler: Callable[[NetStream], None]) -> bool: + def set_stream_handler(self, protocol_id: str, stream_handler: Callable[[NetStream], Coroutine[Any, Any, None]]) -> bool: """ :param protocol_id: protocol id used on stream :param stream_handler: a stream handler instance diff --git a/libp2p/pubsub/floodsub.py b/libp2p/pubsub/floodsub.py index 8b10eb4..c556ef9 100644 --- a/libp2p/pubsub/floodsub.py +++ b/libp2p/pubsub/floodsub.py @@ -42,8 +42,7 @@ class FloodSub(IPubsubRouter): """ self.pubsub = pubsub - # FIXME: Should be changed to type 'peer.ID' - def add_peer(self, peer_id: str, protocol_id: str) -> None: + def add_peer(self, peer_id: ID, protocol_id: str) -> None: """ Notifies the router that a new peer has been connected :param peer_id: id of peer to add diff --git a/libp2p/pubsub/gossipsub.py b/libp2p/pubsub/gossipsub.py index c10dc76..91517a6 100644 --- a/libp2p/pubsub/gossipsub.py +++ b/libp2p/pubsub/gossipsub.py @@ -109,8 +109,7 @@ class GossipSub(IPubsubRouter): # TODO: Start after delay asyncio.ensure_future(self.heartbeat()) - # FIXME: Shoudl be changed to type 'peer.ID' - def add_peer(self, peer_id: str, protocol_id: str) -> None: + def add_peer(self, peer_id: ID, protocol_id: str) -> None: """ Notifies the router that a new peer has been connected :param peer_id: id of peer to add diff --git a/libp2p/pubsub/pubsub.py b/libp2p/pubsub/pubsub.py index f63fa6a..5887fb4 100644 --- a/libp2p/pubsub/pubsub.py +++ b/libp2p/pubsub/pubsub.py @@ -22,8 +22,8 @@ from libp2p.host.host_interface import ( from libp2p.peer.id import ( ID, ) -from libp2p.network.stream.net_stream import ( - NetStream, +from libp2p.network.stream.net_stream_interface import ( + INetStream, ) @@ -40,8 +40,7 @@ class Pubsub: router: IPubsubRouter - # FIXME: Should be changed to `asyncio.Queue[ID]` - peer_queue: asyncio.Queue[str] + peer_queue: asyncio.Queue[ID] protocols: List[str] @@ -78,7 +77,6 @@ class Pubsub: self.router.attach(self) # Register a notifee - # FIXME: Should be changed to `asyncio.Queue[ID]` self.peer_queue = asyncio.Queue() self.host.get_network().notify(PubsubNotifee(self.peer_queue)) @@ -109,7 +107,7 @@ class Pubsub: self.peer_topics = {} # Create peers map, which maps peer_id (as string) to stream (to a given peer) - # FIXME: Should be changed to `Dict[ID, NetStream]` + # FIXME: Should be changed to `Dict[ID, INetStream]` self.peers = {} self.counter = time.time_ns() @@ -130,7 +128,7 @@ class Pubsub: return packet.SerializeToString() - async def continuously_read_stream(self, stream: NetStream) -> None: + async def continuously_read_stream(self, stream: INetStream) -> None: """ Read from input stream in an infinite loop. Process messages from other nodes @@ -168,7 +166,7 @@ class Pubsub: # Force context switch await asyncio.sleep(0) - async def stream_handler(self, stream: NetStream) -> None: + async def stream_handler(self, stream: INetStream) -> None: """ Stream handler for pubsub. Gets invoked whenever a new stream is created on one of the supported pubsub protocols. @@ -196,13 +194,12 @@ class Pubsub: """ while True: - # FIXME: Should be changed to type 'ID' - peer_id: str = await self.peer_queue.get() + peer_id: ID = await self.peer_queue.get() # Open a stream to peer on existing connection # (we know connection exists since that's the only way # an element gets added to peer_queue) - stream: NetStream = await self.host.new_stream(peer_id, self.protocols) + stream: INetStream = await self.host.new_stream(peer_id, self.protocols) # Add Peer # Map peer to stream diff --git a/libp2p/pubsub/pubsub_notifee.py b/libp2p/pubsub/pubsub_notifee.py index 2798bb1..bc05f33 100644 --- a/libp2p/pubsub/pubsub_notifee.py +++ b/libp2p/pubsub/pubsub_notifee.py @@ -5,8 +5,11 @@ from typing import ( from multiaddr import Multiaddr -from libp2p.stream_muxer.mplex.mplex import ( - Mplex, +from libp2p.peer.id import ( + ID, +) +from libp2p.stream_muxer.muxed_connection_interface import ( + IMuxedConn, ) from libp2p.network.notifee_interface import ( INotifee, @@ -22,10 +25,9 @@ from libp2p.network.stream.net_stream_interface import ( class PubsubNotifee(INotifee): # pylint: disable=too-many-instance-attributes, cell-var-from-loop - # FIXME: Should be changed to type 'peer.ID' - initiator_peers_queue: asyncio.Queue[str] + initiator_peers_queue: asyncio.Queue[ID] - def __init__(self, initiator_peers_queue: asyncio.Queue[str]) -> None: + def __init__(self, initiator_peers_queue: asyncio.Queue[ID]) -> None: """ :param initiator_peers_queue: queue to add new peers to so that pubsub can process new peers after we connect to them @@ -38,7 +40,7 @@ class PubsubNotifee(INotifee): async def closed_stream(self, network: INetwork, stream: INetStream) -> None: pass - async def connected(self, network: INetwork, conn: Mplex) -> None: + async def connected(self, network: INetwork, conn: IMuxedConn) -> None: """ Add peer_id to initiator_peers_queue, so that this peer_id can be used to create a stream and we only want to have one pubsub stream with each peer. @@ -51,7 +53,7 @@ class PubsubNotifee(INotifee): if conn.initiator: await self.initiator_peers_queue.put(conn.peer_id) - async def disconnected(self, network: INetwork, conn: Mplex) -> None: + async def disconnected(self, network: INetwork, conn: IMuxedConn) -> None: pass async def listen(self, network: INetwork, multiaddr: Multiaddr) -> None: diff --git a/libp2p/pubsub/pubsub_router_interface.py b/libp2p/pubsub/pubsub_router_interface.py index 14ad2f0..d644c39 100644 --- a/libp2p/pubsub/pubsub_router_interface.py +++ b/libp2p/pubsub/pubsub_router_interface.py @@ -25,9 +25,8 @@ class IPubsubRouter(ABC): :param pubsub: pubsub instance to attach to """ - # FIXME: Should be changed to type 'peer.ID' @abstractmethod - def add_peer(self, peer_id: str, protocol_id: str) -> None: + def add_peer(self, peer_id: ID, protocol_id: str) -> None: """ Notifies the router that a new peer has been connected :param peer_id: id of peer to add diff --git a/libp2p/stream_muxer/muxed_connection_interface.py b/libp2p/stream_muxer/muxed_connection_interface.py index 0faf770..b7bd4e6 100644 --- a/libp2p/stream_muxer/muxed_connection_interface.py +++ b/libp2p/stream_muxer/muxed_connection_interface.py @@ -1,11 +1,16 @@ from abc import ABC, abstractmethod +from libp2p.peer.id import ID + class IMuxedConn(ABC): """ reference: https://github.com/libp2p/go-stream-muxer/blob/master/muxer.go """ + initiator: bool + peer_id: ID + @abstractmethod def __init__(self, conn, generic_protocol_handler, peer_id): """ From e7ac09cb9444000c6ab1f638b86c6a88f8ae1b75 Mon Sep 17 00:00:00 2001 From: NIC619 Date: Sun, 28 Jul 2019 14:52:02 +0800 Subject: [PATCH 12/20] Fix: Add Gossipsub attribute `peers_protocol` and do cleanup when peer removed --- libp2p/pubsub/gossipsub.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/libp2p/pubsub/gossipsub.py b/libp2p/pubsub/gossipsub.py index 91517a6..d7fe84a 100644 --- a/libp2p/pubsub/gossipsub.py +++ b/libp2p/pubsub/gossipsub.py @@ -41,6 +41,9 @@ class GossipSub(IPubsubRouter): # FIXME: Should be changed to `Dict[str, List[ID]]` fanout: Dict[str, List[str]] + # FIXME: Should be changed to `Dict[ID, str]` + peers_to_protocol: Dict[str, str] + time_since_last_publish: Dict[str, int] #FIXME: Should be changed to List[ID] @@ -119,6 +122,9 @@ class GossipSub(IPubsubRouter): # Add peer to the correct peer list peer_type = GossipSub.get_peer_type(protocol_id) peer_id_str = str(peer_id) + + self.peers_to_protocol[peer_id_str] = protocol_id + if peer_type == "gossip": self.peers_gossipsub.append(peer_id_str) elif peer_type == "flood": @@ -130,7 +136,12 @@ class GossipSub(IPubsubRouter): :param peer_id: id of peer to remove """ peer_id_str = str(peer_id) - self.peers_to_protocol.remove(peer_id_str) + del self.peers_to_protocol[peer_id_str] + + if peer_id_str in self.peers_gossipsub: + self.peers_gossipsub.remove(peer_id_str) + if peer_id_str in self.peers_gossipsub: + self.peers_floodsub.remove(peer_id_str) # FIXME: type of `sender_peer_id` should be changed to `ID` async def handle_rpc(self, rpc: rpc_pb2.Message, sender_peer_id: str) -> None: From a4a0d79f6ddae9ca2cd3c5c2ff93cc3af98da4a1 Mon Sep 17 00:00:00 2001 From: NIC619 Date: Sun, 28 Jul 2019 22:30:51 +0800 Subject: [PATCH 13/20] Improve import layout --- libp2p/host/basic_host.py | 30 +++++++----------------- libp2p/host/host_interface.py | 25 ++++++-------------- libp2p/network/network_interface.py | 7 +++--- libp2p/network/notifee_interface.py | 3 ++- libp2p/network/stream/net_stream.py | 5 ++-- libp2p/network/swarm.py | 12 ++++++---- libp2p/peer/id.py | 8 +++---- libp2p/peer/peerstore.py | 2 +- libp2p/peer/peerstore_interface.py | 2 +- libp2p/pubsub/floodsub.py | 3 --- libp2p/pubsub/gossipsub.py | 3 +-- libp2p/pubsub/pubsub.py | 17 ++++---------- libp2p/pubsub/pubsub_notifee.py | 21 +++++------------ libp2p/pubsub/pubsub_router_interface.py | 5 ++-- libp2p/routing/interfaces.py | 8 ++----- 15 files changed, 53 insertions(+), 98 deletions(-) diff --git a/libp2p/host/basic_host.py b/libp2p/host/basic_host.py index 5e83f5c..d5e5fd1 100644 --- a/libp2p/host/basic_host.py +++ b/libp2p/host/basic_host.py @@ -8,27 +8,15 @@ from typing import ( import multiaddr -from .host_interface import ( - IHost, -) -from libp2p.network.swarm import ( - Swarm -) -from libp2p.peer.id import ( - ID, -) -from libp2p.peer.peerinfo import ( - PeerInfo, -) -from libp2p.peer.peerstore import ( - PeerStore, -) -from libp2p.network.stream.net_stream_interface import ( - INetStream, -) -from libp2p.routing.kademlia.kademlia_peer_router import ( - KadmeliaPeerRouter, -) +from libp2p.network.swarm import Swarm +from libp2p.peer.id import ID +from libp2p.peer.peerinfo import PeerInfo +from libp2p.peer.peerstore import PeerStore + +from libp2p.network.stream.net_stream_interface import INetStream +from libp2p.routing.kademlia.kademlia_peer_router import KadmeliaPeerRouter + +from .host_interface import IHost # Upon host creation, host takes in options, # including the list of addresses on which to listen. diff --git a/libp2p/host/host_interface.py b/libp2p/host/host_interface.py index b14c11d..d158a9b 100644 --- a/libp2p/host/host_interface.py +++ b/libp2p/host/host_interface.py @@ -9,24 +9,13 @@ from typing import ( import multiaddr -from libp2p.network.swarm import ( - Swarm -) -from libp2p.peer.id import ( - ID, -) -from libp2p.peer.peerinfo import ( - PeerInfo, -) -from libp2p.peer.peerstore import ( - PeerStore, -) -from libp2p.network.stream.net_stream_interface import ( - INetStream, -) -from libp2p.routing.kademlia.kademlia_peer_router import ( - KadmeliaPeerRouter, -) +from libp2p.network.swarm import Swarm +from libp2p.peer.id import ID +from libp2p.peer.peerinfo import PeerInfo +from libp2p.peer.peerstore import PeerStore + +from libp2p.network.stream.net_stream_interface import INetStream +from libp2p.routing.kademlia.kademlia_peer_router import KadmeliaPeerRouter class IHost(ABC): diff --git a/libp2p/network/network_interface.py b/libp2p/network/network_interface.py index 41ab900..d3dffde 100644 --- a/libp2p/network/network_interface.py +++ b/libp2p/network/network_interface.py @@ -8,12 +8,11 @@ from typing import ( from multiaddr import Multiaddr +from libp2p.peer.id import ID +from libp2p.stream_muxer.muxed_connection_interface import IMuxedConn + from .notifee_interface import INotifee from .stream.net_stream import NetStream -from libp2p.peer.id import ( - ID, -) -from libp2p.stream_muxer.muxed_connection_interface import IMuxedConn class INetwork(ABC): diff --git a/libp2p/network/notifee_interface.py b/libp2p/network/notifee_interface.py index 168de42..00ca157 100644 --- a/libp2p/network/notifee_interface.py +++ b/libp2p/network/notifee_interface.py @@ -3,9 +3,10 @@ from abc import ABC, abstractmethod from multiaddr import Multiaddr from libp2p.network.network_interface import INetwork -from libp2p.network.stream.net_stream_interface import INetStream from libp2p.stream_muxer.muxed_connection_interface import IMuxedConn +from libp2p.network.stream.net_stream_interface import INetStream + class INotifee(ABC): diff --git a/libp2p/network/stream/net_stream.py b/libp2p/network/stream/net_stream.py index c1be2e9..237ad73 100644 --- a/libp2p/network/stream/net_stream.py +++ b/libp2p/network/stream/net_stream.py @@ -1,6 +1,7 @@ -from .net_stream_interface import INetStream -from libp2p.stream_muxer.mplex.mplex_stream import MplexStream from libp2p.stream_muxer.mplex.mplex import Mplex +from libp2p.stream_muxer.mplex.mplex_stream import MplexStream + +from .net_stream_interface import INetStream class NetStream(INetStream): diff --git a/libp2p/network/swarm.py b/libp2p/network/swarm.py index 50fd603..e457981 100644 --- a/libp2p/network/swarm.py +++ b/libp2p/network/swarm.py @@ -10,10 +10,6 @@ from typing import ( from multiaddr import Multiaddr -from .network_interface import INetwork -from .notifee_interface import INotifee -from .connection.raw_connection import RawConnection -from .stream.net_stream import NetStream from libp2p.peer.id import ( ID, id_b58_decode, @@ -25,9 +21,15 @@ from libp2p.routing.interfaces import IPeerRouting from libp2p.transport.upgrader import TransportUpgrader from libp2p.transport.transport_interface import ITransport from libp2p.transport.listener_interface import IListener -from libp2p.stream_muxer.mplex.mplex_stream import MplexStream from libp2p.stream_muxer.muxed_connection_interface import IMuxedConn +from libp2p.stream_muxer.mplex.mplex_stream import MplexStream + +from .network_interface import INetwork +from .notifee_interface import INotifee +from .connection.raw_connection import RawConnection +from .stream.net_stream import NetStream + class Swarm(INetwork): # pylint: disable=too-many-instance-attributes,cell-var-from-loop,too-many-arguments diff --git a/libp2p/peer/id.py b/libp2p/peer/id.py index 578f67f..461787f 100644 --- a/libp2p/peer/id.py +++ b/libp2p/peer/id.py @@ -1,14 +1,14 @@ +import hashlib from typing import ( Union, ) -from Crypto.PublicKey.RSA import ( - RsaKey, -) -import hashlib import base58 + import multihash +from Crypto.PublicKey.RSA import RsaKey + # MaxInlineKeyLength is the maximum length a key can be for it to be inlined in # the peer ID. # * When `len(pubKey.Bytes()) <= MaxInlineKeyLength`, the peer ID is the diff --git a/libp2p/peer/peerstore.py b/libp2p/peer/peerstore.py index 5d32db1..3a59495 100644 --- a/libp2p/peer/peerstore.py +++ b/libp2p/peer/peerstore.py @@ -9,9 +9,9 @@ from typing import ( from multiaddr import Multiaddr from .id import ID -from .peerstore_interface import IPeerStore from .peerdata import PeerData from .peerinfo import PeerInfo +from .peerstore_interface import IPeerStore class PeerStore(IPeerStore): diff --git a/libp2p/peer/peerstore_interface.py b/libp2p/peer/peerstore_interface.py index 01b5b83..db6dbde 100644 --- a/libp2p/peer/peerstore_interface.py +++ b/libp2p/peer/peerstore_interface.py @@ -5,9 +5,9 @@ from typing import ( ) +from .addrbook_interface import IAddrBook from .id import ID from .peerinfo import PeerInfo -from .addrbook_interface import IAddrBook from .peermetadata_interface import IPeerMetadata diff --git a/libp2p/pubsub/floodsub.py b/libp2p/pubsub/floodsub.py index c556ef9..f2eb79f 100644 --- a/libp2p/pubsub/floodsub.py +++ b/libp2p/pubsub/floodsub.py @@ -12,9 +12,6 @@ from libp2p.peer.id import ( from .pb import rpc_pb2 from .pubsub import Pubsub from .pubsub_router_interface import IPubsubRouter -from libp2p.network.stream.net_stream_interface import ( - INetStream, -) class FloodSub(IPubsubRouter): diff --git a/libp2p/pubsub/gossipsub.py b/libp2p/pubsub/gossipsub.py index d7fe84a..e4b105c 100644 --- a/libp2p/pubsub/gossipsub.py +++ b/libp2p/pubsub/gossipsub.py @@ -1,3 +1,4 @@ +from ast import literal_eval import asyncio import random from typing import ( @@ -9,8 +10,6 @@ from typing import ( Sequence, ) -from ast import literal_eval - from libp2p.peer.id import ( ID, id_b58_decode, diff --git a/libp2p/pubsub/pubsub.py b/libp2p/pubsub/pubsub.py index 5887fb4..f7fab10 100644 --- a/libp2p/pubsub/pubsub.py +++ b/libp2p/pubsub/pubsub.py @@ -10,21 +10,14 @@ from typing import ( from lru import LRU +from libp2p.host.host_interface import IHost +from libp2p.peer.id import ID + +from libp2p.network.stream.net_stream_interface import INetStream from .pb import rpc_pb2 from .pubsub_notifee import PubsubNotifee -from .pubsub_router_interface import ( - IPubsubRouter, -) -from libp2p.host.host_interface import ( - IHost, -) -from libp2p.peer.id import ( - ID, -) -from libp2p.network.stream.net_stream_interface import ( - INetStream, -) +from .pubsub_router_interface import IPubsubRouter def get_msg_id(msg: rpc_pb2.Message) -> Tuple[bytes, bytes]: diff --git a/libp2p/pubsub/pubsub_notifee.py b/libp2p/pubsub/pubsub_notifee.py index bc05f33..880c06d 100644 --- a/libp2p/pubsub/pubsub_notifee.py +++ b/libp2p/pubsub/pubsub_notifee.py @@ -5,21 +5,12 @@ from typing import ( from multiaddr import Multiaddr -from libp2p.peer.id import ( - ID, -) -from libp2p.stream_muxer.muxed_connection_interface import ( - IMuxedConn, -) -from libp2p.network.notifee_interface import ( - INotifee, -) -from libp2p.network.network_interface import ( - INetwork, -) -from libp2p.network.stream.net_stream_interface import ( - INetStream, -) +from libp2p.peer.id import ID +from libp2p.network.network_interface import INetwork +from libp2p.network.notifee_interface import INotifee +from libp2p.stream_muxer.muxed_connection_interface import IMuxedConn + +from libp2p.network.stream.net_stream_interface import INetStream class PubsubNotifee(INotifee): diff --git a/libp2p/pubsub/pubsub_router_interface.py b/libp2p/pubsub/pubsub_router_interface.py index d644c39..ce49b4f 100644 --- a/libp2p/pubsub/pubsub_router_interface.py +++ b/libp2p/pubsub/pubsub_router_interface.py @@ -3,11 +3,10 @@ from typing import ( List, ) +from libp2p.peer.id import ID + from .pb import rpc_pb2 from .pubsub import Pubsub -from libp2p.peer.id import ( - ID, -) class IPubsubRouter(ABC): diff --git a/libp2p/routing/interfaces.py b/libp2p/routing/interfaces.py index 8f3f841..5f7dfc2 100644 --- a/libp2p/routing/interfaces.py +++ b/libp2p/routing/interfaces.py @@ -5,12 +5,8 @@ from typing import ( Iterable, ) -from libp2p.peer.id import ( - ID, -) -from libp2p.peer.peerinfo import ( - PeerInfo, -) +from libp2p.peer.id import ID +from libp2p.peer.peerinfo import PeerInfo # pylint: disable=too-few-public-methods From 2d4e23cfe20443d0d8f559a02a6bd5d1bb1d0516 Mon Sep 17 00:00:00 2001 From: NIC619 Date: Mon, 29 Jul 2019 12:42:13 +0800 Subject: [PATCH 14/20] Fix cyclic import and lint --- libp2p/host/basic_host.py | 7 ++++-- libp2p/host/host_interface.py | 11 ++++++---- libp2p/network/network_interface.py | 21 +++++++++++++----- libp2p/network/notifee_interface.py | 22 ++++++++++++------- libp2p/network/swarm.py | 13 ++++++++--- libp2p/peer/id.py | 4 ++-- libp2p/peer/peerinfo.py | 4 ++-- libp2p/pubsub/gossipsub.py | 15 ++++++++----- libp2p/pubsub/pubsub.py | 19 ++++++++++------ libp2p/pubsub/pubsub_notifee.py | 8 ++----- libp2p/pubsub/pubsub_router_interface.py | 7 ++++-- libp2p/routing/interfaces.py | 2 +- .../kademlia/kademlia_content_router.py | 2 +- 13 files changed, 87 insertions(+), 48 deletions(-) diff --git a/libp2p/host/basic_host.py b/libp2p/host/basic_host.py index d5e5fd1..e4a54cf 100644 --- a/libp2p/host/basic_host.py +++ b/libp2p/host/basic_host.py @@ -24,6 +24,9 @@ from .host_interface import IHost # telling it to listen on the given listen addresses. +StreamHandlerFn = Callable[[INetStream], Coroutine[Any, Any, None]] + + class BasicHost(IHost): _network: Swarm @@ -31,7 +34,7 @@ class BasicHost(IHost): peerstore: PeerStore # default options constructor - def __init__(self, network: Swarm, router: KadmeliaPeerRouter=None) -> None: + def __init__(self, network: Swarm, router: KadmeliaPeerRouter = None) -> None: self._network = network self._router = router self.peerstore = self._network.peerstore @@ -72,7 +75,7 @@ class BasicHost(IHost): addrs.append(addr.encapsulate(p2p_part)) return addrs - def set_stream_handler(self, protocol_id: str, stream_handler: Callable[[INetStream], Coroutine[Any, Any, None]]) -> bool: + def set_stream_handler(self, protocol_id: str, stream_handler: StreamHandlerFn) -> bool: """ set stream handler for host :param protocol_id: protocol id used on stream diff --git a/libp2p/host/host_interface.py b/libp2p/host/host_interface.py index d158a9b..bf4fe5f 100644 --- a/libp2p/host/host_interface.py +++ b/libp2p/host/host_interface.py @@ -12,10 +12,11 @@ import multiaddr from libp2p.network.swarm import Swarm from libp2p.peer.id import ID from libp2p.peer.peerinfo import PeerInfo -from libp2p.peer.peerstore import PeerStore from libp2p.network.stream.net_stream_interface import INetStream -from libp2p.routing.kademlia.kademlia_peer_router import KadmeliaPeerRouter + + +StreamHandlerFn = Callable[[INetStream], Coroutine[Any, Any, None]] class IHost(ABC): @@ -46,7 +47,7 @@ class IHost(ABC): """ @abstractmethod - def set_stream_handler(self, protocol_id: str, stream_handler: Callable[[INetStream], Coroutine[Any, Any, None]]) -> bool: + def set_stream_handler(self, protocol_id: str, stream_handler: StreamHandlerFn) -> bool: """ set stream handler for host :param protocol_id: protocol id used on stream @@ -57,7 +58,9 @@ class IHost(ABC): # protocol_id can be a list of protocol_ids # stream will decide which protocol_id to run on @abstractmethod - def new_stream(self, peer_id: ID, protocol_ids: Sequence[str]) -> Coroutine[Any, Any, INetStream]: + def new_stream(self, + peer_id: ID, + protocol_ids: Sequence[str]) -> Coroutine[Any, Any, INetStream]: """ :param peer_id: peer_id that host is connecting :param protocol_ids: protocol ids that stream can run on diff --git a/libp2p/network/network_interface.py b/libp2p/network/network_interface.py index d3dffde..0a4a8eb 100644 --- a/libp2p/network/network_interface.py +++ b/libp2p/network/network_interface.py @@ -1,9 +1,13 @@ -from abc import ABC, abstractmethod +from abc import ( + ABC, + abstractmethod, +) from typing import ( Any, Callable, Coroutine, Sequence, + TYPE_CHECKING, ) from multiaddr import Multiaddr @@ -11,9 +15,14 @@ from multiaddr import Multiaddr from libp2p.peer.id import ID from libp2p.stream_muxer.muxed_connection_interface import IMuxedConn -from .notifee_interface import INotifee from .stream.net_stream import NetStream +if TYPE_CHECKING: + from .notifee_interface import INotifee + + +StreamHandlerFn = Callable[[NetStream], Coroutine[Any, Any, None]] + class INetwork(ABC): @@ -34,7 +43,7 @@ class INetwork(ABC): """ @abstractmethod - def set_stream_handler(self, protocol_id: str, stream_handler: Callable[[NetStream], Coroutine[Any, Any, None]]) -> bool: + def set_stream_handler(self, protocol_id: str, stream_handler: StreamHandlerFn) -> bool: """ :param protocol_id: protocol id used on stream :param stream_handler: a stream handler instance @@ -42,7 +51,9 @@ class INetwork(ABC): """ @abstractmethod - def new_stream(self, peer_id: ID, protocol_ids: Sequence[str]) -> Coroutine[Any, Any, NetStream]: + def new_stream(self, + peer_id: ID, + protocol_ids: Sequence[str]) -> Coroutine[Any, Any, NetStream]: """ :param peer_id: peer_id of destination :param protocol_ids: available protocol ids to use for stream @@ -57,7 +68,7 @@ class INetwork(ABC): """ @abstractmethod - def notify(self, notifee: INotifee) -> bool: + def notify(self, notifee: 'INotifee') -> bool: """ :param notifee: object implementing Notifee interface :return: true if notifee registered successfully, false otherwise diff --git a/libp2p/network/notifee_interface.py b/libp2p/network/notifee_interface.py index 00ca157..65ad448 100644 --- a/libp2p/network/notifee_interface.py +++ b/libp2p/network/notifee_interface.py @@ -1,52 +1,58 @@ -from abc import ABC, abstractmethod +from abc import ( + ABC, + abstractmethod, +) +from typing import TYPE_CHECKING from multiaddr import Multiaddr -from libp2p.network.network_interface import INetwork from libp2p.stream_muxer.muxed_connection_interface import IMuxedConn from libp2p.network.stream.net_stream_interface import INetStream +if TYPE_CHECKING: + from .network_interface import INetwork + class INotifee(ABC): @abstractmethod - async def opened_stream(self, network: INetwork, stream: INetStream) -> None: + async def opened_stream(self, network: 'INetwork', stream: INetStream) -> None: """ :param network: network the stream was opened on :param stream: stream that was opened """ @abstractmethod - async def closed_stream(self, network: INetwork, stream: INetStream) -> None: + async def closed_stream(self, network: 'INetwork', stream: INetStream) -> None: """ :param network: network the stream was closed on :param stream: stream that was closed """ @abstractmethod - async def connected(self, network: INetwork, conn: IMuxedConn) -> None: + async def connected(self, network: 'INetwork', conn: IMuxedConn) -> None: """ :param network: network the connection was opened on :param conn: connection that was opened """ @abstractmethod - async def disconnected(self, network: INetwork, conn: IMuxedConn) -> None: + async def disconnected(self, network: 'INetwork', conn: IMuxedConn) -> None: """ :param network: network the connection was closed on :param conn: connection that was closed """ @abstractmethod - async def listen(self, network: INetwork, multiaddr: Multiaddr) -> None: + async def listen(self, network: 'INetwork', multiaddr: Multiaddr) -> None: """ :param network: network the listener is listening on :param multiaddr: multiaddress listener is listening on """ @abstractmethod - async def listen_close(self, network: INetwork, multiaddr: Multiaddr) -> None: + async def listen_close(self, network: 'INetwork', multiaddr: Multiaddr) -> None: """ :param network: network the connection was opened on :param multiaddr: multiaddress listener is no longer listening on diff --git a/libp2p/network/swarm.py b/libp2p/network/swarm.py index e457981..68b1892 100644 --- a/libp2p/network/swarm.py +++ b/libp2p/network/swarm.py @@ -31,6 +31,9 @@ from .connection.raw_connection import RawConnection from .stream.net_stream import NetStream +StreamHandlerFn = Callable[[NetStream], Coroutine[Any, Any, None]] + + class Swarm(INetwork): # pylint: disable=too-many-instance-attributes,cell-var-from-loop,too-many-arguments @@ -76,7 +79,7 @@ class Swarm(INetwork): def get_peer_id(self) -> ID: return self.self_id - def set_stream_handler(self, protocol_id: str, stream_handler: Callable[[NetStream], Coroutine[Any, Any, None]]) -> bool: + def set_stream_handler(self, protocol_id: str, stream_handler: StreamHandlerFn) -> bool: """ :param protocol_id: protocol id used on stream :param stream_handler: a stream handler instance @@ -150,7 +153,10 @@ class Swarm(INetwork): muxed_stream = await muxed_conn.open_stream(protocol_ids[0], multiaddr) # Perform protocol muxing to determine protocol to use - selected_protocol = await self.multiselect_client.select_one_of(list(protocol_ids), muxed_stream) + selected_protocol = await self.multiselect_client.select_one_of( + list(protocol_ids), + muxed_stream, + ) # Create a net stream with the selected protocol net_stream = NetStream(muxed_stream) @@ -180,7 +186,8 @@ class Swarm(INetwork): if str(multiaddr) in self.listeners: return True - async def conn_handler(reader: asyncio.StreamReader, writer: asyncio.StreamWriter) -> None: + async def conn_handler(reader: asyncio.StreamReader, + writer: asyncio.StreamWriter) -> None: # Read in first message (should be peer_id of initiator) and ack peer_id = id_b58_decode((await reader.read(1024)).decode()) diff --git a/libp2p/peer/id.py b/libp2p/peer/id.py index 461787f..3a4c4a3 100644 --- a/libp2p/peer/id.py +++ b/libp2p/peer/id.py @@ -86,5 +86,5 @@ def id_from_private_key(key: RsaKey) -> ID: def digest(data: Union[str, bytes]) -> bytes: if not isinstance(data, bytes): - data_bytes = str(data).encode('utf8') - return hashlib.sha1(data_bytes).digest() + data = str(data).encode('utf8') + return hashlib.sha1(data).digest() diff --git a/libp2p/peer/peerinfo.py b/libp2p/peer/peerinfo.py index 6271f18..6ccd023 100644 --- a/libp2p/peer/peerinfo.py +++ b/libp2p/peer/peerinfo.py @@ -17,7 +17,7 @@ class PeerInfo: peer_id: ID addrs: List[multiaddr.Multiaddr] - def __init__(self, peer_id: ID, peer_data: PeerData=None) -> None: + def __init__(self, peer_id: ID, peer_data: PeerData = None) -> None: self.peer_id = peer_id self.addrs = peer_data.get_addrs() if peer_data else None @@ -49,7 +49,7 @@ def info_from_p2p_addr(addr: multiaddr.Multiaddr) -> PeerInfo: addr = multiaddr.Multiaddr.join(*parts[:-1]) peer_data = PeerData() - peer_data.add_addrs(addr) + peer_data.add_addrs([addr]) peer_data.set_protocols([p.code for p in addr.protocols()]) return PeerInfo(peer_id, peer_data) diff --git a/libp2p/pubsub/gossipsub.py b/libp2p/pubsub/gossipsub.py index e4b105c..c778d66 100644 --- a/libp2p/pubsub/gossipsub.py +++ b/libp2p/pubsub/gossipsub.py @@ -60,9 +60,9 @@ class GossipSub(IPubsubRouter): degree_low: int, degree_high: int, time_to_live: int, - gossip_window: int=3, - gossip_history: int=5, - heartbeat_interval: int=120) -> None: + gossip_window: int = 3, + gossip_history: int = 5, + heartbeat_interval: int = 120) -> None: # pylint: disable=too-many-arguments self.protocols = list(protocols) self.pubsub = None @@ -79,6 +79,9 @@ class GossipSub(IPubsubRouter): self.mesh = {} self.fanout = {} + # Create peer --> protocol mapping + self.peers_to_protocol = {} + # Create topic --> time since last publish map self.time_since_last_publish = {} @@ -449,7 +452,9 @@ class GossipSub(IPubsubRouter): self.mcache.shift() @staticmethod - def select_from_minus(num_to_select: int, pool: Sequence[Any], minus: Sequence[Any]) -> List[Any]: + def select_from_minus(num_to_select: int, + pool: Sequence[Any], + minus: Sequence[Any]) -> List[Any]: """ Select at most num_to_select subset of elements from the set (pool - minus) randomly. :param num_to_select: number of elements to randomly select @@ -510,7 +515,7 @@ class GossipSub(IPubsubRouter): # Add all unknown message ids (ids that appear in ihave_msg but not in seen_seqnos) to list # of messages we want to request # FIXME: Update type of message ID - msg_ids_wanted = [ + msg_ids_wanted: List[Any] = [ msg_id for msg_id in ihave_msg.messageIDs if literal_eval(msg_id) not in seen_seqnos_and_peers diff --git a/libp2p/pubsub/pubsub.py b/libp2p/pubsub/pubsub.py index f7fab10..5e86015 100644 --- a/libp2p/pubsub/pubsub.py +++ b/libp2p/pubsub/pubsub.py @@ -6,6 +6,7 @@ from typing import ( Dict, List, Tuple, + TYPE_CHECKING, ) from lru import LRU @@ -17,7 +18,9 @@ from libp2p.network.stream.net_stream_interface import INetStream from .pb import rpc_pb2 from .pubsub_notifee import PubsubNotifee -from .pubsub_router_interface import IPubsubRouter + +if TYPE_CHECKING: + from .pubsub_router_interface import IPubsubRouter def get_msg_id(msg: rpc_pb2.Message) -> Tuple[bytes, bytes]: @@ -31,17 +34,19 @@ class Pubsub: host: IHost my_id: ID - router: IPubsubRouter + router: 'IPubsubRouter' - peer_queue: asyncio.Queue[ID] + peer_queue: asyncio.Queue protocols: List[str] - incoming_msgs_from_peers: asyncio.Queue[rpc_pb2.Message] - outgoing_messages: asyncio.Queue[rpc_pb2.Message] + incoming_msgs_from_peers: asyncio.Queue + outgoing_messages: asyncio.Queue seen_messages: LRU + my_topics: Dict[str, asyncio.Queue] + # FIXME: Should be changed to `Dict[str, List[ID]]` peer_topics: Dict[str, List[str]] # FIXME: Should be changed to `Dict[ID, INetStream]` @@ -52,7 +57,7 @@ class Pubsub: def __init__(self, host: IHost, - router: IPubsubRouter, + router: 'IPubsubRouter', my_id: ID, cache_size: int = None) -> None: """ @@ -247,7 +252,7 @@ class Pubsub: # for each topic await self.my_topics[topic].put(publish_message) - async def subscribe(self, topic_id: str) -> asyncio.Queue[rpc_pb2.Message]: + async def subscribe(self, topic_id: str) -> asyncio.Queue: """ Subscribe ourself to a topic :param topic_id: topic_id to subscribe to diff --git a/libp2p/pubsub/pubsub_notifee.py b/libp2p/pubsub/pubsub_notifee.py index 880c06d..cdaec4b 100644 --- a/libp2p/pubsub/pubsub_notifee.py +++ b/libp2p/pubsub/pubsub_notifee.py @@ -1,11 +1,7 @@ import asyncio -from typing import ( - Sequence, -) from multiaddr import Multiaddr -from libp2p.peer.id import ID from libp2p.network.network_interface import INetwork from libp2p.network.notifee_interface import INotifee from libp2p.stream_muxer.muxed_connection_interface import IMuxedConn @@ -16,9 +12,9 @@ from libp2p.network.stream.net_stream_interface import INetStream class PubsubNotifee(INotifee): # pylint: disable=too-many-instance-attributes, cell-var-from-loop - initiator_peers_queue: asyncio.Queue[ID] + initiator_peers_queue: asyncio.Queue - def __init__(self, initiator_peers_queue: asyncio.Queue[ID]) -> None: + def __init__(self, initiator_peers_queue: asyncio.Queue) -> None: """ :param initiator_peers_queue: queue to add new peers to so that pubsub can process new peers after we connect to them diff --git a/libp2p/pubsub/pubsub_router_interface.py b/libp2p/pubsub/pubsub_router_interface.py index ce49b4f..0cc6c12 100644 --- a/libp2p/pubsub/pubsub_router_interface.py +++ b/libp2p/pubsub/pubsub_router_interface.py @@ -1,12 +1,15 @@ from abc import ABC, abstractmethod from typing import ( List, + TYPE_CHECKING, ) from libp2p.peer.id import ID from .pb import rpc_pb2 -from .pubsub import Pubsub + +if TYPE_CHECKING: + from .pubsub import Pubsub class IPubsubRouter(ABC): @@ -17,7 +20,7 @@ class IPubsubRouter(ABC): """ @abstractmethod - def attach(self, pubsub: Pubsub) -> None: + def attach(self, pubsub: 'Pubsub') -> None: """ Attach is invoked by the PubSub constructor to attach the router to a freshly initialized PubSub instance. diff --git a/libp2p/routing/interfaces.py b/libp2p/routing/interfaces.py index 5f7dfc2..de2ec7a 100644 --- a/libp2p/routing/interfaces.py +++ b/libp2p/routing/interfaces.py @@ -13,7 +13,7 @@ from libp2p.peer.peerinfo import PeerInfo class IContentRouting(ABC): @abstractmethod - def provide(self, cid: bytes, announce: bool=True) -> None: + def provide(self, cid: bytes, announce: bool = True) -> None: """ Provide adds the given cid to the content routing system. If announce is True, it also announces it, otherwise it is just kept in the local diff --git a/libp2p/routing/kademlia/kademlia_content_router.py b/libp2p/routing/kademlia/kademlia_content_router.py index e5f31d8..468ca3a 100644 --- a/libp2p/routing/kademlia/kademlia_content_router.py +++ b/libp2p/routing/kademlia/kademlia_content_router.py @@ -8,7 +8,7 @@ from libp2p.routing.interfaces import IContentRouting class KadmeliaContentRouter(IContentRouting): - def provide(self, cid: bytes, announce: bool=True) -> None: + def provide(self, cid: bytes, announce: bool = True) -> None: """ Provide adds the given cid to the content routing system. If announce is True, it also announces it, otherwise it is just kept in the local From e53727d301b0b2a683956e97c310facf0a063752 Mon Sep 17 00:00:00 2001 From: NIC619 Date: Tue, 30 Jul 2019 15:31:02 +0800 Subject: [PATCH 15/20] Apply PR feedback: fix type hints --- libp2p/host/basic_host.py | 18 ++++++++-------- libp2p/host/host_interface.py | 14 ++++++------- libp2p/network/network_interface.py | 21 ++++++++++++------- libp2p/network/stream/net_stream.py | 10 ++++----- libp2p/network/stream/net_stream_interface.py | 11 +++++----- libp2p/network/swarm.py | 11 +++++----- libp2p/peer/peerdata.py | 4 ++-- libp2p/peer/peerdata_interface.py | 6 ++++-- libp2p/peer/peermetadata_interface.py | 4 ++-- libp2p/peer/peerstore.py | 4 ++-- libp2p/pubsub/pubsub.py | 3 +-- libp2p/routing/interfaces.py | 3 +-- .../routing/kademlia/kademlia_peer_router.py | 1 + libp2p/stream_muxer/mplex/mplex_stream.py | 3 ++- libp2p/stream_muxer/muxed_stream_interface.py | 4 ++++ 15 files changed, 65 insertions(+), 52 deletions(-) diff --git a/libp2p/host/basic_host.py b/libp2p/host/basic_host.py index e4a54cf..99ee832 100644 --- a/libp2p/host/basic_host.py +++ b/libp2p/host/basic_host.py @@ -1,17 +1,17 @@ from typing import ( Any, + Awaitable, Callable, - Coroutine, List, Sequence, ) import multiaddr -from libp2p.network.swarm import Swarm +from libp2p.network.network_interface import INetwork from libp2p.peer.id import ID from libp2p.peer.peerinfo import PeerInfo -from libp2p.peer.peerstore import PeerStore +from libp2p.peer.peerstore_interface import IPeerStore from libp2p.network.stream.net_stream_interface import INetStream from libp2p.routing.kademlia.kademlia_peer_router import KadmeliaPeerRouter @@ -24,17 +24,17 @@ from .host_interface import IHost # telling it to listen on the given listen addresses. -StreamHandlerFn = Callable[[INetStream], Coroutine[Any, Any, None]] +StreamHandlerFn = Callable[[INetStream], Awaitable[None]] class BasicHost(IHost): - _network: Swarm + _network: INetwork router: KadmeliaPeerRouter - peerstore: PeerStore + peerstore: IPeerStore # default options constructor - def __init__(self, network: Swarm, router: KadmeliaPeerRouter = None) -> None: + def __init__(self, network: INetwork, router: KadmeliaPeerRouter = None) -> None: self._network = network self._router = router self.peerstore = self._network.peerstore @@ -45,13 +45,13 @@ class BasicHost(IHost): """ return self._network.get_peer_id() - def get_network(self) -> Swarm: + def get_network(self) -> INetwork: """ :return: network instance of host """ return self._network - def get_peerstore(self) -> PeerStore: + def get_peerstore(self) -> IPeerStore: """ :return: peerstore of the host (same one as in its network instance) """ diff --git a/libp2p/host/host_interface.py b/libp2p/host/host_interface.py index bf4fe5f..1607d0f 100644 --- a/libp2p/host/host_interface.py +++ b/libp2p/host/host_interface.py @@ -1,22 +1,22 @@ from abc import ABC, abstractmethod from typing import ( Any, + Awaitable, Callable, - Coroutine, List, Sequence, ) import multiaddr -from libp2p.network.swarm import Swarm +from libp2p.network.network_interface import INetwork from libp2p.peer.id import ID from libp2p.peer.peerinfo import PeerInfo from libp2p.network.stream.net_stream_interface import INetStream -StreamHandlerFn = Callable[[INetStream], Coroutine[Any, Any, None]] +StreamHandlerFn = Callable[[INetStream], Awaitable[None]] class IHost(ABC): @@ -28,7 +28,7 @@ class IHost(ABC): """ @abstractmethod - def get_network(self) -> Swarm: + def get_network(self) -> INetwork: """ :return: network instance of host """ @@ -58,9 +58,9 @@ class IHost(ABC): # protocol_id can be a list of protocol_ids # stream will decide which protocol_id to run on @abstractmethod - def new_stream(self, + async def new_stream(self, peer_id: ID, - protocol_ids: Sequence[str]) -> Coroutine[Any, Any, INetStream]: + protocol_ids: Sequence[str]) -> INetStream: """ :param peer_id: peer_id that host is connecting :param protocol_ids: protocol ids that stream can run on @@ -68,7 +68,7 @@ class IHost(ABC): """ @abstractmethod - def connect(self, peer_info: PeerInfo) -> Coroutine[Any, Any, None]: + async def connect(self, peer_info: PeerInfo) -> None: """ connect ensures there is a connection between this host and the peer with given peer_info.peer_id. connect will absorb the addresses in peer_info into its internal diff --git a/libp2p/network/network_interface.py b/libp2p/network/network_interface.py index 0a4a8eb..fc1694a 100644 --- a/libp2p/network/network_interface.py +++ b/libp2p/network/network_interface.py @@ -4,8 +4,9 @@ from abc import ( ) from typing import ( Any, + Awaitable, Callable, - Coroutine, + Dict, Sequence, TYPE_CHECKING, ) @@ -13,19 +14,25 @@ from typing import ( from multiaddr import Multiaddr from libp2p.peer.id import ID +from libp2p.peer.peerstore import PeerStore from libp2p.stream_muxer.muxed_connection_interface import IMuxedConn +from libp2p.transport.listener_interface import IListener -from .stream.net_stream import NetStream +from .stream.net_stream_interface import INetStream if TYPE_CHECKING: from .notifee_interface import INotifee -StreamHandlerFn = Callable[[NetStream], Coroutine[Any, Any, None]] +StreamHandlerFn = Callable[[INetStream], Awaitable[None]] class INetwork(ABC): + peerstore: PeerStore + connections: Dict[ID, IMuxedConn] + listeners: Dict[str, IListener] + @abstractmethod def get_peer_id(self) -> ID: """ @@ -33,7 +40,7 @@ class INetwork(ABC): """ @abstractmethod - def dial_peer(self, peer_id: ID) -> Coroutine[Any, Any, IMuxedConn]: + async def dial_peer(self, peer_id: ID) -> IMuxedConn: """ dial_peer try to create a connection to peer_id @@ -51,9 +58,9 @@ class INetwork(ABC): """ @abstractmethod - def new_stream(self, + async def new_stream(self, peer_id: ID, - protocol_ids: Sequence[str]) -> Coroutine[Any, Any, NetStream]: + protocol_ids: Sequence[str]) -> INetStream: """ :param peer_id: peer_id of destination :param protocol_ids: available protocol ids to use for stream @@ -61,7 +68,7 @@ class INetwork(ABC): """ @abstractmethod - def listen(self, *args: Multiaddr) -> Coroutine[Any, Any, bool]: + async def listen(self, *args: Multiaddr) -> bool: """ :param *args: one or many multiaddrs to start listening on :return: True if at least one success diff --git a/libp2p/network/stream/net_stream.py b/libp2p/network/stream/net_stream.py index 237ad73..8ebe791 100644 --- a/libp2p/network/stream/net_stream.py +++ b/libp2p/network/stream/net_stream.py @@ -1,16 +1,16 @@ -from libp2p.stream_muxer.mplex.mplex import Mplex -from libp2p.stream_muxer.mplex.mplex_stream import MplexStream +from libp2p.stream_muxer.muxed_connection_interface import IMuxedConn +from libp2p.stream_muxer.muxed_stream_interface import IMuxedStream from .net_stream_interface import INetStream class NetStream(INetStream): - muxed_stream: MplexStream - mplex_conn: Mplex + muxed_stream: IMuxedStream + mplex_conn: IMuxedConn protocol_id: str - def __init__(self, muxed_stream: MplexStream) -> None: + def __init__(self, muxed_stream: IMuxedStream) -> None: self.muxed_stream = muxed_stream self.mplex_conn = muxed_stream.mplex_conn self.protocol_id = None diff --git a/libp2p/network/stream/net_stream_interface.py b/libp2p/network/stream/net_stream_interface.py index 30d7ac7..31c0080 100644 --- a/libp2p/network/stream/net_stream_interface.py +++ b/libp2p/network/stream/net_stream_interface.py @@ -1,15 +1,14 @@ from abc import ABC, abstractmethod from typing import ( Any, - Coroutine, ) -from libp2p.stream_muxer.mplex.mplex import Mplex +from libp2p.stream_muxer.muxed_connection_interface import IMuxedConn class INetStream(ABC): - mplex_conn: Mplex + mplex_conn: IMuxedConn @abstractmethod def get_protocol(self) -> str: @@ -25,21 +24,21 @@ class INetStream(ABC): """ @abstractmethod - def read(self) -> Coroutine[Any, Any, bytes]: + async def read(self) -> bytes: """ reads from the underlying muxed_stream :return: bytes of input """ @abstractmethod - def write(self, data: bytes) -> Coroutine[Any, Any, int]: + async def write(self, data: bytes) -> int: """ write to the underlying muxed_stream :return: number of bytes written """ @abstractmethod - def close(self) -> Coroutine[Any, Any, bool]: + async def close(self) -> bool: """ close the underlying muxed stream :return: true if successful diff --git a/libp2p/network/swarm.py b/libp2p/network/swarm.py index 68b1892..0038360 100644 --- a/libp2p/network/swarm.py +++ b/libp2p/network/swarm.py @@ -1,8 +1,8 @@ import asyncio from typing import ( Any, + Awaitable, Callable, - Coroutine, Dict, List, Sequence, @@ -18,10 +18,10 @@ from libp2p.peer.peerstore import PeerStore from libp2p.protocol_muxer.multiselect import Multiselect from libp2p.protocol_muxer.multiselect_client import MultiselectClient from libp2p.routing.interfaces import IPeerRouting +from libp2p.stream_muxer.muxed_connection_interface import IMuxedConn from libp2p.transport.upgrader import TransportUpgrader from libp2p.transport.transport_interface import ITransport from libp2p.transport.listener_interface import IListener -from libp2p.stream_muxer.muxed_connection_interface import IMuxedConn from libp2p.stream_muxer.mplex.mplex_stream import MplexStream @@ -29,9 +29,10 @@ from .network_interface import INetwork from .notifee_interface import INotifee from .connection.raw_connection import RawConnection from .stream.net_stream import NetStream +from .stream.net_stream_interface import INetStream -StreamHandlerFn = Callable[[NetStream], Coroutine[Any, Any, None]] +StreamHandlerFn = Callable[[INetStream], Awaitable[None]] class Swarm(INetwork): @@ -44,7 +45,7 @@ class Swarm(INetwork): router: IPeerRouting connections: Dict[ID, IMuxedConn] listeners: Dict[str, IListener] - stream_handlers: Dict[NetStream, Callable[[NetStream], None]] + stream_handlers: Dict[INetStream, Callable[[INetStream], None]] multiselect: Multiselect multiselect_client: MultiselectClient @@ -252,7 +253,7 @@ class Swarm(INetwork): # TODO: `disconnect`? -GenericProtocolHandlerFn = Callable[[MplexStream], Coroutine[Any, Any, None]] +GenericProtocolHandlerFn = Callable[[MplexStream], Awaitable[None]] def create_generic_protocol_handler(swarm: Swarm) -> GenericProtocolHandlerFn: diff --git a/libp2p/peer/peerdata.py b/libp2p/peer/peerdata.py index 696e736..bb4962b 100644 --- a/libp2p/peer/peerdata.py +++ b/libp2p/peer/peerdata.py @@ -39,10 +39,10 @@ class PeerData(IPeerData): def clear_addrs(self) -> None: self.addrs = [] - def put_metadata(self, key: Any, val: Any) -> None: + def put_metadata(self, key: str, val: Any) -> None: self.metadata[key] = val - def get_metadata(self, key: Any) -> Any: + def get_metadata(self, key: str) -> Any: if key in self.metadata: return self.metadata[key] raise PeerDataError("key not found") diff --git a/libp2p/peer/peerdata_interface.py b/libp2p/peer/peerdata_interface.py index e2f150e..cefd56d 100644 --- a/libp2p/peer/peerdata_interface.py +++ b/libp2p/peer/peerdata_interface.py @@ -7,6 +7,8 @@ from typing import ( from multiaddr import Multiaddr +from .peermetadata_interface import IPeerMetadata + class IPeerData(ABC): @@ -47,7 +49,7 @@ class IPeerData(ABC): """ @abstractmethod - def put_metadata(self, key: Any, val: Any) -> None: + def put_metadata(self, key: str, val: Any) -> None: """ :param key: key in KV pair :param val: val to associate with key @@ -55,7 +57,7 @@ class IPeerData(ABC): """ @abstractmethod - def get_metadata(self, key: Any) -> Any: + def get_metadata(self, key: str) -> IPeerMetadata: """ :param key: key in KV pair :return: val for key diff --git a/libp2p/peer/peermetadata_interface.py b/libp2p/peer/peermetadata_interface.py index 6fcf8de..3d60259 100644 --- a/libp2p/peer/peermetadata_interface.py +++ b/libp2p/peer/peermetadata_interface.py @@ -14,7 +14,7 @@ class IPeerMetadata(ABC): pass @abstractmethod - def get(self, peer_id: ID, key: Any) -> Any: + def get(self, peer_id: ID, key: str) -> Any: """ :param peer_id: peer ID to lookup key for :param key: key to look up @@ -23,7 +23,7 @@ class IPeerMetadata(ABC): """ @abstractmethod - def put(self, peer_id: ID, key: Any, val: Any) -> None: + def put(self, peer_id: ID, key: str, val: Any) -> None: """ :param peer_id: peer ID to lookup key for :param key: key to associate with peer diff --git a/libp2p/peer/peerstore.py b/libp2p/peer/peerstore.py index 3a59495..2cd1574 100644 --- a/libp2p/peer/peerstore.py +++ b/libp2p/peer/peerstore.py @@ -58,13 +58,13 @@ class PeerStore(IPeerStore): def peer_ids(self) -> List[ID]: return list(self.peer_map.keys()) - def get(self, peer_id: ID, key: Any) -> Any: + def get(self, peer_id: ID, key: str) -> Any: if peer_id in self.peer_map: val = self.peer_map[peer_id].get_metadata(key) return val raise PeerStoreError("peer ID not found") - def put(self, peer_id: ID, key: Any, val: Any) -> None: + def put(self, peer_id: ID, key: str, val: Any) -> None: # <> # This can output an error, not sure what the possible errors are peer = self.__create_or_get_peer(peer_id) diff --git a/libp2p/pubsub/pubsub.py b/libp2p/pubsub/pubsub.py index 5e86015..ddfa59c 100644 --- a/libp2p/pubsub/pubsub.py +++ b/libp2p/pubsub/pubsub.py @@ -306,8 +306,7 @@ class Pubsub: # Tell router we are leaving this topic await self.router.leave(topic_id) - # FIXME: `raw_msg` can be further type hinted with mypy_protobuf - async def message_all_peers(self, raw_msg: Any) -> None: + async def message_all_peers(self, raw_msg: bytes) -> None: """ Broadcast a message to peers :param raw_msg: raw contents of the message to broadcast diff --git a/libp2p/routing/interfaces.py b/libp2p/routing/interfaces.py index de2ec7a..e656215 100644 --- a/libp2p/routing/interfaces.py +++ b/libp2p/routing/interfaces.py @@ -1,7 +1,6 @@ from abc import ABC, abstractmethod from typing import ( Any, - Coroutine, Iterable, ) @@ -31,7 +30,7 @@ class IContentRouting(ABC): class IPeerRouting(ABC): @abstractmethod - def find_peer(self, peer_id: ID) -> Coroutine[Any, Any, PeerInfo]: + async def find_peer(self, peer_id: ID) -> PeerInfo: """ Find specific Peer FindPeer searches for a peer with given peer_id, returns a peer.PeerInfo diff --git a/libp2p/routing/kademlia/kademlia_peer_router.py b/libp2p/routing/kademlia/kademlia_peer_router.py index 5e426fe..e50ec84 100644 --- a/libp2p/routing/kademlia/kademlia_peer_router.py +++ b/libp2p/routing/kademlia/kademlia_peer_router.py @@ -9,6 +9,7 @@ from libp2p.kademlia.kad_peerinfo import ( ) from libp2p.kademlia.network import KademliaServer from libp2p.peer.id import ID +from libp2p.peer.peerinfo import PeerInfo from libp2p.routing.interfaces import IPeerRouting diff --git a/libp2p/stream_muxer/mplex/mplex_stream.py b/libp2p/stream_muxer/mplex/mplex_stream.py index 3fd6961..08b30d5 100644 --- a/libp2p/stream_muxer/mplex/mplex_stream.py +++ b/libp2p/stream_muxer/mplex/mplex_stream.py @@ -1,7 +1,8 @@ import asyncio +from libp2p.stream_muxer.muxed_stream_interface import IMuxedStream + from .utils import get_flag -from ..muxed_stream_interface import IMuxedStream class MplexStream(IMuxedStream): diff --git a/libp2p/stream_muxer/muxed_stream_interface.py b/libp2p/stream_muxer/muxed_stream_interface.py index b15dba1..b9034ce 100644 --- a/libp2p/stream_muxer/muxed_stream_interface.py +++ b/libp2p/stream_muxer/muxed_stream_interface.py @@ -1,8 +1,12 @@ from abc import ABC, abstractmethod +from libp2p.stream_muxer.mplex.mplex import Mplex + class IMuxedStream(ABC): + mplex_conn: Mplex + @abstractmethod def read(self): """ From c4105688d1744e19f76301bba0c44548ca0d49df Mon Sep 17 00:00:00 2001 From: NIC619 Date: Tue, 30 Jul 2019 17:31:08 +0800 Subject: [PATCH 16/20] Fix after rebase --- libp2p/host/host_interface.py | 4 +-- libp2p/network/network_interface.py | 5 ++-- libp2p/network/stream/net_stream_interface.py | 3 -- libp2p/network/swarm.py | 1 - libp2p/peer/id.py | 3 +- libp2p/pubsub/floodsub.py | 3 +- libp2p/pubsub/gossipsub.py | 23 ++++++++------- libp2p/pubsub/pubsub.py | 29 +++++++++---------- libp2p/pubsub/pubsub_notifee.py | 7 +++-- libp2p/pubsub/pubsub_router_interface.py | 3 +- libp2p/routing/interfaces.py | 8 ++--- .../routing/kademlia/kademlia_peer_router.py | 1 - libp2p/stream_muxer/muxed_stream_interface.py | 13 +++++++-- 13 files changed, 52 insertions(+), 51 deletions(-) diff --git a/libp2p/host/host_interface.py b/libp2p/host/host_interface.py index 1607d0f..5085808 100644 --- a/libp2p/host/host_interface.py +++ b/libp2p/host/host_interface.py @@ -59,8 +59,8 @@ class IHost(ABC): # stream will decide which protocol_id to run on @abstractmethod async def new_stream(self, - peer_id: ID, - protocol_ids: Sequence[str]) -> INetStream: + peer_id: ID, + protocol_ids: Sequence[str]) -> INetStream: """ :param peer_id: peer_id that host is connecting :param protocol_ids: protocol ids that stream can run on diff --git a/libp2p/network/network_interface.py b/libp2p/network/network_interface.py index fc1694a..f34a752 100644 --- a/libp2p/network/network_interface.py +++ b/libp2p/network/network_interface.py @@ -3,7 +3,6 @@ from abc import ( abstractmethod, ) from typing import ( - Any, Awaitable, Callable, Dict, @@ -59,8 +58,8 @@ class INetwork(ABC): @abstractmethod async def new_stream(self, - peer_id: ID, - protocol_ids: Sequence[str]) -> INetStream: + peer_id: ID, + protocol_ids: Sequence[str]) -> INetStream: """ :param peer_id: peer_id of destination :param protocol_ids: available protocol ids to use for stream diff --git a/libp2p/network/stream/net_stream_interface.py b/libp2p/network/stream/net_stream_interface.py index 31c0080..ca3858f 100644 --- a/libp2p/network/stream/net_stream_interface.py +++ b/libp2p/network/stream/net_stream_interface.py @@ -1,7 +1,4 @@ from abc import ABC, abstractmethod -from typing import ( - Any, -) from libp2p.stream_muxer.muxed_connection_interface import IMuxedConn diff --git a/libp2p/network/swarm.py b/libp2p/network/swarm.py index 0038360..bedba40 100644 --- a/libp2p/network/swarm.py +++ b/libp2p/network/swarm.py @@ -1,6 +1,5 @@ import asyncio from typing import ( - Any, Awaitable, Callable, Dict, diff --git a/libp2p/peer/id.py b/libp2p/peer/id.py index 3a4c4a3..4e82ca2 100644 --- a/libp2p/peer/id.py +++ b/libp2p/peer/id.py @@ -25,7 +25,8 @@ class ID: def __init__(self, id_str: str) -> None: self._id_str = id_str - def to_bytes(self) -> bytes: + # FIXME: Should return type `bytes` + def to_bytes(self) -> str: return self._id_str def get_raw_id(self) -> str: diff --git a/libp2p/pubsub/floodsub.py b/libp2p/pubsub/floodsub.py index f2eb79f..e51c831 100644 --- a/libp2p/pubsub/floodsub.py +++ b/libp2p/pubsub/floodsub.py @@ -51,8 +51,7 @@ class FloodSub(IPubsubRouter): :param peer_id: id of peer to remove """ - # FIXME: Should be changed to type 'peer.ID' - async def handle_rpc(self, rpc: rpc_pb2.ControlMessage, sender_peer_id: str) -> None: + async def handle_rpc(self, rpc: rpc_pb2.ControlMessage, sender_peer_id: ID) -> None: """ Invoked to process control messages in the RPC envelope. It is invoked after subscriptions and payload messages have been processed diff --git a/libp2p/pubsub/gossipsub.py b/libp2p/pubsub/gossipsub.py index c778d66..4f2c88e 100644 --- a/libp2p/pubsub/gossipsub.py +++ b/libp2p/pubsub/gossipsub.py @@ -6,7 +6,7 @@ from typing import ( Dict, Iterable, List, - MutableSet, + Set, Sequence, ) @@ -145,8 +145,7 @@ class GossipSub(IPubsubRouter): if peer_id_str in self.peers_gossipsub: self.peers_floodsub.remove(peer_id_str) - # FIXME: type of `sender_peer_id` should be changed to `ID` - async def handle_rpc(self, rpc: rpc_pb2.Message, sender_peer_id: str) -> None: + async def handle_rpc(self, rpc: rpc_pb2.Message, sender_peer_id: ID) -> None: """ Invoked to process control messages in the RPC envelope. It is invoked after subscriptions and payload messages have been processed @@ -154,21 +153,21 @@ class GossipSub(IPubsubRouter): :param sender_peer_id: id of the peer who sent the message """ control_message = rpc.control - sender_peer_id = str(sender_peer_id) + sender_peer_id_str = str(sender_peer_id) # Relay each rpc control message to the appropriate handler if control_message.ihave: for ihave in control_message.ihave: - await self.handle_ihave(ihave, sender_peer_id) + await self.handle_ihave(ihave, sender_peer_id_str) if control_message.iwant: for iwant in control_message.iwant: - await self.handle_iwant(iwant, sender_peer_id) + await self.handle_iwant(iwant, sender_peer_id_str) if control_message.graft: for graft in control_message.graft: - await self.handle_graft(graft, sender_peer_id) + await self.handle_graft(graft, sender_peer_id_str) if control_message.prune: for prune in control_message.prune: - await self.handle_prune(prune, sender_peer_id) + await self.handle_prune(prune, sender_peer_id_str) async def publish(self, msg_forwarder: ID, pubsub_msg: rpc_pb2.Message) -> None: # pylint: disable=too-many-locals @@ -203,7 +202,8 @@ class GossipSub(IPubsubRouter): :param origin: peer id of the peer the message originate from. :return: a generator of the peer ids who we send data to. """ - send_to: MutableSet[ID] = set() + # pylint: disable=len-as-condition + send_to: Set[ID] = set() for topic in topic_ids: if topic not in self.pubsub.peer_topics: continue @@ -228,7 +228,6 @@ class GossipSub(IPubsubRouter): # I assume there could be short periods between heartbeats where topic may not # be but we should check that this path gets hit appropriately - # pylint: disable=len-as-condition if (topic not in self.fanout) or (len(self.fanout[topic]) == 0): # If no peers in fanout, choose some peers from gossipsub peers in topic. self.fanout[topic] = self._get_in_topic_gossipsub_peers_from_minus( @@ -480,11 +479,13 @@ class GossipSub(IPubsubRouter): return selection + # FIXME: type of `minus` should be changed to type `Sequence[ID]` + # FIXME: return type should be changed to type `List[ID]` def _get_in_topic_gossipsub_peers_from_minus( self, topic: str, num_to_select: int, - minus: Sequence[ID]) -> List[ID]: + minus: Sequence[str]) -> List[str]: gossipsub_peers_in_topic = [ peer_str for peer_str in self.pubsub.peer_topics[topic] diff --git a/libp2p/pubsub/pubsub.py b/libp2p/pubsub/pubsub.py index ddfa59c..77f1fa7 100644 --- a/libp2p/pubsub/pubsub.py +++ b/libp2p/pubsub/pubsub.py @@ -29,23 +29,23 @@ def get_msg_id(msg: rpc_pb2.Message) -> Tuple[bytes, bytes]: class Pubsub: - # pylint: disable=too-many-instance-attributes, no-member + # pylint: disable=too-many-instance-attributes, no-member, unsubscriptable-object host: IHost my_id: ID router: 'IPubsubRouter' - peer_queue: asyncio.Queue + peer_queue: asyncio.Queue[ID] protocols: List[str] - incoming_msgs_from_peers: asyncio.Queue - outgoing_messages: asyncio.Queue + incoming_msgs_from_peers: asyncio.Queue[rpc_pb2.Message] + outgoing_messages: asyncio.Queue[rpc_pb2.Message] seen_messages: LRU - my_topics: Dict[str, asyncio.Queue] + my_topics: Dict[str, asyncio.Queue[rpc_pb2.Message]] # FIXME: Should be changed to `Dict[str, List[ID]]` peer_topics: Dict[str, List[str]] @@ -214,9 +214,8 @@ class Pubsub: # Force context switch await asyncio.sleep(0) - # FIXME: type of `origin_id` should be changed to `ID` # FIXME: `sub_message` can be further type hinted with mypy_protobuf - def handle_subscription(self, origin_id: str, sub_message: Any) -> None: + def handle_subscription(self, origin_id: ID, sub_message: Any) -> None: """ Handle an incoming subscription message from a peer. Update internal mapping to mark the peer as subscribed or unsubscribed to topics as @@ -224,17 +223,17 @@ class Pubsub: :param origin_id: id of the peer who subscribe to the message :param sub_message: RPC.SubOpts """ - origin_id = str(origin_id) + origin_id_str = str(origin_id) if sub_message.subscribe: if sub_message.topicid not in self.peer_topics: - self.peer_topics[sub_message.topicid] = [origin_id] - elif origin_id not in self.peer_topics[sub_message.topicid]: + self.peer_topics[sub_message.topicid] = [origin_id_str] + elif origin_id_str not in self.peer_topics[sub_message.topicid]: # Add peer to topic - self.peer_topics[sub_message.topicid].append(origin_id) + self.peer_topics[sub_message.topicid].append(origin_id_str) else: if sub_message.topicid in self.peer_topics: - if origin_id in self.peer_topics[sub_message.topicid]: - self.peer_topics[sub_message.topicid].remove(origin_id) + if origin_id_str in self.peer_topics[sub_message.topicid]: + self.peer_topics[sub_message.topicid].remove(origin_id_str) # FIXME(mhchia): Change the function name? # FIXME(mhchia): `publish_message` can be further type hinted with mypy_protobuf @@ -252,7 +251,7 @@ class Pubsub: # for each topic await self.my_topics[topic].put(publish_message) - async def subscribe(self, topic_id: str) -> asyncio.Queue: + async def subscribe(self, topic_id: str) -> asyncio.Queue[rpc_pb2.Message]: """ Subscribe ourself to a topic :param topic_id: topic_id to subscribe to @@ -374,6 +373,6 @@ class Pubsub: self.seen_messages[msg_id] = 1 def _is_subscribed_to_msg(self, msg: rpc_pb2.Message) -> bool: - if len(self.my_topics) == 0: + if not bool(self.my_topics): return False return all([topic in self.my_topics for topic in msg.topicIDs]) diff --git a/libp2p/pubsub/pubsub_notifee.py b/libp2p/pubsub/pubsub_notifee.py index cdaec4b..5830072 100644 --- a/libp2p/pubsub/pubsub_notifee.py +++ b/libp2p/pubsub/pubsub_notifee.py @@ -4,17 +4,18 @@ from multiaddr import Multiaddr from libp2p.network.network_interface import INetwork from libp2p.network.notifee_interface import INotifee +from libp2p.peer.id import ID from libp2p.stream_muxer.muxed_connection_interface import IMuxedConn from libp2p.network.stream.net_stream_interface import INetStream class PubsubNotifee(INotifee): - # pylint: disable=too-many-instance-attributes, cell-var-from-loop + # pylint: disable=too-many-instance-attributes, cell-var-from-loop, unsubscriptable-object - initiator_peers_queue: asyncio.Queue + initiator_peers_queue: asyncio.Queue[ID] - def __init__(self, initiator_peers_queue: asyncio.Queue) -> None: + def __init__(self, initiator_peers_queue: asyncio.Queue[ID]) -> None: """ :param initiator_peers_queue: queue to add new peers to so that pubsub can process new peers after we connect to them diff --git a/libp2p/pubsub/pubsub_router_interface.py b/libp2p/pubsub/pubsub_router_interface.py index 0cc6c12..8a6a879 100644 --- a/libp2p/pubsub/pubsub_router_interface.py +++ b/libp2p/pubsub/pubsub_router_interface.py @@ -41,9 +41,8 @@ class IPubsubRouter(ABC): :param peer_id: id of peer to remove """ - # FIXME: Should be changed to type 'peer.ID' @abstractmethod - async def handle_rpc(self, rpc: rpc_pb2.ControlMessage, sender_peer_id: str) -> None: + async def handle_rpc(self, rpc: rpc_pb2.ControlMessage, sender_peer_id: ID) -> None: """ Invoked to process control messages in the RPC envelope. It is invoked after subscriptions and payload messages have been processed diff --git a/libp2p/routing/interfaces.py b/libp2p/routing/interfaces.py index e656215..5d0e63f 100644 --- a/libp2p/routing/interfaces.py +++ b/libp2p/routing/interfaces.py @@ -1,8 +1,8 @@ -from abc import ABC, abstractmethod -from typing import ( - Any, - Iterable, +from abc import ( + ABC, + abstractmethod, ) +from typing import Iterable from libp2p.peer.id import ID from libp2p.peer.peerinfo import PeerInfo diff --git a/libp2p/routing/kademlia/kademlia_peer_router.py b/libp2p/routing/kademlia/kademlia_peer_router.py index e50ec84..5e426fe 100644 --- a/libp2p/routing/kademlia/kademlia_peer_router.py +++ b/libp2p/routing/kademlia/kademlia_peer_router.py @@ -9,7 +9,6 @@ from libp2p.kademlia.kad_peerinfo import ( ) from libp2p.kademlia.network import KademliaServer from libp2p.peer.id import ID -from libp2p.peer.peerinfo import PeerInfo from libp2p.routing.interfaces import IPeerRouting diff --git a/libp2p/stream_muxer/muxed_stream_interface.py b/libp2p/stream_muxer/muxed_stream_interface.py index b9034ce..2b4dac3 100644 --- a/libp2p/stream_muxer/muxed_stream_interface.py +++ b/libp2p/stream_muxer/muxed_stream_interface.py @@ -1,11 +1,18 @@ -from abc import ABC, abstractmethod +from abc import ( + ABC, + abstractmethod, +) +from typing import ( + TYPE_CHECKING, +) -from libp2p.stream_muxer.mplex.mplex import Mplex +if TYPE_CHECKING: + from libp2p.stream_muxer.mplex.mplex import Mplex class IMuxedStream(ABC): - mplex_conn: Mplex + mplex_conn: 'Mplex' @abstractmethod def read(self): From 60d6703964ced92173b9152fd050fd592fafd59b Mon Sep 17 00:00:00 2001 From: NIC619 Date: Tue, 30 Jul 2019 17:41:46 +0800 Subject: [PATCH 17/20] Temporary disable pylint on tests folder --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 10302ca..4e87b31 100644 --- a/tox.ini +++ b/tox.ini @@ -19,5 +19,5 @@ basepython = basepython = python3 extras = dev commands = - pylint --rcfile={toxinidir}/.pylintrc libp2p examples tests + pylint --rcfile={toxinidir}/.pylintrc libp2p examples mypy -p libp2p -p examples --config-file {toxinidir}/mypy.ini From 437b7665c4a9b427e86403504e4f3f44af264ed9 Mon Sep 17 00:00:00 2001 From: NIC619 Date: Tue, 30 Jul 2019 18:00:30 +0800 Subject: [PATCH 18/20] Fix: type object not subscriptable --- libp2p/pubsub/pubsub.py | 10 +++++----- libp2p/pubsub/pubsub_notifee.py | 13 +++++++++---- 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/libp2p/pubsub/pubsub.py b/libp2p/pubsub/pubsub.py index 77f1fa7..43fc763 100644 --- a/libp2p/pubsub/pubsub.py +++ b/libp2p/pubsub/pubsub.py @@ -36,16 +36,16 @@ class Pubsub: router: 'IPubsubRouter' - peer_queue: asyncio.Queue[ID] + peer_queue: 'asyncio.Queue[ID]' protocols: List[str] - incoming_msgs_from_peers: asyncio.Queue[rpc_pb2.Message] - outgoing_messages: asyncio.Queue[rpc_pb2.Message] + incoming_msgs_from_peers: 'asyncio.Queue[rpc_pb2.Message]' + outgoing_messages: 'asyncio.Queue[rpc_pb2.Message]' seen_messages: LRU - my_topics: Dict[str, asyncio.Queue[rpc_pb2.Message]] + my_topics: Dict[str, 'asyncio.Queue[rpc_pb2.Message]'] # FIXME: Should be changed to `Dict[str, List[ID]]` peer_topics: Dict[str, List[str]] @@ -251,7 +251,7 @@ class Pubsub: # for each topic await self.my_topics[topic].put(publish_message) - async def subscribe(self, topic_id: str) -> asyncio.Queue[rpc_pb2.Message]: + async def subscribe(self, topic_id: str) -> 'asyncio.Queue[rpc_pb2.Message]': """ Subscribe ourself to a topic :param topic_id: topic_id to subscribe to diff --git a/libp2p/pubsub/pubsub_notifee.py b/libp2p/pubsub/pubsub_notifee.py index 5830072..9f06176 100644 --- a/libp2p/pubsub/pubsub_notifee.py +++ b/libp2p/pubsub/pubsub_notifee.py @@ -1,21 +1,26 @@ -import asyncio +from typing import ( + TYPE_CHECKING, +) from multiaddr import Multiaddr from libp2p.network.network_interface import INetwork from libp2p.network.notifee_interface import INotifee -from libp2p.peer.id import ID from libp2p.stream_muxer.muxed_connection_interface import IMuxedConn from libp2p.network.stream.net_stream_interface import INetStream +if TYPE_CHECKING: + import asyncio + from libp2p.peer.id import ID + class PubsubNotifee(INotifee): # pylint: disable=too-many-instance-attributes, cell-var-from-loop, unsubscriptable-object - initiator_peers_queue: asyncio.Queue[ID] + initiator_peers_queue: 'asyncio.Queue[ID]' - def __init__(self, initiator_peers_queue: asyncio.Queue[ID]) -> None: + def __init__(self, initiator_peers_queue: 'asyncio.Queue[ID]') -> None: """ :param initiator_peers_queue: queue to add new peers to so that pubsub can process new peers after we connect to them From 76de01a17d7b527db87e4cb30c79f8de4ebff5f2 Mon Sep 17 00:00:00 2001 From: NIC619 Date: Tue, 30 Jul 2019 18:01:01 +0800 Subject: [PATCH 19/20] Add `duplicate-code` and `cyclic-import` to pylintrc --- .pylintrc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.pylintrc b/.pylintrc index 4955221..6784a7c 100644 --- a/.pylintrc +++ b/.pylintrc @@ -128,6 +128,8 @@ disable=print-statement, dict-keys-not-iterating, dict-values-not-iterating, missing-docstring, + cyclic-import, + duplicate-code, # Enable the message, report, category or checker with the given id(s). You can # either give multiple identifier separated by comma (,) or put this option From 5e215901c09c29345ff069c1a1791e3070d5e0fc Mon Sep 17 00:00:00 2001 From: NIC619 Date: Tue, 30 Jul 2019 23:41:28 +0800 Subject: [PATCH 20/20] Apply PR feedback --- libp2p/network/network_interface.py | 2 +- libp2p/network/swarm.py | 8 ++++---- libp2p/peer/id.py | 13 ++++++------- libp2p/peer/peerdata.py | 2 +- libp2p/pubsub/gossipsub.py | 1 + libp2p/pubsub/pubsub.py | 2 +- libp2p/stream_muxer/muxed_stream_interface.py | 8 ++------ tox.ini | 1 + 8 files changed, 17 insertions(+), 20 deletions(-) diff --git a/libp2p/network/network_interface.py b/libp2p/network/network_interface.py index f34a752..1a84c0e 100644 --- a/libp2p/network/network_interface.py +++ b/libp2p/network/network_interface.py @@ -67,7 +67,7 @@ class INetwork(ABC): """ @abstractmethod - async def listen(self, *args: Multiaddr) -> bool: + async def listen(self, *args: Sequence[Multiaddr]) -> bool: """ :param *args: one or many multiaddrs to start listening on :return: True if at least one success diff --git a/libp2p/network/swarm.py b/libp2p/network/swarm.py index bedba40..a259fce 100644 --- a/libp2p/network/swarm.py +++ b/libp2p/network/swarm.py @@ -18,11 +18,11 @@ from libp2p.protocol_muxer.multiselect import Multiselect from libp2p.protocol_muxer.multiselect_client import MultiselectClient from libp2p.routing.interfaces import IPeerRouting from libp2p.stream_muxer.muxed_connection_interface import IMuxedConn +from libp2p.stream_muxer.muxed_stream_interface import IMuxedStream from libp2p.transport.upgrader import TransportUpgrader from libp2p.transport.transport_interface import ITransport from libp2p.transport.listener_interface import IListener -from libp2p.stream_muxer.mplex.mplex_stream import MplexStream from .network_interface import INetwork from .notifee_interface import INotifee @@ -168,7 +168,7 @@ class Swarm(INetwork): return net_stream - async def listen(self, *args: Multiaddr) -> bool: + async def listen(self, *args: Sequence[Multiaddr]) -> bool: """ :param *args: one or many multiaddrs to start listening on :return: true if at least one success @@ -252,7 +252,7 @@ class Swarm(INetwork): # TODO: `disconnect`? -GenericProtocolHandlerFn = Callable[[MplexStream], Awaitable[None]] +GenericProtocolHandlerFn = Callable[[IMuxedStream], Awaitable[None]] def create_generic_protocol_handler(swarm: Swarm) -> GenericProtocolHandlerFn: @@ -264,7 +264,7 @@ def create_generic_protocol_handler(swarm: Swarm) -> GenericProtocolHandlerFn: """ multiselect = swarm.multiselect - async def generic_protocol_handler(muxed_stream: MplexStream) -> None: + async def generic_protocol_handler(muxed_stream: IMuxedStream) -> None: # Perform protocol muxing to determine protocol to use protocol, handler = await multiselect.negotiate(muxed_stream) diff --git a/libp2p/peer/id.py b/libp2p/peer/id.py index 4e82ca2..8561e0e 100644 --- a/libp2p/peer/id.py +++ b/libp2p/peer/id.py @@ -20,16 +20,15 @@ MAX_INLINE_KEY_LENGTH = 42 class ID: - _id_str: str + _id_str: bytes - def __init__(self, id_str: str) -> None: + def __init__(self, id_str: bytes) -> None: self._id_str = id_str - # FIXME: Should return type `bytes` - def to_bytes(self) -> str: + def to_bytes(self) -> bytes: return self._id_str - def get_raw_id(self) -> str: + def get_raw_id(self) -> bytes: return self._id_str def pretty(self) -> str: @@ -86,6 +85,6 @@ def id_from_private_key(key: RsaKey) -> ID: return id_from_public_key(key.publickey()) def digest(data: Union[str, bytes]) -> bytes: - if not isinstance(data, bytes): - data = str(data).encode('utf8') + if isinstance(data, str): + data = data.encode('utf8') return hashlib.sha1(data).digest() diff --git a/libp2p/peer/peerdata.py b/libp2p/peer/peerdata.py index bb4962b..68749c8 100644 --- a/libp2p/peer/peerdata.py +++ b/libp2p/peer/peerdata.py @@ -31,7 +31,7 @@ class PeerData(IPeerData): self.protocols = list(protocols) def add_addrs(self, addrs: Sequence[Multiaddr]) -> None: - self.addrs.extend(list(addrs)) + self.addrs.extend(addrs) def get_addrs(self) -> List[Multiaddr]: return self.addrs diff --git a/libp2p/pubsub/gossipsub.py b/libp2p/pubsub/gossipsub.py index 4f2c88e..f0d903b 100644 --- a/libp2p/pubsub/gossipsub.py +++ b/libp2p/pubsub/gossipsub.py @@ -533,6 +533,7 @@ class GossipSub(IPubsubRouter): from_id_str = sender_peer_id # FIXME: Update type of message ID + # FIXME: Find a better way to parse the msg ids msg_ids: List[Any] = [literal_eval(msg) for msg in iwant_msg.messageIDs] msgs_to_forward: List[rpc_pb2.Message] = [] for msg_id_iwant in msg_ids: diff --git a/libp2p/pubsub/pubsub.py b/libp2p/pubsub/pubsub.py index 43fc763..744c37c 100644 --- a/libp2p/pubsub/pubsub.py +++ b/libp2p/pubsub/pubsub.py @@ -373,6 +373,6 @@ class Pubsub: self.seen_messages[msg_id] = 1 def _is_subscribed_to_msg(self, msg: rpc_pb2.Message) -> bool: - if not bool(self.my_topics): + if not self.my_topics: return False return all([topic in self.my_topics for topic in msg.topicIDs]) diff --git a/libp2p/stream_muxer/muxed_stream_interface.py b/libp2p/stream_muxer/muxed_stream_interface.py index 2b4dac3..eb6f267 100644 --- a/libp2p/stream_muxer/muxed_stream_interface.py +++ b/libp2p/stream_muxer/muxed_stream_interface.py @@ -2,17 +2,13 @@ from abc import ( ABC, abstractmethod, ) -from typing import ( - TYPE_CHECKING, -) -if TYPE_CHECKING: - from libp2p.stream_muxer.mplex.mplex import Mplex +from libp2p.stream_muxer.muxed_connection_interface import IMuxedConn class IMuxedStream(ABC): - mplex_conn: 'Mplex' + mplex_conn: IMuxedConn @abstractmethod def read(self): diff --git a/tox.ini b/tox.ini index 4e87b31..ea6af8f 100644 --- a/tox.ini +++ b/tox.ini @@ -19,5 +19,6 @@ basepython = basepython = python3 extras = dev commands = + # TODO: Add the tests/ folder back to pylint pylint --rcfile={toxinidir}/.pylintrc libp2p examples mypy -p libp2p -p examples --config-file {toxinidir}/mypy.ini