py-libp2p/libp2p/pubsub/gossipsub.py

730 lines
28 KiB
Python
Raw Normal View History

2019-07-28 22:30:51 +08:00
from ast import literal_eval
import asyncio
from collections import defaultdict
2019-09-14 21:54:26 +08:00
import logging
2019-07-26 18:35:25 +08:00
import random
from typing import Any, DefaultDict, Dict, Iterable, List, Sequence, Set, Tuple
2019-08-01 06:00:12 +08:00
from libp2p.network.stream.exceptions import StreamClosed
from libp2p.peer.id import ID
2019-09-03 23:37:34 +08:00
from libp2p.pubsub import floodsub
from libp2p.typing import TProtocol
2019-09-03 16:07:44 +08:00
from libp2p.utils import encode_varint_prefixed
2019-07-26 18:35:25 +08:00
from .mcache import MessageCache
from .pb import rpc_pb2
2019-07-24 16:29:14 +08:00
from .pubsub import Pubsub
from .pubsub_router_interface import IPubsubRouter
PROTOCOL_ID = TProtocol("/meshsub/1.0.0")
2019-09-14 21:54:26 +08:00
logger = logging.getLogger("libp2p.pubsub.gossipsub")
class GossipSub(IPubsubRouter):
protocols: List[TProtocol]
2019-07-24 16:29:14 +08:00
pubsub: Pubsub
degree: int
degree_high: int
degree_low: int
time_to_live: int
mesh: Dict[str, Set[ID]]
fanout: Dict[str, Set[ID]]
2019-07-24 16:29:14 +08:00
peers_to_protocol: Dict[ID, str]
2019-07-24 16:29:14 +08:00
time_since_last_publish: Dict[str, int]
peers_gossipsub: Set[ID]
peers_floodsub: Set[ID]
2019-07-24 16:29:14 +08:00
mcache: MessageCache
heartbeat_initial_delay: float
2019-07-24 16:29:14 +08:00
heartbeat_interval: int
2019-08-01 06:00:12 +08:00
def __init__(
self,
protocols: Sequence[TProtocol],
2019-08-01 06:00:12 +08:00
degree: int,
degree_low: int,
degree_high: int,
time_to_live: int,
gossip_window: int = 3,
gossip_history: int = 5,
heartbeat_initial_delay: float = 0.1,
2019-08-01 06:00:12 +08:00
heartbeat_interval: int = 120,
) -> None:
self.protocols = list(protocols)
self.pubsub = None
# Store target degree, upper degree bound, and lower degree bound
self.degree = degree
self.degree_low = degree_low
self.degree_high = degree_high
# Store time to live (for topics in fanout)
self.time_to_live = time_to_live
# Create topic --> list of peers mappings
self.mesh = {}
self.fanout = {}
2019-07-29 12:42:13 +08:00
# Create peer --> protocol mapping
self.peers_to_protocol = {}
# Create topic --> time since last publish map
self.time_since_last_publish = {}
self.peers_gossipsub = set()
self.peers_floodsub = set()
# Create message cache
self.mcache = MessageCache(gossip_window, gossip_history)
# Create heartbeat timer
self.heartbeat_initial_delay = heartbeat_initial_delay
self.heartbeat_interval = heartbeat_interval
# Interface functions
def get_protocols(self) -> List[TProtocol]:
"""
:return: the list of protocols supported by the router
"""
return self.protocols
2019-07-24 16:29:14 +08:00
def attach(self, pubsub: Pubsub) -> None:
"""
Attach is invoked by the PubSub constructor to attach the router to a
freshly initialized PubSub instance.
:param pubsub: pubsub instance to attach to
"""
self.pubsub = pubsub
2019-09-14 21:54:26 +08:00
logger.debug("attached to pusub")
# Start heartbeat now that we have a pubsub instance
asyncio.ensure_future(self.heartbeat())
def add_peer(self, peer_id: ID, protocol_id: TProtocol) -> None:
"""
Notifies the router that a new peer has been connected.
:param peer_id: id of peer to add
2019-07-24 16:29:14 +08:00
:param protocol_id: router protocol the peer speaks, e.g., floodsub, gossipsub
"""
2019-09-14 21:54:26 +08:00
logger.debug("adding peer %s with protocol %s", peer_id, protocol_id)
2019-09-03 23:37:34 +08:00
if protocol_id == PROTOCOL_ID:
self.peers_gossipsub.add(peer_id)
2019-09-03 23:37:34 +08:00
elif protocol_id == floodsub.PROTOCOL_ID:
self.peers_floodsub.add(peer_id)
2019-09-03 23:37:34 +08:00
else:
# We should never enter here. Becuase the `protocol_id` is registered by your pubsub
# instance in multistream-select, but it is not the protocol that gossipsub supports.
# In this case, probably we registered gossipsub to a wrong `protocol_id`
# in multistream-select, or wrong versions.
2019-12-05 17:33:07 +08:00
raise Exception(f"Unreachable: Protocol={protocol_id} is not supported.")
2019-09-14 21:54:26 +08:00
self.peers_to_protocol[peer_id] = protocol_id
2019-07-24 16:29:14 +08:00
def remove_peer(self, peer_id: ID) -> None:
"""
Notifies the router that a peer has been disconnected.
:param peer_id: id of peer to remove
"""
2019-09-14 21:54:26 +08:00
logger.debug("removing peer %s", peer_id)
if peer_id in self.peers_gossipsub:
self.peers_gossipsub.discard(peer_id)
2019-09-14 21:54:26 +08:00
elif peer_id in self.peers_floodsub:
self.peers_floodsub.discard(peer_id)
for topic in self.mesh:
if peer_id in self.mesh[topic]:
self.mesh[topic].discard(peer_id)
for topic in self.fanout:
if peer_id in self.fanout[topic]:
self.fanout[topic].discard(peer_id)
self.peers_to_protocol.pop(peer_id, None)
2019-09-14 21:54:26 +08:00
async def handle_rpc(self, rpc: rpc_pb2.RPC, sender_peer_id: ID) -> None:
"""
Invoked to process control messages in the RPC envelope. It is invoked
after subscriptions and payload messages have been processed.
2019-07-24 16:29:14 +08:00
:param rpc: RPC message
:param sender_peer_id: id of the peer who sent the message
"""
control_message = rpc.control
2019-07-24 16:29:14 +08:00
# Relay each rpc control message to the appropriate handler
if control_message.ihave:
for ihave in control_message.ihave:
await self.handle_ihave(ihave, sender_peer_id)
if control_message.iwant:
for iwant in control_message.iwant:
await self.handle_iwant(iwant, sender_peer_id)
if control_message.graft:
for graft in control_message.graft:
await self.handle_graft(graft, sender_peer_id)
if control_message.prune:
for prune in control_message.prune:
await self.handle_prune(prune, sender_peer_id)
async def publish(self, msg_forwarder: ID, pubsub_msg: rpc_pb2.Message) -> None:
"""Invoked to forward a new message that has been validated."""
2019-07-26 18:35:25 +08:00
self.mcache.put(pubsub_msg)
peers_gen = self._get_peers_to_send(
pubsub_msg.topicIDs,
msg_forwarder=msg_forwarder,
origin=ID(pubsub_msg.from_id),
2019-07-26 18:35:25 +08:00
)
2019-08-01 06:00:12 +08:00
rpc_msg = rpc_pb2.RPC(publish=[pubsub_msg])
2019-09-14 21:54:26 +08:00
logger.debug("publishing message %s", pubsub_msg)
2019-07-26 18:35:25 +08:00
for peer_id in peers_gen:
if peer_id not in self.pubsub.peers:
continue
stream = self.pubsub.peers[peer_id]
2019-07-26 18:35:25 +08:00
# FIXME: We should add a `WriteMsg` similar to write delimited messages.
# Ref: https://github.com/libp2p/go-libp2p-pubsub/blob/master/comm.go#L107
# TODO: Go use `sendRPC`, which possibly piggybacks gossip/control messages.
try:
await stream.write(encode_varint_prefixed(rpc_msg.SerializeToString()))
except StreamClosed:
logger.debug("Fail to publish message to %s: stream closed", peer_id)
self.pubsub._handle_dead_peer(peer_id)
2019-07-26 18:35:25 +08:00
def _get_peers_to_send(
2019-08-01 06:00:12 +08:00
self, topic_ids: Iterable[str], msg_forwarder: ID, origin: ID
) -> Iterable[ID]:
"""
Get the eligible peers to send the data to.
:param msg_forwarder: the peer id of the peer who forwards the message to me.
2019-07-28 18:06:38 +08:00
:param origin: peer id of the peer the message originate from.
2019-07-26 18:35:25 +08:00
:return: a generator of the peer ids who we send data to.
"""
send_to: Set[ID] = set()
2019-07-26 18:35:25 +08:00
for topic in topic_ids:
if topic not in self.pubsub.peer_topics:
continue
2019-07-26 18:35:25 +08:00
# floodsub peers
2019-12-05 14:35:34 +08:00
# FIXME: `gossipsub.peers_floodsub` can be changed to `gossipsub.peers` in go.
# This will improve the efficiency when searching for a peer's protocol id.
floodsub_peers: Set[ID] = set(
2019-12-05 14:35:34 +08:00
peer_id
for peer_id in self.pubsub.peer_topics[topic]
if peer_id in self.peers_floodsub
)
send_to.update(floodsub_peers)
2019-07-26 18:35:25 +08:00
# gossipsub peers
gossipsub_peers: Set[ID] = set()
2019-07-26 18:35:25 +08:00
if topic in self.mesh:
2019-12-05 14:35:34 +08:00
gossipsub_peers = self.mesh[topic]
2019-07-26 18:35:25 +08:00
else:
2019-12-05 14:35:34 +08:00
# When we publish to a topic that we have not subscribe to, we randomly pick
# `self.degree` number of peers who have subscribed to the topic and add them
2019-12-05 14:35:34 +08:00
# as our `fanout` peers.
topic_in_fanout: bool = topic in self.fanout
fanout_peers: Set[ID] = self.fanout[topic] if topic_in_fanout else set()
2019-12-05 14:35:34 +08:00
fanout_size = len(fanout_peers)
2019-12-05 15:10:04 +08:00
if not topic_in_fanout or (
topic_in_fanout and fanout_size < self.degree
):
if topic in self.pubsub.peer_topics:
# Combine fanout peers with selected peers
fanout_peers.update(
self._get_in_topic_gossipsub_peers_from_minus(
2019-12-17 17:49:49 +08:00
topic, self.degree - fanout_size, list(fanout_peers)
)
2019-12-05 15:10:04 +08:00
)
self.fanout[topic] = fanout_peers
2019-12-05 14:35:34 +08:00
gossipsub_peers = fanout_peers
send_to.update(gossipsub_peers)
# Excludes `msg_forwarder` and `origin`
yield from send_to.difference([msg_forwarder, origin])
2019-07-24 16:29:14 +08:00
async def join(self, topic: str) -> None:
"""
Join notifies the router that we want to receive and forward messages
in a topic. It is invoked after the subscription announcement.
:param topic: topic to join
"""
2019-09-14 21:54:26 +08:00
logger.debug("joining topic %s", topic)
if topic in self.mesh:
return
# Create mesh[topic] if it does not yet exist
self.mesh[topic] = set()
2019-07-24 16:29:14 +08:00
topic_in_fanout: bool = topic in self.fanout
fanout_peers: Set[ID] = self.fanout[topic] if topic_in_fanout else set()
2019-07-15 17:13:46 +08:00
fanout_size = len(fanout_peers)
if not topic_in_fanout or (topic_in_fanout and fanout_size < self.degree):
# There are less than D peers (let this number be x)
# in the fanout for a topic (or the topic is not in the fanout).
# Selects the remaining number of peers (D-x) from peers.gossipsub[topic].
if topic in self.pubsub.peer_topics:
2019-07-28 18:06:38 +08:00
selected_peers = self._get_in_topic_gossipsub_peers_from_minus(
2019-12-17 17:49:49 +08:00
topic, self.degree - fanout_size, list(fanout_peers)
2019-07-27 12:06:36 +08:00
)
# Combine fanout peers with selected peers
fanout_peers.update(selected_peers)
2019-07-15 17:13:46 +08:00
# Add fanout peers to mesh and notifies them with a GRAFT(topic) control message.
for peer in fanout_peers:
self.mesh[topic].add(peer)
2019-12-05 14:35:34 +08:00
await self.emit_graft(topic, peer)
2019-07-15 17:13:46 +08:00
self.fanout.pop(topic, None)
2019-07-24 16:29:14 +08:00
async def leave(self, topic: str) -> None:
# Note: the comments here are the near-exact algorithm description from the spec
"""
Leave notifies the router that we are no longer interested in a topic.
It is invoked after the unsubscription announcement.
:param topic: topic to leave
"""
2019-09-14 21:54:26 +08:00
logger.debug("leaving topic %s", topic)
if topic not in self.mesh:
return
# Notify the peers in mesh[topic] with a PRUNE(topic) message
for peer in self.mesh[topic]:
await self.emit_prune(topic, peer)
# Forget mesh[topic]
2019-12-05 14:35:34 +08:00
self.mesh.pop(topic, None)
2019-12-03 15:48:23 +08:00
async def _emit_control_msgs(
self,
peers_to_graft: Dict[ID, List[str]],
peers_to_prune: Dict[ID, List[str]],
peers_to_gossip: Dict[ID, Dict[str, List[str]]],
) -> None:
2019-12-03 22:14:45 +08:00
graft_msgs: List[rpc_pb2.ControlGraft] = []
prune_msgs: List[rpc_pb2.ControlPrune] = []
ihave_msgs: List[rpc_pb2.ControlIHave] = []
2019-12-03 15:48:23 +08:00
# Starting with GRAFT messages
for peer, topics in peers_to_graft.items():
for topic in topics:
graft_msg: rpc_pb2.ControlGraft = rpc_pb2.ControlGraft(topicID=topic)
2019-12-03 15:48:23 +08:00
graft_msgs.append(graft_msg)
# If there are also PRUNE messages to send to this peer
if peer in peers_to_prune:
for topic in peers_to_prune[peer]:
prune_msg: rpc_pb2.ControlPrune = rpc_pb2.ControlPrune(
topicID=topic
)
2019-12-03 15:48:23 +08:00
prune_msgs.append(prune_msg)
del peers_to_prune[peer]
# If there are also IHAVE messages to send to this peer
if peer in peers_to_gossip:
for topic in peers_to_gossip[peer]:
ihave_msg: rpc_pb2.ControlIHave = rpc_pb2.ControlIHave(
messageIDs=peers_to_gossip[peer][topic], topicID=topic
)
2019-12-03 15:48:23 +08:00
ihave_msgs.append(ihave_msg)
del peers_to_gossip[peer]
control_msg = self.pack_control_msgs(ihave_msgs, graft_msgs, prune_msgs)
await self.emit_control_message(control_msg, peer)
# Next with PRUNE messages
for peer, topics in peers_to_prune.items():
prune_msgs = []
for topic in topics:
prune_msg = rpc_pb2.ControlPrune(topicID=topic)
2019-12-03 15:48:23 +08:00
prune_msgs.append(prune_msg)
# If there are also IHAVE messages to send to this peer
if peer in peers_to_gossip:
ihave_msgs = []
for topic in peers_to_gossip[peer]:
ihave_msg = rpc_pb2.ControlIHave(
messageIDs=peers_to_gossip[peer][topic], topicID=topic
)
2019-12-03 15:48:23 +08:00
ihave_msgs.append(ihave_msg)
del peers_to_gossip[peer]
control_msg = self.pack_control_msgs(ihave_msgs, None, prune_msgs)
await self.emit_control_message(control_msg, peer)
# Fianlly IHAVE messages
for peer in peers_to_gossip:
ihave_msgs = []
for topic in peers_to_gossip[peer]:
ihave_msg = rpc_pb2.ControlIHave(
messageIDs=peers_to_gossip[peer][topic], topicID=topic
)
2019-12-03 15:48:23 +08:00
ihave_msgs.append(ihave_msg)
control_msg = self.pack_control_msgs(ihave_msgs, None, None)
await self.emit_control_message(control_msg, peer)
# Heartbeat
2019-07-24 16:29:14 +08:00
async def heartbeat(self) -> None:
"""
Call individual heartbeats.
Note: the heartbeats are called with awaits because each heartbeat depends on the
state changes in the preceding heartbeat
"""
# Start after a delay. Ref: https://github.com/libp2p/go-libp2p-pubsub/blob/01b9825fbee1848751d90a8469e3f5f43bac8466/gossipsub.go#L410 # Noqa: E501
await asyncio.sleep(self.heartbeat_initial_delay)
while True:
2019-12-03 15:48:23 +08:00
# Maintain mesh and keep track of which peers to send GRAFT or PRUNE to
peers_to_graft, peers_to_prune = self.mesh_heartbeat()
# Maintain fanout
self.fanout_heartbeat()
# Get the peers to send IHAVE to
peers_to_gossip = self.gossip_heartbeat()
# Pack GRAFT, PRUNE and IHAVE for the same peer into one control message and send it
await self._emit_control_msgs(
peers_to_graft, peers_to_prune, peers_to_gossip
)
2019-12-03 15:48:23 +08:00
self.mcache.shift()
await asyncio.sleep(self.heartbeat_interval)
def mesh_heartbeat(
self
) -> Tuple[DefaultDict[ID, List[str]], DefaultDict[ID, List[str]]]:
peers_to_graft: DefaultDict[ID, List[str]] = defaultdict(list)
peers_to_prune: DefaultDict[ID, List[str]] = defaultdict(list)
for topic in self.mesh:
2019-07-22 23:22:07 +08:00
# Skip if no peers have subscribed to the topic
if topic not in self.pubsub.peer_topics:
continue
num_mesh_peers_in_topic = len(self.mesh[topic])
if num_mesh_peers_in_topic < self.degree_low:
2019-07-22 23:22:07 +08:00
# Select D - |mesh[topic]| peers from peers.gossipsub[topic] - mesh[topic]
2019-07-28 18:06:38 +08:00
selected_peers = self._get_in_topic_gossipsub_peers_from_minus(
2019-12-17 17:49:49 +08:00
topic, self.degree - num_mesh_peers_in_topic, list(self.mesh[topic])
2019-07-22 23:22:07 +08:00
)
for peer in selected_peers:
2019-07-22 23:22:07 +08:00
# Add peer to mesh[topic]
self.mesh[topic].add(peer)
2019-07-22 23:22:07 +08:00
# Emit GRAFT(topic) control message to peer
peers_to_graft[peer].append(topic)
if num_mesh_peers_in_topic > self.degree_high:
# Select |mesh[topic]| - D peers from mesh[topic]
selected_peers = GossipSub.select_from_minus(
2019-12-17 17:49:49 +08:00
num_mesh_peers_in_topic - self.degree, list(self.mesh[topic]), []
2019-07-24 16:29:14 +08:00
)
for peer in selected_peers:
# Remove peer from mesh[topic]
self.mesh[topic].discard(peer)
# Emit PRUNE(topic) control message to peer
peers_to_prune[peer].append(topic)
2019-12-03 15:48:23 +08:00
return peers_to_graft, peers_to_prune
2019-12-03 15:48:23 +08:00
def fanout_heartbeat(self) -> None:
# Note: the comments here are the exact pseudocode from the spec
for topic in self.fanout:
# Delete topic entry if it's not in `pubsub.peer_topics`
# or if it's time-since-last-published > ttl
# TODO: there's no way time_since_last_publish gets set anywhere yet
if (
topic not in self.pubsub.peer_topics
or self.time_since_last_publish[topic] > self.time_to_live
):
# Remove topic from fanout
del self.fanout[topic]
del self.time_since_last_publish[topic]
else:
2019-12-05 14:35:34 +08:00
# Check if fanout peers are still in the topic and remove the ones that are not
# ref: https://github.com/libp2p/go-libp2p-pubsub/blob/01b9825fbee1848751d90a8469e3f5f43bac8466/gossipsub.go#L498-L504 # noqa: E501
in_topic_fanout_peers = [
peer
for peer in self.fanout[topic]
if peer in self.pubsub.peer_topics[topic]
]
self.fanout[topic] = set(in_topic_fanout_peers)
num_fanout_peers_in_topic = len(self.fanout[topic])
# If |fanout[topic]| < D
if num_fanout_peers_in_topic < self.degree:
# Select D - |fanout[topic]| peers from peers.gossipsub[topic] - fanout[topic]
2019-07-28 18:06:38 +08:00
selected_peers = self._get_in_topic_gossipsub_peers_from_minus(
topic,
self.degree - num_fanout_peers_in_topic,
2019-12-17 17:49:49 +08:00
list(self.fanout[topic]),
2019-07-28 18:06:38 +08:00
)
# Add the peers to fanout[topic]
2019-12-17 17:49:49 +08:00
self.fanout[topic].update(selected_peers)
def gossip_heartbeat(self) -> DefaultDict[ID, Dict[str, List[str]]]:
peers_to_gossip: DefaultDict[ID, Dict[str, List[str]]] = defaultdict(dict)
for topic in self.mesh:
msg_ids = self.mcache.window(topic)
if msg_ids:
# Get all pubsub peers in a topic and only add them if they are gossipsub peers too
if topic in self.pubsub.peer_topics:
# Select D peers from peers.gossipsub[topic]
2019-07-28 18:06:38 +08:00
peers_to_emit_ihave_to = self._get_in_topic_gossipsub_peers_from_minus(
2019-12-17 17:49:49 +08:00
topic, self.degree, list(self.mesh[topic])
2019-07-28 18:06:38 +08:00
)
msg_id_strs = [str(msg_id) for msg_id in msg_ids]
for peer in peers_to_emit_ihave_to:
peers_to_gossip[peer][topic] = msg_id_strs
2019-07-28 18:06:38 +08:00
# TODO: Refactor and Dedup. This section is the roughly the same as the above.
# Do the same for fanout, for all topics not already hit in mesh
for topic in self.fanout:
2019-12-05 15:10:04 +08:00
msg_ids = self.mcache.window(topic)
if msg_ids:
# Get all pubsub peers in topic and only add if they are gossipsub peers also
if topic in self.pubsub.peer_topics:
# Select D peers from peers.gossipsub[topic]
peers_to_emit_ihave_to = self._get_in_topic_gossipsub_peers_from_minus(
2019-12-17 17:49:49 +08:00
topic, self.degree, list(self.fanout[topic])
2019-12-05 15:10:04 +08:00
)
msg_id_strs = [str(msg) for msg in msg_ids]
for peer in peers_to_emit_ihave_to:
peers_to_gossip[peer][topic] = msg_id_strs
2019-12-03 15:48:23 +08:00
return peers_to_gossip
@staticmethod
2019-08-01 06:00:12 +08:00
def select_from_minus(
num_to_select: int, pool: Sequence[Any], minus: Sequence[Any]
) -> List[Any]:
"""
Select at most num_to_select subset of elements from the set (pool - minus) randomly.
:param num_to_select: number of elements to randomly select
:param pool: list of items to select from (excluding elements in minus)
:param minus: elements to be excluded from selection pool
:return: list of selected elements
"""
# Create selection pool, which is selection_pool = pool - minus
if minus:
# Create a new selection pool by removing elements of minus
2019-07-24 16:29:14 +08:00
selection_pool: List[Any] = [x for x in pool if x not in minus]
else:
# Don't create a new selection_pool if we are not subbing anything
selection_pool = list(pool)
# If num_to_select > size(selection_pool), then return selection_pool (which has the most
# possible elements s.t. the number of elements is less than num_to_select)
2019-12-17 17:49:49 +08:00
if num_to_select >= len(selection_pool):
return selection_pool
# Random selection
2019-07-24 16:29:14 +08:00
selection: List[Any] = random.sample(selection_pool, num_to_select)
return selection
2019-07-28 18:06:38 +08:00
def _get_in_topic_gossipsub_peers_from_minus(
self, topic: str, num_to_select: int, minus: Sequence[ID]
) -> List[ID]:
2019-07-26 18:35:25 +08:00
gossipsub_peers_in_topic = [
peer_id
for peer_id in self.pubsub.peer_topics[topic]
if peer_id in self.peers_gossipsub
2019-07-26 18:35:25 +08:00
]
return self.select_from_minus(
num_to_select, gossipsub_peers_in_topic, list(minus)
)
2019-07-26 18:35:25 +08:00
# RPC handlers
async def handle_ihave(
self, ihave_msg: rpc_pb2.ControlIHave, sender_peer_id: ID
) -> None:
"""Checks the seen set and requests unknown messages with an IWANT
message."""
# Get list of all seen (seqnos, from) from the (seqno, from) tuples in seen_messages cache
2019-07-24 16:29:14 +08:00
seen_seqnos_and_peers = [
2019-08-01 06:00:12 +08:00
seqno_and_from for seqno_and_from in self.pubsub.seen_messages.keys()
2019-07-24 16:29:14 +08:00
]
# Add all unknown message ids (ids that appear in ihave_msg but not in seen_seqnos) to list
# of messages we want to request
2019-07-24 16:29:14 +08:00
# FIXME: Update type of message ID
2019-07-29 12:42:13 +08:00
msg_ids_wanted: List[Any] = [
2019-07-24 16:29:14 +08:00
msg_id
for msg_id in ihave_msg.messageIDs
if literal_eval(msg_id) not in seen_seqnos_and_peers
]
# Request messages with IWANT message
if msg_ids_wanted:
await self.emit_iwant(msg_ids_wanted, sender_peer_id)
async def handle_iwant(
self, iwant_msg: rpc_pb2.ControlIWant, sender_peer_id: ID
) -> None:
"""Forwards all request messages that are present in mcache to the
requesting peer."""
2019-07-24 16:29:14 +08:00
# FIXME: Update type of message ID
2019-07-30 23:41:28 +08:00
# FIXME: Find a better way to parse the msg ids
2019-07-24 16:29:14 +08:00
msg_ids: List[Any] = [literal_eval(msg) for msg in iwant_msg.messageIDs]
msgs_to_forward: List[rpc_pb2.Message] = []
for msg_id_iwant in msg_ids:
# Check if the wanted message ID is present in mcache
2019-07-24 16:29:14 +08:00
msg: rpc_pb2.Message = self.mcache.get(msg_id_iwant)
# Cache hit
if msg:
# Add message to list of messages to forward to requesting peers
msgs_to_forward.append(msg)
# Forward messages to requesting peer
# Should this just be publishing? No
# because then the message will forwarded to peers in the topics contained in the messages.
# We should
# 1) Package these messages into a single packet
2019-07-24 16:29:14 +08:00
packet: rpc_pb2.RPC = rpc_pb2.RPC()
packet.publish.extend(msgs_to_forward)
# 2) Serialize that packet
2019-07-24 16:29:14 +08:00
rpc_msg: bytes = packet.SerializeToString()
# 3) Get the stream to this peer
if sender_peer_id not in self.pubsub.peers:
logger.debug(
"Fail to responed to iwant request from %s: peer disconnected",
sender_peer_id,
)
peer_stream = self.pubsub.peers[sender_peer_id]
# 4) And write the packet to the stream
try:
await peer_stream.write(encode_varint_prefixed(rpc_msg))
except StreamClosed:
logger.debug(
"Fail to responed to iwant request from %s: stream closed",
sender_peer_id,
)
self.pubsub._handle_dead_peer(sender_peer_id)
async def handle_graft(
self, graft_msg: rpc_pb2.ControlGraft, sender_peer_id: ID
) -> None:
2019-07-24 16:29:14 +08:00
topic: str = graft_msg.topicID
# Add peer to mesh for topic
if topic in self.mesh:
if sender_peer_id not in self.mesh[topic]:
self.mesh[topic].add(sender_peer_id)
else:
# Respond with PRUNE if not subscribed to the topic
await self.emit_prune(topic, sender_peer_id)
async def handle_prune(
self, prune_msg: rpc_pb2.ControlPrune, sender_peer_id: ID
) -> None:
2019-07-24 16:29:14 +08:00
topic: str = prune_msg.topicID
# Remove peer from mesh for topic, if peer is in topic
if topic in self.mesh and sender_peer_id in self.mesh[topic]:
self.mesh[topic].discard(sender_peer_id)
# RPC emitters
2019-12-03 15:48:23 +08:00
def pack_control_msgs(
self,
ihave_msgs: List[rpc_pb2.ControlIHave],
graft_msgs: List[rpc_pb2.ControlGraft],
prune_msgs: List[rpc_pb2.ControlPrune],
) -> rpc_pb2.ControlMessage:
control_msg: rpc_pb2.ControlMessage = rpc_pb2.ControlMessage()
if ihave_msgs:
control_msg.ihave.extend(ihave_msgs)
if graft_msgs:
control_msg.graft.extend(graft_msgs)
if prune_msgs:
control_msg.prune.extend(prune_msgs)
return control_msg
async def emit_ihave(self, topic: str, msg_ids: Any, to_peer: ID) -> None:
"""Emit ihave message, sent to to_peer, for topic and msg_ids."""
2019-07-24 16:29:14 +08:00
ihave_msg: rpc_pb2.ControlIHave = rpc_pb2.ControlIHave()
ihave_msg.messageIDs.extend(msg_ids)
ihave_msg.topicID = topic
2019-07-24 16:29:14 +08:00
control_msg: rpc_pb2.ControlMessage = rpc_pb2.ControlMessage()
control_msg.ihave.extend([ihave_msg])
await self.emit_control_message(control_msg, to_peer)
async def emit_iwant(self, msg_ids: Any, to_peer: ID) -> None:
"""Emit iwant message, sent to to_peer, for msg_ids."""
2019-07-24 16:29:14 +08:00
iwant_msg: rpc_pb2.ControlIWant = rpc_pb2.ControlIWant()
iwant_msg.messageIDs.extend(msg_ids)
2019-07-24 16:29:14 +08:00
control_msg: rpc_pb2.ControlMessage = rpc_pb2.ControlMessage()
control_msg.iwant.extend([iwant_msg])
await self.emit_control_message(control_msg, to_peer)
async def emit_graft(self, topic: str, to_peer: ID) -> None:
"""Emit graft message, sent to to_peer, for topic."""
2019-07-24 16:29:14 +08:00
graft_msg: rpc_pb2.ControlGraft = rpc_pb2.ControlGraft()
graft_msg.topicID = topic
2019-07-24 16:29:14 +08:00
control_msg: rpc_pb2.ControlMessage = rpc_pb2.ControlMessage()
control_msg.graft.extend([graft_msg])
await self.emit_control_message(control_msg, to_peer)
async def emit_prune(self, topic: str, to_peer: ID) -> None:
"""Emit graft message, sent to to_peer, for topic."""
2019-07-24 16:29:14 +08:00
prune_msg: rpc_pb2.ControlPrune = rpc_pb2.ControlPrune()
prune_msg.topicID = topic
2019-07-24 16:29:14 +08:00
control_msg: rpc_pb2.ControlMessage = rpc_pb2.ControlMessage()
control_msg.prune.extend([prune_msg])
await self.emit_control_message(control_msg, to_peer)
async def emit_control_message(
self, control_msg: rpc_pb2.ControlMessage, to_peer: ID
) -> None:
# Add control message to packet
2019-07-24 16:29:14 +08:00
packet: rpc_pb2.RPC = rpc_pb2.RPC()
packet.control.CopyFrom(control_msg)
2019-07-24 16:29:14 +08:00
rpc_msg: bytes = packet.SerializeToString()
# Get stream for peer from pubsub
if to_peer not in self.pubsub.peers:
logger.debug("Fail to emit control message to %s: peer disconnected", to_peer)
peer_stream = self.pubsub.peers[to_peer]
# Write rpc to stream
try:
await peer_stream.write(encode_varint_prefixed(rpc_msg))
except StreamClosed:
logger.debug("Fail to emit control message to %s: stream closed", to_peer)
self.pubsub._handle_dead_peer(to_peer)