StarMirror/py-libp2p
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Modify files for debugging. Nothing will hit master from here

Browse Source
This commit is contained in:
Stuckinaboot 2019-04-10 17:44:00 -04:00
parent c552134c7c
commit d50027c1ca
13 changed files with 203 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
examples/sharding/receiver.py
View File

@ -34,15 +34,15 @@ class ReceiverNode():
id_opt = ID("peer-" + node_id)
libp2p_node = await new_node(id_opt=id_opt, transport_opt=[transport_opt_str])
libp2p_node = await new_node(transport_opt=[transport_opt_str])
await libp2p_node.get_network().listen(multiaddr.Multiaddr(transport_opt_str))
self.libp2p_node = libp2p_node
self.floodsub = FloodSub(SUPPORTED_PUBSUB_PROTOCOLS)
self.pubsub = Pubsub(self.libp2p_node, self.floodsub, "a")
self.floodsub = None #FloodSub(SUPPORTED_PUBSUB_PROTOCOLS)
self.pubsub = None #Pubsub(self.libp2p_node, self.floodsub, "a")
self.pubsub_messages = await self.pubsub.subscribe(topic)
self.pubsub_messages = None #await self.pubsub.subscribe(topic)
self.topic = topic
self.ack_protocol = ack_protocol
@ -63,7 +63,11 @@ class ReceiverNode():
print("Receiving started")
await self.libp2p_node.connect(sender_node_info)
print("Connection to sender confirmed")
print("Creating ack stream with ack protocol " + self.ack_protocol \
+ ", peer_id " + sender_node_info.peer_id.pretty())
ack_stream = await self.libp2p_node.new_stream(sender_node_info.peer_id, [self.ack_protocol])
print("Ack stream created")
asyncio.ensure_future(self.wait_for_end(ack_stream))

31
examples/sharding/receiver_driver.py
View File

@ -2,19 +2,22 @@ import asyncio
import json
import multiaddr
import sys
import time
from libp2p.peer.id import ID
from sender import SenderNode
from receiver import ReceiverNode
from libp2p.peer.peerinfo import info_from_p2p_addr
from tests.utils import cleanup
ACK_PROTOCOL = "/ack/1.0.0"
from Crypto.PublicKey import RSA
from libp2p.peer.id import id_from_public_key
"""
Driver is called in the following way
python receiver_driver.py topology_config.json "my_node_id"
"""
SLEEP_TIME = 5
async def connect(node1, node2_addr):
# node1 connects to node2
info = info_from_p2p_addr(node2_addr)
@ -77,33 +80,45 @@ async def main():
# Get my topic
my_topic = topology_config_dict["topic_map"][my_node_id]
ack_protocol = topology_config_dict["ACK_PROTOCOL"]
# Create Receiver Node
print("Creating receiver")
my_transport_opt_str = topology_config_dict["node_id_map"][my_node_id]
receiver_node = await ReceiverNode.create(my_node_id, my_transport_opt_str, ACK_PROTOCOL, my_topic)
receiver_node = await ReceiverNode.create(my_node_id, my_transport_opt_str, ack_protocol, my_topic)
print("Receiver created")
# TODO: sleep for like 15 seconds to let other nodes start up
# Allow for all nodes to start up
# await asyncio.sleep(SLEEP_TIME)
new_key = RSA.generate(2048, e=65537)
id_opt = id_from_public_key(new_key.publickey())
# Connect receiver node to all other relevant receiver nodes
for neighbor in topology_config_dict["topology"][my_node_id]:
neighbor_addr_str = topology_config_dict["node_id_map"][neighbor]
# Add p2p part
neighbor_addr_str += "/p2p/" + ID("peer-" + neighbor).pretty()
neighbor_addr_str += "/p2p/" + id_opt.pretty()
print(neighbor_addr_str)
# Convert neighbor_addr_str to multiaddr
neighbor_addr = multiaddr.Multiaddr(neighbor_addr_str)
await connect(receiver_node.libp2p_node, neighbor_addr)
return
# Get sender info as multiaddr
sender_addr_str = topology_config_dict["node_id_map"]["sender"]
sender_addr_str = topology_config_dict["node_id_map"]["sender"] + "/p2p/" + id_opt.pretty()
# Convert sender_info_str to multiaddr
sender_addr = multiaddr.Multiaddr(sender_addr_str)
# Convert sender_addr to sender_info
sender_info = info_from_p2p_addr(sender_addr)
# Start listening for messages from sender
print("Start receiving called")
asyncio.ensure_future(receiver.start_receiving(sender_addr))
asyncio.ensure_future(receiver_node.start_receiving(sender_info))
if __name__ == "__main__":
loop = asyncio.get_event_loop()

14
examples/sharding/sender.py
View File

@ -6,6 +6,7 @@ from timeit import default_timer as timer
from tests.utils import cleanup
from tests.pubsub.utils import generate_RPC_packet, message_id_generator
from libp2p import new_node
from libp2p.peer.id import ID
from libp2p.pubsub.pubsub import Pubsub
from libp2p.pubsub.floodsub import FloodSub
@ -22,7 +23,7 @@ class SenderNode():
self.ack_queue = asyncio.Queue()
@classmethod
async def create(cls, ack_protocol):
async def create(cls, node_id, transport_opt_str, ack_protocol):
"""
Create a new DummyAccountNode and attach a libp2p node, a floodsub, and a pubsub
instance to this new node
@ -32,8 +33,13 @@ class SenderNode():
"""
self = SenderNode()
libp2p_node = await new_node(transport_opt=["/ip4/127.0.0.1/tcp/0"])
await libp2p_node.get_network().listen(multiaddr.Multiaddr("/ip4/127.0.0.1/tcp/0"))
id_opt = ID("peer-" + node_id)
print("Sender id: " + id_opt.pretty())
print("Transport opt is " + transport_opt_str)
libp2p_node = await new_node(transport_opt=[transport_opt_str])
await libp2p_node.get_network().listen(multiaddr.Multiaddr(transport_opt_str))
self.libp2p_node = libp2p_node
@ -60,6 +66,8 @@ class SenderNode():
print("Writing end")
await stream.write("end".encode())
print("Sender ack protocol: " + ack_protocol)
# Set handler for acks
self.ack_protocol = ack_protocol
self.libp2p_node.set_stream_handler(self.ack_protocol, ack_stream_handler)

70
examples/sharding/sender_driver.py Normal file
View File

@ -0,0 +1,70 @@
import asyncio
import json
import multiaddr
import sys
import time
from libp2p.peer.id import ID
from sender import SenderNode
from receiver import ReceiverNode
from libp2p.peer.peerinfo import info_from_p2p_addr
from tests.utils import cleanup
SLEEP_TIME = 5
async def connect(node1, node2_addr):
# node1 connects to node2
info = info_from_p2p_addr(node2_addr)
await node1.connect(info)
async def main():
"""
Read in topology config file, which contains
a map of node IDs to peer IDs, an adjacency list (named topology) using node IDs,
a map of node IDs to topics, and ACK_PROTOCOL
"""
topology_config_dict = json.loads(open(sys.argv[1]).read())
my_node_id = sys.argv[2]
ack_protocol = topology_config_dict["ACK_PROTOCOL"]
# Create sender
print("Creating sender")
my_transport_opt_str = topology_config_dict["node_id_map"][my_node_id]
sender_node = await SenderNode.create(my_node_id, my_transport_opt_str, ack_protocol)
print("Sender created")
# Allow for all nodes to start up
# await asyncio.sleep(SLEEP_TIME)
return
new_key = RSA.generate(2048, e=65537)
id_opt = id_from_public_key(new_key.publickey())
# Connect sender node to all other relevant sender nodes
for neighbor in topology_config_dict["topology"][my_node_id]:
neighbor_addr_str = topology_config_dict["node_id_map"][neighbor]
# Add p2p part
neighbor_addr_str += "/p2p/" + id_opt.pretty()
# Convert neighbor_addr_str to multiaddr
neighbor_addr = multiaddr.Multiaddr(neighbor_addr_str)
await connect(sender_node.libp2p_node, neighbor_addr)
# Perform throughput test
# Start sending messages and perform throughput test
# Determine number of receivers in each topic
topic_map = topology_config_dict["topic_map"]
topics = topic_map.keys()
num_receivers_in_each_topic = {}
for topic in topic_map:
num_receivers_in_each_topic[topic] = len(topic_map[topic])
print("Performing test")
await sender_node.perform_test(num_receivers_in_each_topic, topics, 10)
if __name__ == "__main__":
loop = asyncio.get_event_loop()
loop.run_until_complete(main())
loop.close()

14
examples/sharding/topologies_isolated/simple_1_chain.json Normal file
View File

@ -0,0 +1,14 @@
{
"node_id_map": {
"sender": "/ip4/127.0.0.1/tcp/8004",
"0": "/ip4/127.0.0.1/tcp/8005"
},
"topology": {
"sender": ["0"],
"0": ["sender"]
},
"topic_map": {
"0": "topic1"
},
"ACK_PROTOCOL": "/ack/1.0.0"
}

30
libp2p/network/swarm.py
View File

@ -65,21 +65,22 @@ class Swarm(INetwork):
# set muxed connection equal to existing muxed connection
muxed_conn = self.connections[peer_id]
else:
print("Dialing " + str(peer_id))
# Dial peer (connection to peer does not yet exist)
# Transport dials peer (gets back a raw conn)
raw_conn = await self.transport.dial(multiaddr, self.self_id)
print("raw conn created")
# Use upgrader to upgrade raw conn to muxed conn
muxed_conn = self.upgrader.upgrade_connection(raw_conn, \
self.generic_protocol_handler, peer_id)
print("mux conn created")
# Store muxed connection in connections
self.connections[peer_id] = muxed_conn
# Call notifiers since event occurred
for notifee in self.notifees:
await notifee.connected(self, muxed_conn)
# for notifee in self.notifees:
# await notifee.connected(self, muxed_conn)
print("Muxed conn returned")
return muxed_conn
async def new_stream(self, peer_id, protocol_ids):
@ -88,6 +89,7 @@ class Swarm(INetwork):
:param protocol_id: protocol id
:return: net stream instance
"""
print("New stream")
# Get peer info from peer store
addrs = self.peerstore.addrs(peer_id)
@ -96,23 +98,34 @@ class Swarm(INetwork):
multiaddr = addrs[0]
print("Dialing peer")
muxed_conn = await self.dial_peer(peer_id)
print("Opening stream")
# Use muxed conn to open stream, which returns
# a muxed stream
# TODO: Remove protocol id from being passed into muxed_conn
muxed_stream = await muxed_conn.open_stream(protocol_ids[0], multiaddr)
print("Selecting protocol " + str(protocol_ids))
# Perform protocol muxing to determine protocol to use
selected_protocol = await self.multiselect_client.select_one_of(protocol_ids, muxed_stream)
print("Creating net stream")
# Create a net stream with the selected protocol
net_stream = NetStream(muxed_stream)
net_stream.set_protocol(selected_protocol)
print("Calling notifees")
# Call notifiers since event occurred
for notifee in self.notifees:
await notifee.opened_stream(self, net_stream)
print("Returning net stream")
return net_stream
@ -135,8 +148,11 @@ class Swarm(INetwork):
return True
async def conn_handler(reader, writer):
print("conn handler hit on listen")
print(multiaddr)
# Read in first message (should be peer_id of initiator) and ack
peer_id = id_b58_decode((await reader.read(1024)).decode())
print("Conn handler hit peer_id " + str(peer_id))
writer.write("received peer id".encode())
await writer.drain()
@ -152,8 +168,8 @@ class Swarm(INetwork):
self.connections[peer_id] = muxed_conn
# Call notifiers since event occurred
for notifee in self.notifees:
await notifee.connected(self, muxed_conn)
# for notifee in self.notifees:
# await notifee.connected(self, muxed_conn)
try:
# Success

3
libp2p/peer/id.py
View File

@ -24,6 +24,9 @@ class ID:
return "<peer.ID %s>" % pid
return "<peer.ID %s*%s>" % (pid[:2], pid[len(pid)-6:])
def __len__(self):
return len(self._id_str)
__repr__ = __str__
def __eq__(self, other):

17
libp2p/protocol_muxer/multiselect.py
View File

@ -25,6 +25,7 @@ class Multiselect(IMultiselectMuxer):
self.handlers[protocol] = handler
async def negotiate(self, stream):
print("negotiate")
"""
Negotiate performs protocol selection
:param stream: stream to negotiate on
@ -34,13 +35,18 @@ class Multiselect(IMultiselectMuxer):
# Create a communicator to handle all communication across the stream
communicator = MultiselectCommunicator(stream)
print("sender pre handshake")
# Perform handshake to ensure multiselect protocol IDs match
await self.handshake(communicator)
print("sender post handshake")
# Read and respond to commands until a valid protocol ID is sent
while True:
# Read message
command = await communicator.read_stream_until_eof()
print("sender command " + command)
# Command is ls or a protocol
if command == "ls":
@ -48,12 +54,16 @@ class Multiselect(IMultiselectMuxer):
pass
else:
protocol = command
print("sender checking protocol in handlers")
if protocol in self.handlers:
# Tell counterparty we have decided on a protocol
print("sender writing protocol " + protocol)
await communicator.write(protocol)
print("sender protocol written")
# Return the decided on protocol
return protocol, self.handlers[protocol]
print("sender protocol not found")
# Tell counterparty this protocol was not found
await communicator.write(PROTOCOL_NOT_FOUND_MSG)
@ -66,14 +76,21 @@ class Multiselect(IMultiselectMuxer):
# TODO: Use format used by go repo for messages
print("sender pre write " + MULTISELECT_PROTOCOL_ID)
# Send our MULTISELECT_PROTOCOL_ID to other party
await communicator.write(MULTISELECT_PROTOCOL_ID)
print("sender pre read")
# Read in the protocol ID from other party
handshake_contents = await communicator.read_stream_until_eof()
print("sender pre validate " + handshake_contents)
# Confirm that the protocols are the same
if not validate_handshake(handshake_contents):
print("sender multiselect protocol ID mismatch")
raise MultiselectError("multiselect protocol ID mismatch")
# Handshake succeeded if this point is reached

12
libp2p/protocol_muxer/multiselect_client.py
View File

@ -26,13 +26,16 @@ class MultiselectClient(IMultiselectClient):
# TODO: Use format used by go repo for messages
# Send our MULTISELECT_PROTOCOL_ID to counterparty
print("handshake entered")
await communicator.write(MULTISELECT_PROTOCOL_ID)
print("MULTISELECT_PROTOCOL_ID written " + MULTISELECT_PROTOCOL_ID)
# Read in the protocol ID from other party
handshake_contents = await communicator.read_stream_until_eof()
print("handshake read " + handshake_contents)
# Confirm that the protocols are the same
if not validate_handshake(handshake_contents):
print("multiselect protocol ID mismatch")
raise MultiselectClientError("multiselect protocol ID mismatch")
# Handshake succeeded if this point is reached
@ -67,12 +70,18 @@ class MultiselectClient(IMultiselectClient):
:return: selected protocol
"""
print("select_one_of")
# Create a communicator to handle all communication across the stream
communicator = MultiselectCommunicator(stream)
print("Pre handshake")
# Perform handshake to ensure multiselect protocol IDs match
await self.handshake(communicator)
print("Post handshake")
# For each protocol, attempt to select that protocol
# and return the first protocol selected
for protocol in protocols:
@ -80,6 +89,7 @@ class MultiselectClient(IMultiselectClient):
selected_protocol = await self.try_select(communicator, protocol)
return selected_protocol
except MultiselectClientError:
print("MultiselectClientError")
pass
# No protocols were found, so return no protocols supported error

12
libp2p/stream_muxer/mplex/mplex.py
View File

@ -58,9 +58,12 @@ class Mplex(IMuxedConn):
"""
# TODO: propagate up timeout exception and catch
# TODO: pass down timeout from user and use that
print("read buffer hit")
print(self.buffers)
if stream_id in self.buffers:
try:
data = await asyncio.wait_for(self.buffers[stream_id].get(), timeout=8)
data = await asyncio.wait_for(self.buffers[stream_id].get(), timeout=30)
print("data received")
return data
except asyncio.TimeoutError:
return None
@ -99,6 +102,7 @@ class Mplex(IMuxedConn):
:return: True if success
"""
# << by 3, then or with flag
print("sending message")
header = (stream_id << 3) | flag
header = encode_uvarint(header)
@ -128,7 +132,9 @@ class Mplex(IMuxedConn):
# TODO Deal with other types of messages using flag (currently _)
while True:
print("Message read waiting")
stream_id, flag, message = await self.read_message()
print("Message read occ")
if stream_id is not None and flag is not None and message is not None:
if stream_id not in self.buffers:
@ -153,9 +159,11 @@ class Mplex(IMuxedConn):
# Timeout is set to a relatively small value to alleviate wait time to exit
# loop in handle_incoming
timeout = 0.1
timeout = 10
try:
print("Getting header")
header = await decode_uvarint_from_stream(self.raw_conn.reader, timeout)
print("Got header")
length = await decode_uvarint_from_stream(self.raw_conn.reader, timeout)
message = await asyncio.wait_for(self.raw_conn.reader.read(length), timeout=timeout)
except asyncio.TimeoutError:

3
libp2p/stream_muxer/mplex/mplex_stream.py
View File

@ -38,6 +38,9 @@ class MplexStream(IMuxedStream):
write to stream
:return: number of bytes written
"""
print("Writing message")
print(self.stream_id)
print(self.mplex_conn.peer_id)
return await self.mplex_conn.send_message(
get_flag(self.initiator, "MESSAGE"), data, self.stream_id)

9
libp2p/stream_muxer/mplex/utils.py
View File

@ -34,7 +34,14 @@ async def decode_uvarint_from_stream(reader, timeout):
shift = 0
result = 0
while True:
byte = await asyncio.wait_for(reader.read(1), timeout=timeout)
print("Decoding byte")
print("Trying byte")
print(reader)
byte = await asyncio.wait_for(reader.read(1), timeout=2000)
print("HIT me")
print("Byte is ")
print(type(byte))
print(len(byte))
i = struct.unpack('>H', b'\x00' + byte)[0]
result |= (i & 0x7f) << shift
shift += 7

2
libp2p/transport/tcp/tcp.py
View File

@ -79,10 +79,12 @@ class TCP(ITransport):
# First: send our peer ID so receiver knows it
writer.write(id_b58_encode(self_id).encode())
print("Length of written id is " + str(len(id_b58_encode(self_id).encode())))
await writer.drain()
# Await ack for peer id
ack = (await reader.read(1024)).decode()
print(ack)
if ack != "received peer id":
raise Exception("Receiver did not receive peer id")